rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-08-26 07:58:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into ticket/ent-1961/strip-request-creation-keys

Browse Source
This commit is contained in:
Xuanwo
2026-08-25 15:29:45 +08:00
committed by GitHub
parent b9ad17a54d 2fea7cd48d
commit 505bd2ef99
103 changed files with 13492 additions and 1001 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bumpversion.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.38.0-beta.3"
current_version = "0.38.0-beta.8"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
Cargo.lock
Generated
+49 -45
View File
@@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arc-swap",
"arrow",
@@ -4888,8 +4888,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4911,7 +4911,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4925,7 +4925,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -4934,8 +4934,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrayref",
"crunchy",
@@ -4945,8 +4945,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4983,8 +4983,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5013,8 +5013,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5031,8 +5031,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"proc-macro2",
"quote",
@@ -5041,8 +5041,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5075,8 +5075,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5107,8 +5107,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arc-swap",
"arrow",
@@ -5172,8 +5172,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5195,8 +5195,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5222,7 +5222,11 @@ dependencies = [
"pin-project",
"prost",
"rand 0.9.5",
"reqsign-core",
"reqsign-file-read-tokio",
"reqsign-google",
"serde",
"serde_json",
"tempfile",
"tokio",
"tracing",
@@ -5232,8 +5236,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5247,8 +5251,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"async-trait",
@@ -5260,8 +5264,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5314,8 +5318,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5329,8 +5333,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5370,8 +5374,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5384,8 +5388,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"frostem",
"icu_segmenter",
@@ -5398,7 +5402,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.38.0-beta.3"
version = "0.38.0-beta.8"
dependencies = [
"ahash",
"anyhow",
@@ -5486,7 +5490,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.38.0-beta.3"
version = "0.38.0-beta.8"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5511,7 +5515,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.38.0-beta.3"
version = "0.38.0-beta.8"
dependencies = [
"arrow",
"async-trait",
Cargo.toml
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
docs/src/java/java.md
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.38.0-beta.3</version>
<version>0.38.0-beta.8</version>
</dependency>
```
docs/src/js/classes/Branches.md
+25 -25
View File
@@ -37,6 +37,31 @@ latest and stays writable.
***
### cherryPick()
```ts
cherryPick(fromBranch, dryRun): Promise<CherryPickResult>
```
Cherry-pick a branch onto main.
Set `dryRun` to `true` to preview. A failed cherry-pick resolves
with `status: "failed"` instead of throwing.
#### Parameters
* **fromBranch**: `string`
Branch to cherry-pick from.
* **dryRun**: `boolean` = `false`
When true, only preview. Defaults to false.
#### Returns
`Promise`&lt;[`CherryPickResult`](../interfaces/CherryPickResult.md)&gt;
***
### create()
```ts
@@ -112,28 +137,3 @@ List all branches, mapping name to branch metadata.
#### Returns
`Promise`&lt;`Record`&lt;`string`, [`BranchContents`](BranchContents.md)&gt;&gt;
***
### merge()
```ts
merge(fromBranch, dryRun): Promise<MergeBranchResult>
```
Merge a branch into main.
Set `dryRun` to `true` to preview the merge. A rejected merge resolves
with `status: "rejected"` instead of throwing.
#### Parameters
* **fromBranch**: `string`
Branch to merge from.
* **dryRun**: `boolean` = `false`
When true, only preview the merge. Defaults to false.
#### Returns
`Promise`&lt;[`MergeBranchResult`](../interfaces/MergeBranchResult.md)&gt;
docs/src/js/classes/Connection.md
+148 -6
View File
@@ -169,6 +169,45 @@ Creates a new empty Table
***
### createMaterializedView()
```ts
abstract createMaterializedView(
name,
source,
options?): Promise<MaterializedView>
```
Define a materialized view named `name` over the table `source`.
The view is created empty, with the query recorded in its schema
metadata; `view.refresh()` computes the rows. The view is a normal
table: it can be queried, indexed and searched, and it appears in
`tableNames`. The source table must have stable row ids (create it with
the `newTableEnableStableRowIds` storage option); they keep the view's
provenance valid across source compactions and cannot be enabled after
a table exists. Local databases only.
#### Parameters
* **name**: `string`
* **source**: `string`
* **options?**
* **options.limit?**: `number`
* **options.select?**: [`MaterializedViewSelect`](../type-aliases/MaterializedViewSelect.md)
* **options.where?**: `string`
#### Returns
`Promise`&lt;[`MaterializedView`](MaterializedView.md)&gt;
***
### createNamespace()
```ts
@@ -499,6 +538,22 @@ List server-side jobs across the database's tables.
***
### listMaterializedViews()
```ts
abstract listMaterializedViews(): Promise<string[]>
```
The names of the materialized views in this database.
Found by reading every table's schema, so this costs an open per table.
#### Returns
`Promise`&lt;`string`[]&gt;
***
### listNamespaces()
```ts
@@ -529,6 +584,90 @@ Child namespace names and
***
### listTables()
#### listTables(options)
```ts
abstract listTables(options?): Promise<ListTablesResponse>
```
List a page of the tables in this database.
To retrieve the tables after the page, pass the `pageToken` the response
carries back in. A page can be shorter than `limit` without being the last
one, so walk until a response carries no page token:
```ts
const names = [];
let pageToken = undefined;
do {
const page = await conn.listTables({ pageToken, limit: 100 });
names.push(...page.tables);
pageToken = page.pageToken;
} while (pageToken);
```
##### Parameters
* **options?**: `Partial`&lt;[`ListTablesOptions`](../interfaces/ListTablesOptions.md)&gt;
Pagination options
(`pageToken`, `limit`).
##### Returns
`Promise`&lt;[`ListTablesResponse`](../interfaces/ListTablesResponse.md)&gt;
A page of table names and an
optional token for the tables after it.
#### listTables(namespacePath, options)
```ts
abstract listTables(namespacePath?, options?): Promise<ListTablesResponse>
```
List a page of the tables in this database.
##### Parameters
* **namespacePath?**: `string`[]
The namespace path to list tables from
(defaults to root namespace)
* **options?**: `Partial`&lt;[`ListTablesOptions`](../interfaces/ListTablesOptions.md)&gt;
Pagination options
(`pageToken`, `limit`).
##### Returns
`Promise`&lt;[`ListTablesResponse`](../interfaces/ListTablesResponse.md)&gt;
A page of table names and an
optional token for the tables after it.
***
### openMaterializedView()
```ts
abstract openMaterializedView(name): Promise<MaterializedView>
```
Open the materialized view named `name`.
Rejects a table that exists but is not a materialized view.
#### Parameters
* **name**: `string`
#### Returns
`Promise`&lt;[`MaterializedView`](MaterializedView.md)&gt;
***
### openTable()
```ts
@@ -538,18 +677,13 @@ abstract openTable(
options?): Promise<Table>
```
Open a table in the database.
#### Parameters
* **name**: `string`
The name of the table
* **namespacePath?**: `string`[]
The namespace path of the table (defaults to root namespace)
* **options?**: `Partial`&lt;[`OpenTableOptions`](../interfaces/OpenTableOptions.md)&gt;
Additional options
#### Returns
@@ -590,7 +724,7 @@ a "not supported" error.
***
### tableNames()
### ~~tableNames()~~
#### tableNames(options)
@@ -612,6 +746,10 @@ Tables will be returned in lexicographical order.
`Promise`&lt;`string`[]&gt;
##### Deprecated
Use [Connection.listTables](Connection.md#listtables) instead.
#### tableNames(namespacePath, options)
```ts
@@ -634,3 +772,7 @@ Tables will be returned in lexicographical order.
##### Returns
`Promise`&lt;`string`[]&gt;
##### Deprecated
Use [Connection.listTables](Connection.md#listtables) instead.
docs/src/js/classes/MaterializedView.md
+101
View File
@@ -0,0 +1,101 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / MaterializedView
# Class: MaterializedView
A handle on a materialized view: its table plus its definition.
Obtained from [Connection#createMaterializedView](Connection.md#creatematerializedview) or
[Connection#openMaterializedView](Connection.md#openmaterializedview). The view is a normal table --
queries, indexes and search all apply through [MaterializedView#table](MaterializedView.md#table)
-- whose contents are maintained by [MaterializedView#refresh](MaterializedView.md#refresh).
## Constructors
### new MaterializedView()
```ts
new MaterializedView(table): MaterializedView
```
#### Parameters
* **table**: [`Table`](Table.md)
#### Returns
[`MaterializedView`](MaterializedView.md)
## Accessors
### name
```ts
get name(): string
```
#### Returns
`string`
## Methods
### definition()
```ts
definition(): Promise<MaterializedViewDefinition>
```
The query that defines the view, read from its stored schema.
#### Returns
`Promise`&lt;[`MaterializedViewDefinition`](../interfaces/MaterializedViewDefinition.md)&gt;
***
### refresh()
```ts
refresh(options?): Promise<RefreshMaterializedViewResult>
```
Recompute the view from its source.
The refresh is incremental when the source's changes can be reconciled
into the view -- rows added, changed or removed since the last one --
and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
refreshes to that source version instead of the latest.
Concurrent refreshes of one view do not duplicate its rows. Two that
plan the same source rows conflict on commit, and the loser throws
rather than writing them a second time.
#### Parameters
* **options?**
* **options.full?**: `boolean`
* **options.sourceVersion?**: `number`
#### Returns
`Promise`&lt;[`RefreshMaterializedViewResult`](../interfaces/RefreshMaterializedViewResult.md)&gt;
***
### table()
```ts
table(): Table
```
The view, as the table it is.
#### Returns
[`Table`](Table.md)
docs/src/js/globals.md
+9 -3
View File
@@ -28,6 +28,7 @@
- [Job](classes/Job.md)
- [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
- [MatchQuery](classes/MatchQuery.md)
- [MaterializedView](classes/MaterializedView.md)
- [MergeInsertBuilder](classes/MergeInsertBuilder.md)
- [MultiMatchQuery](classes/MultiMatchQuery.md)
- [NativeJsHeaderProvider](classes/NativeJsHeaderProvider.md)
@@ -59,6 +60,9 @@
- [BranchIndexSummary](interfaces/BranchIndexSummary.md)
- [BranchRowCountSummary](interfaces/BranchRowCountSummary.md)
- [BucketStats](interfaces/BucketStats.md)
- [CherryPickError](interfaces/CherryPickError.md)
- [CherryPickPreview](interfaces/CherryPickPreview.md)
- [CherryPickResult](interfaces/CherryPickResult.md)
- [ClientConfig](interfaces/ClientConfig.md)
- [ColumnAlteration](interfaces/ColumnAlteration.md)
- [ColumnOrdering](interfaces/ColumnOrdering.md)
@@ -96,12 +100,12 @@
- [JobInfo](interfaces/JobInfo.md)
- [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
- [ListTablesOptions](interfaces/ListTablesOptions.md)
- [ListTablesResponse](interfaces/ListTablesResponse.md)
- [LsmStats](interfaces/LsmStats.md)
- [LsmWriteSpec](interfaces/LsmWriteSpec.md)
- [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md)
- [MemtableStats](interfaces/MemtableStats.md)
- [MergeBlocker](interfaces/MergeBlocker.md)
- [MergeBranchResult](interfaces/MergeBranchResult.md)
- [MergePreview](interfaces/MergePreview.md)
- [MergeResult](interfaces/MergeResult.md)
- [NativeOAuthConfig](interfaces/NativeOAuthConfig.md)
- [OAuthConfig](interfaces/OAuthConfig.md)
@@ -110,6 +114,7 @@
- [OptimizeStats](interfaces/OptimizeStats.md)
- [QueryExecutionOptions](interfaces/QueryExecutionOptions.md)
- [RefreshColumnResult](interfaces/RefreshColumnResult.md)
- [RefreshMaterializedViewResult](interfaces/RefreshMaterializedViewResult.md)
- [RemovalStats](interfaces/RemovalStats.md)
- [RenameTableOptions](interfaces/RenameTableOptions.md)
- [RestNamespaceConfig](interfaces/RestNamespaceConfig.md)
@@ -142,6 +147,7 @@
- [FieldLike](type-aliases/FieldLike.md)
- [IntoSql](type-aliases/IntoSql.md)
- [IntoVector](type-aliases/IntoVector.md)
- [MaterializedViewSelect](type-aliases/MaterializedViewSelect.md)
- [MultiVector](type-aliases/MultiVector.md)
- [RecordBatchLike](type-aliases/RecordBatchLike.md)
- [SchemaLike](type-aliases/SchemaLike.md)
docs/src/js/interfaces/BranchDiff.md
+8 -16
View File
@@ -50,6 +50,14 @@ changedColumns: BranchColumnChange[];
***
### errors
```ts
errors: CherryPickError[];
```
***
### fromBranch
```ts
@@ -66,22 +74,6 @@ mainVersion: number;
***
### mergeBlockers
```ts
mergeBlockers: MergeBlocker[];
```
***
### mergeable
```ts
mergeable: boolean;
```
***
### parentVersion
```ts
docs/src/js/interfaces/MergeBlocker.md → docs/src/js/interfaces/CherryPickError.md
+3 -3
View File
@@ -2,11 +2,11 @@
***
[@lancedb/lancedb](../globals.md) / MergeBlocker
[@lancedb/lancedb](../globals.md) / CherryPickError
# Interface: MergeBlocker
# Interface: CherryPickError
A reason why a branch cannot currently be merged.
A reason why a cherry-pick cannot currently land.
## Properties
docs/src/js/interfaces/CherryPickPreview.md
+17
View File
@@ -0,0 +1,17 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / CherryPickPreview
# Interface: CherryPickPreview
Changes that would be, or were, promoted by a cherry-pick.
## Properties
### promotedColumns
```ts
promotedColumns: string[];
```
docs/src/js/interfaces/MergeBranchResult.md → docs/src/js/interfaces/CherryPickResult.md
+6 -6
View File
@@ -2,11 +2,11 @@
***
[@lancedb/lancedb](../globals.md) / MergeBranchResult
[@lancedb/lancedb](../globals.md) / CherryPickResult
# Interface: MergeBranchResult
# Interface: CherryPickResult
Result of previewing or attempting a branch merge.
Result of previewing or attempting a cherry-pick.
## Properties
@@ -29,7 +29,7 @@ optional mainVersionAfter: number;
### preview
```ts
preview: MergePreview;
preview: CherryPickPreview;
```
***
@@ -38,9 +38,9 @@ preview: MergePreview;
```ts
status:
| "failed"
| "unknown"
| "rejected"
| "ready"
| "notImplemented"
| "merged";
| "cherryPicked";
```
docs/src/js/interfaces/ListTablesOptions.md
+34
View File
@@ -0,0 +1,34 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ListTablesOptions
# Interface: ListTablesOptions
## Properties
### limit?
```ts
optional limit: number;
```
An upper bound on how many tables to return.
A page may hold fewer than this and still not be the last one, so keep
going while the response carries a page token rather than while pages are
full.
***
### pageToken?
```ts
optional pageToken: string;
```
Token from a previous response, to resume listing where it left off.
The token is opaque: it carries whatever the database needs to resume, and
callers should not construct or interpret one.
docs/src/js/interfaces/ListTablesResponse.md
+23
View File
@@ -0,0 +1,23 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ListTablesResponse
# Interface: ListTablesResponse
## Properties
### pageToken?
```ts
optional pageToken: string;
```
***
### tables
```ts
tables: string[];
```
docs/src/js/interfaces/MaterializedViewDefinition.md
+59
View File
@@ -0,0 +1,59 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / MaterializedViewDefinition
# Interface: MaterializedViewDefinition
The query that defines a materialized view.
## Properties
### filter?
```ts
optional filter: string;
```
SQL predicate selecting the source rows the view holds.
***
### inputs
```ts
inputs: string[];
```
Source columns the projections and filter read.
***
### limit?
```ts
optional limit: number;
```
Cap on the number of rows the view holds.
***
### projections
```ts
projections: [string, string][];
```
`[output column, SQL expression]` pairs, in view schema order.
***
### sourceTable
```ts
sourceTable: string;
```
Name of the source table, in the same database as the view.
docs/src/js/interfaces/MergePreview.md
-17
View File
@@ -1,17 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / MergePreview
# Interface: MergePreview
Changes that would be, or were, promoted by a branch merge.
## Properties
### promotedColumns
```ts
promotedColumns: string[];
```
docs/src/js/interfaces/RefreshMaterializedViewResult.md
+41
View File
@@ -0,0 +1,41 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / RefreshMaterializedViewResult
# Interface: RefreshMaterializedViewResult
## Properties
### mode
```ts
mode: string;
```
How the view was brought up to date: "rebuild", "incremental" or "no_op".
***
### rowsWritten
```ts
rowsWritten: number;
```
***
### sourceVersion
```ts
sourceVersion: number;
```
***
### version
```ts
version: number;
```
docs/src/js/interfaces/TableNamesOptions.md
+8 -3
View File
@@ -4,11 +4,16 @@
[@lancedb/lancedb](../globals.md) / TableNamesOptions
# Interface: TableNamesOptions
# Interface: ~~TableNamesOptions~~
## Deprecated
Use [ListTablesOptions](ListTablesOptions.md) with [Connection.listTables](../classes/Connection.md#listtables)
instead.
## Properties
### limit?
### ~~limit?~~
```ts
optional limit: number;
@@ -18,7 +23,7 @@ An optional limit to the number of results to return.
***
### startAfter?
### ~~startAfter?~~
```ts
optional startAfter: string;
docs/src/js/type-aliases/MaterializedViewSelect.md
+14
View File
@@ -0,0 +1,14 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / MaterializedViewSelect
# Type Alias: MaterializedViewSelect
```ts
type MaterializedViewSelect: (string | [string, string])[] | Record<string, string>;
```
The view's columns: column names, `[alias, SQL expression]` pairs, or a
record of the same. A bare name projects itself.
docs/src/python/python.md
+12
View File
@@ -102,6 +102,12 @@ listing a storage directory.
::: lancedb.job.AsyncJob
## Materialized Views (Synchronous)
::: lancedb.materialized_view.MaterializedView
::: lancedb.materialized_view.MaterializedViewDefinition
## Expressions
Type-safe expression builder for filters and projections. Use these instead
@@ -255,6 +261,8 @@ instead of being materialized with the rest of the row.
::: lancedb.streaming.StreamingDataset
::: lancedb.streaming.StreamingDataLoader
::: lancedb.permutation.permutation_builder
::: lancedb.permutation.PermutationBuilder
@@ -295,6 +303,10 @@ Table hold your actual data as a collection of records / rows.
::: lancedb.table.AsyncBranches
## Materialized Views (Asynchronous)
::: lancedb.materialized_view.AsyncMaterializedView
## Indices (Asynchronous)
Indices can be created on a table to speed up queries. This section
java/lancedb-core/pom.xml
+1 -1
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.38.0-beta.3</version>
<version>0.38.0-beta.8</version>
<relativePath>../pom.xml</relativePath>
</parent>
java/pom.xml
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.38.0-beta.3</version>
<version>0.38.0-beta.8</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>11.0.0-beta.18</lance-core.version>
<lance-core.version>11.0.0-beta.22</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
nodejs/Cargo.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.38.0-beta.3"
version = "0.38.0-beta.8"
publish = false
license.workspace = true
description.workspace = true
nodejs/__test__/arrow.test.ts
+131
View File
@@ -515,6 +515,137 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
);
});
it("will allow matching inferred types across records", function () {
expect(() =>
makeArrowTable([{ value: 1 }, { value: 2 }]),
).not.toThrow();
});
it("will reject mismatched inferred types across records", function () {
expect(() => makeArrowTable([{ value: 1 }, { value: "two" }])).toThrow(
"Failed to infer schema for data. Previously inferred type Float64 but found Utf8 for field value at row 1. Consider providing an explicit schema.",
);
});
it("will ignore generated dictionary IDs when comparing inferred types", function () {
const table = makeArrowTable([{ str: "a" }, { str: "b" }], {
dictionaryEncodeStrings: true,
});
expect(table.getChild("str")?.toJSON()).toEqual(["a", "b"]);
});
it("will preserve null values without treating them as type mismatches", function () {
for (const records of [
[{ vector: [1, 2, 3] }, { vector: null }],
[{ vector: null }, { vector: [1, 2, 3] }],
]) {
const table = makeArrowTable(records);
expect(table.numRows).toBe(2);
expect(table.getChild("vector")?.nullCount).toBe(1);
}
});
it("will preserve empty variable-size lists", function () {
for (const records of [
[{ items: [1] }, { items: [] }],
[{ items: [] }, { items: [1] }],
]) {
const table = makeArrowTable(records);
expect(
table
.getChild("items")
?.toJSON()
.map((value) => value.toJSON()),
).toEqual(records.map((record) => record.items));
}
});
it("will propagate deferred evidence through nested lists", function () {
for (const records of [
[{ items: [1] }, { items: [null] }],
[{ items: [null] }, { items: [1] }],
[{ items: [null, 1] }, { items: [2, null] }],
]) {
const table = makeArrowTable(records);
expect(
table
.getChild("items")
?.toJSON()
.map((value) => value.toJSON()),
).toEqual(records.map((record) => record.items));
}
const nestedRecords = [{ items: [[1]] }, { items: [[null]] }];
const nestedTable = makeArrowTable(nestedRecords);
expect(
nestedTable
.getChild("items")
?.toJSON()
.map((value) =>
value
.toJSON()
.map((nestedValue: { toJSON: () => unknown[] }) =>
nestedValue.toJSON(),
),
),
).toEqual(nestedRecords.map((record) => record.items));
});
it("will reject incompatible deferred evidence within a list", function () {
for (const items of [
[[], 1],
[1, []],
[[null], 1],
[1, [null]],
]) {
expect(() => makeArrowTable([{ items }])).toThrow(
"Failed to infer data type for field items at row 0.",
);
}
});
it("will reject empty fixed-size lists", function () {
expect(() =>
makeArrowTable([{ vector: [1, 2, 3] }, { vector: [] }]),
).toThrow(
"Failed to infer schema for data. Previously inferred type FixedSizeList[3]<Float32> but found List[0] for field vector at row 1.",
);
});
it("will reject inferred leaf and branch shape changes", function () {
expect(() =>
makeArrowTable([{ value: 1 }, { value: { nested: 2 } }]),
).toThrow(
"Failed to infer schema for data. Previously inferred type Float64 but found Struct for field value at row 1.",
);
expect(() =>
makeArrowTable([{ value: { nested: 1 } }, { value: 2 }]),
).toThrow(
"Failed to infer schema for data. Previously inferred type Struct but found Float64 for field value at row 1.",
);
});
it("will allow null values around inferred struct values", function () {
for (const { records, nullIndex } of [
{
records: [{ value: null }, { value: { nested: 2 } }],
nullIndex: 0,
},
{
records: [{ value: { nested: 1 } }, { value: null }],
nullIndex: 1,
},
]) {
const table = makeArrowTable(records);
const values = table.getChild("value");
expect(values?.nullCount).toBe(1);
expect(values?.get(nullIndex)).toBeNull();
}
});
it("will allow a schema to be provided", async function () {
await checkTableCreation(
async (records, _, schema) =>
nodejs/__test__/connection.test.ts
+68 -1
View File
@@ -4,7 +4,13 @@
import { readdirSync } from "fs";
import { Field, Float64, Schema } from "apache-arrow";
import * as tmp from "tmp";
import { Connection, Table, connect, connectNamespace } from "../lancedb";
import {
Connection,
ListTablesResponse,
Table,
connect,
connectNamespace,
} from "../lancedb";
import { LocalTable } from "../lancedb/table";
describe("when connecting", () => {
@@ -47,6 +53,7 @@ describe("given a connection", () => {
await db.close();
expect(db.isOpen()).toBe(false);
await expect(db.tableNames()).rejects.toThrow("Connection is closed");
await expect(db.listTables()).rejects.toThrow("Connection is closed");
await expect(db.renameTable("a", "b")).rejects.toThrow(
"Connection is closed",
);
@@ -129,6 +136,66 @@ describe("given a connection", () => {
expect(tables).toEqual(["b", "c"]);
});
it("should respect limit and page token when listing tables", async () => {
const db = await connect(tmpDir.name);
await db.createTable("b", [{ id: 1 }]);
await db.createTable("a", [{ id: 1 }]);
await db.createTable("c", [{ id: 1 }]);
const all = await db.listTables();
expect(all.tables).toEqual(["a", "b", "c"]);
expect(all.pageToken).toBeUndefined();
const first = await db.listTables({ limit: 1 });
expect(first.tables).toEqual(["a"]);
expect(first.pageToken).toBeDefined();
const second = await db.listTables({
limit: 1,
pageToken: first.pageToken,
});
expect(second.tables).toEqual(["b"]);
});
it("should visit every table exactly once when walking pages", async () => {
const db = await connect(tmpDir.name);
const created = ["a", "b", "c", "d", "e"];
for (const name of created) {
await db.createTable(name, [{ id: 1 }]);
}
const seen: string[] = [];
let pageToken: string | undefined = undefined;
do {
const page: ListTablesResponse = await db.listTables({
limit: 2,
pageToken,
});
seen.push(...page.tables);
pageToken = page.pageToken;
} while (pageToken);
expect(seen).toEqual(created);
});
it("should list tables in a namespace", async () => {
const db = await connect(tmpDir.name, {
// biome-ignore lint/style/useNamingConvention: opaque backend property key, must match Rust
namespaceClientProperties: { manifest_enabled: "true" },
});
await db.createNamespace(["child"]);
await db.createTable("nested", [{ id: 1 }], ["child"]);
await expect(db.listTables(["child"])).resolves.toEqual(
expect.objectContaining({ tables: ["nested"] }),
);
await expect(db.listTables()).resolves.toEqual(
expect.objectContaining({ tables: [] }),
);
});
it("should create tables in v2 mode", async () => {
const db = await connect(tmpDir.name);
const data = [...Array(10000).keys()].map((i) => ({ id: i }));
nodejs/__test__/embedding.test.ts
+48
View File
@@ -487,4 +487,52 @@ describe("embedding functions", () => {
expect(stringSchema3).toEqual(stringExpectedSchema);
},
);
test("parses one function writing several vector columns", async () => {
class MockEmbeddingFunction extends EmbeddingFunction<string> {
ndims() {
return 3;
}
embeddingDataType(): Float {
return new Float32();
}
async computeQueryEmbeddings(_data: string) {
return [1, 2, 3];
}
async computeSourceEmbeddings(data: string[]) {
return Array.from({ length: data.length }).fill([
1, 2, 3,
]) as number[][];
}
}
const registry = getRegistry();
registry.register("multi_output_mock")(MockEmbeddingFunction);
// A materialized view can project one source vector column under two
// names, so a table's configuration names the same function twice.
const parsed = await registry.parseFunctions(
new Map([
[
"embedding_functions",
JSON.stringify([
{
name: "multi_output_mock",
sourceColumn: "text",
vectorColumn: "vector_a",
model: {},
},
{
name: "multi_output_mock",
sourceColumn: "text",
vectorColumn: "vector_b",
model: {},
},
]),
],
]),
);
expect(
[...parsed.values()].map(({ vectorColumn }) => vectorColumn).sort(),
).toEqual(["vector_a", "vector_b"]);
});
});
nodejs/__test__/materialized_view.test.ts
+147
View File
@@ -0,0 +1,147 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as tmp from "tmp";
import { Connection, connect } from "../lancedb";
import {
DEFINITION_META_KEY,
definitionFromMetadata,
} from "../lancedb/materialized_view";
describe("materialized views", () => {
let tmpDir: tmp.DirResult;
let db: Connection;
beforeEach(async () => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
db = await connect(tmpDir.name);
await db.createTable(
"people",
[
{ name: "ada", age: 36 },
{ name: "kid", age: 7 },
{ name: "grace", age: 85 },
],
{ storageOptions: { newTableEnableStableRowIds: "true" } },
);
});
afterEach(() => tmpDir.removeCallback());
it("rejects a stored limit a number cannot carry", () => {
const big = new Map([
[
DEFINITION_META_KEY,
'{"kind":"select","source_table":"people","limit":9007199254740993}',
],
]);
expect(() => definitionFromMetadata(big, "v")).toThrow(
/too large to represent exactly/,
);
const safe = new Map([
[
DEFINITION_META_KEY,
'{"kind":"select","source_table":"people","limit":42}',
],
]);
expect(definitionFromMetadata(safe, "v").limit).toBe(42);
});
it("creates, refreshes and queries a view", async () => {
const view = await db.createMaterializedView("adults", "people", {
select: ["name", ["shout", "upper(name)"]],
where: "age >= 18",
});
expect(view.name).toBe("adults");
expect(await view.table().countRows()).toBe(0);
const result = await view.refresh();
expect(result.mode).toBe("rebuild");
expect(Number(result.rowsWritten)).toBe(2);
const rows = await view.table().query().toArray();
expect(rows.map((r) => r.shout).sort()).toEqual(["ADA", "GRACE"]);
});
it("round-trips the definition", async () => {
await db.createMaterializedView("adults", "people", {
where: "age >= 18",
});
const view = await db.openMaterializedView("adults");
const definition = await view.definition();
expect(definition.sourceTable).toBe("people");
expect(definition.filter).toBe("age >= 18");
expect(definition.projections).toEqual([
["name", "`name`"],
["age", "`age`"],
]);
expect(definition.inputs).toEqual(["age", "name"]);
});
it("refreshes incrementally after an append", async () => {
const view = await db.createMaterializedView("copy", "people");
await view.refresh();
const people = await db.openTable("people");
await people.add([{ name: "alan", age: 41 }]);
const result = await view.refresh();
expect(result.mode).toBe("incremental");
expect(Number(result.rowsWritten)).toBe(1);
expect(await view.table().countRows()).toBe(4);
expect((await view.refresh()).mode).toBe("no_op");
});
it("lists views and rejects non-views", async () => {
await db.createMaterializedView("adults", "people", {
where: "age >= 18",
});
expect(await db.listMaterializedViews()).toEqual(["adults"]);
await expect(db.openMaterializedView("people")).rejects.toThrow(
"not a materialized view",
);
});
it("rejects an invalid expression at create time", async () => {
await expect(
db.createMaterializedView("bad", "people", {
select: [["x", "missing + 1"]],
}),
).rejects.toThrow("missing");
});
it("rejects invalid numeric options before creating anything", async () => {
for (const limit of [-5, 1.5, Infinity, NaN]) {
await expect(
db.createMaterializedView("bad", "people", { limit }),
).rejects.toThrow("non-negative integer");
}
expect(await db.listMaterializedViews()).toEqual([]);
const view = await db.createMaterializedView("copy", "people");
for (const sourceVersion of [-1, 1.5, Infinity, NaN]) {
await expect(view.refresh({ sourceVersion })).rejects.toThrow(
"non-negative integer",
);
}
});
it("quotes bare select names", async () => {
await db.createTable("odd_names", [{ "order item": "widget" }], {
storageOptions: { newTableEnableStableRowIds: "true" },
});
const view = await db.createMaterializedView("quoted", "odd_names", {
select: ["order item"],
});
const result = await view.refresh();
expect(Number(result.rowsWritten)).toBe(1);
});
it("requires stable row ids on the source", async () => {
await db.createTable("plain", [{ x: 1 }]);
await expect(db.createMaterializedView("v", "plain")).rejects.toThrow(
"stable row ids",
);
});
});
nodejs/__test__/remote.test.ts
+31 -14
View File
@@ -75,6 +75,25 @@ async function withMockDatabase(
}
describe("remote connection", () => {
it("refuses materialized views before issuing any request", async () => {
const paths: string[] = [];
await withMockDatabase(
(req, res) => {
paths.push(req.url ?? "");
res.writeHead(404).end();
},
async (db) => {
await expect(db.openMaterializedView("secret_table")).rejects.toThrow(
/only on local databases/,
);
await expect(db.listMaterializedViews()).rejects.toThrow(
/only on local databases/,
);
expect(paths).toEqual([]);
},
);
});
it("should accept partial connection options", async () => {
await connect("db://test", {
apiKey: "fake",
@@ -311,7 +330,7 @@ describe("remote connection", () => {
expect(createIndexBody?.["custom_stop_words"]).toEqual(["the"]);
});
it("diffs and merges remote branches", async () => {
it("diffs and cherry-picks remote branches", async () => {
const sampleDiff = {
fromBranch: "exp",
parentVersion: 1,
@@ -333,10 +352,9 @@ describe("remote connection", () => {
changedColumns: [],
addedIndexes: [],
removedIndexes: [],
mergeable: true,
mergeBlockers: [],
errors: [],
};
const mergeBodies: Record<string, unknown>[] = [];
const cherryPickBodies: Record<string, unknown>[] = [];
await withMockDatabase(
(req, res) => {
@@ -366,17 +384,16 @@ describe("remote connection", () => {
.end(JSON.stringify(sampleDiff));
return;
}
if (path.endsWith("/branches/merge/")) {
mergeBodies.push(body);
if (path.endsWith("/branches/cherry_pick/")) {
cherryPickBodies.push(body);
const dryRun = body["dry_run"] === true;
const response = {
status: dryRun ? "ready" : "rejected",
status: dryRun ? "ready" : "failed",
diff: dryRun
? sampleDiff
: {
...sampleDiff,
mergeable: false,
mergeBlockers: [
errors: [
{ code: "baseMoved", message: "main has advanced" },
],
},
@@ -398,19 +415,19 @@ describe("remote connection", () => {
await expect(branches.diff("exp")).resolves.toEqual(sampleDiff);
const rejected = await branches.merge("exp");
expect(rejected.status).toBe("rejected");
expect(rejected.diff.mergeBlockers).toEqual([
const failed = await branches.cherryPick("exp");
expect(failed.status).toBe("failed");
expect(failed.diff.errors).toEqual([
{ code: "baseMoved", message: "main has advanced" },
]);
const preview = await branches.merge("exp", true);
const preview = await branches.cherryPick("exp", true);
expect(preview.status).toBe("ready");
expect(preview.preview.promotedColumns).toEqual(["tag"]);
},
);
expect(mergeBodies).toEqual([
expect(cherryPickBodies).toEqual([
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
{ from_branch: "exp", dry_run: false },
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
nodejs/__test__/table.test.ts
+1 -1
View File
@@ -2953,7 +2953,7 @@ describe("column name options", () => {
.limit(10)
.toArray();
expect(results2.length).toBe(10);
});
}, 30_000);
});
describe("when creating an empty table", () => {
nodejs/lancedb/arrow.ts
+33 -307
View File
@@ -5,7 +5,6 @@ import {
Data as ArrowData,
Table as ArrowTable,
Binary,
Bool,
BufferType,
DataType,
DateUnit,
@@ -18,12 +17,7 @@ import {
FixedSizeList,
Float,
Float32,
Float64,
Int,
Int8,
Int16,
Int32,
Int64,
LargeBinary,
List,
Null,
@@ -36,17 +30,16 @@ import {
Struct,
Timestamp,
Type,
Uint8,
Uint16,
Uint32,
Utf8,
Vector,
makeVector as arrowMakeVector,
util as arrowUtil,
vectorFromArray as badVectorFromArray,
makeBuilder,
makeData,
} from "apache-arrow";
import { Buffers } from "apache-arrow/data";
import { typedArrayToArrowType } from "./arrow_type";
import { type EmbeddingFunction } from "./embedding/embedding_function";
import {
EmbeddingFunctionConfig,
@@ -59,14 +52,7 @@ import {
sanitizeTable,
sanitizeType,
} from "./sanitize";
/**
* Check if a field name indicates a vector column.
*/
function nameSuggestsVectorColumn(fieldName: string): boolean {
const nameLower = fieldName.toLowerCase();
return nameLower.includes("vector") || nameLower.includes("embedding");
}
import { inferSchema } from "./schema";
export * from "apache-arrow";
export type SchemaLike =
@@ -459,110 +445,6 @@ export function makeArrowTable(
return new ArrowTable(inferredSchema, finalColumns);
}
function inferSchema(
data: Array<Record<string, unknown>>,
schema: Schema | undefined,
opts: MakeArrowTableOptions,
): Schema {
// We will collect all fields we see in the data.
const pathTree = new PathTree<DataType>();
for (const [rowI, row] of data.entries()) {
for (const [path, value] of rowPathsAndValues(row)) {
if (!pathTree.has(path)) {
// First time seeing this field.
if (schema !== undefined) {
const field = getFieldForPath(schema, path);
if (field === undefined) {
throw new Error(
`Found field not in schema: ${path.join(".")} at row ${rowI}`,
);
} else {
pathTree.set(path, field.type);
}
} else {
const inferredType = inferType(value, path, opts);
if (inferredType === undefined) {
throw new Error(`Failed to infer data type for field ${path.join(
".",
)} at row ${rowI}. \
Consider providing an explicit schema.`);
}
pathTree.set(path, inferredType);
}
} else if (schema === undefined) {
const currentType = pathTree.get(path);
const newType = inferType(value, path, opts);
if (currentType !== newType) {
new Error(`Failed to infer schema for data. Previously inferred type \
${currentType} but found ${newType} at row ${rowI}. Consider \
providing an explicit schema.`);
}
}
}
}
if (schema === undefined) {
function fieldsFromPathTree(pathTree: PathTree<DataType>): Field[] {
const fields = [];
for (const [name, value] of pathTree.map.entries()) {
if (value instanceof PathTree) {
const children = fieldsFromPathTree(value);
fields.push(new Field(name, new Struct(children), true));
} else {
fields.push(new Field(name, value, true));
}
}
return fields;
}
const fields = fieldsFromPathTree(pathTree);
return new Schema(fields);
} else {
function takeMatchingFields(
fields: Field[],
pathTree: PathTree<DataType>,
): Field[] {
const outFields = [];
for (const field of fields) {
if (pathTree.map.has(field.name)) {
const value = pathTree.get([field.name]);
if (value instanceof PathTree) {
const struct = field.type as Struct;
const children = takeMatchingFields(struct.children, value);
outFields.push(
new Field(field.name, new Struct(children), field.nullable),
);
} else {
outFields.push(
new Field(field.name, value as DataType, field.nullable),
);
}
}
}
return outFields;
}
const fields = takeMatchingFields(schema.fields, pathTree);
return new Schema(fields);
}
}
function* rowPathsAndValues(
row: Record<string, unknown>,
basePath: string[] = [],
): Generator<[string[], unknown]> {
for (const [key, value] of Object.entries(row)) {
if (isObject(value)) {
yield* rowPathsAndValues(value, [...basePath, key]);
} else {
// Skip undefined values - they should be treated the same as missing fields
// for embedding function purposes
if (value !== undefined) {
yield [[...basePath, key], value];
}
}
}
}
function isObject(value: unknown): value is Record<string, unknown> {
return (
typeof value === "object" &&
@@ -577,146 +459,19 @@ function isObject(value: unknown): value is Record<string, unknown> {
);
}
function getFieldForPath(schema: Schema, path: string[]): Field | undefined {
let current: Field | Schema = schema;
function valueAtPath(datum: Record<string, unknown>, path: string[]): unknown {
let current: unknown = datum;
for (const key of path) {
if (current instanceof Schema) {
const field: Field | undefined = current.fields.find(
(f) => f.name === key,
);
if (field === undefined) {
return undefined;
}
current = field;
} else if (current instanceof Field && DataType.isStruct(current.type)) {
const struct: Struct = current.type;
const field = struct.children.find((f) => f.name === key);
if (field === undefined) {
return undefined;
}
current = field;
if (current == null) {
return null;
}
if (isObject(current) && (Object.hasOwn(current, key) || key in current)) {
current = current[key];
} else {
return undefined;
}
}
if (current instanceof Field) {
return current;
} else {
return undefined;
}
}
/**
* Try to infer which Arrow type to use for a given value.
*
* May return undefined if the type cannot be inferred.
*/
function inferType(
value: unknown,
path: string[],
opts: MakeArrowTableOptions,
): DataType | undefined {
if (typeof value === "bigint") {
return new Int64();
} else if (typeof value === "number") {
// Even if it's an integer, it's safer to assume Float64. Users can
// always provide an explicit schema or use BigInt if they mean integer.
return new Float64();
} else if (typeof value === "string") {
if (opts.dictionaryEncodeStrings) {
return new Dictionary(new Utf8(), new Int32());
} else {
return new Utf8();
}
} else if (typeof value === "boolean") {
return new Bool();
} else if (value instanceof Buffer) {
return new Binary();
} else if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
const info = typedArrayToArrowType(value);
if (info !== undefined) {
const child = new Field("item", info.elementType, true);
return new FixedSizeList(info.length, child);
}
return undefined;
} else if (Array.isArray(value)) {
if (value.length === 0) {
return undefined; // Without any values we can't infer the type
}
if (path.length === 1 && Object.hasOwn(opts.vectorColumns, path[0])) {
const floatType = sanitizeType(opts.vectorColumns[path[0]].type);
return new FixedSizeList(
value.length,
new Field("item", floatType, true),
);
}
const valueType = inferType(value[0], path, opts);
if (valueType === undefined) {
return undefined;
}
// Try to automatically detect embedding columns.
if (nameSuggestsVectorColumn(path[path.length - 1])) {
// Check if value is a Uint8Array for integer vector type determination
if (value instanceof Uint8Array) {
// For integer vectors, we default to Uint8 (matching Python implementation)
const child = new Field("item", new Uint8(), true);
return new FixedSizeList(value.length, child);
} else {
// For float vectors, we default to Float32
const child = new Field("item", new Float32(), true);
return new FixedSizeList(value.length, child);
}
} else {
const child = new Field("item", valueType, true);
return new List(child);
}
} else {
// TODO: timestamp
return undefined;
}
}
class PathTree<V> {
map: Map<string, V | PathTree<V>>;
constructor(entries?: [string[], V][]) {
this.map = new Map();
if (entries !== undefined) {
for (const [path, value] of entries) {
this.set(path, value);
}
}
}
has(path: string[]): boolean {
let ref: PathTree<V> = this;
for (const part of path) {
if (!(ref instanceof PathTree) || !ref.map.has(part)) {
return false;
}
ref = ref.map.get(part) as PathTree<V>;
}
return true;
}
get(path: string[]): V | undefined {
let ref: PathTree<V> = this;
for (const part of path) {
if (!(ref instanceof PathTree) || !ref.map.has(part)) {
return undefined;
}
ref = ref.map.get(part) as PathTree<V>;
}
return ref as V;
}
set(path: string[], value: V): void {
let ref: PathTree<V> = this;
for (const part of path.slice(0, path.length - 1)) {
if (!ref.map.has(part)) {
ref.map.set(part, new PathTree<V>());
}
ref = ref.map.get(part) as PathTree<V>;
}
ref.map.set(path[path.length - 1], value);
}
return current;
}
function transposeData(
@@ -724,37 +479,26 @@ function transposeData(
field: Field,
path: string[] = [],
): Vector {
const valuesPath = [...path, field.name];
const values = data.map((datum) => valueAtPath(datum, valuesPath));
if (field.type instanceof Struct) {
const childFields = field.type.children;
const fullPath = [...path, field.name];
const childVectors = childFields.map((child) => {
return transposeData(data, child, fullPath);
return transposeData(data, child, valuesPath);
});
const nullCount = values.filter((value) => value === null).length;
const structData = makeData({
type: field.type,
length: values.length,
nullCount,
nullBitmap:
nullCount > 0
? arrowUtil.packBools(values.map((value) => value !== null))
: undefined,
children: childVectors as unknown as ArrowData<DataType>[],
});
return arrowMakeVector(structData);
} else {
const valuesPath = [...path, field.name];
const values = data.map((datum) => {
let current: unknown = datum;
for (const key of valuesPath) {
if (current == null) {
return null;
}
if (
isObject(current) &&
(Object.hasOwn(current, key) || key in current)
) {
current = current[key];
} else {
return null;
}
}
return current;
});
return makeVector(values, field.type, undefined, field.nullable);
}
}
@@ -797,32 +541,6 @@ function makeListVector(lists: unknown[][]): Vector<unknown> {
return listBuilder.finish().toVector();
}
/**
* Map a JS TypedArray instance to the corresponding Arrow element DataType
* and its length. Returns undefined if the value is not a recognized TypedArray.
*/
function typedArrayToArrowType(
value: ArrayBufferView,
): { elementType: DataType; length: number } | undefined {
if (value instanceof Float32Array)
return { elementType: new Float32(), length: value.length };
if (value instanceof Float64Array)
return { elementType: new Float64(), length: value.length };
if (value instanceof Uint8Array)
return { elementType: new Uint8(), length: value.length };
if (value instanceof Uint16Array)
return { elementType: new Uint16(), length: value.length };
if (value instanceof Uint32Array)
return { elementType: new Uint32(), length: value.length };
if (value instanceof Int8Array)
return { elementType: new Int8(), length: value.length };
if (value instanceof Int16Array)
return { elementType: new Int16(), length: value.length };
if (value instanceof Int32Array)
return { elementType: new Int32(), length: value.length };
return undefined;
}
/** Helper function to convert an Array of JS values to an Arrow Vector */
function makeVector(
values: unknown[],
@@ -1462,8 +1180,12 @@ export function ensureNestedFieldsExist(
completeRow[field.name] = row[field.name];
}
} else {
// Field is missing from the data - set to null
completeRow[field.name] = null;
// Keep a missing struct valid while filling each of its children with
// null. This is distinct from an explicitly null struct value.
completeRow[field.name] =
field.type.constructor.name === "Struct"
? ensureStructFieldsExist({}, field.type as Struct)
: null;
}
}
@@ -1498,8 +1220,12 @@ function ensureStructFieldsExist(
completeStruct[childField.name] = data[childField.name];
}
} else {
// Field is missing - set to null
completeStruct[childField.name] = null;
// Keep a missing struct valid while filling each of its children with
// null. This is distinct from an explicitly null struct value.
completeStruct[childField.name] =
childField.type.constructor.name === "Struct"
? ensureStructFieldsExist({}, childField.type as Struct)
: null;
}
}
nodejs/lancedb/arrow_type.ts
+40
View File
@@ -0,0 +1,40 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import {
type DataType,
Float32,
Float64,
Int8,
Int16,
Int32,
Uint8,
Uint16,
Uint32,
} from "apache-arrow";
/**
* Map a JS TypedArray instance to the corresponding Arrow element type and
* length. Returns undefined when the view is not a supported TypedArray.
*/
export function typedArrayToArrowType(
value: ArrayBufferView,
): { elementType: DataType; length: number } | undefined {
if (value instanceof Float32Array)
return { elementType: new Float32(), length: value.length };
if (value instanceof Float64Array)
return { elementType: new Float64(), length: value.length };
if (value instanceof Uint8Array)
return { elementType: new Uint8(), length: value.length };
if (value instanceof Uint16Array)
return { elementType: new Uint16(), length: value.length };
if (value instanceof Uint32Array)
return { elementType: new Uint32(), length: value.length };
if (value instanceof Int8Array)
return { elementType: new Int8(), length: value.length };
if (value instanceof Int16Array)
return { elementType: new Int16(), length: value.length };
if (value instanceof Int32Array)
return { elementType: new Int32(), length: value.length };
return undefined;
}
nodejs/lancedb/connection.ts
+155
View File
@@ -16,6 +16,12 @@ import {
makeEmptyTable,
} from "./arrow";
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
import {
MaterializedView,
MaterializedViewSelect,
normalizeSelect,
validateNonNegativeInteger,
} from "./materialized_view";
import { Connection as LanceDbConnection } from "./native";
import type {
CreateNamespaceResponse,
@@ -25,12 +31,14 @@ import type {
JobDescription,
JobInfo,
ListNamespacesResponse,
ListTablesResponse,
} from "./native";
export type {
CreateNamespaceResponse,
DescribeNamespaceResponse,
DropNamespaceResponse,
ListNamespacesResponse,
ListTablesResponse,
};
import { sanitizeTable } from "./sanitize";
import { LocalTable, Table } from "./table";
@@ -128,6 +136,10 @@ export interface OpenTableOptions {
indexCacheSize?: number;
}
/**
* @deprecated Use {@link ListTablesOptions} with {@link Connection.listTables}
* instead.
*/
export interface TableNamesOptions {
/**
* If present, only return names that come lexicographically after the
@@ -141,6 +153,24 @@ export interface TableNamesOptions {
limit?: number;
}
export interface ListTablesOptions {
/**
* Token from a previous response, to resume listing where it left off.
*
* The token is opaque: it carries whatever the database needs to resume, and
* callers should not construct or interpret one.
*/
pageToken?: string;
/**
* An upper bound on how many tables to return.
*
* A page may hold fewer than this and still not be the last one, so keep
* going while the response carries a page token rather than while pages are
* full.
*/
limit?: number;
}
export interface ListNamespacesOptions {
/** Token from a previous response for pagination. */
pageToken?: string;
@@ -225,6 +255,7 @@ export abstract class Connection {
* @param {Partial<TableNamesOptions>} options - options to control the
* paging / start point (backwards compatibility)
*
* @deprecated Use {@link Connection.listTables} instead.
*/
abstract tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
/**
@@ -235,18 +266,94 @@ export abstract class Connection {
* @param {Partial<TableNamesOptions>} options - options to control the
* paging / start point
*
* @deprecated Use {@link Connection.listTables} instead.
*/
abstract tableNames(
namespacePath?: string[],
options?: Partial<TableNamesOptions>,
): Promise<string[]>;
/**
* List a page of the tables in this database.
*
* To retrieve the tables after the page, pass the `pageToken` the response
* carries back in. A page can be shorter than `limit` without being the last
* one, so walk until a response carries no page token:
*
* ```ts
* const names = [];
* let pageToken = undefined;
* do {
* const page = await conn.listTables({ pageToken, limit: 100 });
* names.push(...page.tables);
* pageToken = page.pageToken;
* } while (pageToken);
* ```
*
* @param {Partial<ListTablesOptions>} options - Pagination options
* (`pageToken`, `limit`).
* @returns {Promise<ListTablesResponse>} A page of table names and an
* optional token for the tables after it.
*/
abstract listTables(
options?: Partial<ListTablesOptions>,
): Promise<ListTablesResponse>;
/**
* List a page of the tables in this database.
*
* @param {string[]} namespacePath - The namespace path to list tables from
* (defaults to root namespace)
* @param {Partial<ListTablesOptions>} options - Pagination options
* (`pageToken`, `limit`).
* @returns {Promise<ListTablesResponse>} A page of table names and an
* optional token for the tables after it.
*/
abstract listTables(
namespacePath?: string[],
options?: Partial<ListTablesOptions>,
): Promise<ListTablesResponse>;
/**
* Open a table in the database.
* @param {string} name - The name of the table
* @param {string[]} namespacePath - The namespace path of the table (defaults to root namespace)
* @param {Partial<OpenTableOptions>} options - Additional options
*/
/**
* Define a materialized view named `name` over the table `source`.
*
* The view is created empty, with the query recorded in its schema
* metadata; `view.refresh()` computes the rows. The view is a normal
* table: it can be queried, indexed and searched, and it appears in
* `tableNames`. The source table must have stable row ids (create it with
* the `newTableEnableStableRowIds` storage option); they keep the view's
* provenance valid across source compactions and cannot be enabled after
* a table exists. Local databases only.
*/
abstract createMaterializedView(
name: string,
source: string,
options?: {
select?: MaterializedViewSelect;
where?: string;
limit?: number;
},
): Promise<MaterializedView>;
/**
* Open the materialized view named `name`.
*
* Rejects a table that exists but is not a materialized view.
*/
abstract openMaterializedView(name: string): Promise<MaterializedView>;
/**
* The names of the materialized views in this database.
*
* Found by reading every table's schema, so this costs an open per table.
*/
abstract listMaterializedViews(): Promise<string[]>;
abstract openTable(
name: string,
namespacePath?: string[],
@@ -531,6 +638,54 @@ export class LocalConnection extends Connection {
);
}
async createMaterializedView(
name: string,
source: string,
options?: {
select?: MaterializedViewSelect;
where?: string;
limit?: number;
},
): Promise<MaterializedView> {
validateNonNegativeInteger(options?.limit, "limit");
const innerTable = await this.inner.createMaterializedView(
name,
source,
normalizeSelect(options?.select),
options?.where,
options?.limit,
);
return new MaterializedView(new LocalTable(innerTable));
}
async openMaterializedView(name: string): Promise<MaterializedView> {
const innerTable = await this.inner.openMaterializedView(name);
return new MaterializedView(new LocalTable(innerTable));
}
async listMaterializedViews(): Promise<string[]> {
return await this.inner.listMaterializedViews();
}
async listTables(
namespacePathOrOptions?: string[] | Partial<ListTablesOptions>,
options?: Partial<ListTablesOptions>,
): Promise<ListTablesResponse> {
// Detect if first argument is namespacePath array or options object
const namespacePath = Array.isArray(namespacePathOrOptions)
? namespacePathOrOptions
: undefined;
const listTablesOptions = Array.isArray(namespacePathOrOptions)
? options
: namespacePathOrOptions;
return this.inner.listTables(
namespacePath ?? [],
listTablesOptions?.pageToken,
listTablesOptions?.limit,
);
}
async openTable(
name: string,
namespacePath?: string[],
nodejs/lancedb/index.ts
+11 -3
View File
@@ -21,6 +21,11 @@ import type { BaseTokenizer } from "./indices";
import type { FtsToken } from "./table";
// Re-export native header provider for use with connectWithHeaderProvider
export {
MaterializedView,
MaterializedViewDefinition,
MaterializedViewSelect,
} from "./materialized_view";
export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
// OpenTelemetry metrics bridge. Only the high-level entry point is public; the
@@ -51,6 +56,7 @@ export {
AddResult,
AddColumnsResult,
RefreshColumnResult,
RefreshMaterializedViewResult,
AlterColumnsResult,
UpdateFieldMetadataResult,
DeleteResult,
@@ -75,11 +81,13 @@ export {
Connection,
CreateTableOptions,
TableNamesOptions,
ListTablesOptions,
OpenTableOptions,
ListNamespacesOptions,
CreateNamespaceOptions,
DropNamespaceOptions,
ListNamespacesResponse,
ListTablesResponse,
CreateNamespaceResponse,
DropNamespaceResponse,
DescribeNamespaceResponse,
@@ -135,10 +143,10 @@ export {
BranchColumnChange,
BranchIndexSummary,
BranchRowCountSummary,
MergeBlocker,
CherryPickError,
BranchDiff,
MergePreview,
MergeBranchResult,
CherryPickPreview,
CherryPickResult,
AddDataOptions,
UpdateOptions,
OptimizeOptions,
nodejs/lancedb/materialized_view.ts
+161
View File
@@ -0,0 +1,161 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { RefreshMaterializedViewResult } from "./native";
import { Table } from "./table";
/** Schema metadata key holding a materialized view's definition. */
export const DEFINITION_META_KEY = "mv.definition";
/** The query that defines a materialized view. */
export interface MaterializedViewDefinition {
/** Name of the source table, in the same database as the view. */
sourceTable: string;
/** `[output column, SQL expression]` pairs, in view schema order. */
projections: [string, string][];
/** SQL predicate selecting the source rows the view holds. */
filter?: string;
/** Cap on the number of rows the view holds. */
limit?: number;
/** Source columns the projections and filter read. */
inputs: string[];
}
/**
* The view's columns: column names, `[alias, SQL expression]` pairs, or a
* record of the same. A bare name projects itself.
*/
export type MaterializedViewSelect =
| (string | [string, string])[]
| Record<string, string>;
/**
* @internal Reject a numeric option N-API would otherwise silently coerce:
* `Infinity` reaches Rust as 0, `1.5` as 1.
*/
export function validateNonNegativeInteger(
value: number | undefined,
name: string,
): void {
if (value !== undefined && !(Number.isSafeInteger(value) && value >= 0)) {
throw new Error(`${name} must be a non-negative integer`);
}
}
/** @internal Quote a column name as a Lance SQL identifier (backticks). */
function quoteIdentifier(name: string): string {
return "`" + name.replace(/`/g, "``") + "`";
}
/**
* @internal Normalize a select argument into `[alias, expression]` pairs.
* A bare name projects itself and is quoted, so any valid column name works;
* pair and record entries are kept verbatim because their right side is an
* expression.
*/
export function normalizeSelect(
select?: MaterializedViewSelect,
): [string, string][] | undefined {
if (select === undefined) {
return undefined;
}
if (Array.isArray(select)) {
return select.map((item) =>
typeof item === "string" ? [item, quoteIdentifier(item)] : item,
);
}
return Object.entries(select);
}
/** @internal Parse a definition off a table's stored schema metadata. */
export function definitionFromMetadata(
metadata: Map<string, string>,
name: string,
): MaterializedViewDefinition {
const raw = metadata.get(DEFINITION_META_KEY);
if (raw === undefined) {
throw new Error(`Table '${name}' is not a materialized view`);
}
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
const value: any = JSON.parse(raw);
if (value.kind !== "select") {
throw new Error(
`materialized view '${name}' is defined by '${value.kind}', which this ` +
"version of lancedb cannot refresh",
);
}
const limit = value.limit ?? undefined;
// JSON.parse rounds integers past 2^53; every exact u64 parses to a safe
// integer and every rounded one does not, so this rejects precisely the
// values a number cannot carry.
if (limit !== undefined && !Number.isSafeInteger(limit)) {
throw new Error(
`materialized view '${name}' has a stored limit too large to represent exactly`,
);
}
return {
sourceTable: value.source_table,
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
projections: (value.projections ?? []).map((p: any) => [
p.output,
p.expression,
]),
filter: value.filter ?? undefined,
limit,
inputs: value.inputs ?? [],
};
}
/**
* A handle on a materialized view: its table plus its definition.
*
* Obtained from {@link Connection#createMaterializedView} or
* {@link Connection#openMaterializedView}. The view is a normal table --
* queries, indexes and search all apply through {@link MaterializedView#table}
* -- whose contents are maintained by {@link MaterializedView#refresh}.
*/
export class MaterializedView {
private readonly inner: Table;
constructor(table: Table) {
this.inner = table;
}
get name(): string {
return this.inner.name;
}
/** The view, as the table it is. */
table(): Table {
return this.inner;
}
/** The query that defines the view, read from its stored schema. */
async definition(): Promise<MaterializedViewDefinition> {
const schema = await this.inner.schema();
return definitionFromMetadata(schema.metadata, this.name);
}
/**
* Recompute the view from its source.
*
* The refresh is incremental when the source's changes can be reconciled
* into the view -- rows added, changed or removed since the last one --
* and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
* refreshes to that source version instead of the latest.
*
* Concurrent refreshes of one view do not duplicate its rows. Two that
* plan the same source rows conflict on commit, and the loser throws
* rather than writing them a second time.
*/
async refresh(options?: {
full?: boolean;
sourceVersion?: number;
}): Promise<RefreshMaterializedViewResult> {
validateNonNegativeInteger(options?.sourceVersion, "sourceVersion");
return await this.inner.refreshMaterializedView(
options?.full,
options?.sourceVersion,
);
}
}
nodejs/lancedb/schema.ts
+566
View File
@@ -0,0 +1,566 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import {
Binary,
Bool,
DataType,
Dictionary,
Field,
FixedSizeList,
Float32,
Float64,
Int32,
Int64,
List,
Schema,
Struct,
Utf8,
util as arrowUtil,
} from "apache-arrow";
import { typedArrayToArrowType } from "./arrow_type";
import { sanitizeType } from "./sanitize";
type InferenceOptions = {
dictionaryEncodeStrings: boolean;
vectorColumns: Record<string, { type: unknown }>;
};
/**
* Infer the Arrow schema represented by a set of records.
*
* This is the intentionally small interface to schema inference. The stateful
* details of combining partial type evidence are encapsulated below so callers
* only need to provide records, an optional schema, and inference options.
*/
export function inferSchema(
data: Array<Record<string, unknown>>,
schema: Schema | undefined,
options: InferenceOptions,
): Schema {
return new SchemaInferrer(schema, options).infer(data);
}
class SchemaInferrer {
private readonly fields = new FieldTree();
constructor(
private readonly providedSchema: Schema | undefined,
private readonly options: InferenceOptions,
) {}
infer(data: Array<Record<string, unknown>>): Schema {
for (const [row, record] of data.entries()) {
for (const [path, value] of recordPathsAndValues(record)) {
this.observe(path, value, row);
}
}
return this.providedSchema === undefined
? new Schema(fieldsFromTree(this.fields))
: new Schema(matchingFields(this.providedSchema.fields, this.fields));
}
private observe(path: string[], value: unknown, row: number): void {
const current = this.fields.get(path);
if (current === undefined) {
this.addField(path, value, row);
} else if (this.providedSchema === undefined) {
this.updateInferredField(path, value, row, current);
}
}
private addField(path: string[], value: unknown, row: number): void {
if (this.providedSchema !== undefined) {
this.addSchemaField(this.providedSchema, path, row);
return;
}
const evidence =
this.inferType(value, path) ?? DeferredTypeEvidence.from(value, row);
if (evidence === undefined) {
throw typeInferenceError(path, row);
}
const conflict = this.fields.set(
path,
evidence,
(existing) =>
existing instanceof DeferredTypeEvidence && existing.isOnlyNulls(),
);
if (conflict !== undefined) {
throw branchConflictError(conflict, row, "Struct");
}
}
private addSchemaField(schema: Schema, path: string[], row: number): void {
const field = fieldAtPath(schema, path);
if (field === undefined) {
throw new Error(
`Found field not in schema: ${path.join(".")} at row ${row}`,
);
}
const conflict = this.fields.set(path, field.type);
if (conflict !== undefined) {
throw branchConflictError(conflict, row, "Struct");
}
}
private updateInferredField(
path: string[],
value: unknown,
row: number,
current: FieldNode,
): void {
const newType = this.inferType(value, path);
const deferred = DeferredTypeEvidence.from(value, row);
if (current instanceof FieldTree) {
if (deferred?.isOnlyNulls()) {
return;
}
throw schemaInferenceError(
path,
row,
"Struct",
describeEvidence(newType ?? deferred),
);
}
if (current instanceof DeferredTypeEvidence) {
this.resolveDeferredField(path, row, current, newType, deferred);
return;
}
if (newType !== undefined) {
if (!inferredTypesEqual(current, newType)) {
throw schemaInferenceError(
path,
row,
describeEvidence(current),
describeEvidence(newType),
);
}
return;
}
if (deferred === undefined || !deferred.matches(current)) {
throw schemaInferenceError(
path,
row,
describeEvidence(current),
describeEvidence(deferred),
);
}
}
private resolveDeferredField(
path: string[],
row: number,
current: DeferredTypeEvidence,
newType: DataType | undefined,
deferred: DeferredTypeEvidence | undefined,
): void {
if (newType !== undefined) {
if (!current.matches(newType)) {
throw schemaInferenceError(
path,
row,
current.describe(),
describeEvidence(newType),
);
}
this.fields.set(path, newType);
return;
}
if (deferred !== undefined) {
this.fields.set(path, current.merge(deferred));
return;
}
throw schemaInferenceError(
path,
row,
current.describe(),
describeEvidence(newType),
);
}
private inferType(value: unknown, path: string[]): DataType | undefined {
if (typeof value === "bigint") {
return new Int64();
}
if (typeof value === "number") {
return new Float64();
}
if (typeof value === "string") {
return this.options.dictionaryEncodeStrings
? new Dictionary(new Utf8(), new Int32())
: new Utf8();
}
if (typeof value === "boolean") {
return new Bool();
}
if (value instanceof Buffer) {
return new Binary();
}
if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
const typedArray = typedArrayToArrowType(value);
return typedArray === undefined
? undefined
: new FixedSizeList(
typedArray.length,
new Field("item", typedArray.elementType, true),
);
}
if (!Array.isArray(value) || value.length === 0) {
return undefined;
}
const configuredVector =
path.length === 1 ? this.options.vectorColumns[path[0]] : undefined;
if (configuredVector !== undefined) {
return new FixedSizeList(
value.length,
new Field("item", sanitizeType(configuredVector.type), true),
);
}
const itemType = this.inferArrayItemType(value, path);
if (itemType === undefined) {
return undefined;
}
return nameSuggestsVectorColumn(path[path.length - 1])
? new FixedSizeList(value.length, new Field("item", new Float32(), true))
: new List(new Field("item", itemType, true));
}
private inferArrayItemType(
values: unknown[],
path: string[],
): DataType | undefined {
let itemType: DataType | undefined;
const deferredItems: unknown[] = [];
for (const value of values) {
const candidate = this.inferType(value, path);
if (candidate === undefined) {
if (!isDeferredValue(value)) {
return undefined;
}
deferredItems.push(value);
} else if (itemType === undefined) {
itemType = candidate;
} else if (!inferredTypesEqual(itemType, candidate)) {
return undefined;
}
}
if (itemType === undefined) {
return undefined;
}
return deferredItems.every((value) =>
deferredValueMatchesType(value, itemType),
)
? itemType
: undefined;
}
}
/** Nulls and empty/all-null lists that do not determine a type by themselves. */
class DeferredTypeEvidence {
private constructor(
private readonly values: Array<{ value: unknown; row: number }>,
) {}
static from(value: unknown, row: number): DeferredTypeEvidence | undefined {
return isDeferredValue(value)
? new DeferredTypeEvidence([{ value, row }])
: undefined;
}
isOnlyNulls(): boolean {
return this.values.every(({ value }) => value == null);
}
matches(type: DataType): boolean {
return this.values.every(({ value }) =>
deferredValueMatchesType(value, type),
);
}
merge(other: DeferredTypeEvidence): DeferredTypeEvidence {
return new DeferredTypeEvidence([...this.values, ...other.values]);
}
describe(): string {
const list = this.values.find(({ value }) => Array.isArray(value));
return list === undefined
? "null"
: `List[${(list.value as unknown[]).length}]`;
}
firstRow(): number {
return this.values[0].row;
}
}
type FieldNode = DataType | DeferredTypeEvidence | FieldTree;
type LeafNode = Exclude<FieldNode, FieldTree>;
type FieldConflict = { path: string[]; value: FieldNode };
/** Nested field state, kept separate from Arrow's eventual Struct types. */
class FieldTree {
private readonly children = new Map<string, FieldNode>();
get(path: string[]): FieldNode | undefined {
let current: FieldNode = this;
for (const part of path) {
if (!(current instanceof FieldTree)) {
return undefined;
}
const child = current.children.get(part);
if (child === undefined) {
return undefined;
}
current = child;
}
return current;
}
set(
path: string[],
value: LeafNode,
canReplaceLeaf: (value: LeafNode) => boolean = () => false,
): FieldConflict | undefined {
let branch: FieldTree = this;
for (const [index, part] of path.slice(0, -1).entries()) {
const child = branch.children.get(part);
if (child === undefined || (isLeaf(child) && canReplaceLeaf(child))) {
const nextBranch = new FieldTree();
branch.children.set(part, nextBranch);
branch = nextBranch;
} else if (child instanceof FieldTree) {
branch = child;
} else {
return { path: path.slice(0, index + 1), value: child };
}
}
const name = path[path.length - 1];
const current = branch.children.get(name);
if (current instanceof FieldTree) {
return { path, value: current };
}
branch.children.set(name, value);
return undefined;
}
entries(): IterableIterator<[string, FieldNode]> {
return this.children.entries();
}
has(name: string): boolean {
return this.children.has(name);
}
}
function isLeaf(value: FieldNode): value is LeafNode {
return !(value instanceof FieldTree);
}
function fieldsFromTree(tree: FieldTree, path: string[] = []): Field[] {
const fields: Field[] = [];
for (const [name, value] of tree.entries()) {
if (value instanceof FieldTree) {
fields.push(
new Field(
name,
new Struct(fieldsFromTree(value, [...path, name])),
true,
),
);
} else if (value instanceof DeferredTypeEvidence) {
throw typeInferenceError([...path, name], value.firstRow());
} else {
fields.push(new Field(name, value, true));
}
}
return fields;
}
function matchingFields(fields: Field[], tree: FieldTree): Field[] {
const matches: Field[] = [];
for (const field of fields) {
if (!tree.has(field.name)) {
continue;
}
const value = tree.get([field.name]);
if (value instanceof FieldTree) {
const struct = field.type as Struct;
matches.push(
new Field(
field.name,
new Struct(matchingFields(struct.children, value)),
field.nullable,
),
);
} else {
matches.push(new Field(field.name, value as DataType, field.nullable));
}
}
return matches;
}
function* recordPathsAndValues(
record: Record<string, unknown>,
path: string[] = [],
): Generator<[string[], unknown]> {
for (const [name, value] of Object.entries(record)) {
if (isRecord(value)) {
yield* recordPathsAndValues(value, [...path, name]);
} else if (value !== undefined) {
yield [[...path, name], value];
}
}
}
function isRecord(value: unknown): value is Record<string, unknown> {
return (
typeof value === "object" &&
value !== null &&
!Array.isArray(value) &&
!(value instanceof RegExp) &&
!(value instanceof Date) &&
!(value instanceof Set) &&
!(value instanceof Map) &&
!(value instanceof Buffer) &&
!ArrayBuffer.isView(value)
);
}
function fieldAtPath(schema: Schema, path: string[]): Field | undefined {
let fields = schema.fields;
let field: Field | undefined;
for (const [index, name] of path.entries()) {
field = fields.find((candidate) => candidate.name === name);
if (field === undefined || index === path.length - 1) {
return field;
}
if (!DataType.isStruct(field.type)) {
return undefined;
}
fields = field.type.children;
}
return field;
}
function isDeferredValue(value: unknown): boolean {
return (
value == null || (Array.isArray(value) && value.every(isDeferredValue))
);
}
function deferredValueMatchesType(value: unknown, type: DataType): boolean {
if (value == null) {
return true;
}
if (!Array.isArray(value)) {
return false;
}
if (DataType.isList(type)) {
return value.every((item) =>
deferredValueMatchesType(item, type.valueType),
);
}
if (DataType.isFixedSizeList(type)) {
return (
value.length === type.listSize &&
value.every((item) => deferredValueMatchesType(item, type.valueType))
);
}
return false;
}
function inferredTypesEqual(current: DataType, candidate: DataType): boolean {
if (DataType.isDictionary(current)) {
return (
DataType.isDictionary(candidate) &&
current.isOrdered === candidate.isOrdered &&
inferredTypesEqual(current.indices, candidate.indices) &&
inferredTypesEqual(current.dictionary, candidate.dictionary)
);
}
if (DataType.isList(current)) {
return (
DataType.isList(candidate) &&
current.valueField.name === candidate.valueField.name &&
current.valueField.nullable === candidate.valueField.nullable &&
inferredTypesEqual(current.valueType, candidate.valueType)
);
}
if (DataType.isFixedSizeList(current)) {
return (
DataType.isFixedSizeList(candidate) &&
current.listSize === candidate.listSize &&
current.valueField.name === candidate.valueField.name &&
current.valueField.nullable === candidate.valueField.nullable &&
inferredTypesEqual(current.valueType, candidate.valueType)
);
}
return arrowUtil.compareTypes(current, candidate);
}
function describeEvidence(
evidence: DataType | DeferredTypeEvidence | undefined,
): string {
if (evidence === undefined) {
return "an unsupported value";
}
return evidence instanceof DeferredTypeEvidence
? evidence.describe()
: evidence.toString();
}
function branchConflictError(
conflict: FieldConflict,
row: number,
candidate: string,
): Error {
return schemaInferenceError(
conflict.path,
row,
conflict.value instanceof FieldTree
? "Struct"
: describeEvidence(conflict.value),
candidate,
);
}
function schemaInferenceError(
path: string[],
row: number,
currentType: string,
newType: string,
): Error {
return new Error(
`Failed to infer schema for data. Previously inferred type ${currentType} ` +
`but found ${newType} for field ${path.join(".")} at row ${row}. ` +
"Consider providing an explicit schema.",
);
}
function typeInferenceError(path: string[], row: number): Error {
return new Error(
`Failed to infer data type for field ${path.join(".")} at row ${row}. ` +
"Consider providing an explicit schema.",
);
}
function nameSuggestsVectorColumn(name: string): boolean {
const normalized = name.toLowerCase();
return normalized.includes("vector") || normalized.includes("embedding");
}
nodejs/lancedb/table.ts
+38 -19
View File
@@ -35,6 +35,7 @@ import {
Branches as NativeBranches,
OptimizeStats,
RefreshColumnResult,
RefreshMaterializedViewResult,
TableStatistics,
Tags,
UpdateFieldMetadataResult,
@@ -602,6 +603,18 @@ export abstract class Table {
*/
abstract refreshColumnAsync(column: string): Promise<Job>;
/**
* Recompute this table's contents from its materialized-view definition.
*
* Plumbing for {@link MaterializedView.refresh}, which is the way to call
* it: rejects tables that carry no view definition. Local tables only.
* @ignore
*/
abstract refreshMaterializedView(
full?: boolean,
sourceVersion?: number,
): Promise<RefreshMaterializedViewResult>;
/**
* Alter the name or nullability of columns.
* @param {ColumnAlteration[]} columnAlterations One or more alterations to
@@ -1264,6 +1277,13 @@ export class LocalTable extends Table {
return await this.inner.refreshColumnAsync(column);
}
async refreshMaterializedView(
full?: boolean,
sourceVersion?: number,
): Promise<RefreshMaterializedViewResult> {
return await this.inner.refreshMaterializedView(full, sourceVersion);
}
async alterColumns(
columnAlterations: ColumnAlteration[],
): Promise<AlterColumnsResult> {
@@ -1557,8 +1577,8 @@ export interface BranchRowCountSummary {
deltaAvailable: boolean;
}
/** A reason why a branch cannot currently be merged. */
export interface MergeBlocker {
/** A reason why a cherry-pick cannot currently land. */
export interface CherryPickError {
code: string;
message: string;
}
@@ -1578,20 +1598,19 @@ export interface BranchDiff {
changedColumns: BranchColumnChange[];
addedIndexes: BranchIndexSummary[];
removedIndexes: BranchIndexSummary[];
mergeable: boolean;
mergeBlockers: MergeBlocker[];
errors: CherryPickError[];
}
/** Changes that would be, or were, promoted by a branch merge. */
export interface MergePreview {
/** Changes that would be, or were, promoted by a cherry-pick. */
export interface CherryPickPreview {
promotedColumns: string[];
}
/** Result of previewing or attempting a branch merge. */
export interface MergeBranchResult {
status: "ready" | "rejected" | "notImplemented" | "merged" | "unknown";
/** Result of previewing or attempting a cherry-pick. */
export interface CherryPickResult {
status: "ready" | "failed" | "notImplemented" | "cherryPicked" | "unknown";
diff: BranchDiff;
preview: MergePreview;
preview: CherryPickPreview;
mainVersionAfter?: number;
}
@@ -1654,21 +1673,21 @@ export class Branches {
}
/**
* Merge a branch into main.
* Cherry-pick a branch onto main.
*
* Set `dryRun` to `true` to preview the merge. A rejected merge resolves
* with `status: "rejected"` instead of throwing.
* Set `dryRun` to `true` to preview. A failed cherry-pick resolves
* with `status: "failed"` instead of throwing.
*
* @param fromBranch Branch to merge from.
* @param dryRun When true, only preview the merge. Defaults to false.
* @param fromBranch Branch to cherry-pick from.
* @param dryRun When true, only preview. Defaults to false.
*/
async merge(
async cherryPick(
fromBranch: string,
dryRun: boolean = false,
): Promise<MergeBranchResult> {
return (await this.#inner.merge(
): Promise<CherryPickResult> {
return (await this.#inner.cherryPick(
fromBranch,
dryRun,
)) as unknown as MergeBranchResult;
)) as unknown as CherryPickResult;
}
}
nodejs/npm/darwin-arm64/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.8",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
nodejs/npm/linux-arm64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.8",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
nodejs/npm/linux-arm64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.8",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
nodejs/npm/linux-x64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.8",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
nodejs/npm/linux-x64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.8",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
nodejs/npm/win32-arm64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.8",
"os": [
"win32"
],
nodejs/npm/win32-x64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.8",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
nodejs/package-lock.json
Generated
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "@lancedb/lancedb",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.8",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@lancedb/lancedb",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.8",
"cpu": [
"x64",
"arm64"
nodejs/package.json
+1 -1
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.8",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
nodejs/src/connection.rs
+86
View File
@@ -17,6 +17,7 @@ use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection, conne
use lance_namespace::models::{
CreateNamespaceRequest, DescribeNamespaceRequest, DropNamespaceRequest, ListNamespacesRequest,
ListTablesRequest,
};
use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
@@ -36,6 +37,12 @@ pub struct ListNamespacesResponse {
pub page_token: Option<String>,
}
#[napi(object)]
pub struct ListTablesResponse {
pub tables: Vec<String>,
pub page_token: Option<String>,
}
#[napi(object)]
pub struct CreateNamespaceResponse {
pub properties: Option<HashMap<String, String>>,
@@ -206,6 +213,33 @@ impl Connection {
op.execute().await.default_error()
}
/// List a page of tables in the database.
#[napi(catch_unwind)]
pub async fn list_tables(
&self,
namespace_path: Option<Vec<String>>,
page_token: Option<String>,
limit: Option<u32>,
) -> napi::Result<ListTablesResponse> {
let request = ListTablesRequest {
// The root namespace is an empty path, not an absent one: a namespace-backed
// database rejects a request that names no namespace.
id: Some(namespace_path.unwrap_or_default()),
page_token,
limit: limit.map(|limit| i32::try_from(limit).unwrap_or(i32::MAX)),
..Default::default()
};
let response = self
.get_inner()?
.list_tables(request)
.await
.default_error()?;
Ok(ListTablesResponse {
tables: response.tables,
page_token: response.page_token,
})
}
/// Create table from a Apache Arrow IPC (file) buffer.
///
/// Parameters:
@@ -266,6 +300,58 @@ impl Connection {
Ok(Table::new(tbl))
}
#[napi(catch_unwind)]
pub async fn create_materialized_view(
&self,
name: String,
source: String,
projections: Option<Vec<Vec<String>>>,
filter: Option<String>,
limit: Option<i64>,
) -> napi::Result<Table> {
let mut builder = self.get_inner()?.create_materialized_view(name, source);
if let Some(projections) = projections {
let mut pairs = Vec::with_capacity(projections.len());
for pair in projections {
let [output, expression]: [String; 2] = pair.try_into().map_err(|_| {
napi::Error::from_reason("each projection must be an [output, expression] pair")
})?;
pairs.push((output, expression));
}
builder = builder.select(pairs);
}
if let Some(filter) = filter {
builder = builder.only_if(filter);
}
if let Some(limit) = limit {
let limit = u64::try_from(limit)
.map_err(|_| napi::Error::from_reason("limit must be a non-negative integer"))?;
builder = builder.limit(limit);
}
let view = builder.execute().await.default_error()?;
Ok(Table::new(view.table().clone()))
}
#[napi(catch_unwind)]
pub async fn open_materialized_view(&self, name: String) -> napi::Result<Table> {
let view = self
.get_inner()?
.open_materialized_view(&name)
.await
.default_error()?;
Ok(Table::new(view.table().clone()))
}
#[napi(catch_unwind)]
pub async fn list_materialized_views(&self) -> napi::Result<Vec<String>> {
let views = self
.get_inner()?
.list_materialized_views()
.await
.default_error()?;
Ok(views.into_iter().map(|v| v.name).collect())
}
#[napi(catch_unwind)]
pub async fn open_table(
&self,
nodejs/src/job.rs
+5 -2
View File
@@ -14,9 +14,12 @@ pub struct Job {
}
impl Job {
pub(crate) fn new(inner: lancedb::Job) -> Self {
pub(crate) fn new<T>(inner: lancedb::Job<T>) -> Self
where
T: Clone + Send + Sync + 'static,
{
Self {
inner: Arc::new(inner),
inner: Arc::new(inner.map(|_| ())),
}
}
}
nodejs/src/lib.rs
+4
View File
@@ -1,6 +1,10 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// The materialized-view refresh future deepens the type graph past the
// default trait-recursion depth; same raise as the core crate applies.
#![recursion_limit = "256"]
use std::collections::HashMap;
use env_logger::Env;
nodejs/src/table.rs
+48 -3
View File
@@ -381,6 +381,26 @@ impl Table {
Ok(crate::job::Job::new(job))
}
#[napi(catch_unwind)]
pub async fn refresh_materialized_view(
&self,
full: Option<bool>,
source_version: Option<i64>,
) -> napi::Result<RefreshMaterializedViewResult> {
let view = lancedb::MaterializedView::from_table(self.inner_ref()?.clone())
.await
.default_error()?;
let mut builder = view.refresh().full(full.unwrap_or(false));
if let Some(version) = source_version {
let version = u64::try_from(version).map_err(|_| {
napi::Error::from_reason("sourceVersion must be a non-negative integer")
})?;
builder = builder.source_version(version);
}
let result = builder.execute().await.default_error()?;
Ok(result.into())
}
#[napi(catch_unwind)]
pub async fn add_columns_with_schema(
&self,
@@ -1387,6 +1407,31 @@ pub struct RefreshColumnResult {
pub version: i64,
}
#[napi(object)]
pub struct RefreshMaterializedViewResult {
/// How the view was brought up to date: "rebuild", "incremental" or "no_op".
pub mode: String,
pub rows_written: i64,
pub source_version: i64,
pub version: i64,
}
impl From<lancedb::RefreshMaterializedViewResult> for RefreshMaterializedViewResult {
fn from(value: lancedb::RefreshMaterializedViewResult) -> Self {
let mode = match value.mode {
lancedb::RefreshMode::Rebuild => "rebuild",
lancedb::RefreshMode::Incremental => "incremental",
lancedb::RefreshMode::NoOp => "no_op",
};
Self {
mode: mode.to_string(),
rows_written: value.rows_written as i64,
source_version: value.source_version as i64,
version: value.version as i64,
}
}
}
impl From<lancedb::table::RefreshColumnResult> for RefreshColumnResult {
fn from(value: lancedb::table::RefreshColumnResult) -> Self {
Self {
@@ -1605,18 +1650,18 @@ impl Branches {
}
#[napi(ts_return_type = "Promise<Record<string, unknown>>")]
pub async fn merge(
pub async fn cherry_pick(
&self,
from_branch: String,
dry_run: Option<bool>,
) -> napi::Result<serde_json::Value> {
let result = self
.inner
.merge_branch(&from_branch, dry_run.unwrap_or(false))
.cherry_pick(&from_branch, dry_run.unwrap_or(false))
.await
.default_error()?;
serde_json::to_value(result).map_err(|err| {
napi::Error::from_reason(format!("failed to serialize branch merge result: {err}"))
napi::Error::from_reason(format!("failed to serialize cherry-pick result: {err}"))
})
}
}
python/Cargo.toml
+5 -6
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.38.0-beta.3"
version = "0.38.0-beta.8"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
@@ -26,7 +26,9 @@ lance-namespace-impls.workspace = true
lance-io.workspace = true
env_logger.workspace = true
log.workspace = true
pyo3 = { version = "0.28", features = ["extension-module", "abi3-py310", "chrono"] }
# Maturin enables extension-module mode for Python builds. Keeping it out of
# Cargo features lets Rust unit tests link against libpython.
pyo3 = { version = "0.28", features = ["abi3-py310", "chrono"] }
chrono.workspace = true
pyo3-async-runtimes = { version = "0.28", features = [
"attributes",
@@ -41,10 +43,7 @@ tokio.workspace = true
libc = "0.2"
[build-dependencies]
pyo3-build-config = { version = "0.28", features = [
"extension-module",
"abi3-py310",
] }
pyo3-build-config = { version = "0.28", features = ["abi3-py310"] }
[features]
default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/cos", "lancedb/goosefs", "lancedb/metrics-otel"]
python/README.md
+19
View File
@@ -38,6 +38,25 @@ Stable releases are created about every 2 weeks. For the latest features and bug
pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb
```
### Threading in CPU-limited containers
LanceDB uses separate pools for compute work and storage I/O. On a container with
two visible CPUs, current releases intentionally use one compute worker by default;
no manual configuration is needed. If every query logs an I/O core reservation
warning on a two-CPU container, upgrade from LanceDB 0.21.1 or earlier.
The two commonly tuned environment variables control different resources:
- `LANCE_CPU_THREADS` overrides the number of compute workers. One worker is the
appropriate setting for a two-CPU container when an explicit override is needed.
- `LANCE_IO_THREADS` controls concurrent storage operations, not reserved CPU
cores. Its default can be greater than the number of CPUs because I/O workers
spend much of their time waiting for storage.
Keep the defaults unless measurements show that the workload benefits from an
override. See the [Lance threading model](https://lance.org/guide/performance/#threading-model)
for the current defaults and tuning guidance.
## Usage
### Basic Example
python/pyproject.toml
+1 -1
View File
@@ -103,7 +103,7 @@ python-source = "python"
module-name = "lancedb._lancedb"
[build-system]
requires = ["maturin>=1.4"]
requires = ["maturin>=1.9.4"]
build-backend = "maturin"
[tool.ruff.lint]
python/python/lancedb/__init__.py
+9
View File
@@ -29,9 +29,15 @@ from .functions import (
FunctionRegistrationRequest as FunctionRegistrationRequest,
FunctionVersion as FunctionVersion,
PythonRuntimeSpec as PythonRuntimeSpec,
RefreshColumnResult as RefreshColumnResult,
UdfDefinition as UdfDefinition,
udf as udf,
)
from .materialized_view import (
AsyncMaterializedView,
MaterializedView,
MaterializedViewDefinition,
)
from .table import AsyncTable, Table
from .types import BaseTokenizerType
from ._lancedb import Session
@@ -506,6 +512,9 @@ async def connect_async(
__all__ = [
"AsyncMaterializedView",
"MaterializedView",
"MaterializedViewDefinition",
"connect",
"connect_async",
"tokenize",
python/python/lancedb/_lancedb.pyi
+21 -10
View File
@@ -147,7 +147,7 @@ class Connection(object):
limit: Optional[int],
) -> list[str]: ... # Deprecated: Use list_tables instead
def job(self, job_id: str) -> Job: ...
async def create_function_async(self, request_json: str) -> FunctionJob: ...
async def create_function_async(self, request_json: str) -> Job: ...
async def get_function(self, name: str, version: str) -> str: ...
async def list_jobs(self) -> List[JobInfo]: ...
async def get_job(self, job_id: str) -> Optional[JobDescription]: ...
@@ -197,6 +197,15 @@ class Connection(object):
cur_namespace_path: Optional[List[str]] = None,
new_namespace_path: Optional[List[str]] = None,
) -> None: ...
async def create_materialized_view(
self,
name: str,
source: str,
projections: Optional[List[Tuple[str, str]]] = None,
filter: Optional[str] = None,
limit: Optional[int] = None,
) -> Table: ...
async def list_materialized_views(self) -> List[str]: ...
async def drop_table(
self, name: str, namespace_path: Optional[List[str]] = None
) -> None: ...
@@ -225,14 +234,7 @@ class Job:
@property
def id(self) -> Optional[str]: ...
async def status(self) -> str: ...
async def wait(self) -> None: ...
async def cancel(self) -> None: ...
class FunctionJob:
@property
def id(self) -> Optional[str]: ...
async def status(self) -> str: ...
async def wait(self) -> str: ...
async def wait(self) -> Optional[str]: ...
async def cancel(self) -> None: ...
class JobInfo:
@@ -355,6 +357,9 @@ class Table:
) -> AddColumnsResult: ...
async def refresh_column(self, column: str) -> RefreshColumnResult: ...
async def refresh_column_async(self, column: str) -> Job: ...
async def refresh_materialized_view(
self, full: bool = False, source_version: Optional[int] = None
) -> RefreshMaterializedViewResult: ...
async def add_columns_with_schema(self, schema: pa.Schema) -> AddColumnsResult: ...
async def alter_columns(
self, columns: list[dict[str, Any]]
@@ -420,7 +425,7 @@ class Branches:
async def checkout(self, name: str, version: Optional[int] = None) -> Table: ...
async def delete(self, name: str) -> None: ...
async def diff(self, from_branch: str) -> Dict[str, Any]: ...
async def merge(
async def cherry_pick(
self, from_branch: str, dry_run: bool = False
) -> Dict[str, Any]: ...
@@ -704,6 +709,12 @@ class RefreshColumnResult:
rows_filled: int
version: int
class RefreshMaterializedViewResult:
mode: str
rows_written: int
source_version: int
version: int
class AlterColumnsResult:
version: int
python/python/lancedb/db.py
+168 -2
View File
@@ -46,7 +46,13 @@ from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
from . import __version__
from ._lancedb import connect as lancedb_connect # type: ignore
from .functions import FunctionVersion, UdfDefinition
from .job import AsyncJob, Job, _function_job
from .job import AsyncJob, Job, _typed_job
from .materialized_view import (
AsyncMaterializedView,
MaterializedView,
SelectArg,
normalize_select,
)
from .table import (
AsyncTable,
LanceTable,
@@ -510,6 +516,70 @@ class DBConnection(EnforceOverrides):
"""
raise NotImplementedError
def create_materialized_view(
self,
name: str,
source: str,
*,
select: SelectArg = None,
where: Optional[str] = None,
limit: Optional[int] = None,
) -> MaterializedView:
"""Define a materialized view named ``name`` over the table ``source``.
The view is created empty, with the query recorded in its schema
metadata; ``view.refresh()`` computes the rows. The view is a normal
table: it can be queried, indexed and searched, and it appears in
``table_names``. Local databases only.
The source table must have stable row ids (create it with the
``new_table_enable_stable_row_ids`` storage option): they keep the
view's provenance valid across source compactions, and cannot be
enabled after a table exists.
Parameters
----------
name: str
The name of the view.
source: str
The name of the source table, in this database.
select: list or dict, optional
The view's columns: column names, ``(alias, SQL expression)``
pairs, or a dict of the same. Omitting it selects every source
column, expanded against the source schema at creation time.
where: str, optional
SQL predicate; only matching source rows appear in the view.
limit: int, optional
Cap the view at this many rows, in materialization order.
Returns
-------
MaterializedView
"""
raise NotImplementedError(
"materialized views are not supported on this connection type"
)
def open_materialized_view(self, name: str) -> MaterializedView:
"""Open the materialized view named ``name``.
Raises ``ValueError`` if the table exists but is not a materialized
view.
"""
raise NotImplementedError(
"materialized views are not supported on this connection type"
)
def list_materialized_views(self) -> List[str]:
"""The names of the materialized views in this database.
Found by reading every table's schema, so this costs an open per
table.
"""
raise NotImplementedError(
"materialized views are not supported on this connection type"
)
def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
"""Drop a table from the database.
@@ -1136,6 +1206,58 @@ class LanceDBConnection(DBConnection):
tbl.checkout(version)
return tbl
@override
def create_materialized_view(
self,
name: str,
source: str,
*,
select: SelectArg = None,
where: Optional[str] = None,
limit: Optional[int] = None,
) -> MaterializedView:
"""Define a materialized view named ``name`` over the table ``source``.
See
[DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view].
Examples
--------
>>> import lancedb
>>> db = lancedb.connect(
... "./.lancedb",
... storage_options={"new_table_enable_stable_row_ids": "true"},
... )
>>> data = [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]
>>> table = db.create_table("people", data)
>>> view = db.create_materialized_view(
... "adults",
... "people",
... select=["name", ("shout", "upper(name)")],
... where="age >= 18",
... )
>>> result = view.refresh()
>>> result.rows_written
1
"""
LOOP.run(
self._conn.create_materialized_view(
name, source, select=select, where=where, limit=limit
)
)
return MaterializedView(self.open_table(name))
@override
def open_materialized_view(self, name: str) -> MaterializedView:
"""Open the materialized view named ``name``."""
view = MaterializedView(self.open_table(name))
view.definition
return view
@override
def list_materialized_views(self) -> List[str]:
"""The names of the materialized views in this database."""
return LOOP.run(self._conn.list_materialized_views())
def clone_table(
self,
target_table_name: str,
@@ -1906,6 +2028,50 @@ class AsyncConnection(object):
await tbl.checkout(version)
return tbl
async def create_materialized_view(
self,
name: str,
source: str,
*,
select: SelectArg = None,
where: Optional[str] = None,
limit: Optional[int] = None,
) -> AsyncMaterializedView:
"""Define a materialized view named ``name`` over the table ``source``.
See
[DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view].
"""
inner = await self._inner.create_materialized_view(
name,
source,
projections=normalize_select(select),
filter=where,
limit=limit,
)
return AsyncMaterializedView(AsyncTable(inner))
async def open_materialized_view(self, name: str) -> AsyncMaterializedView:
"""Open the materialized view named ``name``.
Raises ``ValueError`` if the table exists but is not a materialized
view.
"""
if self.uri.startswith("db://"):
raise NotImplementedError(
"materialized views are supported only on local databases"
)
view = AsyncMaterializedView(await self.open_table(name))
await view.definition()
return view
async def list_materialized_views(self) -> List[str]:
"""The names of the materialized views in this database.
Found by reading every table's schema, so this costs an open per
table.
"""
return await self._inner.list_materialized_views()
async def clone_table(
self,
target_table_name: str,
@@ -2071,7 +2237,7 @@ class AsyncConnection(object):
inner = await self._inner.create_function_async(
definition.registration_request.to_canonical_json()
)
return _function_job(inner)
return _typed_job(inner, FunctionVersion.from_json)
async def get_function(self, name: str, *, version: str) -> FunctionVersion:
"""Open one exact immutable Function version from the remote catalog."""
python/python/lancedb/functions.py
+197 -61
View File
@@ -1,19 +1,24 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Canonical values exchanged with LanceDB Enterprise Function services.
"""Canonical Function values exchanged with LanceDB Enterprise services.
These immutable models contain client/wire state only. Catalog persistence,
environment bake, secret resolution, and execution are owned by Sophon.
``RefreshColumnResult`` is also the backend-neutral result of a local
expression-backed refresh job.
"""
from __future__ import annotations
import ast
import builtins
import base64
import functools
import hashlib
import importlib
import inspect
import symtable
import json
import math
import re
@@ -460,7 +465,11 @@ class FunctionBinding(_RemoteValue):
class RefreshColumnResult(_RemoteValue):
"""Terminal result of a remote Function-column refresh Job."""
"""Terminal result of an expression-backed or Function-backed refresh Job.
Local jobs produce this value in process. LanceDB Cloud and Enterprise
decode the same value from the durable server-job terminal payload.
"""
rows_assigned: _UInt64
rows_failed: _UInt64
@@ -483,59 +492,58 @@ _FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$")
_SECRET_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
_GRAMMAR_PRIMITIVES = (
(pa.bool_(), "bool"),
(pa.int8(), "int8"),
(pa.int16(), "int16"),
(pa.int32(), "int32"),
(pa.int64(), "int64"),
(pa.uint8(), "uint8"),
(pa.uint16(), "uint16"),
(pa.uint32(), "uint32"),
(pa.uint64(), "uint64"),
(pa.float16(), "float16"),
(pa.float32(), "float32"),
(pa.float64(), "float64"),
(pa.string(), "utf8"),
(pa.binary(), "binary"),
(pa.date32(), "date32"),
(pa.date64(), "date64"),
)
def _canonical_arrow_type(data_type: pa.DataType) -> str:
primitive_types = (
(pa.bool_(), "bool"),
(pa.int8(), "int8"),
(pa.int16(), "int16"),
(pa.int32(), "int32"),
(pa.int64(), "int64"),
(pa.uint8(), "uint8"),
(pa.uint16(), "uint16"),
(pa.uint32(), "uint32"),
(pa.uint64(), "uint64"),
(pa.float16(), "float16"),
(pa.float32(), "float32"),
(pa.float64(), "float64"),
(pa.string(), "utf8"),
(pa.large_utf8(), "large_utf8"),
(pa.binary(), "binary"),
(pa.large_binary(), "large_binary"),
(pa.date32(), "date32"),
(pa.date64(), "date64"),
)
for candidate, name in primitive_types:
"""The server's V1 Function type grammar. Anything outside it is rejected
here rather than at registration."""
for candidate, name in _GRAMMAR_PRIMITIVES:
if data_type == candidate:
return name
if pa.types.is_fixed_size_binary(data_type):
return f"fixed_size_binary[{data_type.byte_width}]"
if pa.types.is_list(data_type):
return f"list<{_canonical_arrow_type(data_type.value_type)}>"
if pa.types.is_large_list(data_type):
return f"large_list<{_canonical_arrow_type(data_type.value_type)}>"
if pa.types.is_fixed_size_list(data_type):
if pa.types.is_list(data_type) or pa.types.is_large_list(data_type):
prefix = "list" if pa.types.is_list(data_type) else "large_list"
return f"{prefix}<{_canonical_list_item(data_type)}>"
if pa.types.is_fixed_size_list(data_type) and data_type.list_size > 0:
return (
f"fixed_size_list<{_canonical_arrow_type(data_type.value_type)}>"
f"[{data_type.list_size}]"
f"fixed_size_list<{_canonical_list_item(data_type)}, {data_type.list_size}>"
)
if pa.types.is_struct(data_type):
fields = ",".join(
f"{field.name}:{_canonical_arrow_type(field.type)}" for field in data_type
)
return f"struct<{fields}>"
if pa.types.is_timestamp(data_type):
timezone = f",tz={data_type.tz}" if data_type.tz is not None else ""
return f"timestamp[{data_type.unit}{timezone}]"
if pa.types.is_time32(data_type) or pa.types.is_time64(data_type):
return f"time[{data_type.unit}]"
if pa.types.is_duration(data_type):
return f"duration[{data_type.unit}]"
if pa.types.is_decimal(data_type):
bit_width = data_type.bit_width
return f"decimal{bit_width}({data_type.precision},{data_type.scale})"
raise TypeError(f"unsupported Arrow type for Function signature: {data_type}")
def _canonical_list_item(data_type: pa.DataType) -> str:
"""The grammar names only the item type; it always means a non-nullable
child called `item`, so any other child metadata cannot be represented."""
child = data_type.value_field
if child.name != "item" or child.nullable or child.metadata:
raise TypeError(
"unsupported Arrow type for Function signature: list items must be a "
f"non-nullable field named 'item', got {child}"
)
return _canonical_arrow_type(child.type)
def _list_of(item: pa.DataType) -> pa.DataType:
return pa.list_(pa.field("item", item, nullable=False))
def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]:
nullable = False
origin = get_origin(annotation)
@@ -583,7 +591,7 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]:
value_type, value_nullable = _annotation_type(arguments[0])
if value_nullable:
raise TypeError("nullable Function list elements are not supported")
return pa.list_(value_type), nullable
return _list_of(value_type), nullable
raise TypeError(f"unsupported Function annotation: {annotation!r}")
@@ -730,6 +738,104 @@ def _literal_source(value: Any) -> str:
)
_DYNAMIC_NAMESPACE_ACCESS = frozenset(
{"globals", "locals", "vars", "eval", "exec", "compile", "__import__"}
)
# Modules that hand out namespaces (`sys.modules`, `builtins`, importers,
# introspection). The artifact's module namespace holds only the names it was
# packaged with, so reaching around it cannot be represented.
_NAMESPACE_MODULES = frozenset(
{"sys", "builtins", "importlib", "inspect", "gc", "ctypes", "types"}
)
def _namespace_acquisition(
definition: ast.FunctionDef, references: set[str]
) -> list[str]:
found = set(references & _DYNAMIC_NAMESPACE_ACCESS)
for node in ast.walk(definition):
if isinstance(node, ast.Import):
found.update(
alias.name
for alias in node.names
if alias.name.split(".")[0] in _NAMESPACE_MODULES
)
elif isinstance(node, ast.ImportFrom) and node.module:
if node.module.split(".")[0] in _NAMESPACE_MODULES:
found.add(node.module)
return sorted(found)
def _module_references(module_source: str) -> set[str]:
"""Names any scope in `module_source` binds or loads at module scope.
Python's own scope analysis on the exact text that ships: free variables
belong to an enclosing scope inside the function, and postponed
annotations are not runtime loads."""
def visit(table: symtable.SymbolTable, found: set[str]) -> None:
for symbol in table.get_symbols():
if symbol.is_global() and (
symbol.is_referenced() or symbol.is_declared_global()
):
found.add(symbol.get_name())
for child in table.get_children():
visit(child, found)
found: set[str] = set()
for table in symtable.symtable(module_source, "<udf>", "exec").get_children():
visit(table, found)
return found
def _global_source(name: str, value: Any) -> str:
"""One module-level line that rebinds `name` to `value` in the artifact:
an import for modules and importable classes/functions, a literal otherwise."""
if isinstance(value, types.ModuleType):
if value.__name__.split(".")[0] in _NAMESPACE_MODULES:
raise ValueError(
f"@udf cannot package dynamic namespace access: {value.__name__!r}"
)
try:
imported = importlib.import_module(value.__name__)
except ImportError:
imported = None
if imported is not value:
raise TypeError(
f"Function source references module {name!r} that does not import "
f"as {value.__name__!r}"
)
return f"import {value.__name__} as {name}"
module_name = getattr(value, "__module__", None)
qualname = getattr(value, "__qualname__", None)
if (
isinstance(module_name, str)
and isinstance(qualname, str)
and module_name != "__main__"
and "." not in qualname
and "<" not in qualname
):
try:
imported = getattr(importlib.import_module(module_name), qualname)
except (ImportError, AttributeError):
imported = None
if imported is value:
return f"from {module_name} import {qualname} as {name}"
return f"{name} = {_literal_source(value)}"
def _is_recursive_reference(function: Callable[..., Any], name: str) -> bool:
"""`name` inside the body means the function itself unless the module has
since bound it to something else."""
if name != function.__name__:
return False
bound = function.__globals__.get(name, function)
if bound is function:
return True
# The decorator's own result is the one wrapper known to call `function`
# unchanged; any other binding may behave differently from a self-call.
return type(bound) is UdfDefinition and bound._function is function
def _package_source(function: Callable[..., Any]) -> bytes:
if not inspect.isfunction(function) or inspect.iscoroutinefunction(function):
raise TypeError("@udf requires a synchronous Python function")
@@ -754,23 +860,46 @@ def _package_source(function: Callable[..., Any]) -> bytes:
closure = inspect.getclosurevars(function)
if closure.nonlocals:
raise ValueError("@udf cannot package functions that capture closure values")
if closure.unbound:
raise ValueError(
f"@udf source contains unresolved global names: {sorted(closure.unbound)!r}"
)
globals_source = []
for name, value in sorted(closure.globals.items()):
if isinstance(value, types.ModuleType):
globals_source.append(f"import {value.__name__} as {name}")
else:
globals_source.append(f"{name} = {_literal_source(value)}")
function_source = ast.unparse(definition)
parts = ["from __future__ import annotations"]
module_header = "from __future__ import annotations"
references = _module_references(f"{module_header}\n\n{function_source}\n")
dynamic = _namespace_acquisition(definition, references)
if dynamic:
raise ValueError(f"@udf cannot package dynamic namespace access: {dynamic!r}")
# Resolve every module-scope reference the way the interpreter would: the
# function's own globals first (a module global may shadow a builtin, and
# nested scopes are not visible to getclosurevars), then its builtins.
# The artifact runs under the standard builtins; only the exact mapping is
# provably equivalent (a subclass or copy can change lookups and hooks).
if function.__builtins__ is not vars(builtins):
raise ValueError("@udf cannot package a non-standard builtins environment")
globals_source = []
unresolved = []
for name in sorted(references):
if name == function.__name__:
if not _is_recursive_reference(function, name):
raise ValueError(
f"@udf cannot package {name!r}: the module binds that name to "
"another value, which the artifact's own definition would shadow"
)
continue
if name in function.__globals__:
globals_source.append(_global_source(name, function.__globals__[name]))
elif hasattr(builtins, name):
pass
else:
unresolved.append(name)
if unresolved:
raise ValueError(
f"@udf source contains unresolved global names: {unresolved!r}"
)
parts = [module_header]
if globals_source:
parts.extend(["", *globals_source])
parts.extend(["", function_source, ""])
return "\n".join(parts).encode("utf-8")
packaged = "\n".join(parts)
return packaged.encode("utf-8")
class UdfDefinition:
@@ -917,6 +1046,13 @@ def udf(
python_version : str, optional
Remote Python major/minor version. Defaults to the client version.
The packaged artifact is a snapshot: the function source plus exactly
the module-level names it references (modules as imports, importable
classes and functions as imports, literals inline). Code that reaches the
module namespace another way -- ``globals()``/``eval``, ``sys.modules``,
``builtins`` -- is rejected where it can be seen and otherwise
unsupported; closures and a non-standard ``__builtins__`` are rejected.
Returns
-------
UdfDefinition
python/python/lancedb/index.py
+11
View File
@@ -163,6 +163,15 @@ class FTS:
The number of documents per compressed posting block. Supported values
are 128 and 256. A value of 256 uses the experimental FTS V3 format
and may introduce breaking changes.
memory_limit : int, optional
The total memory limit in MiB for the local FTS build stage. The limit
is divided evenly among indexing workers. This build-only setting is
not persisted with the index and does not apply to remote tables.
num_workers : int, optional
The number of workers for a local FTS build. By default Lance uses
roughly half of the available CPU cores. The effective value is
limited by the available compute capacity. This build-only setting is
not persisted with the index and does not apply to remote tables.
Notes
-----
@@ -185,6 +194,8 @@ class FTS:
prefix_only: bool = False
block_size: int = 128
custom_stop_words: Optional[List[str]] = None
memory_limit: Optional[int] = None
num_workers: Optional[int] = None
@dataclass
python/python/lancedb/job.py
+28 -30
View File
@@ -5,12 +5,11 @@
import asyncio
from datetime import timedelta
from typing import Any, Generic, Optional, TypeVar, cast
from typing import Any, Callable, Generic, Optional, TypeVar, cast
from lancedb.background_loop import LOOP
from . import _lancedb
from .functions import FunctionVersion
T = TypeVar("T")
@@ -18,11 +17,18 @@ T = TypeVar("T")
class AsyncJob(Generic[T]):
"""A handle to an operation that may still be running.
The operation may already be complete when the handle is created.
The operation may already be complete when the handle is created. ``T``
is the endpoint's terminal result type; unit-result jobs resolve to
``None``.
"""
def __init__(self, inner: Optional[Any]):
def __init__(
self,
inner: Optional[Any],
result_decoder: Optional[Callable[[Any], T]] = None,
):
self._inner = inner
self._result_decoder = result_decoder
@property
def id(self) -> Optional[str]:
@@ -50,17 +56,21 @@ class AsyncJob(Generic[T]):
async def wait(self, timeout: Optional[timedelta] = None) -> T:
"""Wait until the operation reaches a terminal state.
Returns the endpoint's typed result, or ``None`` for a unit-result
job.
Raises `JobFailedError` if the operation failed, `JobCancelledError`
if it was cancelled, and `TimeoutError` if `timeout` elapses first.
"""
if self._inner is None:
return cast(T, None)
if timeout is None:
return cast(T, await self._inner.wait())
return cast(
T,
await asyncio.wait_for(self._inner.wait(), timeout.total_seconds()),
)
result = await self._inner.wait()
else:
result = await asyncio.wait_for(self._inner.wait(), timeout.total_seconds())
if self._result_decoder is not None:
return self._result_decoder(result)
return cast(T, result)
async def cancel(self):
"""Request cancellation. Cancelling a finished operation is a no-op."""
@@ -70,7 +80,7 @@ class AsyncJob(Generic[T]):
class Job(Generic[T]):
"""Synchronous counterpart of `AsyncJob`."""
"""Synchronous counterpart of `AsyncJob` with the same result type."""
def __init__(self, inner: Optional[AsyncJob[T]]):
self._inner = inner
@@ -96,6 +106,9 @@ class Job(Generic[T]):
def wait(self, timeout: Optional[timedelta] = None) -> T:
"""Block until the operation reaches a terminal state.
Returns the endpoint's typed result, or ``None`` for a unit-result
job.
Raises `JobFailedError` if the operation failed, `JobCancelledError`
if it was cancelled, and `TimeoutError` if `timeout` elapses first.
"""
@@ -110,23 +123,8 @@ class Job(Generic[T]):
LOOP.run(self._inner.cancel())
class _FunctionJobAdapter:
def __init__(self, inner: "_lancedb.FunctionJob"):
self._inner = inner
@property
def id(self) -> Optional[str]:
return self._inner.id
async def status(self) -> str:
return await self._inner.status()
async def wait(self) -> FunctionVersion:
return FunctionVersion.from_json(await self._inner.wait())
async def cancel(self):
await self._inner.cancel()
def _function_job(inner: "_lancedb.FunctionJob") -> AsyncJob[FunctionVersion]:
return AsyncJob(_FunctionJobAdapter(inner))
def _typed_job(
inner: "_lancedb.Job", result_decoder: Callable[[str], T]
) -> AsyncJob[T]:
"""Bind an internal JSON-producing job to its public result model."""
return AsyncJob(inner, result_decoder)
python/python/lancedb/materialized_view.py
+178
View File
@@ -0,0 +1,178 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Materialized views: tables defined by a query over a source table and
maintained by refresh. See ``DBConnection.create_materialized_view``."""
from __future__ import annotations
import json
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Tuple, Union
from .background_loop import LOOP
if TYPE_CHECKING:
import pyarrow as pa
from ._lancedb import RefreshMaterializedViewResult
from .table import AsyncTable, LanceTable
DEFINITION_META_KEY = b"mv.definition"
SelectArg = Union[
str,
Sequence[Union[str, Tuple[str, str]]],
Dict[str, str],
None,
]
@dataclass
class MaterializedViewDefinition:
"""The query that defines a materialized view."""
source_table: str
"""Name of the source table, in the same database as the view."""
projections: List[Tuple[str, str]]
"""``(output column, SQL expression)`` pairs, in view schema order."""
filter: Optional[str] = None
"""SQL predicate selecting the source rows the view holds."""
limit: Optional[int] = None
"""Cap on the number of rows the view holds."""
inputs: List[str] = field(default_factory=list)
"""Source columns the projections and filter read."""
def _definition_from_schema(
schema: "pa.Schema", name: str
) -> MaterializedViewDefinition:
metadata = schema.metadata or {}
raw = metadata.get(DEFINITION_META_KEY)
if raw is None:
raise ValueError(f"Table '{name}' is not a materialized view")
value = json.loads(raw)
kind = value.get("kind")
if kind != "select":
raise NotImplementedError(
f"materialized view '{name}' is defined by '{kind}', which this "
"version of lancedb cannot refresh"
)
return MaterializedViewDefinition(
source_table=value["source_table"],
projections=[
(p["output"], p["expression"]) for p in value.get("projections", [])
],
filter=value.get("filter"),
limit=value.get("limit"),
inputs=value.get("inputs", []),
)
def _quote_identifier(name: str) -> str:
"""Quote a column name as a Lance SQL identifier (backticks)."""
escaped = name.replace("`", "``")
return f"`{escaped}`"
def normalize_select(select: SelectArg) -> Optional[List[Tuple[str, str]]]:
"""``select`` items may be a column name, an ``(alias, expression)`` pair,
or a dict of the same. A bare name projects itself and is quoted, so any
valid column name works; dict and pair entries are kept verbatim because
their right side is an expression.
A lone string is one column, not a sequence of its characters."""
if select is None:
return None
if isinstance(select, str):
select = [select]
if isinstance(select, dict):
return list(select.items())
normalized = []
for item in select:
if isinstance(item, str):
normalized.append((item, _quote_identifier(item)))
else:
alias, expression = item
normalized.append((alias, expression))
return normalized
class AsyncMaterializedView:
"""A handle on a materialized view: its table plus its definition.
Obtained from ``AsyncConnection.create_materialized_view`` or
``AsyncConnection.open_materialized_view``.
"""
def __init__(self, table: "AsyncTable"):
self._table = table
def __repr__(self) -> str:
return f"AsyncMaterializedView(name={self.name!r})"
@property
def name(self) -> str:
return self._table.name
@property
def table(self) -> "AsyncTable":
"""The view, as the table it is. Queries, indexes and search all
apply; writes are not blocked, but a rebuild replaces them."""
return self._table
async def definition(self) -> MaterializedViewDefinition:
"""The query that defines the view, read from its stored schema."""
return _definition_from_schema(await self._table.schema(), self.name)
async def refresh(
self, *, full: bool = False, source_version: Optional[int] = None
) -> "RefreshMaterializedViewResult":
"""Recompute the view from its source.
The refresh is incremental when the source's changes can be
reconciled into the view -- rows added, changed or removed since the
last one -- and otherwise rebuilds. ``full=True`` forces a rebuild;
``source_version`` refreshes to that source version instead of the
latest.
Concurrent refreshes of one view do not duplicate its rows. Two that
plan the same source rows conflict on commit, and the loser raises
rather than writing them a second time.
"""
return await self._table._inner.refresh_materialized_view(
full=full, source_version=source_version
)
class MaterializedView:
"""Synchronous variant of
[AsyncMaterializedView][lancedb.materialized_view.AsyncMaterializedView]."""
def __init__(self, table: "LanceTable"):
self._table = table
self._async = AsyncMaterializedView(table._table)
def __repr__(self) -> str:
return f"MaterializedView(name={self.name!r})"
@property
def name(self) -> str:
return self._table.name
@property
def table(self) -> "LanceTable":
"""The view, as the table it is."""
return self._table
@property
def definition(self) -> MaterializedViewDefinition:
"""The query that defines the view, read from its stored schema."""
return _definition_from_schema(self._table.schema, self.name)
def refresh(
self, *, full: bool = False, source_version: Optional[int] = None
) -> "RefreshMaterializedViewResult":
"""Recompute the view from its source. See
[AsyncMaterializedView.refresh][lancedb.materialized_view.AsyncMaterializedView.refresh]."""
return LOOP.run(self._async.refresh(full=full, source_version=source_version))
python/python/lancedb/namespace.py
+68
View File
@@ -61,6 +61,11 @@ from lance_namespace import (
NamespaceExistsRequest,
TableExistsRequest,
)
from lancedb.materialized_view import (
AsyncMaterializedView,
MaterializedView,
SelectArg,
)
from lancedb.table import AsyncTable, LanceTable, Table
from lancedb.util import validate_table_name
from lancedb.common import DATA
@@ -619,6 +624,42 @@ class LanceNamespaceDBConnection(DBConnection):
tbl.checkout(version)
return tbl
@override
def create_materialized_view(
self,
name: str,
source: str,
*,
select: "SelectArg" = None,
where: Optional[str] = None,
limit: Optional[int] = None,
) -> "MaterializedView":
"""Define a materialized view over a table in the root namespace.
See
[DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view].
"""
return MaterializedView(
self.open_table(
LOOP.run(
self._inner.create_materialized_view(
name, source, select=select, where=where, limit=limit
)
).name
)
)
@override
def open_materialized_view(self, name: str) -> "MaterializedView":
"""Open the materialized view named ``name``."""
view = MaterializedView(self.open_table(name))
view.definition
return view
@override
def list_materialized_views(self) -> List[str]:
"""The names of the materialized views in the root namespace."""
return LOOP.run(self._inner.list_materialized_views())
@override
def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
if namespace_path is None:
@@ -1141,6 +1182,33 @@ class AsyncLanceNamespaceDBConnection:
route_pushdown_to_rust=self._route_pushdown_to_rust,
)
async def create_materialized_view(
self,
name: str,
source: str,
*,
select: "SelectArg" = None,
where: Optional[str] = None,
limit: Optional[int] = None,
) -> "AsyncMaterializedView":
"""Define a materialized view over a table in the root namespace."""
view = await self._inner.create_materialized_view(
name, source, select=select, where=where, limit=limit
)
# Reopen through the namespace so the view's table carries the
# namespace client and pushdown configuration a bare inner table lacks.
return AsyncMaterializedView(await self.open_table(view.name))
async def open_materialized_view(self, name: str) -> "AsyncMaterializedView":
"""Open the materialized view named ``name``."""
view = AsyncMaterializedView(await self.open_table(name))
await view.definition()
return view
async def list_materialized_views(self) -> List[str]:
"""The names of the materialized views in the root namespace."""
return await self._inner.list_materialized_views()
async def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
"""Drop a table from the namespace."""
if namespace_path is None:
python/python/lancedb/permutation.py
+21 -5
View File
@@ -391,6 +391,15 @@ def _table_to_pickle_state(table: Table) -> dict[str, Any]:
}
def _drop_base_version(permutation_data: pa.Table) -> pa.Table:
"""Strip the recorded base version so the reader leaves the base table unpinned."""
metadata = dict(permutation_data.schema.metadata or {})
if metadata.pop(b"base_version", None) is None:
return permutation_data
metadata.pop(b"base_branch", None)
return permutation_data.replace_schema_metadata(metadata)
def _table_from_pickle_state(state: dict[str, Any]) -> Table:
from . import connect
@@ -679,11 +688,15 @@ class Permutation:
from . import connect
connection_factory = state["connection_factory"]
rebuilt_base = False
if connection_factory is not None:
base_table = connection_factory(state["base_table_name"])
elif "base_table_state" in state:
base_table = _table_from_pickle_state(state["base_table_state"])
base_state = state["base_table_state"]
rebuilt_base = base_state["kind"] == "memory"
base_table = _table_from_pickle_state(base_state)
elif "base_table_data" in state:
rebuilt_base = True
# In-memory base table inlined into the pickle; rebuild the same
# way we rebuild the in-memory permutation table.
mem_db = connect("memory://")
@@ -701,11 +714,14 @@ class Permutation:
)
permutation_table: Optional[Table] = None
if state["permutation_data"] is not None:
permutation_data = state["permutation_data"]
if permutation_data is not None:
if rebuilt_base:
# The base table was materialized from Arrow, so it is a fresh
# single-version dataset and the recorded pin cannot resolve on it.
permutation_data = _drop_base_version(permutation_data)
mem_db = connect("memory://")
permutation_table = mem_db.create_table(
"permutation", state["permutation_data"]
)
permutation_table = mem_db.create_table("permutation", permutation_data)
self.base_table = base_table
self.permutation_table = permutation_table
python/python/lancedb/remote/db.py
+27
View File
@@ -25,6 +25,7 @@ from ..common import DATA
from ..db import DBConnection, LOOP
from ..functions import FunctionVersion, UdfDefinition
from ..job import AsyncJob, Job
from ..materialized_view import MaterializedView, SelectArg
if TYPE_CHECKING:
from .._lancedb import JobDescription, JobInfo
@@ -648,6 +649,32 @@ class RemoteDBConnection(DBConnection):
namespace_path=namespace_path,
)
@override
def create_materialized_view(
self,
name: str,
source: str,
*,
select: SelectArg = None,
where: Optional[str] = None,
limit: Optional[int] = None,
) -> MaterializedView:
raise NotImplementedError(
"materialized views are supported only on local databases"
)
@override
def open_materialized_view(self, name: str) -> MaterializedView:
raise NotImplementedError(
"materialized views are supported only on local databases"
)
@override
def list_materialized_views(self) -> List[str]:
raise NotImplementedError(
"materialized views are supported only on local databases"
)
@override
def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
"""Drop a table from the database.
python/python/lancedb/remote/table.py
+2 -2
View File
@@ -49,7 +49,7 @@ from lancedb.index import (
LabelList,
)
from lancedb.job import Job
from lancedb.functions import FunctionApplication
from lancedb.functions import FunctionApplication, RefreshColumnResult
from lancedb.remote.db import LOOP
from lancedb.table import IndexConfigType, KNOWN_METRICS
import pyarrow as pa
@@ -972,7 +972,7 @@ class RemoteTable(Table):
def refresh_column(self, column: str):
return LOOP.run(self._table.refresh_column(column))
def refresh_column_async(self, column: str) -> Job:
def refresh_column_async(self, column: str) -> Job[RefreshColumnResult]:
return Job(LOOP.run(self._table.refresh_column_async(column)))
def alter_columns(
python/python/lancedb/streaming.py
+1037 -91
View File
File diff suppressed because it is too large Load Diff
python/python/lancedb/table.py
+43 -18
View File
@@ -40,7 +40,7 @@ from ._blob import (
from .types import BlobMode
from lancedb.arrow import peek_reader
from lancedb.background_loop import LOOP, embedding_executor
from lancedb.job import AsyncJob, Job
from lancedb.job import AsyncJob, Job, _typed_job
from .dependencies import (
_check_for_hugging_face,
_check_for_lance,
@@ -72,7 +72,10 @@ from .index import (
FTS,
)
from .expr import Expr
from .functions import FunctionApplication
from .functions import (
FunctionApplication,
RefreshColumnResult as RefreshColumnJobResult,
)
from .merge import LanceMergeInsertBuilder
from .pydantic import LanceModel, model_to_dict
from .query import (
@@ -2039,7 +2042,7 @@ class Table(ABC):
"""
@abstractmethod
def refresh_column_async(self, column: str) -> Job:
def refresh_column_async(self, column: str) -> Job[RefreshColumnJobResult]:
"""
Like :meth:`refresh_column`, but returns a handle to the refresh job
instead of blocking until it completes.
@@ -2050,6 +2053,12 @@ class Table(ABC):
than failing the job. On local tables the job runs in-process; on
LanceDB Cloud and Enterprise it is the server's backfill job.
Returns
-------
Job[RefreshColumnResult]
A job whose successful ``wait`` returns row counts plus the source
and published table versions.
Examples
--------
>>> import lancedb
@@ -2058,7 +2067,9 @@ class Table(ABC):
>>> table.add_columns(computed={"doubled": "x * 2"})
AddColumnsResult(version=2)
>>> job = table.refresh_column_async("doubled")
>>> job.wait()
>>> result = job.wait()
>>> result.rows_assigned
2
>>> job.status()
'finished'
"""
@@ -4082,7 +4093,7 @@ class LanceTable(Table):
[`AsyncTable.refresh_column`][lancedb.AsyncTable.refresh_column]."""
return LOOP.run(self._table.refresh_column(column))
def refresh_column_async(self, column: str) -> Job:
def refresh_column_async(self, column: str) -> Job[RefreshColumnJobResult]:
"""Fill a computed column's unfilled rows, returning a handle to the
refresh job. See
[`Table.refresh_column_async`][lancedb.table.Table.refresh_column_async].
@@ -6122,7 +6133,9 @@ class AsyncTable:
"""
return await self._inner.refresh_column(column)
async def refresh_column_async(self, column: str) -> AsyncJob:
async def refresh_column_async(
self, column: str
) -> AsyncJob[RefreshColumnJobResult]:
"""
Like :meth:`refresh_column`, but returns a handle to the refresh job
instead of blocking until it completes.
@@ -6134,6 +6147,12 @@ class AsyncTable:
in-process; on LanceDB Cloud and Enterprise it is the server's
backfill job.
Returns
-------
AsyncJob[RefreshColumnResult]
A job whose successful ``wait`` returns row counts plus the source
and published table versions.
Examples
--------
>>> import asyncio
@@ -6143,12 +6162,16 @@ class AsyncTable:
... table = await db.create_table("computed_job_async_demo", [{"x": 1}])
... await table.add_columns(computed={"doubled": "x * 2"})
... job = await table.refresh_column_async("doubled")
... await job.wait()
... result = await job.wait()
... assert result.rows_assigned == 1
... return await job.status()
>>> asyncio.run(refresh_in_background())
'finished'
"""
return AsyncJob(await self._inner.refresh_column_async(column))
return _typed_job(
await self._inner.refresh_column_async(column),
RefreshColumnJobResult.from_json,
)
async def alter_columns(
self, *alterations: Iterable[dict[str, Any]]
@@ -6801,21 +6824,21 @@ class Branches:
"""Diff a branch against main."""
return LOOP.run(self._table.branches.diff(from_branch))
def merge(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]:
"""Merge a branch into main, or dry-run.
def cherry_pick(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]:
"""Cherry-pick a branch onto main, or dry-run.
Parameters
----------
from_branch: str
Branch to merge from.
Branch to cherry-pick from.
dry_run: bool, default False
When True, only preview. When False, attempt the merge.
When True, only preview. When False, attempt the cherry-pick.
Notes
-----
A rejected merge returns ``status="rejected"`` instead of raising.
A failed cherry-pick returns ``status="failed"`` instead of raising.
"""
return LOOP.run(self._table.branches.merge(from_branch, dry_run))
return LOOP.run(self._table.branches.cherry_pick(from_branch, dry_run))
def _wrap(
self, async_table: "AsyncTable", version: Optional[int] = None
@@ -6951,9 +6974,11 @@ class AsyncBranches:
"""Diff a branch against main."""
return await self._table.branches.diff(from_branch)
async def merge(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]:
"""Merge a branch into main, or dry-run.
async def cherry_pick(
self, from_branch: str, dry_run: bool = False
) -> Dict[str, Any]:
"""Cherry-pick a branch onto main, or dry-run.
A rejected merge returns ``status="rejected"`` instead of raising.
A failed cherry-pick returns ``status="failed"`` instead of raising.
"""
return await self._table.branches.merge(from_branch, dry_run)
return await self._table.branches.cherry_pick(from_branch, dry_run)
python/python/tests/test_db.py
+2 -2
View File
@@ -774,7 +774,7 @@ def test_drop_table_async(tmp_db: lancedb.DBConnection):
job = tmp_db.drop_table_async("test")
assert job.id is None
assert job.status() == "finished"
job.wait()
assert job.wait() is None
assert tmp_db.table_names() == []
tmp_db.create_table("test", data=data)
@@ -790,7 +790,7 @@ async def test_drop_table_async_connection(tmp_db_async: lancedb.AsyncConnection
job = await tmp_db_async.drop_table_async("test")
assert job.id is None
assert await job.status() == "finished"
await job.wait()
assert await job.wait() is None
assert await tmp_db_async.table_names() == []
python/python/tests/test_elastic_dataloader.py
+1050 -2
View File
File diff suppressed because it is too large Load Diff
python/python/tests/test_first_class_function_slice2.py
+398 -3
View File
@@ -3,7 +3,12 @@
from __future__ import annotations
import base64
import contextlib
import functools
import importlib.util
import types
from datetime import date
import http.server
import json
from pathlib import Path
@@ -16,6 +21,9 @@ import pytest
import lancedb
from lancedb.functions import UdfDefinition, udf
THRESHOLD = 20
_CACHE = None
FIXTURES = (
Path(__file__).parents[3]
@@ -71,9 +79,396 @@ def test_scalar_udf_matches_shared_registration_golden_and_remains_callable():
_assert_no_secret_values(request)
def _run_packaged(definition, *args):
"""Execute the shipped artifact in a fresh namespace, as a worker would."""
source = base64.b64decode(definition.registration_request.artifact.content.data)
namespace: dict = {}
exec(compile(source, "<udf>", "exec"), namespace)
return namespace[definition.registration_request.artifact.entrypoint](*args)
def test_udf_packages_attribute_access_and_body_imports():
@udf
def word_norm(body: str) -> float:
import numpy as np
try:
words = body.split()
except AttributeError as error:
raise ValueError(str(error)) from error
return float(np.linalg.norm([len(w) for w in words]))
assert _run_packaged(word_norm, "aa bb") == pytest.approx(8**0.5)
def test_udf_packages_module_globals_and_global_caches():
@udf
def label(value: int) -> str:
return "big" if value >= THRESHOLD else "small"
assert _run_packaged(label, 21) == "big"
@udf
def cached(value: int) -> int:
global _CACHE
if _CACHE is None:
_CACHE = 40
return _CACHE + value
assert _run_packaged(cached, 2) == 42
def test_udf_annotations_are_not_runtime_names():
@udf
def identity(value: date) -> date:
return value
assert _run_packaged(identity, date(2026, 8, 25)) == date(2026, 8, 25)
def test_udf_nested_scopes_resolve_lexically():
@udf
def score(value: int) -> int:
offset = 2
def add_offset() -> int:
return value + offset
return add_offset() + sum(v for v in [0])
assert _run_packaged(score, 3) == 5
def test_udf_resolves_module_globals_before_builtins(tmp_path):
module_path = tmp_path / "shadowing_udfs.py"
module_path.write_text(
"max = 7\n"
"len = lambda _: 99\n"
"\n"
"def uses_literal_shadow(value: int) -> int:\n"
" def nested() -> int:\n"
" return max\n"
" return nested() + value\n"
"\n"
"def uses_callable_shadow(value: int) -> int:\n"
" def nested() -> int:\n"
" return len([1])\n"
" return nested() + value\n"
)
spec = importlib.util.spec_from_file_location("shadowing_udfs", module_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
# The module's `max = 7` is what the interpreter would use, so it ships.
assert _run_packaged(udf(module.uses_literal_shadow), 1) == 8
# A callable global cannot ship; it must not be silently swapped for the builtin.
with pytest.raises(TypeError, match="unsupported global value of type function"):
udf(module.uses_callable_shadow)
def test_canonical_arrow_type_is_exactly_the_grammar():
from lancedb.functions import _GRAMMAR_PRIMITIVES, _canonical_arrow_type
golden = json.loads(
(
Path(__file__).parents[3]
/ "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json"
).read_text()
)
primitives = [
case["arrow_type"] for case in golden["valid"] if "<" not in case["arrow_type"]
]
assert [name for _, name in _GRAMMAR_PRIMITIVES] == primitives
for outside in [
pa.timestamp("us"),
pa.decimal128(10, 2),
pa.large_string(),
pa.large_binary(),
pa.binary(4),
pa.duration("s"),
pa.struct([pa.field("a", pa.int32())]),
pa.list_(pa.float32(), 0),
pa.list_(pa.timestamp("us")),
]:
with pytest.raises(TypeError, match="unsupported Arrow type"):
_canonical_arrow_type(outside)
def test_udf_nested_annotations_are_postponed_in_the_artifact():
@udf
def score(value: int) -> int:
def identity(item: date) -> date:
return item
identity(date(2026, 8, 25))
return value
assert _run_packaged(score, 3) == 3
def test_udf_ships_globals_the_body_deletes():
@udf
def clear(value: int) -> int:
global _CACHE
del _CACHE
return value
assert _run_packaged(clear, 3) == 3
def test_udf_rejects_a_module_global_that_does_not_import_as_itself(tmp_path):
module_path = tmp_path / "fake_module_udfs.py"
module_path.write_text(
"import types\n"
"np = types.ModuleType('numpy')\n"
"np.sqrt = lambda x: 0\n"
"\n"
"def score(value: int) -> int:\n"
" return int(np.sqrt(value))\n"
)
spec = importlib.util.spec_from_file_location("fake_module_udfs", module_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
with pytest.raises(TypeError, match="does not import as 'numpy'"):
udf(module.score)
def test_udf_rejects_a_module_level_namespace_alias(tmp_path):
module_path = tmp_path / "aliasing_udfs.py"
module_path.write_text(
"import builtins as b\n"
"THRESHOLD = 5\n"
"\n"
"def score(value: int) -> int:\n"
" return value + b.vars(b.__import__('aliasing_udfs'))['THRESHOLD']\n"
)
spec = importlib.util.spec_from_file_location("aliasing_udfs", module_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
with pytest.raises(ValueError, match="dynamic namespace access"):
udf(module.score)
@pytest.mark.parametrize(
"access",
[
"globals()['THRESHOLD']",
"eval('THRESHOLD')",
"(lambda g: g()['THRESHOLD'])(globals)",
"__import__('sys').modules[__name__].THRESHOLD",
"sys.modules[__name__].THRESHOLD",
],
)
def test_udf_rejects_dynamic_namespace_access(access):
namespace: dict = {}
exec(
f"def score(value: int) -> int:\n return value + {access}\n",
{"THRESHOLD": 5},
namespace,
)
with pytest.raises(ValueError, match="dynamic namespace access"):
_package_from_text(
"def score(value: int) -> int:\n"
" import sys\n"
f" return value + {access}\n"
)
def _package_from_text(source: str, module_globals: dict | None = None):
"""Load `source` as a real module file so the packager can inspect it."""
import tempfile
directory = tempfile.mkdtemp()
path = Path(directory) / "generated_udf_module.py"
path.write_text(source)
spec = importlib.util.spec_from_file_location(f"generated_udf_{id(source)}", path)
module = importlib.util.module_from_spec(spec)
if module_globals:
module.__dict__.update(module_globals)
spec.loader.exec_module(module)
functions = [
value
for value in vars(module).values()
if callable(value) and getattr(value, "__module__", None) == module.__name__
]
return udf(functions[0])
def test_udf_rejects_a_non_standard_builtins_environment():
def score(value: int) -> int:
return len([1]) + value
score.__globals__ # noqa: B018 -- real function, real globals
import builtins
patched = types.FunctionType(
score.__code__,
{"__builtins__": {**vars(builtins), "len": lambda _: 99}},
"score",
)
patched.__annotations__ = score.__annotations__
assert patched(3) == 102
with pytest.raises(ValueError, match="non-standard builtins environment"):
udf(patched)
class ReportingDict(dict): # reports standard entries, resolves differently
def __missing__(self, key):
return vars(builtins)[key]
disguised = types.FunctionType(
score.__code__, {"__builtins__": ReportingDict(len=lambda _: 99)}, "score"
)
disguised.__annotations__ = score.__annotations__
assert disguised(3) == 102
with pytest.raises(ValueError, match="non-standard builtins environment"):
udf(disguised)
hooked = types.FunctionType(
score.__code__,
{"__builtins__": {**vars(builtins), "__import__": lambda *a, **k: None}},
"score",
)
hooked.__annotations__ = score.__annotations__
with pytest.raises(ValueError, match="non-standard builtins environment"):
udf(hooked)
def test_udf_recursion_versus_a_rebound_module_name(tmp_path):
module_path = tmp_path / "rebound_udfs.py"
module_path.write_text(
"def fact(value: int) -> int:\n"
" return 1 if value <= 1 else value * fact(value - 1)\n"
"\n"
"def score(value: int) -> int:\n"
" return score + value\n"
)
spec = importlib.util.spec_from_file_location("rebound_udfs", module_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
assert _run_packaged(udf(module.fact), 5) == 120
raw = module.score
module.score = 10
with pytest.raises(ValueError, match="binds that name to another value"):
udf(raw)
# A wrapper that merely exposes __wrapped__ is not the function.
module.score = functools.wraps(raw)(lambda value: 41)
with pytest.raises(ValueError, match="binds that name to another value"):
udf(raw)
# The decorator's own result is; a subclass of it is not.
module.fact = udf(module.fact)
assert _run_packaged(module.fact, 4) == 24
class Twisted(UdfDefinition):
def __call__(self, *args, **kwargs):
return 41
raw_fact = module.fact._function
module.fact = Twisted(
raw_fact,
name=None,
input_schema=None,
output_schema=None,
pip=(),
env={},
secrets=(),
python_version=None,
)
with pytest.raises(ValueError, match="binds that name to another value"):
udf(raw_fact)
def test_canonical_arrow_type_rejects_unrepresentable_list_children():
from lancedb.functions import _canonical_arrow_type
for outside in [
pa.list_(pa.float32()), # pyarrow default: nullable child
pa.list_(pa.field("custom", pa.float32(), nullable=False)),
pa.list_(pa.field("item", pa.float32(), nullable=False, metadata={"k": "v"})),
pa.list_(pa.field("item", pa.float32(), nullable=False), 0),
]:
with pytest.raises(TypeError, match="unsupported Arrow type"):
_canonical_arrow_type(outside)
assert (
_canonical_arrow_type(
pa.list_(pa.field("item", pa.float32(), nullable=False), 3)
)
== "fixed_size_list<float32, 3>"
)
def _calls_missing(value: int) -> int:
return missing(value) # noqa: F821
def _shadows_missing_in_a_comprehension(value: int) -> int:
return missing(value) + sum(missing for missing in ()) # noqa: F821
def _shadows_missing_in_a_lambda(value: int) -> int:
return (lambda missing: missing)(value) + missing # noqa: F821
@pytest.mark.parametrize(
"function",
[_calls_missing, _shadows_missing_in_a_comprehension, _shadows_missing_in_a_lambda],
)
def test_udf_rejects_a_truly_unresolved_global(function):
with pytest.raises(ValueError, match=r"unresolved global names: \['missing'\]"):
udf(function)
def _arrow_type_from_golden(spec: dict) -> pa.DataType:
kind = spec["type"]
if kind in ("list", "large_list", "fixed_size_list"):
item = _arrow_type_from_golden(spec["fields"][0]["type"])
field = pa.field("item", item, nullable=False)
if kind == "list":
return pa.list_(field)
if kind == "large_list":
return pa.large_list(field)
return pa.list_(field, spec["length"])
return {
"null": pa.null(),
"bool": pa.bool_(),
"utf8": pa.string(),
"binary": pa.binary(),
"float16": pa.float16(),
"float32": pa.float32(),
"float64": pa.float64(),
"date32": pa.date32(),
"date64": pa.date64(),
}.get(kind) or getattr(pa, kind)()
def test_arrow_type_grammar_matches_the_shared_golden():
golden = json.loads(
(
Path(__file__).parents[3]
/ "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json"
).read_text()
)
from lancedb.functions import _canonical_arrow_type
emitted = {
case["arrow_type"]: _canonical_arrow_type(_arrow_type_from_golden(case["json"]))
for case in golden["valid"]
}
assert emitted == {
case["arrow_type"]: case["arrow_type"] for case in golden["valid"]
}
assert not set(emitted) & set(golden["invalid"])
for case in golden["server_only"]:
with pytest.raises(TypeError, match="unsupported Arrow type"):
_canonical_arrow_type(_arrow_type_from_golden(case["json"]))
def test_explicit_arrow_schema_is_deterministic():
input_schema = pa.schema([pa.field("value", pa.float32(), nullable=True)])
output_schema = pa.field("embedding", pa.list_(pa.float32(), 3), nullable=False)
output_schema = pa.field(
"embedding",
pa.list_(pa.field("item", pa.float32(), nullable=False), 3),
nullable=False,
)
@udf(input_schema=input_schema, output_schema=output_schema)
def explicit(value):
@@ -82,7 +477,7 @@ def test_explicit_arrow_schema_is_deterministic():
signature = explicit.registration_request.signature
assert signature.inputs[0].arrow_type == "float32"
assert signature.inputs[0].nullable is True
assert signature.output.arrow_type == "fixed_size_list<float32>[3]"
assert signature.output.arrow_type == "fixed_size_list<float32, 3>"
assert signature.output.nullable is False
@@ -187,7 +582,7 @@ def _mock_remote_function_catalog():
"job_state": "DONE",
"result": state["version"],
}
elif self.path == "/v1/functions/get":
elif self.path == "/v1/functions/describe":
assert body == {
"name": "normalize_score",
"version": "fv_exact",
python/python/tests/test_fts.py
+8
View File
@@ -245,6 +245,14 @@ def test_create_inverted_index_rejects_invalid_block_size(table):
table.create_index("text", config=FTS(block_size=129))
def test_create_inverted_index_respects_build_memory_limit(table):
with pytest.raises(ValueError, match="exceeds worker memory limit"):
table.create_index(
"text",
config=FTS(memory_limit=0, num_workers=1),
)
def test_custom_stop_words_list(table):
table.create_index(
"text",
python/python/tests/test_index.py
+1 -1
View File
@@ -88,7 +88,7 @@ async def binary_table(db_async):
async def test_create_index_async_returns_done_job(some_table: AsyncTable):
job = await some_table.create_index_async("id", config=BTree())
assert job.id is None
await job.wait()
assert await job.wait() is None
assert len(await some_table.list_indices()) == 1
await job.cancel()
python/python/tests/test_materialized_views.py
+268
View File
@@ -0,0 +1,268 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import lancedb
import pytest
from lancedb.materialized_view import MaterializedViewDefinition
STABLE_ROW_IDS = {"new_table_enable_stable_row_ids": "true"}
def make_db(tmp_path):
db = lancedb.connect(tmp_path, storage_options=STABLE_ROW_IDS)
db.create_table(
"people",
[
{"name": "ada", "age": 36},
{"name": "kid", "age": 7},
{"name": "grace", "age": 85},
],
)
return db
def test_create_refresh_and_query(tmp_path):
db = make_db(tmp_path)
view = db.create_materialized_view(
"adults",
"people",
select=["name", ("shout", "upper(name)")],
where="age >= 18",
)
assert view.name == "adults"
assert view.table.count_rows() == 0
result = view.refresh()
assert result.mode == "rebuild"
assert result.rows_written == 2
rows = view.table.search().to_list()
assert sorted(row["shout"] for row in rows) == ["ADA", "GRACE"]
def test_definition_round_trips(tmp_path):
db = make_db(tmp_path)
db.create_materialized_view("adults", "people", where="age >= 18")
view = db.open_materialized_view("adults")
assert view.definition == MaterializedViewDefinition(
source_table="people",
projections=[("name", "`name`"), ("age", "`age`")],
filter="age >= 18",
inputs=["age", "name"],
)
def test_incremental_refresh_after_append(tmp_path):
db = make_db(tmp_path)
view = db.create_materialized_view("copy", "people")
view.refresh()
db.open_table("people").add([{"name": "alan", "age": 41}])
result = view.refresh()
assert result.mode == "incremental"
assert result.rows_written == 1
assert view.table.count_rows() == 4
assert view.refresh().mode == "no_op"
def test_incremental_refresh_after_update(tmp_path):
db = make_db(tmp_path)
view = db.create_materialized_view("copy", "people")
view.refresh()
db.open_table("people").update(where="name = 'kid'", values={"age": 8})
result = view.refresh()
assert result.mode == "incremental"
assert result.rows_written == 1
rows = view.table.search().to_list()
assert sorted(row["age"] for row in rows) == [8, 36, 85]
def test_legacy_storage_source_update_rebuilds(tmp_path):
db = lancedb.connect(
tmp_path,
storage_options={**STABLE_ROW_IDS, "new_table_data_storage_version": "legacy"},
)
db.create_table("people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}])
view = db.create_materialized_view("copy", "people")
view.refresh()
db.open_table("people").update(where="name = 'kid'", values={"age": 8})
result = view.refresh()
assert result.mode == "rebuild"
rows = view.table.search().to_list()
assert sorted(row["age"] for row in rows) == [8, 36]
def test_list_and_not_a_view(tmp_path):
db = make_db(tmp_path)
db.create_materialized_view("adults", "people", where="age >= 18")
assert db.list_materialized_views() == ["adults"]
with pytest.raises(ValueError, match="not a materialized view"):
db.open_materialized_view("people")
def test_invalid_expression_fails_at_create(tmp_path):
db = make_db(tmp_path)
with pytest.raises(Exception, match="missing"):
db.create_materialized_view("bad", "people", select=[("x", "missing + 1")])
assert "bad" not in db.list_tables().tables
@pytest.mark.asyncio
async def test_async_create_refresh_and_open(tmp_path):
db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS)
await db.create_table("people", [{"name": "ada", "age": 36}])
view = await db.create_materialized_view(
"shouts", "people", select=[("shout", "upper(name)")]
)
result = await view.refresh()
assert result.mode == "rebuild"
assert result.rows_written == 1
reopened = await db.open_materialized_view("shouts")
definition = await reopened.definition()
assert definition.projections == [("shout", "upper(name)")]
assert await db.list_materialized_views() == ["shouts"]
@pytest.mark.asyncio
async def test_async_incremental(tmp_path):
db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS)
await db.create_table("people", [{"name": "ada", "age": 36}])
view = await db.create_materialized_view("copy", "people")
await view.refresh()
table = await db.open_table("people")
await table.add([{"name": "alan", "age": 41}])
result = await view.refresh()
assert result.mode == "incremental"
assert result.rows_written == 1
def test_source_requires_stable_row_ids(tmp_path):
db = lancedb.connect(tmp_path)
db.create_table("plain", [{"x": 1}])
with pytest.raises(Exception, match="stable row ids"):
db.create_materialized_view("v", "plain")
def test_bare_select_names_are_quoted(tmp_path):
db = lancedb.connect(tmp_path, storage_options=STABLE_ROW_IDS)
db.create_table("odd_names", [{"order item": "widget", "select": 2}])
view = db.create_materialized_view(
"quoted", "odd_names", select=["order item", "select"]
)
result = view.refresh()
assert result.rows_written == 1
rows = view.table.search().to_list()
assert rows[0]["order item"] == "widget"
assert rows[0]["select"] == 2
@pytest.mark.asyncio
async def test_async_remote_is_refused_without_network():
db = await lancedb.connect_async(
"db://nowhere", api_key="sk_test", region="us-east-1"
)
with pytest.raises(NotImplementedError, match="local"):
await db.create_materialized_view("v", "src")
with pytest.raises(NotImplementedError, match="local"):
await db.open_materialized_view("v")
with pytest.raises(NotImplementedError, match="local"):
await db.list_materialized_views()
def test_scalar_select_is_one_column(tmp_path):
db = make_db(tmp_path)
view = db.create_materialized_view("just_name", "people", select="name")
view.refresh()
rows = view.table.search().to_list()
assert set(rows[0]) - {"__source_row_id"} == {"name"}
assert sorted(row["name"] for row in rows) == ["ada", "grace", "kid"]
@pytest.mark.asyncio
async def test_async_scalar_select_is_one_column(tmp_path):
db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS)
await db.create_table("people", [{"name": "ada", "age": 36}])
view = await db.create_materialized_view("just_name", "people", select="name")
await view.refresh()
rows = await view.table.query().to_list()
assert set(rows[0]) - {"__source_row_id"} == {"name"}
def test_limit_above_i64_max_is_refused(tmp_path):
db = make_db(tmp_path)
with pytest.raises(ValueError, match="exceeds the maximum"):
db.create_materialized_view("too_big", "people", limit=2**63)
# The boundary is fine, and zero still means an empty view.
db.create_materialized_view("at_max", "people", limit=2**63 - 1)
empty = db.create_materialized_view("none", "people", limit=0)
empty.refresh()
assert empty.table.count_rows() == 0
def _namespace_db(tmp_path):
return lancedb.connect_namespace(
"dir",
{"root": str(tmp_path)},
storage_options=STABLE_ROW_IDS,
)
def test_namespace_connection_materialized_views(tmp_path):
db = _namespace_db(tmp_path)
db.create_table(
"people",
[{"name": "ada", "age": 36}, {"name": "kid", "age": 7}],
storage_options=STABLE_ROW_IDS,
)
view = db.create_materialized_view("adults", "people", where="age >= 18")
view.refresh()
assert view.table.count_rows() == 1
assert db.list_materialized_views() == ["adults"]
reopened = db.open_materialized_view("adults")
assert reopened.definition.source_table == "people"
with pytest.raises(ValueError, match="not a materialized view"):
db.open_materialized_view("people")
@pytest.mark.asyncio
async def test_async_namespace_connection_materialized_views(tmp_path):
db = lancedb.connect_namespace_async(
"dir",
{"root": str(tmp_path)},
storage_options=STABLE_ROW_IDS,
)
await db.create_table(
"people",
[{"name": "ada", "age": 36}, {"name": "kid", "age": 7}],
storage_options=STABLE_ROW_IDS,
)
view = await db.create_materialized_view("adults", "people", where="age >= 18")
await view.refresh()
assert await view.table.count_rows() == 1
assert await db.list_materialized_views() == ["adults"]
reopened = await db.open_materialized_view("adults")
assert (await reopened.definition()).source_table == "people"
# The view's table came through the namespace, not straight from the
# inner connection: a bare inner table carries no namespace context, so
# its pushdown routing differs from a table the namespace opened.
through_namespace = await db.open_table("adults")
for handle in (view.table, reopened.table):
assert (
handle._route_pushdown_to_rust == through_namespace._route_pushdown_to_rust
)
assert handle._namespace_path == through_namespace._namespace_path
python/python/tests/test_permutation.py
+25
View File
@@ -56,6 +56,31 @@ def test_execute_does_not_reenter_background_loop(tmp_path, monkeypatch):
assert permutation_tbl._conn.read_consistency_interval is None
def test_pickled_permutation_reads_pinned_version(tmp_path):
"""An unpickled copy must still read the pinned version, which also covers the
version surviving the ``to_arrow()`` round trip in ``__getstate__``."""
import pickle
db = connect(tmp_path)
tbl = db.create_table("base", pa.table({"idx": range(20)}))
permutation_tbl = permutation_builder(tbl).execute()
perm = Permutation.from_tables(tbl, permutation_tbl)
payload = pickle.dumps(perm)
# Compact so the stored row addresses no longer describe these rows at latest.
tbl.delete("true")
tbl.optimize()
assert tbl.count_rows() == 0
# Unpickle after the mutation: __setstate__ reopens at latest, so this only
# passes if the recorded version is applied on reopen.
restored = pickle.loads(payload)
assert len(restored) == 20
rows = restored.__getitems__(list(range(20)))
assert sorted(row["idx"] for row in rows) == list(range(20))
def test_split_random_counts(mem_db):
"""Test random splitting with absolute counts."""
tbl = mem_db.create_table(
python/python/tests/test_remote_db.py
+83 -10
View File
@@ -242,8 +242,8 @@ def test_remote_table_branches_sync():
table.branches.delete("exp")
def test_remote_table_branch_merge_defaults_to_execute():
merge_bodies = []
def test_remote_table_cherry_pick_defaults_to_execute():
cherry_pick_bodies = []
diff = {
"fromBranch": "exp",
"parentVersion": 1,
@@ -265,8 +265,7 @@ def test_remote_table_branch_merge_defaults_to_execute():
"changedColumns": [],
"addedIndexes": [],
"removedIndexes": [],
"mergeable": True,
"mergeBlockers": [],
"errors": [],
}
def handler(request):
@@ -276,11 +275,11 @@ def test_remote_table_branch_merge_defaults_to_execute():
else:
content_len = int(request.headers.get("Content-Length"))
request_body = json.loads(request.rfile.read(content_len))
merge_bodies.append(request_body)
cherry_pick_bodies.append(request_body)
dry_run = request_body["dry_run"]
status = 200 if dry_run else 409
body = {
"status": "ready" if dry_run else "rejected",
"status": "ready" if dry_run else "failed",
"diff": diff,
"preview": {"promotedColumns": []},
}
@@ -292,10 +291,10 @@ def test_remote_table_branch_merge_defaults_to_execute():
with mock_lancedb_connection(handler) as db:
branches = db.open_table("test").branches
assert branches.merge("exp")["status"] == "rejected"
assert branches.merge("exp", dry_run=True)["status"] == "ready"
assert branches.cherry_pick("exp")["status"] == "failed"
assert branches.cherry_pick("exp", dry_run=True)["status"] == "ready"
assert merge_bodies == [
assert cherry_pick_bodies == [
{"from_branch": "exp", "dry_run": False},
{"from_branch": "exp", "dry_run": True},
]
@@ -876,11 +875,85 @@ def test_remote_create_index_async_returns_job():
table = db.create_table("test", [{"id": 1}])
job = table.create_index_async("id", config=BTree())
assert job.id == "job-1"
job.wait(timeout=timedelta(seconds=30))
assert job.wait(timeout=timedelta(seconds=30)) is None
assert len(describe_calls) == 2
job.cancel()
def test_remote_refresh_async_returns_typed_terminal_result():
terminal_result = {
"rows_assigned": 12,
"rows_failed": 0,
"rows_remaining": 0,
"source_version": 7,
"published_version": 8,
}
def handler(request):
content_len = int(request.headers.get("Content-Length", 0))
body = request.rfile.read(content_len) if content_len > 0 else b""
if request.path == "/v1/table/test/backfill_column":
assert json.loads(body)["column"] == "derived"
request.send_response(202)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b'{"job_id": "refresh-1"}')
elif request.path == "/v1/jobs/describe":
assert json.loads(body)["job_id"] == "refresh-1"
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(
json.dumps(
{
"job_id": "refresh-1",
"job_type": "function_refresh",
"job_state": "DONE",
"result": terminal_result,
}
).encode()
)
elif request.path == "/v1/table/test/create/?mode=create":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b"{}")
elif request.path == "/v1/table/test/describe/":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(
json.dumps(
{
"version": 1,
"schema": {
"fields": [
{
"name": "id",
"type": {"type": "int64"},
"nullable": False,
}
]
},
}
).encode()
)
else:
request.send_response(404)
request.end_headers()
with mock_lancedb_connection(handler) as db:
table = db.create_table("test", [{"id": 1}])
job = table.refresh_column_async("derived")
assert job.id == "refresh-1"
result = job.wait(timeout=timedelta(seconds=30))
assert isinstance(result, lancedb.RefreshColumnResult)
assert result.model_dump() == terminal_result
assert result.rows_filled == 12
assert result.version == 8
def test_remote_job_wait_raises_on_failure():
from lancedb.exceptions import JobFailedError
from lancedb.index import BTree
python/python/tests/test_table.py
+18 -3
View File
@@ -1467,7 +1467,7 @@ def test_create_index_async_returns_done_job(mem_db: DBConnection):
table = mem_db.create_table("job_test", [{"id": i} for i in range(10)])
job = table.create_index_async("id", config=BTree())
assert job.id is None
job.wait()
assert job.wait() is None
assert len(table.list_indices()) == 1
job.cancel()
@@ -3947,10 +3947,21 @@ def test_refresh_column_async_returns_job(tmp_path):
job = table.refresh_column_async("doubled")
assert job.id is None # in-process jobs have no server id
assert job.wait() is None
result = job.wait()
assert isinstance(result, lancedb.RefreshColumnResult)
assert result.rows_assigned == 2
assert result.rows_failed == 0
assert result.rows_remaining == 0
assert result.source_version == 2
assert result.published_version == 3
assert job.status() == "finished"
assert sorted(table.to_arrow()["doubled"].to_pylist()) == [2, 4]
no_op = table.refresh_column_async("doubled").wait()
assert no_op.rows_assigned == 0
assert no_op.source_version == 3
assert no_op.published_version is None
# Bad input raises at the call, not through the job.
with pytest.raises(Exception, match="not a computed column"):
table.refresh_column_async("x")
@@ -3963,6 +3974,10 @@ async def test_refresh_column_async_job_async_table(tmp_path):
await table.add_columns(computed={"tripled": "x * 3"})
job = await table.refresh_column_async("tripled")
assert await job.wait() is None
result = await job.wait()
assert isinstance(result, lancedb.RefreshColumnResult)
assert result.rows_assigned == 1
assert result.source_version == 2
assert result.published_version == 3
assert await job.status() == "finished"
assert (await table.to_arrow())["tripled"].to_pylist() == [9]
python/src/connection.rs
+35 -1
View File
@@ -333,6 +333,40 @@ impl Connection {
})
}
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None))]
pub fn create_materialized_view(
self_: PyRef<'_, Self>,
name: String,
source: String,
projections: Option<Vec<(String, String)>>,
filter: Option<String>,
limit: Option<u64>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let mut builder = inner.create_materialized_view(name, source);
if let Some(projections) = projections {
builder = builder.select(projections);
}
if let Some(filter) = filter {
builder = builder.only_if(filter);
}
if let Some(limit) = limit {
builder = builder.limit(limit);
}
let view = builder.execute().await.infer_error()?;
Ok(Table::new(view.table().clone()))
})
}
pub fn list_materialized_views(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let views = inner.list_materialized_views().await.infer_error()?;
Ok(views.into_iter().map(|view| view.name).collect::<Vec<_>>())
})
}
#[pyo3(signature = (name, namespace_path=None))]
pub fn drop_table(
self_: PyRef<'_, Self>,
@@ -575,7 +609,7 @@ impl Connection {
.create_function_async(request)
.await
.infer_error()
.map(crate::job::FunctionJob::new)
.map(crate::job::Job::new_typed)
})
}
python/src/index.rs
+57 -1
View File
@@ -42,7 +42,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
"Fm" => Ok(LanceDbIndex::Fm(FmIndexBuilder::default())),
"FTS" => {
let params = source.extract::<FtsParams>()?;
let inner_opts = FtsIndexBuilder::default()
let mut inner_opts = FtsIndexBuilder::default()
.base_tokenizer(params.base_tokenizer)
.language(&params.language)
.map_err(|_| {
@@ -61,6 +61,12 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
.ngram_max_length(params.ngram_max_length)
.ngram_prefix_only(params.prefix_only)
.custom_stop_words(params.custom_stop_words);
if let Some(memory_limit) = params.memory_limit {
inner_opts = inner_opts.memory_limit_mb(memory_limit);
}
if let Some(num_workers) = params.num_workers {
inner_opts = inner_opts.num_workers(num_workers);
}
let inner_opts = inner_opts
.block_size(params.block_size)
.map_err(|err| PyValueError::new_err(err.to_string()))?;
@@ -213,6 +219,8 @@ struct FtsParams {
ngram_max_length: u32,
prefix_only: bool,
block_size: usize,
memory_limit: Option<u64>,
num_workers: Option<usize>,
}
#[derive(FromPyObject)]
@@ -444,3 +452,51 @@ impl IndexConfig {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use pyo3::types::{PyDict, PyDictMethods};
use serde_json::json;
#[test]
fn fts_build_controls_are_forwarded() {
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
c"class FTS:
with_position = True
base_tokenizer = 'simple'
language = 'English'
max_token_length = None
lower_case = True
stem = False
remove_stop_words = False
custom_stop_words = None
ascii_folding = False
ngram_min_length = 3
ngram_max_length = 3
prefix_only = False
block_size = 128
memory_limit = 2048
num_workers = 7
config = FTS()",
None,
Some(&locals),
)
.unwrap();
let config = locals.get_item("config").unwrap().unwrap();
let index = extract_index_params(&Some(config)).unwrap();
let LanceDbIndex::FTS(params) = index else {
panic!("expected FTS index parameters");
};
let training_json = params.to_training_json().unwrap();
assert_eq!(training_json.get("memory_limit"), Some(&json!(2048)));
assert_eq!(training_json.get("num_workers"), Some(&json!(7)));
});
}
}
python/src/job.rs
+18 -55
View File
@@ -5,72 +5,33 @@ use std::sync::Arc;
use crate::runtime::future_into_py;
use pyo3::{Bound, PyAny, PyRef, PyResult, pyclass, pymethods};
use serde::Serialize;
use crate::error::PythonErrorExt;
#[pyclass]
pub struct Job {
inner: Arc<lancedb::Job>,
}
/// Python bridge for a typed remote Function registration job.
///
/// The public Python layer decodes the canonical JSON returned by `wait`
/// into its immutable `FunctionVersion` model.
#[pyclass]
pub struct FunctionJob {
inner: Arc<lancedb::Job<lancedb::function::FunctionVersion>>,
}
impl FunctionJob {
pub(crate) fn new(inner: lancedb::Job<lancedb::function::FunctionVersion>) -> Self {
Self {
inner: Arc::new(inner),
}
}
inner: Arc<lancedb::Job<std::result::Result<Option<String>, String>>>,
}
impl Job {
pub(crate) fn new(inner: lancedb::Job) -> Self {
Self {
inner: Arc::new(inner),
inner: Arc::new(inner.map(|()| Ok(None))),
}
}
}
#[pymethods]
impl FunctionJob {
#[getter]
pub fn id(&self) -> Option<String> {
self.inner.id().map(str::to_string)
}
pub fn status(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(
self_.py(),
async move { inner.status().await.infer_error() },
)
}
pub fn wait(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
inner
.wait()
.await
.infer_error()?
.to_canonical_json()
.infer_error()
})
}
pub fn cancel(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
inner.cancel().await.infer_error()?;
Ok(())
})
pub(crate) fn new_typed<T>(inner: lancedb::Job<T>) -> Self
where
T: Clone + Serialize + Send + Sync + 'static,
{
Self {
inner: Arc::new(inner.map(|result| {
serde_json::to_string(&result)
.map(Some)
.map_err(|error| format!("failed to serialize typed job result: {error}"))
})),
}
}
}
@@ -92,8 +53,10 @@ impl Job {
pub fn wait(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
inner.wait().await.infer_error()?;
Ok(None::<()>)
let result = inner.wait().await.infer_error()?;
result
.map_err(|message| lancedb::Error::Runtime { message })
.infer_error()
})
}
python/src/lib.rs
+3 -3
View File
@@ -16,8 +16,8 @@ use query::{FTSQuery, HybridQuery, Query, VectorQuery};
use session::Session;
use table::{
AddColumnsResult, AddResult, AlterColumnsResult, DeleteResult, DropColumnsResult, FtsToken,
LsmWriteSpec, MergeResult, PyBlobFile, RefreshColumnResult, Table, UpdateFieldMetadataResult,
UpdateResult,
LsmWriteSpec, MergeResult, PyBlobFile, RefreshColumnResult, RefreshMaterializedViewResult,
Table, UpdateFieldMetadataResult, UpdateResult,
};
pub mod arrow;
@@ -47,7 +47,6 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<Session>()?;
m.add_class::<Table>()?;
m.add_class::<crate::job::Job>()?;
m.add_class::<crate::job::FunctionJob>()?;
m.add_class::<crate::job::JobInfo>()?;
m.add_class::<crate::job::JobDescription>()?;
m.add_class::<crate::job::JobFailureInfo>()?;
@@ -60,6 +59,7 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<RecordBatchStream>()?;
m.add_class::<AddColumnsResult>()?;
m.add_class::<RefreshColumnResult>()?;
m.add_class::<RefreshMaterializedViewResult>()?;
m.add_class::<AlterColumnsResult>()?;
m.add_class::<UpdateFieldMetadataResult>()?;
m.add_class::<AddResult>()?;
python/src/table.rs
+58 -3
View File
@@ -441,6 +441,41 @@ impl From<lancedb::table::RefreshColumnResult> for RefreshColumnResult {
}
}
#[pyclass(get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct RefreshMaterializedViewResult {
pub mode: String,
pub rows_written: u64,
pub source_version: u64,
pub version: u64,
}
#[pymethods]
impl RefreshMaterializedViewResult {
pub fn __repr__(&self) -> String {
format!(
"RefreshMaterializedViewResult(mode={}, rows_written={}, source_version={}, version={})",
self.mode, self.rows_written, self.source_version, self.version
)
}
}
impl From<lancedb::RefreshMaterializedViewResult> for RefreshMaterializedViewResult {
fn from(result: lancedb::RefreshMaterializedViewResult) -> Self {
let mode = match result.mode {
lancedb::RefreshMode::Rebuild => "rebuild",
lancedb::RefreshMode::Incremental => "incremental",
lancedb::RefreshMode::NoOp => "no_op",
};
Self {
mode: mode.to_string(),
rows_written: result.rows_written,
source_version: result.source_version,
version: result.version,
}
}
}
#[pymethods]
impl AddColumnsResult {
pub fn __repr__(&self) -> String {
@@ -1584,7 +1619,27 @@ impl Table {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let job = inner.refresh_column_async(column).await.infer_error()?;
Ok(crate::job::Job::new(job))
Ok(crate::job::Job::new_typed(job))
})
}
#[pyo3(signature = (full=false, source_version=None))]
pub fn refresh_materialized_view(
self_: PyRef<'_, Self>,
full: bool,
source_version: Option<u64>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let view = lancedb::MaterializedView::from_table(inner)
.await
.infer_error()?;
let mut builder = view.refresh().full(full);
if let Some(version) = source_version {
builder = builder.source_version(version);
}
let result = builder.execute().await.infer_error()?;
Ok(RefreshMaterializedViewResult::from(result))
})
}
@@ -1885,7 +1940,7 @@ impl Branches {
}
#[pyo3(signature = (from_branch, dry_run=false))]
pub fn merge(
pub fn cherry_pick(
self_: PyRef<'_, Self>,
from_branch: String,
dry_run: bool,
@@ -1893,7 +1948,7 @@ impl Branches {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let result = inner
.merge_branch(&from_branch, dry_run)
.cherry_pick(&from_branch, dry_run)
.await
.infer_error()?;
Python::attach(|py| struct_to_wire_py(py, &result))
rust/lancedb/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb"
version = "0.38.0-beta.3"
version = "0.38.0-beta.8"
edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true
rust/lancedb/src/connection.rs
+45 -1
View File
@@ -41,7 +41,7 @@ use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
mod create_table;
fn merge_storage_options(
pub(crate) fn merge_storage_options(
store_params: &mut ObjectStoreParams,
pairs: impl IntoIterator<Item = (String, String)>,
) {
@@ -1679,6 +1679,50 @@ mod tests {
assert_eq!(tables, names[..7]);
}
#[tokio::test]
async fn test_list_tables_walks_page_boundaries() {
let tc = new_test_connection().await.unwrap();
if tc.is_remote {
// What resumes a page is the server's to decide, and asserting it here would be
// asserting the server's contract rather than this one.
return;
}
let db = tc.connection;
let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)]));
let mut names = Vec::with_capacity(5);
for _ in 0..5 {
let name = uuid::Uuid::new_v4().to_string();
names.push(name.clone());
db.create_empty_table(name, schema.clone())
.execute()
.await
.unwrap();
}
names.sort();
// Walking in pages has to reach every table exactly once, with nothing lost at a
// page boundary.
let mut seen = Vec::with_capacity(names.len());
let mut page_token = None;
loop {
let page = db
.list_tables(ListTablesRequest {
id: Some(Vec::new()),
limit: Some(2),
page_token,
..Default::default()
})
.await
.unwrap();
seen.extend(page.tables);
page_token = page.page_token.filter(|token| !token.is_empty());
if page_token.is_none() {
break;
}
}
assert_eq!(seen, names);
}
#[tokio::test]
async fn test_open_table() {
let tc = new_test_connection().await.unwrap();
rust/lancedb/src/database/listing.rs
+22 -9
View File
@@ -974,17 +974,15 @@ impl Database for ListingDatabase {
f.drain(0..index);
}
// Determine if there's a next page
let next_page_token = if let Some(limit) = request.limit {
if f.len() > limit as usize {
let token = f[limit as usize].clone();
// Determine if there's a next page. The token is the last name of this page,
// not the first of the next one: the next page resumes strictly after the
// token, so naming the next page's first entry would skip it.
let next_page_token = match request.limit {
Some(limit) if f.len() > limit as usize => {
f.truncate(limit as usize);
Some(token)
} else {
None
f.last().cloned()
}
} else {
None
_ => None,
};
Ok(ListTablesResponse {
@@ -2747,6 +2745,21 @@ mod tests {
}
}
/// Regression test for https://github.com/lancedb/lancedb/issues/2283.
///
/// Object-store URIs must use `/` on every platform. In particular, joining
/// with `std::path::Path` used to insert a `\\` into Azure blob keys on
/// Windows.
#[tokio::test]
async fn test_table_uri_uses_forward_slashes_for_azure() {
let (_tempdir, mut db) = setup_database().await;
db.uri = "az://test/db/test".to_string();
let uri = db.table_uri("test").unwrap();
assert_eq!(uri, "az://test/db/test/test.lance");
}
/// Regression: connecting via a URL-style URI (which goes through
/// `url::Url::parse` and the `query_pairs_mut()` path) must not
/// append a trailing `?` to per-table URIs when the input URI has
rust/lancedb/src/dataloader/permutation/builder.rs
+294 -20
View File
@@ -27,6 +27,12 @@ pub const SRC_ROW_ID_COL: &str = "row_id";
pub const SPLIT_NAMES_CONFIG_KEY: &str = "split_names";
/// Base table version the permutation was built against.
pub const BASE_VERSION_CONFIG_KEY: &str = "base_version";
/// Base table branch the permutation was built against. Absent means main.
pub const BASE_BRANCH_CONFIG_KEY: &str = "base_branch";
pub const DEFAULT_MEMORY_LIMIT: usize = 100 * 1024 * 1024;
/// Where to store the permutation table
@@ -214,21 +220,11 @@ impl PermutationBuilder {
Ok(Box::pin(SimpleRecordBatchStream { schema, stream }))
}
fn add_split_names(
fn add_config_metadata(
data: SendableRecordBatchStream,
split_names: &[String],
metadata: HashMap<String, String>,
) -> Result<SendableRecordBatchStream> {
let schema = data
.schema()
.as_ref()
.clone()
.with_metadata(HashMap::from([(
SPLIT_NAMES_CONFIG_KEY.to_string(),
serde_json::to_string(split_names).map_err(|e| Error::Other {
message: format!("Failed to serialize split names: {}", e),
source: Some(e.into()),
})?,
)]));
let schema = data.schema().as_ref().clone().with_metadata(metadata);
let schema = Arc::new(schema);
let schema_clone = schema.clone();
let stream = data.map_ok(move |batch| batch.with_schema(schema.clone()).unwrap());
@@ -239,7 +235,20 @@ impl PermutationBuilder {
}
/// Builds the permutation table and stores it in the given database.
pub async fn build(self) -> Result<Table> {
pub async fn build(mut self) -> Result<Table> {
// Remote tables resolve latest independently for each request. Use a
// separate pinned handle so count, projection, and scan all refer to one
// snapshot without changing the caller's table checkout state. Native
// tables return `None` here and retain their existing behavior.
if let Some(snapshot) = self
.base_table
.base_table()
.snapshot_at_current_version()
.await?
{
self.base_table = Table::from(snapshot);
}
// Unflushed rows have no row id, so a permutation cannot address them.
match self.base_table.base_table().get_lsm_write_spec().await {
Ok(Some(_)) => {
@@ -256,9 +265,14 @@ impl PermutationBuilder {
Err(err) => return Err(err),
}
// The handle above is already pinned to one version. Record which one, so a
// reader -- in a DataLoader worker, against a table that has since moved --
// resolves these row addresses against the same snapshot.
let base_version = self.base_table.version().await?;
let base_branch = self.base_table.current_branch();
// First pass, apply filter and load row ids. `Shuffler` permutes positions, so
// every rank must scan the rows in the same order to build the same permutation.
// TODO: pin the version resolved here; remote does not implement Lazy.
let mut rows = self.base_table.query().select(Select::columns(&[ROW_ID]));
if let Some(filter) = &self.config.filter {
@@ -318,11 +332,24 @@ impl PermutationBuilder {
// Rename _rowid to row_id
let renamed = rename_column(sorted, ROW_ID, SRC_ROW_ID_COL)?;
let streaming_data = if let Some(split_names) = &self.config.split_names {
Self::add_split_names(renamed, split_names)?
} else {
renamed
};
let mut metadata = HashMap::from([(
BASE_VERSION_CONFIG_KEY.to_string(),
base_version.to_string(),
)]);
// Version numbers are per-branch, so the branch is part of the coordinate.
if let Some(branch) = &base_branch {
metadata.insert(BASE_BRANCH_CONFIG_KEY.to_string(), branch.clone());
}
if let Some(split_names) = &self.config.split_names {
metadata.insert(
SPLIT_NAMES_CONFIG_KEY.to_string(),
serde_json::to_string(split_names).map_err(|e| Error::Other {
message: format!("Failed to serialize split names: {}", e),
source: Some(e.into()),
})?,
);
}
let streaming_data = Self::add_config_metadata(renamed, metadata)?;
let (name, database) = match &self.config.destination {
PermutationDestination::Permanent(database, table_name) => {
@@ -409,6 +436,253 @@ mod tests {
assert!(table.base_table().scan_order_is_deterministic());
}
#[cfg(feature = "remote")]
#[tokio::test]
async fn test_remote_permutation_builder_pins_snapshot() {
use std::sync::{
Mutex,
atomic::{AtomicU64, Ordering},
};
use arrow_array::{RecordBatch, UInt64Array};
use arrow_schema::{DataType, Field, Schema};
let row_ids = RecordBatch::try_new(
Arc::new(Schema::new(vec![Field::new(
ROW_ID,
DataType::UInt64,
false,
)])),
vec![Arc::new(UInt64Array::from(vec![100]))],
)
.unwrap();
let mut query_body = Vec::new();
{
let mut writer =
arrow_ipc::writer::FileWriter::try_new(&mut query_body, &row_ids.schema()).unwrap();
writer.write(&row_ids).unwrap();
writer.finish().unwrap();
}
let latest = Arc::new(AtomicU64::new(7));
let expected_snapshot = Arc::new(AtomicU64::new(7));
let planning_versions = Arc::new(Mutex::new(Vec::new()));
let latest_ref = latest.clone();
let expected_snapshot_ref = expected_snapshot.clone();
let planning_versions_ref = planning_versions.clone();
let table = Table::new_with_handler("remote_base", move |request| {
let path = request.url().path();
let body = request
.body()
.and_then(|body| body.as_bytes())
.map(|body| serde_json::from_slice::<serde_json::Value>(body).unwrap());
match path {
"/v1/table/remote_base/describe/" => {
let requested = body.as_ref().and_then(|body| body["version"].as_u64());
let version = requested.unwrap_or_else(|| latest_ref.load(Ordering::SeqCst));
http::Response::builder()
.status(200)
.body(
format!(r#"{{"version":{version},"schema":{{"fields":[]}}}}"#)
.into_bytes(),
)
.unwrap()
}
"/v1/table/remote_base/get_lsm_write_spec/" => http::Response::builder()
.status(200)
.body(br#"{"lsm_write_spec":null}"#.to_vec())
.unwrap(),
"/v1/table/remote_base/count_rows/" => {
let body = body.unwrap();
let version = body["version"].as_u64().unwrap();
assert_eq!(version, expected_snapshot_ref.load(Ordering::SeqCst));
assert_eq!(body["predicate"], "value > 0");
planning_versions_ref.lock().unwrap().push(version);
// Simulate a concurrent append after count_rows. An unpinned
// scan would now resolve version 8 and include different rows.
latest_ref.store(8, Ordering::SeqCst);
http::Response::builder()
.status(200)
.body(b"1".to_vec())
.unwrap()
}
"/v1/table/remote_base/query/" => {
let body = body.unwrap();
let version = body["version"].as_u64().unwrap();
assert_eq!(version, expected_snapshot_ref.load(Ordering::SeqCst));
assert_eq!(body["filter"], "value > 0");
assert_eq!(body["columns"], serde_json::json!([ROW_ID]));
planning_versions_ref.lock().unwrap().push(version);
http::Response::builder()
.status(200)
.header("content-type", "application/vnd.apache.arrow.file")
.body(query_body.clone())
.unwrap()
}
_ => panic!("unexpected request: {path}"),
}
});
let permutation = PermutationBuilder::new(table.clone())
.with_filter("value > 0".to_string())
.build()
.await
.unwrap();
assert_eq!(permutation.count_rows(None).await.unwrap(), 1);
// Building uses a separate handle and must not pin the caller's table.
assert_eq!(table.version().await.unwrap(), 8);
// An explicit checkout is copied as-is and remains checked out afterward.
expected_snapshot.store(6, Ordering::SeqCst);
table.checkout(6).await.unwrap();
let permutation = PermutationBuilder::new(table.clone())
.with_filter("value > 0".to_string())
.build()
.await
.unwrap();
assert_eq!(permutation.count_rows(None).await.unwrap(), 1);
assert_eq!(table.version().await.unwrap(), 6);
assert_eq!(*planning_versions.lock().unwrap(), vec![7, 7, 6, 6]);
}
#[tokio::test]
async fn test_permutation_records_base_version() {
let temp_dir = tempfile::tempdir().unwrap();
let db = connect(temp_dir.path().to_str().unwrap())
.execute()
.await
.unwrap();
let initial_data = lance_datagen::gen_batch()
.col("col_a", lance_datagen::array::step::<Int32Type>())
.into_ldb_stream(RowCount::from(100), BatchCount::from(2));
let data_table = db
.create_table("base_tbl", initial_data)
.execute()
.await
.unwrap();
let build_version = data_table.version().await.unwrap();
let permutation_table = PermutationBuilder::new(data_table.clone())
.build()
.await
.unwrap();
let recorded = permutation_table
.schema()
.await
.unwrap()
.metadata
.get(BASE_VERSION_CONFIG_KEY)
.expect("permutation should record the base version")
.parse::<u64>()
.unwrap();
assert_eq!(recorded, build_version);
// Advancing the base table must not move the recorded version.
let more_data = lance_datagen::gen_batch()
.col("col_a", lance_datagen::array::step::<Int32Type>())
.into_ldb_stream(RowCount::from(50), BatchCount::from(1));
data_table.add(more_data).execute().await.unwrap();
assert!(data_table.version().await.unwrap() > recorded);
assert_eq!(
permutation_table
.schema()
.await
.unwrap()
.metadata
.get(BASE_VERSION_CONFIG_KEY)
.unwrap()
.parse::<u64>()
.unwrap(),
recorded,
);
}
/// Version numbers are per-branch, so a permutation built on a branch must record
/// it -- a worker reopens by name and lands on main at the same number.
#[tokio::test]
async fn test_permutation_records_base_branch() {
let temp_dir = tempfile::tempdir().unwrap();
let db = connect(temp_dir.path().to_str().unwrap())
.execute()
.await
.unwrap();
let initial_data = lance_datagen::gen_batch()
.col("col_a", lance_datagen::array::step::<Int32Type>())
.into_ldb_stream(RowCount::from(10), BatchCount::from(1));
let data_table = db
.create_table("base_tbl", initial_data)
.execute()
.await
.unwrap();
let branch = data_table
.create_branch("exp", lance::dataset::refs::Ref::from(("main", 1)))
.await
.unwrap();
let permutation_table = PermutationBuilder::new(branch.clone())
.build()
.await
.unwrap();
let metadata = permutation_table.schema().await.unwrap().metadata.clone();
assert_eq!(
metadata.get(BASE_BRANCH_CONFIG_KEY).map(String::as_str),
Some("exp")
);
// Main records nothing, so an absent key keeps meaning main.
let main_permutation = PermutationBuilder::new(data_table.clone())
.build()
.await
.unwrap();
assert!(
!main_permutation
.schema()
.await
.unwrap()
.metadata
.contains_key(BASE_BRANCH_CONFIG_KEY)
);
}
#[tokio::test]
async fn test_build_does_not_pin_the_callers_table() {
let temp_dir = tempfile::tempdir().unwrap();
let db = connect(temp_dir.path().to_str().unwrap())
.execute()
.await
.unwrap();
let initial_data = lance_datagen::gen_batch()
.col("col_a", lance_datagen::array::step::<Int32Type>())
.into_ldb_stream(RowCount::from(100), BatchCount::from(1));
let data_table = db
.create_table("base_tbl", initial_data)
.execute()
.await
.unwrap();
PermutationBuilder::new(data_table.clone())
.build()
.await
.unwrap();
// The builder pins its own handle; the caller's must still track latest.
let more_data = lance_datagen::gen_batch()
.col("col_a", lance_datagen::array::step::<Int32Type>())
.into_ldb_stream(RowCount::from(50), BatchCount::from(1));
data_table.add(more_data).execute().await.unwrap();
assert_eq!(data_table.count_rows(None).await.unwrap(), 150);
}
#[tokio::test]
async fn test_permutation_builder() {
let temp_dir = tempfile::tempdir().unwrap();
rust/lancedb/src/dataloader/permutation/reader.rs
+92 -1
View File
@@ -8,7 +8,9 @@
//! the rows from a source table that correspond to row IDs stored in a separate table.
use crate::arrow::{SendableRecordBatchStream, SimpleRecordBatchStream};
use crate::dataloader::permutation::builder::SRC_ROW_ID_COL;
use crate::dataloader::permutation::builder::{
BASE_BRANCH_CONFIG_KEY, BASE_VERSION_CONFIG_KEY, SRC_ROW_ID_COL,
};
use crate::dataloader::permutation::split::SPLIT_ID_COLUMN;
use crate::error::Error;
use crate::query::{
@@ -23,6 +25,7 @@ use arrow_array::{RecordBatch, UInt64Array};
use arrow_schema::SchemaRef;
use datafusion_expr::{Expr, col, lit};
use futures::{StreamExt, TryStreamExt};
use lance::dataset::refs::MAIN_BRANCH;
use lance::dataset::scanner::DatasetRecordBatchStream;
use lance::io::RecordBatchStream;
use lance_arrow::RecordBatchExt;
@@ -69,6 +72,10 @@ impl PermutationReader {
permutation_table: Option<Arc<dyn BaseTable>>,
split: u64,
) -> Result<Self> {
let base_table = match &permutation_table {
Some(permutation_table) => Self::pin_base_table(base_table, permutation_table).await?,
None => base_table,
};
let mut slf = Self {
base_table,
permutation_table,
@@ -89,6 +96,34 @@ impl PermutationReader {
Ok(slf)
}
/// Pins the base table to the version the permutation was built against.
/// Permutations written before that was recorded carry no key and stay unpinned.
async fn pin_base_table(
base_table: Arc<dyn BaseTable>,
permutation_table: &Arc<dyn BaseTable>,
) -> Result<Arc<dyn BaseTable>> {
let schema = permutation_table.schema().await?;
let Some(raw) = schema.metadata.get(BASE_VERSION_CONFIG_KEY) else {
return Ok(base_table);
};
let version = raw.parse::<u64>().map_err(|e| Error::InvalidInput {
message: format!(
"Permutation table has an unreadable {} of {:?}: {}",
BASE_VERSION_CONFIG_KEY, raw, e
),
})?;
// The recorded branch, not the handle's: a worker reopens by name and lands
// on main, and version numbers are per-branch.
let branch = schema
.metadata
.get(BASE_BRANCH_CONFIG_KEY)
.map(String::as_str)
.unwrap_or(MAIN_BRANCH);
base_table
.checkout_branch_version(branch, Some(version))
.await
}
pub async fn try_from_tables(
base_table: Arc<dyn BaseTable>,
permutation_table: Arc<dyn BaseTable>,
@@ -511,9 +546,13 @@ mod tests {
use lance_datagen::{BatchCount, RowCount};
use rand::seq::SliceRandom;
// Aliased: `test_utils::datagen` exports a trait of the same name.
use crate::arrow::LanceDbDatagenExt as _;
use crate::{
Table,
arrow::SendableRecordBatchStream,
connect,
dataloader::permutation::builder::PermutationBuilder,
query::{ExecutableQuery, QueryBase},
test_utils::datagen::{LanceDbDatagenExt, virtual_table},
};
@@ -545,6 +584,58 @@ mod tests {
.await
}
/// Compaction moves row addresses, so the reader must read the pinned version.
#[tokio::test]
async fn test_reader_pins_base_version() {
let temp_dir = tempfile::tempdir().unwrap();
let db = connect(temp_dir.path().to_str().unwrap())
.execute()
.await
.unwrap();
let data = lance_datagen::gen_batch()
.col("idx", lance_datagen::array::step::<Int32Type>())
.into_ldb_stream(RowCount::from(20), BatchCount::from(1));
let base_table = db.create_table("base_tbl", data).execute().await.unwrap();
let permutation_table = PermutationBuilder::new(base_table.clone())
.build()
.await
.unwrap();
base_table.delete("true").await.unwrap();
base_table
.optimize(crate::table::OptimizeAction::All)
.await
.unwrap();
assert_eq!(base_table.count_rows(None).await.unwrap(), 0);
let reader = PermutationReader::try_from_tables(
base_table.base_table().clone(),
permutation_table.base_table().clone(),
0,
)
.await
.unwrap();
let values = collect_from_stream::<Int32Type>(
reader
.read(
Select::Columns(vec!["idx".to_string()]),
QueryExecutionOptions::default(),
)
.await
.unwrap(),
"idx",
)
.await;
assert_eq!(
values.len(),
20,
"reader should still see the pinned version"
);
}
#[tokio::test]
async fn test_permutation_reader() {
let base_table = lance_datagen::gen_batch()
rust/lancedb/src/error.rs
+2
View File
@@ -77,6 +77,8 @@ pub enum Error {
ColumnAlreadyExists { name: String },
#[snafu(display("Column '{name}' is not a computed column"))]
NotAComputedColumn { name: String },
#[snafu(display("Table '{name}' is not a materialized view"))]
NotAMaterializedView { name: String },
#[snafu(display("Invalid expression for column '{column}': {message}"))]
InvalidExpression { column: String, message: String },
rust/lancedb/src/function.rs
+12 -2
View File
@@ -1,7 +1,8 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! Canonical values exchanged with the Enterprise Function service.
//! Canonical Function values exchanged with the Enterprise service, plus the
//! backend-neutral terminal result of a computed-column refresh.
//!
//! This module contains client/wire values only. Catalog persistence,
//! environment bake, secret resolution, and execution are owned by Sophon.
@@ -580,13 +581,22 @@ impl FunctionBinding {
impl_json!(FunctionBinding);
/// Stable terminal result of a remote Function-column refresh Job.
/// Stable terminal result of an expression-backed or Function-backed column
/// refresh [`crate::Job`].
///
/// Local refresh jobs produce this value in process. LanceDB Cloud and
/// Enterprise decode the same value from the durable job's terminal payload.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RefreshColumnResult {
/// Rows assigned a value by this refresh.
pub rows_assigned: u64,
/// Rows whose computation failed.
pub rows_failed: u64,
/// Rows that still need a value when the job completes.
pub rows_remaining: u64,
/// Exact table version the refresh read.
pub source_version: u64,
/// Table version made visible by the refresh, when one was published.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub published_version: Option<u64>,
}
rust/lancedb/src/job.rs
+115 -37
View File
@@ -6,7 +6,7 @@
use std::sync::Arc;
use async_trait::async_trait;
use serde::de::DeserializeOwned;
use serde::{Serialize, de::DeserializeOwned};
use serde_json::Value;
use tokio::sync::watch;
use tokio::task::{AbortHandle, JoinHandle};
@@ -26,20 +26,16 @@ pub(crate) trait JobHandle: Send + Sync {
}
/// A backend-neutral successful terminal result.
///
/// Local operations do not carry a value. Remote operations may carry JSON
/// that the public [`Job`] decodes according to its result type.
#[derive(Clone)]
pub(crate) struct TerminalResult {
#[allow(dead_code)] // Typed remote submit endpoints consume this after Slice 1.
value: Option<Value>,
#[allow(dead_code)] // Preserved so typed decode errors retain request correlation.
request_id: Option<String>,
}
impl TerminalResult {
pub(crate) fn local() -> Self {
fn local(value: Value) -> Self {
Self {
value: None,
value: Some(value),
request_id: None,
}
}
@@ -51,23 +47,31 @@ impl TerminalResult {
}
}
#[allow(dead_code)] // Exercised by the remote typed-result fixtures in Slice 1.
fn decode<T: DeserializeOwned>(self) -> Result<T> {
let request_id = self.request_id.unwrap_or_default();
let value = self.value.ok_or_else(|| Error::Http {
source: "successful typed job response did not contain a result".into(),
request_id: request_id.clone(),
status_code: None,
let value = self.value.ok_or_else(|| match &self.request_id {
Some(request_id) => Error::Http {
source: "successful typed job response did not contain a result".into(),
request_id: request_id.clone(),
status_code: None,
},
None => Error::Runtime {
message: "successful typed job did not contain a result".to_string(),
},
})?;
serde_json::from_value(value).map_err(|error| Error::Http {
source: format!("failed to parse typed job result: {error}").into(),
request_id,
status_code: None,
serde_json::from_value(value).map_err(|error| match self.request_id {
Some(request_id) => Error::Http {
source: format!("failed to parse typed job result: {error}").into(),
request_id,
status_code: None,
},
None => Error::Runtime {
message: format!("failed to parse typed job result: {error}"),
},
})
}
}
type ResultDecoder<T> = fn(TerminalResult) -> Result<T>;
type ResultDecoder<T> = Arc<dyn Fn(TerminalResult) -> Result<T> + Send + Sync>;
enum JobInner<T> {
Handle {
@@ -79,7 +83,9 @@ enum JobInner<T> {
/// A handle to an operation that may still be running.
///
/// The operation may already be complete when the handle is created.
/// The operation may already be complete when the handle is created. `T` is
/// the endpoint's successful terminal result; unit-result operations use the
/// default `Job<()>`.
pub struct Job<T = ()>
where
T: Clone + Send + Sync + 'static,
@@ -111,15 +117,10 @@ impl Job<()> {
Self {
inner: JobInner::Handle {
handle,
decode: |_| Ok(()),
decode: Arc::new(|_| Ok(())),
},
}
}
/// A unit-result job running as a task in this process.
pub(crate) fn spawned(task: JoinHandle<Result<()>>) -> Self {
Self::new(Box::new(SpawnedJob::new(task)))
}
}
impl<T> Job<T>
@@ -131,12 +132,22 @@ where
Self {
inner: JobInner::Handle {
handle,
decode: TerminalResult::decode::<T>,
decode: Arc::new(TerminalResult::decode::<T>),
},
}
}
}
impl<T> Job<T>
where
T: Clone + Serialize + DeserializeOwned + Send + Sync + 'static,
{
/// A typed job running as a task in this process.
pub(crate) fn spawned(task: JoinHandle<Result<T>>) -> Self {
Self::new_typed(Box::new(SpawnedJob::new(task)))
}
}
impl<T> Job<T>
where
T: Clone + Send + Sync + 'static,
@@ -169,11 +180,13 @@ where
/// Waits until the operation reaches a terminal state.
///
/// Returns the endpoint's typed result. Unit-result jobs return `()`.
///
/// Returns [`crate::Error::JobFailed`] if the operation failed and
/// [`crate::Error::JobCancelled`] if it was cancelled.
pub async fn wait(&self) -> Result<T> {
match &self.inner {
JobInner::Handle { handle, decode } => decode(handle.wait().await?),
JobInner::Handle { handle, decode } => (decode)(handle.wait().await?),
JobInner::Completed(result) => Ok(result.clone()),
}
}
@@ -187,21 +200,53 @@ where
JobInner::Completed(_) => Ok(()),
}
}
/// Maps a successful terminal result without changing the job lifecycle.
/// The mapping may run once for each call to [`Job::wait`], so it should
/// be deterministic and free of externally visible side effects.
///
/// ```
/// use lancedb::{Job, function::RefreshColumnResult};
///
/// # async fn rows_assigned(
/// # job: Job<RefreshColumnResult>,
/// # ) -> lancedb::Result<u64> {
/// let job = job.map(|result| result.rows_assigned);
/// job.wait().await
/// # }
/// ```
pub fn map<U, F>(self, map: F) -> Job<U>
where
U: Clone + Send + Sync + 'static,
F: Fn(T) -> U + Send + Sync + 'static,
{
match self.inner {
JobInner::Handle { handle, decode } => Job {
inner: JobInner::Handle {
handle,
decode: Arc::new(move |result| Ok(map((decode)(result)?))),
},
},
JobInner::Completed(result) => Job {
inner: JobInner::Completed(map(result)),
},
}
}
}
/// How an in-process operation ended. Cloneable so every waiter can be given
/// the outcome; [`Error`] is not, so failures share one behind an [`Arc`].
#[derive(Clone)]
enum Outcome {
Succeeded,
Succeeded(TerminalResult),
Failed(Arc<Error>),
Cancelled,
}
impl Outcome {
fn into_result(self) -> Result<()> {
fn into_result(self) -> Result<TerminalResult> {
match self {
Self::Succeeded => Ok(()),
Self::Succeeded(result) => Ok(result),
Self::Failed(source) => Err(Error::JobFailed {
job_id: None,
failure: JobFailure::from_source(source),
@@ -220,12 +265,20 @@ struct SpawnedJob {
}
impl SpawnedJob {
fn new(task: JoinHandle<Result<()>>) -> Self {
fn new<T>(task: JoinHandle<Result<T>>) -> Self
where
T: Serialize + Send + 'static,
{
let abort = task.abort_handle();
let (tx, outcome) = watch::channel(None);
tokio::spawn(async move {
let outcome = match task.await {
Ok(Ok(())) => Outcome::Succeeded,
Ok(Ok(result)) => match serde_json::to_value(result) {
Ok(value) => Outcome::Succeeded(TerminalResult::local(value)),
Err(err) => Outcome::Failed(Arc::new(Error::Runtime {
message: format!("failed to serialize job result: {err}"),
})),
},
Ok(Err(err)) => Outcome::Failed(Arc::new(err)),
Err(err) if err.is_cancelled() => Outcome::Cancelled,
Err(err) => Outcome::Failed(Arc::new(Error::Runtime {
@@ -243,7 +296,7 @@ impl JobHandle for SpawnedJob {
async fn status(&self) -> Result<String> {
let label = match &*self.outcome.borrow() {
None => "running",
Some(Outcome::Succeeded) => "finished",
Some(Outcome::Succeeded(_)) => "finished",
Some(Outcome::Failed(_)) => "failed",
Some(Outcome::Cancelled) => "cancelled",
};
@@ -256,12 +309,11 @@ impl JobHandle for SpawnedJob {
.wait_for(|outcome| outcome.is_some())
.await
.map_err(|_| Error::Runtime {
message: "index job outcome was dropped before it completed".to_string(),
message: "job outcome was dropped before it completed".to_string(),
})?
.clone()
.expect("wait_for returns once an outcome is set");
settled.into_result()?;
Ok(TerminalResult::local())
settled.into_result()
}
async fn cancel(&self) -> Result<()> {
@@ -269,3 +321,29 @@ impl JobHandle for SpawnedJob {
Ok(())
}
}
#[cfg(test)]
mod tests {
use std::future::pending;
use super::*;
#[tokio::test]
async fn mapped_spawned_job_reuses_outcome() {
let job = Job::spawned(tokio::spawn(async { Ok(41_u64) })).map(|value| value + 1);
assert_eq!(job.wait().await.unwrap(), 42);
assert_eq!(job.wait().await.unwrap(), 42);
assert_eq!(job.status().await.unwrap(), "finished");
}
#[tokio::test]
async fn mapped_spawned_job_preserves_cancellation() {
let job = Job::spawned(tokio::spawn(async { pending::<Result<u64>>().await }))
.map(|value| value.to_string());
job.cancel().await.unwrap();
assert!(matches!(job.wait().await, Err(Error::JobCancelled { .. })));
assert_eq!(job.status().await.unwrap(), "cancelled");
}
}
rust/lancedb/src/lib.rs
+4
View File
@@ -186,6 +186,7 @@ pub mod index;
pub mod io;
pub mod ipc;
pub mod job;
pub mod materialized_view;
#[cfg(feature = "metrics-otel")]
pub mod metrics_otel;
#[cfg(feature = "polars")]
@@ -210,6 +211,9 @@ pub use function::FunctionVersion;
pub use job::Job;
use lance_index::vector::ApproxMode as LanceApproxMode;
use lance_linalg::distance::DistanceType as LanceDistanceType;
pub use materialized_view::{
MaterializedView, MaterializedViewDefinition, RefreshMaterializedViewResult, RefreshMode,
};
/// Re-export of the [`metrics`](https://docs.rs/metrics) crate facade. Enable
/// the `metrics` feature to publish LanceDB's internal metrics; install any
/// `metrics`-compatible recorder to collect them. See also [`metrics_otel`] for
rust/lancedb/src/materialized_view.rs
+2106
View File
File diff suppressed because it is too large Load Diff
rust/lancedb/src/materialized_view/differential.rs
+730
View File
@@ -0,0 +1,730 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! Differential refresh testing.
//!
//! The refresh contract is a property: after any sequence of source
//! mutations, a view maintained by default (incremental-where-possible)
//! refreshes equals the definition evaluated against the source directly,
//! and so does a forced rebuild. The oracle is an independent read of the
//! source -- plain column scan, filter applied in Rust -- so it shares
//! nothing with the refresh path it checks.
//!
//! The oracle runs after every step, not just at the end: a later mutation
//! that forces a rebuild would silently heal an incremental error, and those
//! transient errors are exactly the bugs this exists to catch.
use arrow_array::{Float32Array, Int32Array, RecordBatch};
use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
use futures::{StreamExt, TryStreamExt};
use lance::dataset::NewColumnTransform;
use std::sync::Arc;
use super::MaterializedView;
use super::refresh::RefreshMode;
use crate::connect;
use crate::connection::Connection;
use crate::query::{ExecutableQuery, QueryBase, Select};
use crate::table::{CompactionOptions, OptimizeAction, Table};
/// One source mutation, one per correctness-relevant class.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum SrcOp {
/// Fresh non-colliding ids of both parities, so every other op has
/// view-resident rows to act on: the only op that should refresh
/// incrementally.
AppendNew,
/// Deletion in surviving fragments must break the pure-append check.
DeleteEven,
/// An in-place update; on the filtered shape it crosses the predicate,
/// so rows must leave the view.
UpdateOddScore,
/// Fragment rewrite/renumber must break the pure-append check.
Compact,
/// A column the view does not read must NOT force a rebuild.
AddColumn,
/// merge_insert commits an Update whose by-source arm deletes rows, so a
/// classifier that reads Update as "changed only" loses those deletions.
MergeDropLargest,
/// merge_insert that both changes existing rows and inserts new ones in
/// one transaction.
MergeUpsert,
}
const ALL_OPS: [SrcOp; 7] = [
SrcOp::AppendNew,
SrcOp::DeleteEven,
SrcOp::UpdateOddScore,
SrcOp::Compact,
SrcOp::AddColumn,
SrcOp::MergeDropLargest,
SrcOp::MergeUpsert,
];
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Shape {
/// SELECT id, score.
Identity,
/// SELECT id, score WHERE score > 50: additionally sensitive to rows
/// crossing the predicate.
Filtered,
/// SELECT id, score LIMIT 4. Which rows are held depends on the order
/// they were first materialized, so the oracle checks containment and
/// the cap rather than equality.
Limited,
}
impl Shape {
fn filter(&self) -> Option<&'static str> {
match self {
Self::Identity | Self::Limited => None,
Self::Filtered => Some("score > 50"),
}
}
fn matches(&self, score: f32) -> bool {
match self {
Self::Identity | Self::Limited => true,
Self::Filtered => score > 50.0,
}
}
fn limit(&self) -> Option<usize> {
match self {
Self::Limited => Some(4),
_ => None,
}
}
}
struct Case {
conn: Connection,
source: Table,
view: MaterializedView,
shape: Shape,
next_id: i32,
added_columns: u32,
}
fn rows_batch(ids: &[i32]) -> RecordBatch {
let scores: Vec<f32> = ids.iter().map(|id| (*id * 10) as f32).collect();
RecordBatch::try_new(
Arc::new(ArrowSchema::new(vec![
ArrowField::new("id", DataType::Int32, true),
ArrowField::new("score", DataType::Float32, true),
])),
vec![
Arc::new(Int32Array::from(ids.to_vec())),
Arc::new(Float32Array::from(scores)),
],
)
.unwrap()
}
fn merge_batch(ids: &[i32]) -> RecordBatch {
let scores: Vec<f32> = ids.iter().map(|id| (*id * 10 + 5) as f32).collect();
RecordBatch::try_new(
Arc::new(ArrowSchema::new(vec![
ArrowField::new("id", DataType::Int32, true),
ArrowField::new("score", DataType::Float32, true),
])),
vec![
Arc::new(Int32Array::from(ids.to_vec())),
Arc::new(Float32Array::from(scores)),
],
)
.unwrap()
}
impl Case {
async fn new(shape: Shape) -> Self {
let conn = connect("memory://").execute().await.unwrap();
let source = conn
.create_table("src", rows_batch(&[1, 2, 3, 4]))
.write_options(crate::materialized_view::tests::stable_row_ids())
.execute()
.await
.unwrap();
let mut builder = conn
.create_materialized_view("view", "src")
.select([("id", "id"), ("score", "score")]);
if let Some(filter) = shape.filter() {
builder = builder.only_if(filter);
}
if let Some(limit) = shape.limit() {
builder = builder.limit(limit as u64);
}
let view = builder.execute().await.unwrap();
Self {
conn,
source,
view,
shape,
next_id: 100,
added_columns: 0,
}
}
async fn apply(&mut self, op: SrcOp) {
match op {
SrcOp::AppendNew => {
// Mixed parity: the middle id is odd, so UpdateOddScore always
// has a filter-matching appended row to evict.
let ids = vec![self.next_id, self.next_id + 101, self.next_id + 202];
self.next_id += 303;
self.source.add(rows_batch(&ids)).execute().await.unwrap();
}
SrcOp::DeleteEven => {
self.source.delete("id % 2 = 0").await.unwrap();
}
SrcOp::UpdateOddScore => {
self.source
.update()
.column("score", "-1.0")
.only_if("id % 2 = 1")
.execute()
.await
.unwrap();
}
SrcOp::Compact => {
self.source
.optimize(OptimizeAction::Compact {
options: CompactionOptions::default(),
remap_options: None,
})
.await
.unwrap();
}
SrcOp::MergeDropLargest => {
let mut ids = self.source_ids().await;
ids.sort_unstable();
ids.pop();
if ids.is_empty() {
return;
}
let batch = rows_batch(&ids);
let reader =
arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema());
let mut merge = self.source.merge_insert(&["id"]);
merge.when_not_matched_by_source_delete(None);
merge.execute(Box::new(reader)).await.unwrap();
}
SrcOp::MergeUpsert => {
let mut ids = self.source_ids().await;
ids.sort_unstable();
// One row that exists (updated in place) and one that does not.
let existing = ids.first().copied().unwrap_or(self.next_id);
let fresh = self.next_id;
self.next_id += 1;
let batch = merge_batch(&[existing, fresh]);
let reader =
arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema());
let mut merge = self.source.merge_insert(&["id"]);
merge
.when_matched_update_all(None)
.when_not_matched_insert_all();
merge.execute(Box::new(reader)).await.unwrap();
}
SrcOp::AddColumn => {
self.added_columns += 1;
let field = ArrowField::new(
format!("extra_{}", self.added_columns),
DataType::Int32,
true,
);
self.source
.add_columns()
.transform(NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new(
vec![field],
))))
.execute()
.await
.unwrap();
}
}
}
async fn source_ids(&self) -> Vec<i32> {
read_rows(
self.source
.query()
.select(Select::columns(&["id", "score"])),
)
.await
.into_iter()
.map(|(id, _)| id)
.collect()
}
/// The definition's result, read independently of the refresh path:
/// plain column scan, filter applied here, sorted.
async fn oracle(&self) -> Vec<(i32, i32)> {
let mut rows = read_rows(
self.source
.query()
.select(Select::columns(&["id", "score"])),
)
.await
.into_iter()
.filter(|(_, score)| self.shape.matches(*score as f32))
.collect::<Vec<_>>();
rows.sort_unstable();
rows
}
async fn view_rows(&self) -> Vec<(i32, i32)> {
let mut rows = read_rows(
self.view
.table()
.query()
.select(Select::columns(&["id", "score"])),
)
.await;
rows.sort_unstable();
rows
}
async fn check(&self, label: &str) -> Result<(), String> {
let expected = self.oracle().await;
let actual = self.view_rows().await;
let Some(cap) = self.shape.limit() else {
if expected != actual {
return Err(format!(
"{label}: view diverged from oracle\n expected: {expected:?}\n actual: {actual:?}"
));
}
return Ok(());
};
// A capped view holds some subset of the definition's result, never
// more than the cap, and never the same row twice.
if actual.len() > cap {
return Err(format!(
"{label}: view holds {} rows, over its cap of {cap}: {actual:?}",
actual.len()
));
}
let mut unique = actual.clone();
unique.dedup();
if unique.len() != actual.len() {
return Err(format!("{label}: view holds a row twice: {actual:?}"));
}
if let Some(stray) = actual.iter().find(|row| !expected.contains(row)) {
return Err(format!(
"{label}: view holds {stray:?}, which the definition does not select: {expected:?}"
));
}
// Below the cap the view must be complete, or a row was lost.
if actual.len() < cap.min(expected.len()) {
return Err(format!(
"{label}: view holds {} of {} selectable rows under a cap of {cap}: {actual:?}",
actual.len(),
expected.len()
));
}
Ok(())
}
}
async fn read_rows(query: impl ExecutableQuery) -> Vec<(i32, i32)> {
let batches = query
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
batches
.iter()
.flat_map(|batch| {
let ids = batch["id"].as_any().downcast_ref::<Int32Array>().unwrap();
let scores = batch["score"]
.as_any()
.downcast_ref::<Float32Array>()
.unwrap();
// Scores are integer-valued by construction; compare exactly.
(0..batch.num_rows())
.map(|i| (ids.value(i), scores.value(i) as i32))
.collect::<Vec<_>>()
})
.collect()
}
/// Drive one mutation sequence: refresh + oracle-check after every step,
/// then a forced rebuild checked against the same oracle.
async fn run_sequence(ops: &[SrcOp], shape: Shape) -> Result<(), String> {
let label = format!("{shape:?} {ops:?}");
let mut case = Case::new(shape).await;
case.view
.refresh()
.execute()
.await
.map_err(|e| format!("{label}: initial refresh failed: {e}"))?;
case.check(&format!("{label} (initial)")).await?;
for (step, op) in ops.iter().enumerate() {
case.apply(*op).await;
case.view
.refresh()
.execute()
.await
.map_err(|e| format!("{label}: refresh at step {step} failed: {e}"))?;
case.check(&format!("{label} (step {step}, {op:?})"))
.await?;
}
case.view
.refresh()
.full(true)
.execute()
.await
.map_err(|e| format!("{label}: final full refresh failed: {e}"))?;
case.check(&format!("{label} (final rebuild)")).await?;
// Silence the unused-connection lint without dropping it mid-case.
let _ = &case.conn;
Ok(())
}
/// Every op sequence up to `max_len`.
fn all_sequences(max_len: u32) -> Vec<Vec<SrcOp>> {
let mut sequences = Vec::new();
for len in 1..=max_len {
for mut index in 0..ALL_OPS.len().pow(len) {
let mut ops = Vec::with_capacity(len as usize);
for _ in 0..len {
ops.push(ALL_OPS[index % ALL_OPS.len()]);
index /= ALL_OPS.len();
}
sequences.push(ops);
}
}
sequences
}
async fn run_exhaustive(max_len: u32) {
let mut cases = Vec::new();
for shape in [Shape::Identity, Shape::Filtered, Shape::Limited] {
for ops in all_sequences(max_len) {
cases.push((ops, shape));
}
}
let failures: Vec<String> = futures::stream::iter(cases)
.map(|(ops, shape)| async move { run_sequence(&ops, shape).await.err() })
.buffer_unordered(8)
.filter_map(|failure| async move { failure })
.collect()
.await;
assert!(
failures.is_empty(),
"{} sequences diverged; first: {}",
failures.len(),
failures[0]
);
}
#[tokio::test(flavor = "multi_thread")]
async fn differential_exhaustive() {
run_exhaustive(3).await;
}
#[tokio::test(flavor = "multi_thread")]
#[ignore = "longer sweep; run manually"]
async fn differential_exhaustive_deep() {
run_exhaustive(4).await;
}
/// Named interleavings that double as repro handles. The mode assertions pin
/// the classifier, which value comparison alone cannot: a wrongly rebuilt
/// view still matches the oracle.
#[tokio::test(flavor = "multi_thread")]
async fn differential_named_regressions() {
// An append is the one op that must stay incremental.
let mut case = Case::new(Shape::Identity).await;
case.view.refresh().execute().await.unwrap();
case.apply(SrcOp::AppendNew).await;
let result = case.view.refresh().execute().await.unwrap();
assert_eq!(result.mode, RefreshMode::Incremental);
case.check("append stays incremental").await.unwrap();
// A column the view does not read must not force a rebuild.
let mut case = Case::new(Shape::Identity).await;
case.view.refresh().execute().await.unwrap();
case.apply(SrcOp::AddColumn).await;
let result = case.view.refresh().execute().await.unwrap();
assert_eq!(result.mode, RefreshMode::Incremental);
assert_eq!(result.rows_written, 0);
// Compaction rearranges rows without changing them: the watermark
// advances and nothing rebuilds.
let mut case = Case::new(Shape::Identity).await;
case.view.refresh().execute().await.unwrap();
case.apply(SrcOp::AppendNew).await;
case.view.refresh().execute().await.unwrap();
case.apply(SrcOp::Compact).await;
let result = case.view.refresh().execute().await.unwrap();
assert_eq!(result.mode, RefreshMode::Incremental);
assert_eq!(result.rows_written, 0);
case.check("compaction alone").await.unwrap();
// Fragment bookkeeping stays coherent across the compaction: the next
// append is separable and computed alone.
case.apply(SrcOp::AppendNew).await;
let result = case.view.refresh().execute().await.unwrap();
assert_eq!(result.mode, RefreshMode::Incremental);
assert_eq!(result.rows_written, 3);
case.check("compact then append").await.unwrap();
// A row updated to no longer match the filter must leave the view --
// and the fixture must prove the eviction happened, not merely that the
// end state matches: an update that never touched a view-resident row
// would also "match".
let mut case = Case::new(Shape::Filtered).await;
case.apply(SrcOp::AppendNew).await;
case.view.refresh().execute().await.unwrap();
let before = case.view_rows().await.len();
case.apply(SrcOp::UpdateOddScore).await;
case.view.refresh().execute().await.unwrap();
let after = case.view_rows().await.len();
assert!(
after < before,
"no view-resident row was evicted ({before} -> {after}); the fixture \
no longer exercises the filtered-update transition"
);
case.check("update crosses the filter").await.unwrap();
}
// ---------------------------------------------------------------------------
// Concurrency
// ---------------------------------------------------------------------------
//
// The sequential cases above cannot observe a cross-process race: the
// per-view refresh lock is process-local, so a second refresh in this
// process queues behind the first. What is missing is not more op
// sequences but a second process. These cases add one, and assert the same
// property the harness always asserts -- the view holds each row once.
/// Rows the definition selects from the source: every id but the first,
/// read straight from the source, sharing nothing with the refresh path.
async fn concurrency_oracle(conn: &Connection) -> Vec<i32> {
let batches: Vec<RecordBatch> = conn
.open_table("src")
.execute()
.await
.unwrap()
.query()
.select(Select::columns(&["id"]))
.execute()
.await
.unwrap()
.try_collect()
.await
.unwrap();
let mut ids = Vec::new();
for batch in &batches {
let column = batch
.column(0)
.as_any()
.downcast_ref::<Int32Array>()
.unwrap();
for i in 0..batch.num_rows() {
if column.value(i) > 1 {
ids.push(column.value(i));
}
}
}
ids.sort_unstable();
ids
}
/// The view's ids, sorted.
async fn concurrency_view_ids(conn: &Connection) -> Vec<i32> {
let batches: Vec<RecordBatch> = conn
.open_table("mv")
.execute()
.await
.unwrap()
.query()
.select(Select::columns(&["id"]))
.execute()
.await
.unwrap()
.try_collect()
.await
.unwrap();
let mut ids = Vec::new();
for batch in &batches {
let column = batch
.column(0)
.as_any()
.downcast_ref::<Int32Array>()
.unwrap();
for i in 0..batch.num_rows() {
ids.push(column.value(i));
}
}
ids.sort_unstable();
ids
}
/// One refresh of the view at `MV_RACE_DIR`, in its own process.
///
/// Setup happens before the start barrier so warm-up does not stagger the
/// two processes. What makes the race certain rather than likely is the
/// second barrier inside `refresh()` itself, which holds every participant
/// between staging and commit.
#[tokio::test]
#[ignore = "spawned as a child process by the concurrency cases"]
async fn cross_process_refresh_child() {
let Ok(dir) = std::env::var("MV_RACE_DIR") else {
return;
};
let dir = std::path::PathBuf::from(dir);
let tag = std::env::var("MV_RACE_TAG").unwrap();
let conn = connect(dir.to_str().unwrap()).execute().await.unwrap();
let table = conn.open_table("mv").execute().await.unwrap();
let _ = table.schema().await.unwrap();
let _ = table.count_rows(None).await.unwrap();
let source = conn.open_table("src").execute().await.unwrap();
let _ = source.count_rows(None).await.unwrap();
let view = MaterializedView::from_table(table).await.unwrap();
std::fs::write(dir.join(format!("ready-{tag}")), b"1").unwrap();
while !dir.join("START").exists() {
std::thread::sleep(std::time::Duration::from_millis(2));
}
let outcome = match view.refresh().execute().await {
Ok(result) => format!("committed rows={}", result.rows_written),
Err(err) if is_commit_conflict(&err) => "conflicted".to_string(),
Err(err) => format!("failed {err}"),
};
std::fs::write(dir.join(format!("outcome-{tag}")), outcome).unwrap();
}
/// Whether a refresh lost its commit to a concurrent one, as opposed to
/// failing for any other reason.
fn is_commit_conflict(err: &crate::Error) -> bool {
let text = err.to_string();
text.contains("Retryable commit conflict") || text.contains("preempted by concurrent")
}
/// Two processes refreshing one view concurrently must leave the view
/// equal to the oracle: each selected row present exactly once.
///
/// Both plan the same incremental delta from one watermark. A refresh is
/// meant to land on the generation it planned or leave nothing behind, so
/// at most one of them may write.
#[tokio::test]
async fn concurrent_refreshes_hold_each_row_once() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().to_str().unwrap().to_string();
let conn = connect(&path).execute().await.unwrap();
conn.create_table("src", rows_batch(&[1, 2, 3, 4]))
.write_options(crate::materialized_view::tests::stable_row_ids())
.execute()
.await
.unwrap();
let view = conn
.create_materialized_view("mv", "src")
.select([("id", "id"), ("score", "score")])
.only_if("id > 1")
.execute()
.await
.unwrap();
// Seed the watermark so the racing refreshes are both incremental.
view.refresh().execute().await.unwrap();
// Large enough that a refresh is real work rather than a formality.
let ids: Vec<i32> = (100..200_100).collect();
conn.open_table("src")
.execute()
.await
.unwrap()
.add(rows_batch(&ids))
.execute()
.await
.unwrap();
let tags = ["a", "b"];
let exe = std::env::current_exe().unwrap();
let children: Vec<std::process::Child> = tags
.iter()
.map(|tag| {
std::process::Command::new(&exe)
.args([
"--exact",
"materialized_view::differential::cross_process_refresh_child",
"--ignored",
"--nocapture",
])
.env("MV_RACE_DIR", dir.path())
.env("MV_RACE_SYNC", dir.path())
.env("MV_RACE_PEERS", "2")
.env("MV_RACE_TAG", tag)
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.spawn()
.unwrap()
})
.collect();
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(180);
while tags
.iter()
.any(|tag| !dir.path().join(format!("ready-{tag}")).exists())
{
assert!(
std::time::Instant::now() < deadline,
"children never became ready"
);
std::thread::sleep(std::time::Duration::from_millis(10));
}
std::fs::write(dir.path().join("START"), b"1").unwrap();
for (tag, mut child) in tags.iter().zip(children) {
let status = loop {
match child.try_wait().unwrap() {
Some(status) => break status,
None if std::time::Instant::now() >= deadline => {
child.kill().unwrap();
panic!("child {tag} never finished");
}
None => std::thread::sleep(std::time::Duration::from_millis(10)),
}
};
assert!(status.success(), "child {tag} exited {status}");
}
// Both refreshes reached the commit boundary before either committed --
// the in-refresh barrier guarantees it -- so exactly one may win.
let outcomes: Vec<String> = tags
.iter()
.map(|tag| {
std::fs::read_to_string(dir.path().join(format!("outcome-{tag}")))
.unwrap_or_else(|_| panic!("child {tag} recorded no outcome"))
})
.collect();
for tag in tags {
assert!(
dir.path().join(format!("planned-{tag}")).exists(),
"child {tag} never reached the commit boundary, so nothing was synchronized"
);
}
let committed = outcomes.iter().filter(|o| o.contains("committed")).count();
let conflicted = outcomes.iter().filter(|o| o.contains("conflicted")).count();
assert_eq!(
(committed, conflicted),
(1, 1),
"exactly one refresh may win the generation both planned: {outcomes:?}"
);
let expected = concurrency_oracle(&conn).await;
let actual = concurrency_view_ids(&conn).await;
assert_eq!(
actual.len(),
expected.len(),
"the view holds {} rows, the oracle {}: a losing refresh left rows behind",
actual.len(),
expected.len()
);
assert_eq!(actual, expected, "the view does not match the oracle");
}
rust/lancedb/src/materialized_view/refresh.rs
+2996
View File
File diff suppressed because it is too large Load Diff
rust/lancedb/src/remote/db.rs
+173 -22
View File
@@ -344,6 +344,62 @@ impl<S: HttpSend> RemoteDatabase<S> {
self.table_cache.remove(&cache_key).await;
Ok((request_id, resp))
}
/// Collect the tables of a namespace in name order, for `table_names`.
///
/// `table_names` promises name order and resumes after a table name, but the namespace
/// route's `page_token` is opaque -- it belongs to the store the listing walks, and a
/// token this client invented would resume from the wrong place. So the whole namespace is
/// walked by handing each response's token straight back, and the name semantics are
/// applied here. Constructing no token is what makes this work against a server on either
/// side of the change: it only ever repeats what the server said.
///
/// This is the cost `table_names` already paid -- the server used to enumerate and sort the
/// namespace on every request -- and it is why `list_tables` replaces it.
async fn table_names_in_namespace(
&self,
request: &TableNamesRequest,
) -> Result<(Vec<String>, ServerVersion)> {
let namespace_id =
build_namespace_identifier(&request.namespace_path, &self.client.id_delimiter);
let path = format!("/v1/namespace/{}/table/list", namespace_id);
let mut names = Vec::new();
// Every page reports the same server, so keep the first page's version.
let mut version: Option<ServerVersion> = None;
let mut page_token: Option<String> = None;
loop {
let mut req = self.client.get(&path);
if let Some(ref token) = page_token {
req = req.query(&[("page_token", token)]);
}
let (request_id, rsp) = self.client.send_with_retry(req, None, true).await?;
let rsp = self.client.check_response(&request_id, rsp).await?;
if version.is_none() {
version = Some(parse_server_version(&request_id, &rsp)?);
}
let response: ListTablesResponse = rsp.json().await.err_to_http(request_id)?;
names.extend(response.tables);
// An empty token is the end of the listing, not a token to send back: a server
// that reads an empty token as "start from the beginning" would hand back the
// first page again.
match response.page_token.filter(|token| !token.is_empty()) {
// A server that repeated a token would never finish; treat that as the end
// rather than looping on it.
Some(token) if Some(&token) != page_token.as_ref() => page_token = Some(token),
_ => break,
}
}
names.sort();
if let Some(ref start_after) = request.start_after {
names.retain(|name| name > start_after);
}
if let Some(limit) = request.limit {
names.truncate(limit as usize);
}
Ok((names, version.unwrap_or_default()))
}
}
#[cfg(all(test, feature = "remote"))]
@@ -513,7 +569,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
async fn get_function(&self, name: &str, version: &str) -> Result<FunctionVersion> {
let req = self
.client
.post("/v1/functions/get")
.post("/v1/functions/describe")
.json(&serde_json::json!({
"name": name,
"version": version,
@@ -621,29 +677,29 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
}
async fn table_names(&self, request: TableNamesRequest) -> Result<Vec<String>> {
let mut req = if !request.namespace_path.is_empty() {
let namespace_id =
build_namespace_identifier(&request.namespace_path, &self.client.id_delimiter);
self.client
.get(&format!("/v1/namespace/{}/table/list", namespace_id))
let (tables, version) = if request.namespace_path.is_empty() {
// The flat route resumes after a table name and orders by name, which is exactly
// what `start_after` means, so the server does the paging.
let mut req = self.client.get("/v1/table/");
if let Some(limit) = request.limit {
req = req.query(&[("limit", limit)]);
}
if let Some(ref start_after) = request.start_after {
req = req.query(&[("page_token", start_after)]);
}
let (request_id, rsp) = self.client.send_with_retry(req, None, true).await?;
let rsp = self.client.check_response(&request_id, rsp).await?;
let version = parse_server_version(&request_id, &rsp)?;
let tables = rsp
.json::<ListTablesResponse>()
.await
.err_to_http(request_id)?
.tables;
(tables, version)
} else {
self.client.get("/v1/table/")
self.table_names_in_namespace(&request).await?
};
if let Some(limit) = request.limit {
req = req.query(&[("limit", limit)]);
}
if let Some(start_after) = request.start_after {
req = req.query(&[("page_token", start_after)]);
}
let (request_id, rsp) = self.client.send_with_retry(req, None, true).await?;
let rsp = self.client.check_response(&request_id, rsp).await?;
let version = parse_server_version(&request_id, &rsp)?;
let tables = rsp
.json::<ListTablesResponse>()
.await
.err_to_http(request_id)?
.tables;
for table in &tables {
let table_identifier =
build_table_identifier(table, &request.namespace_path, &self.client.id_delimiter);
@@ -1227,6 +1283,101 @@ mod tests {
assert_eq!(names, vec!["table1", "table2"]);
}
#[tokio::test]
async fn test_table_names_in_a_namespace_never_invents_a_page_token() {
// The namespace route's token belongs to the store, so `table_names` cannot build one
// from `start_after`. It walks the namespace on the server's own tokens and applies the
// name semantics itself, which is what keeps it working either side of the change.
let page = Arc::new(AtomicUsize::new(0));
let conn = Connection::new_with_handler(move |request| {
assert_eq!(request.url().path(), "/v1/namespace/ns/table/list");
let query = request.url().query().unwrap_or("");
assert!(
!query.contains("page_token=users"),
"a table name must never be sent as a page token: {query}"
);
match page.fetch_add(1, Ordering::SeqCst) {
0 => {
assert!(
!query.contains("page_token"),
"the walk starts with no token"
);
http::Response::builder()
.status(200)
.body(r#"{"tables": ["users", "orders"], "page_token": "opaque-1"}"#)
.unwrap()
}
_ => {
assert!(query.contains("page_token=opaque-1"));
http::Response::builder()
.status(200)
.body(r#"{"tables": ["widgets"]}"#)
.unwrap()
}
}
});
let names = conn
.table_names()
.namespace(vec!["ns".to_string()])
.start_after("users")
.execute()
.await
.unwrap();
// Name order, resumed after "users": "orders" sorts before it and is dropped.
assert_eq!(names, vec!["widgets"]);
}
#[tokio::test]
async fn test_table_names_in_a_namespace_stops_on_a_repeated_token() {
// A server that handed back the token it was given would never finish the walk.
let conn = Connection::new_with_handler(|_request| {
http::Response::builder()
.status(200)
.body(r#"{"tables": ["a"], "page_token": "same"}"#)
.unwrap()
});
let names = conn
.table_names()
.namespace(vec!["ns".to_string()])
.execute()
.await
.unwrap();
// The guard bounds the walk instead of letting it run forever. The repeat is the
// server breaking the token contract and is not papered over here.
assert_eq!(names, vec!["a", "a"]);
}
#[tokio::test]
async fn test_table_names_in_a_namespace_stops_on_an_empty_token() {
// An empty token ends the listing. Sending it back would ask a server that reads it
// as "start from the beginning" for the first page a second time, and every name on
// that page would be collected twice.
let requests = Arc::new(AtomicUsize::new(0));
let seen = requests.clone();
let conn = Connection::new_with_handler(move |request| {
seen.fetch_add(1, Ordering::SeqCst);
assert!(
!request.url().query().unwrap_or("").contains("page_token"),
"an empty token must never be sent back"
);
http::Response::builder()
.status(200)
.body(r#"{"tables": ["a"], "page_token": ""}"#)
.unwrap()
});
let names = conn
.table_names()
.namespace(vec!["ns".to_string()])
.execute()
.await
.unwrap();
assert_eq!(names, vec!["a"]);
assert_eq!(requests.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_table_names_pagination() {
let conn = Connection::new_with_handler(|request| {
@@ -2520,7 +2671,7 @@ mod tests {
);
let conn = Connection::new_with_handler(|request| {
assert_eq!(request.method(), &reqwest::Method::POST);
assert_eq!(request.url().path(), "/v1/functions/get");
assert_eq!(request.url().path(), "/v1/functions/describe");
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
assert_eq!(
rust/lancedb/src/remote/table.rs
+179 -43
View File
@@ -21,11 +21,11 @@ use crate::remote::job::RemoteJob;
use crate::table::AddColumnsResult;
use crate::table::AddResult;
use crate::table::BranchDiff;
use crate::table::CherryPickResult;
use crate::table::DeleteResult;
use crate::table::DropColumnsResult;
use crate::table::LsmStats;
use crate::table::LsmWriteSpec;
use crate::table::MergeBranchResult;
use crate::table::MergeResult;
use crate::table::Tags;
use crate::table::UpdateResult;
@@ -1775,6 +1775,18 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
Ok(())
}
async fn snapshot_at_current_version(&self) -> Result<Option<Arc<dyn BaseTable>>> {
// A checked-out handle already names its snapshot. Otherwise resolve
// latest exactly once before creating the independent pinned handle.
let version = match self.current_version().await {
Some(version) => version,
None => self.describe().await?.version,
};
let snapshot = self.with_branch(self.branch.clone());
*snapshot.version.write().await = Some(version);
Ok(Some(Arc::new(snapshot)))
}
async fn restore(&self) -> Result<()> {
let mut request = self
.client
@@ -2031,7 +2043,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
async fn diff_branch(&self, from_branch: &str) -> Result<BranchDiff> {
if from_branch.trim().is_empty() {
return Err(Error::InvalidInput {
message: "from_branch must be a non-empty string".into(),
message: "Branch name cannot be empty.".into(),
});
}
let request = self
@@ -2058,20 +2070,23 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
})
}
async fn merge_branch(&self, from_branch: &str, dry_run: bool) -> Result<MergeBranchResult> {
async fn cherry_pick(&self, from_branch: &str, dry_run: bool) -> Result<CherryPickResult> {
if from_branch.trim().is_empty() {
return Err(Error::InvalidInput {
message: "from_branch must be a non-empty string".into(),
message: "Branch name cannot be empty.".into(),
});
}
let request = self
.client
.post(&format!("/v1/table/{}/branches/merge/", self.identifier))
.post(&format!(
"/v1/table/{}/branches/cherry_pick/",
self.identifier
))
.json(&serde_json::json!({
"from_branch": from_branch,
"dry_run": dry_run,
}));
// No retry. 409 rejected merge is final and carries a body.
// No retry. HTTP 409 is CherryPickStatus::Failed with a body, not a transport error.
let (request_id, response) = self.send(request, false).await?;
let status = response.status();
if status == StatusCode::NOT_FOUND {
@@ -2080,11 +2095,11 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
source: format!("branch '{}' does not exist", from_branch).into(),
});
}
// 200 and 409 both carry MergeBranchResult.
// 200 and 409 both carry CherryPickResult.
if status != StatusCode::OK && status != StatusCode::CONFLICT {
let body = response.text().await.unwrap_or_default();
return Err(Error::Http {
source: format!("unexpected status {status} from merge_branch: {body}").into(),
source: format!("unexpected status {status} from cherry_pick: {body}").into(),
request_id,
status_code: Some(status),
});
@@ -2092,7 +2107,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
let body = response.text().await.err_to_http(request_id.clone())?;
serde_json::from_str(&body).map_err(|err| Error::Http {
source: format!(
"Failed to parse merge_branch response: {}, body: {}",
"Failed to parse cherry_pick response: {}, body: {}",
err, body
)
.into(),
@@ -2861,7 +2876,10 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
})
}
async fn refresh_column_async(&self, column: &str) -> Result<Job> {
async fn refresh_column_async(
&self,
column: &str,
) -> Result<Job<crate::function::RefreshColumnResult>> {
self.check_mutable().await?;
let mut body = serde_json::json!({ "column": column });
self.apply_branch_body(&mut body);
@@ -2882,7 +2900,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
status_code: None,
})?;
Ok(Job::new(Box::new(FreshnessJob {
Ok(Job::new_typed(Box::new(FreshnessJob {
inner: RemoteJob::new(self.client.clone(), response.job_id),
freshness: self.freshness.clone(),
version: self.version.clone(),
@@ -3277,6 +3295,21 @@ mod tests {
},
};
fn refresh_done(job_id: &str) -> String {
json!({
"job_id": job_id,
"job_state": "DONE",
"result": {
"rows_assigned": 12,
"rows_failed": 0,
"rows_remaining": 0,
"source_version": 7,
"published_version": 8,
}
})
.to_string()
}
#[tokio::test]
async fn test_not_found() {
let table = Table::new_with_handler("my_table", |_| {
@@ -4316,6 +4349,43 @@ mod tests {
assert!(!table.base_table().scan_order_is_deterministic());
}
#[tokio::test]
async fn test_checkout_branch_pins_without_touching_the_original() {
let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
let recorder = seen.clone();
let table = Table::new_with_handler_version(
"my_table",
semver::Version::new(0, 5, 0),
move |request| match request.url().path() {
"/v1/table/my_table/describe/" => http::Response::builder()
.status(200)
.body(br#"{"version": 42, "schema": {"fields": []}}"#.to_vec())
.unwrap(),
"/v1/table/my_table/count_rows/" => {
let body = request_body_json(&request);
recorder.lock().unwrap().push(body["version"].clone());
http::Response::builder()
.status(200)
.body(b"0".to_vec())
.unwrap()
}
path => panic!("unexpected request path: {path}"),
},
);
let pinned = table.checkout_branch("main", Some(42)).await.unwrap();
pinned.count_rows(None).await.unwrap();
table.count_rows(None).await.unwrap();
let seen = seen.lock().unwrap();
assert_eq!(seen[0], 42, "the pinned handle must send its version");
assert!(
seen[1].is_null(),
"the original handle must still track latest, got {:?}",
seen[1]
);
}
#[tokio::test]
async fn test_fetch_blobs_sends_the_checked_out_version() {
let ipc = one_row_blob_ipc_stream("image");
@@ -6732,6 +6802,53 @@ mod tests {
assert_eq!(result.version, 8);
}
#[tokio::test]
async fn test_add_fixed_size_list_function_column_declares_the_vector_type() {
let table = Table::new_with_handler("my_table", |request| {
match request.url().path() {
"/v1/table/my_table/describe/" => http::Response::builder()
.status(200)
.body(
r#"{"version":1,"schema":{"fields":[{"name":"description","nullable":true,"type":{"type":"string"}}]}}"#,
)
.unwrap(),
"/v1/table/my_table/add_columns/" => {
let actual: serde_json::Value = serde_json::from_slice(
request.body().unwrap().as_bytes().unwrap(),
)
.unwrap();
let expected: serde_json::Value = serde_json::from_str(include_str!(
"../../tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json"
))
.unwrap();
assert_eq!(actual, expected);
http::Response::builder()
.status(200)
.body(r#"{"version":8}"#)
.unwrap()
}
path => panic!("Unexpected path: {path}"),
}
});
let application = crate::function::FunctionApplication::from_json(
r#"{
"function":{"name":"embed","version":"fv_01K3EXACT"},
"inputs":[{"parameter":"text","kind":"column","value":{"path":"description"}}],
"output":{"kind":"scalar","arrow_type":"fixed_size_list<float32, 3>","nullable":false},
"group_id":"fg_fixed"
}"#,
)
.unwrap();
let result = table
.add_columns()
.function_as("embedding", application)
.execute()
.await
.unwrap();
assert_eq!(result.version, 8);
}
#[tokio::test]
async fn test_add_named_struct_function_expands_one_atomic_sibling_group() {
let table = Table::new_with_handler("my_table", |request| match request.url().path() {
@@ -6889,7 +7006,7 @@ mod tests {
.unwrap(),
"/v1/jobs/describe" => http::Response::builder()
.status(200)
.body(r#"{"job_id": "j-7", "job_state": "DONE"}"#.to_string())
.body(refresh_done("j-7"))
.unwrap(),
"/v1/table/my_table/count_rows/" => {
saw.store(
@@ -6905,7 +7022,9 @@ mod tests {
});
let job = table.refresh_column_async("doubled").await.unwrap();
job.wait().await.unwrap();
let result = job.wait().await.unwrap();
assert_eq!(result.rows_assigned, 12);
assert_eq!(result.published_version, Some(8));
table.count_rows(None).await.unwrap();
assert!(
saw_min_timestamp.load(std::sync::atomic::Ordering::SeqCst),
@@ -6927,7 +7046,7 @@ mod tests {
.unwrap(),
"/v1/jobs/describe" => http::Response::builder()
.status(200)
.body(r#"{"job_id": "j-8", "job_state": "DONE"}"#.to_string())
.body(refresh_done("j-8"))
.unwrap(),
"/v1/table/my_table/describe/" => {
let schema = Schema::new(vec![Field::new("x", DataType::Int32, true)]);
@@ -6973,7 +7092,7 @@ mod tests {
.unwrap(),
"/v1/jobs/describe" => http::Response::builder()
.status(200)
.body(r#"{"job_id": "j-9", "job_state": "DONE"}"#.to_string())
.body(refresh_done("j-9"))
.unwrap(),
"/v1/table/my_table/tags/version/" => http::Response::builder()
.status(200)
@@ -7037,7 +7156,7 @@ mod tests {
}
"/v1/jobs/describe" => http::Response::builder()
.status(200)
.body(r#"{"job_id": "j-10", "job_state": "DONE"}"#.to_string())
.body(refresh_done("j-10"))
.unwrap(),
"/v1/table/my_table/describe/" => {
let schema = Schema::new(vec![Field::new("x", DataType::Int32, true)]);
@@ -7110,7 +7229,7 @@ mod tests {
}
"/v1/jobs/describe" => http::Response::builder()
.status(200)
.body(r#"{"job_id": "j-11", "job_state": "DONE"}"#.to_string())
.body(refresh_done("j-11"))
.unwrap(),
"/v1/table/my_table/count_rows/" => {
*saw.lock().unwrap() = request
@@ -10410,6 +10529,20 @@ mod tests {
);
}
#[tokio::test]
async fn test_materialized_view_refused_without_a_request() {
// Materialized views are local-only. The table-level entry the
// bindings use must refuse a remote table before reading its schema,
// so the panicking handler is the assertion.
let table = Table::new_with_handler("my_table", |request| -> http::Response<String> {
panic!("unexpected request: {}", request.url().path())
});
let err = crate::MaterializedView::from_table(table)
.await
.unwrap_err();
assert!(matches!(err, Error::NotSupported { .. }), "got {err:?}");
}
#[tokio::test]
async fn test_create_branch_empty_name_rejected_client_side() {
use lance::dataset::refs::Ref;
@@ -10499,8 +10632,7 @@ mod tests {
"changedColumns":[],
"addedIndexes":[],
"removedIndexes":[],
"mergeable":true,
"mergeBlockers":[]
"errors":[]
}"#
}
@@ -10518,15 +10650,18 @@ mod tests {
});
let diff = table.diff_branch("exp").await.unwrap();
assert_eq!(diff.from_branch, "exp");
assert!(diff.mergeable);
assert!(diff.errors.is_empty());
assert_eq!(diff.added_columns.len(), 1);
assert_eq!(diff.added_columns[0].name, "tag");
}
#[tokio::test]
async fn test_merge_branch_dry_run() {
async fn test_cherry_pick_dry_run() {
let table = Table::new_with_handler("my_table", |request| {
assert_eq!(request.url().path(), "/v1/table/my_table/branches/merge/");
assert_eq!(
request.url().path(),
"/v1/table/my_table/branches/cherry_pick/"
);
let body = request_body_json(&request);
assert_eq!(body["from_branch"], "exp");
assert_eq!(body["dry_run"], true);
@@ -10536,27 +10671,29 @@ mod tests {
);
http::Response::builder().status(200).body(resp).unwrap()
});
let result = table.merge_branch("exp", true).await.unwrap();
assert_eq!(result.status, crate::table::MergeBranchStatus::Ready);
let result = table.cherry_pick("exp", true).await.unwrap();
assert_eq!(result.status, crate::table::CherryPickStatus::Ready);
assert_eq!(result.preview.promoted_columns, vec!["tag".to_string()]);
assert!(result.main_version_after.is_none());
}
#[tokio::test]
async fn test_merge_branch_rejected_returns_ok_with_body() {
async fn test_cherry_pick_failed_returns_ok_with_body() {
let table = Table::new_with_handler("my_table", |request| {
assert_eq!(request.url().path(), "/v1/table/my_table/branches/merge/");
assert_eq!(
request.url().path(),
"/v1/table/my_table/branches/cherry_pick/"
);
let body = request_body_json(&request);
assert_eq!(body["dry_run"], false);
let mut diff: serde_json::Value =
serde_json::from_str(sample_branch_diff_json()).unwrap();
diff["mergeable"] = serde_json::json!(false);
diff["mergeBlockers"] = serde_json::json!([{
diff["errors"] = serde_json::json!([{
"code": "baseMoved",
"message": "main has advanced"
}]);
let resp = serde_json::json!({
"status": "rejected",
"status": "failed",
"diff": diff,
"preview": { "promotedColumns": [] }
});
@@ -10565,24 +10702,23 @@ mod tests {
.body(resp.to_string())
.unwrap()
});
let result = table.merge_branch("exp", false).await.unwrap();
assert_eq!(result.status, crate::table::MergeBranchStatus::Rejected);
assert!(!result.diff.mergeable);
assert_eq!(result.diff.merge_blockers.len(), 1);
let result = table.cherry_pick("exp", false).await.unwrap();
assert_eq!(result.status, crate::table::CherryPickStatus::Failed);
assert!(!result.diff.errors.is_empty());
assert_eq!(result.diff.errors.len(), 1);
}
#[tokio::test]
async fn test_merge_branch_unknown_blocker_code_parses() {
async fn test_cherry_pick_unknown_error_code_parses() {
let table = Table::new_with_handler("my_table", |_| {
let mut diff: serde_json::Value =
serde_json::from_str(sample_branch_diff_json()).unwrap();
diff["mergeable"] = serde_json::json!(false);
diff["mergeBlockers"] = serde_json::json!([{
diff["errors"] = serde_json::json!([{
"code": "multipleCommits",
"message": "branch has more than one data commit"
}]);
let resp = serde_json::json!({
"status": "rejected",
"status": "failed",
"diff": diff,
"preview": { "operation": "append", "rowsAdded": 2 }
});
@@ -10591,24 +10727,24 @@ mod tests {
.body(resp.to_string())
.unwrap()
});
let result = table.merge_branch("exp", false).await.unwrap();
assert_eq!(result.status, crate::table::MergeBranchStatus::Rejected);
let result = table.cherry_pick("exp", false).await.unwrap();
assert_eq!(result.status, crate::table::CherryPickStatus::Failed);
assert_eq!(
result.diff.merge_blockers[0].code,
crate::table::MergeBlockerCode::Unknown
result.diff.errors[0].code,
crate::table::CherryPickErrorCode::Unknown
);
assert!(result.preview.promoted_columns.is_empty());
}
#[tokio::test]
async fn test_merge_branch_unexpected_2xx_is_error() {
async fn test_cherry_pick_unexpected_2xx_is_error() {
let table = Table::new_with_handler("my_table", |_| {
http::Response::builder()
.status(204)
.body(String::new())
.unwrap()
});
let err = table.merge_branch("exp", false).await.unwrap_err();
let err = table.cherry_pick("exp", false).await.unwrap_err();
match err {
Error::Http {
status_code: Some(code),
rust/lancedb/src/table.rs
+43 -22
View File
@@ -66,8 +66,8 @@ use self::merge::MergeInsertBuilder;
pub mod add_columns;
mod add_data;
pub mod branch_merge;
pub mod checkpoint;
pub mod cherry_pick;
pub mod computed_columns;
mod create_index;
pub mod datafusion;
@@ -87,9 +87,9 @@ pub use add_columns::AddColumnsBuilder;
#[cfg(feature = "remote")]
pub(crate) use add_data::PreprocessingOutput;
pub use add_data::{AddDataBuilder, AddDataMode, AddResult, NaNVectorBehavior};
pub use branch_merge::{
BranchDiff, ColumnChange, ColumnSummary, IndexSummary, MergeBlocker, MergeBlockerCode,
MergeBranchResult, MergeBranchStatus, MergePreview, RowCountSummary,
pub use cherry_pick::{
BranchDiff, CherryPickError, CherryPickErrorCode, CherryPickPreview, CherryPickResult,
CherryPickStatus, ColumnChange, ColumnSummary, IndexSummary, RowCountSummary,
};
pub use chrono::Duration;
pub use computed_columns::{
@@ -771,7 +771,10 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
}
/// Fill a computed column's unfilled rows, returning a [`Job`] tracking
/// the operation.
async fn refresh_column_async(&self, _column: &str) -> Result<Job> {
async fn refresh_column_async(
&self,
_column: &str,
) -> Result<Job<crate::function::RefreshColumnResult>> {
Err(Error::NotSupported {
message: "computed columns are supported only on local tables".into(),
})
@@ -789,6 +792,14 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
async fn checkout_tag(&self, tag: &str) -> Result<()>;
/// Checkout the latest version of the table.
async fn checkout_latest(&self) -> Result<()>;
/// Return an independent handle pinned to the version currently selected.
///
/// Backends that can advance between requests should override this for
/// multi-request operations that need snapshot consistency. Backends whose
/// existing handles already provide the desired behavior return `None`.
async fn snapshot_at_current_version(&self) -> Result<Option<Arc<dyn BaseTable>>> {
Ok(None)
}
/// Whether repeated identical scans return rows in the same order.
///
/// Callers that assign meaning to a row's position must order the results
@@ -832,14 +843,14 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
/// Diff a branch against main. Remote only.
async fn diff_branch(&self, _from_branch: &str) -> Result<BranchDiff> {
Err(Error::NotSupported {
message: "diff_branch is only supported on remote tables".into(),
message: "Branch diffs are only supported on Enterprise tables.".into(),
})
}
/// Merge a branch into main, or dry-run. Remote only.
/// HTTP 409 still returns [`Ok`] with [`MergeBranchStatus::Rejected`].
async fn merge_branch(&self, _from_branch: &str, _dry_run: bool) -> Result<MergeBranchResult> {
/// Cherry-pick a branch onto main, or dry-run. Remote only.
/// HTTP 409 still returns [`Ok`] with [`CherryPickStatus::Failed`].
async fn cherry_pick(&self, _from_branch: &str, _dry_run: bool) -> Result<CherryPickResult> {
Err(Error::NotSupported {
message: "merge_branch is only supported on remote tables".into(),
message: "Cherry-picking branches is only supported on Enterprise tables.".into(),
})
}
/// The branch this handle is scoped to, or `None` for `main`.
@@ -1066,6 +1077,11 @@ impl Table {
self.database.as_ref().unwrap()
}
/// The database this handle was opened through, when it was.
pub fn database_opt(&self) -> Option<&Arc<dyn Database>> {
self.database.as_ref()
}
pub fn embedding_registry(&self) -> &Arc<dyn EmbeddingRegistry> {
&self.embedding_registry
}
@@ -1703,7 +1719,9 @@ impl Table {
/// operation instead of blocking until it completes.
///
/// The job may already be complete when returned, and callers must not
/// assume the column is filled until [`Job::wait`] returns. Invalid input
/// assume the column is filled until [`Job::wait`] returns. A successful
/// wait returns the durable [`crate::function::RefreshColumnResult`] for
/// both expression-backed and Function-backed columns. Invalid input
/// -- an unknown column, or one that is not computed -- is reported by
/// this call rather than by the job. On local tables the job runs as an
/// in-process task; on LanceDB Cloud and Enterprise it is the server's
@@ -1714,11 +1732,15 @@ impl Table {
/// # async fn refresh_in_background(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
/// let job = table.refresh_column_async("doubled").await?;
/// println!("refresh running: {:?}", job.status().await?);
/// job.wait().await?;
/// let result = job.wait().await?;
/// println!("assigned {} rows", result.rows_assigned);
/// # Ok(())
/// # }
/// ```
pub async fn refresh_column_async(&self, column: impl AsRef<str>) -> Result<Job> {
pub async fn refresh_column_async(
&self,
column: impl AsRef<str>,
) -> Result<Job<crate::function::RefreshColumnResult>> {
self.inner.refresh_column_async(column.as_ref()).await
}
@@ -2258,14 +2280,10 @@ impl Table {
self.inner.diff_branch(from_branch).await
}
/// Merge a branch into main, or dry-run. Remote only.
/// HTTP 409 still returns [`Ok`] with [`MergeBranchStatus::Rejected`].
pub async fn merge_branch(
&self,
from_branch: &str,
dry_run: bool,
) -> Result<MergeBranchResult> {
self.inner.merge_branch(from_branch, dry_run).await
/// Cherry-pick a branch onto main, or dry-run. Remote only.
/// HTTP 409 still returns [`Ok`] with [`CherryPickStatus::Failed`].
pub async fn cherry_pick(&self, from_branch: &str, dry_run: bool) -> Result<CherryPickResult> {
self.inner.cherry_pick(from_branch, dry_run).await
}
/// The branch this handle is scoped to, or `None` for `main`.
@@ -3424,7 +3442,10 @@ impl BaseTable for NativeTable {
Ok(result)
}
async fn refresh_column_async(&self, column: &str) -> Result<Job> {
async fn refresh_column_async(
&self,
column: &str,
) -> Result<Job<crate::function::RefreshColumnResult>> {
refresh::execute_refresh_column_async(self, column).await
}
rust/lancedb/src/table/branch_merge.rs → rust/lancedb/src/table/cherry_pick.rs
+13 -14
View File
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! Types for remote branch diff / merge against main.
//! Types for remote branch diff / cherry-pick onto main.
use serde::{Deserialize, Serialize};
@@ -44,13 +44,13 @@ pub struct RowCountSummary {
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub enum MergeBlockerCode {
pub enum CherryPickErrorCode {
BaseMoved,
RowCountMismatch,
RowsChanged,
ColumnRemoved,
ColumnChanged,
NoMergeableChanges,
NothingToApply,
NoColumnChanges,
InputColumnDependency,
ParentNotMain,
@@ -60,8 +60,8 @@ pub enum MergeBlockerCode {
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct MergeBlocker {
pub code: MergeBlockerCode,
pub struct CherryPickError {
pub code: CherryPickErrorCode,
pub message: String,
}
@@ -81,34 +81,33 @@ pub struct BranchDiff {
pub changed_columns: Vec<ColumnChange>,
pub added_indexes: Vec<IndexSummary>,
pub removed_indexes: Vec<IndexSummary>,
pub mergeable: bool,
pub merge_blockers: Vec<MergeBlocker>,
pub errors: Vec<CherryPickError>,
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct MergePreview {
pub struct CherryPickPreview {
#[serde(default)]
pub promoted_columns: Vec<String>,
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub enum MergeBranchStatus {
pub enum CherryPickStatus {
Ready,
Rejected,
Failed,
NotImplemented,
Merged,
CherryPicked,
#[serde(other)]
Unknown,
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct MergeBranchResult {
pub status: MergeBranchStatus,
pub struct CherryPickResult {
pub status: CherryPickStatus,
pub diff: BranchDiff,
pub preview: MergePreview,
pub preview: CherryPickPreview,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub main_version_after: Option<u64>,
}
rust/lancedb/src/table/computed_columns.rs
+55
View File
@@ -586,6 +586,25 @@ fn canonical_input_arrow_type(field: &JsonArrowField) -> Result<String> {
}
}
/// `fixed_size_list<item, size>` -> (`item`, `size`); the comma must sit outside
/// any nested `<...>`.
fn split_fixed_size_list(raw: &str) -> Option<(&str, i32)> {
let inner = raw.strip_prefix("fixed_size_list<")?.strip_suffix('>')?;
let mut depth = 0_u32;
let mut separator = None;
for (index, byte) in inner.bytes().enumerate() {
match byte {
b'<' => depth += 1,
b'>' => depth = depth.checked_sub(1)?,
b',' if depth == 0 => separator = Some(index),
_ => {}
}
}
let (item, size) = inner.split_at(separator?);
let size: i32 = size[1..].trim().parse().ok()?;
(size > 0).then_some((item.trim(), size))
}
fn parse_output_arrow_type(raw: &str) -> Result<JsonArrowDataType> {
fn parse(raw: &str) -> Result<JsonArrowDataType> {
let raw = raw.trim();
@@ -618,6 +637,16 @@ fn parse_output_arrow_type(raw: &str) -> Result<JsonArrowDataType> {
)]);
return Ok(data_type);
}
if let Some((inner, size)) = split_fixed_size_list(raw) {
let mut data_type = JsonArrowDataType::new("fixed_size_list".to_string());
data_type.fields = Some(vec![JsonArrowField::new(
"item".to_string(),
false,
parse(inner)?,
)]);
data_type.length = Some(i64::from(size));
return Ok(data_type);
}
let normalized = match raw {
"boolean" => "bool",
"string" => "utf8",
@@ -1322,6 +1351,32 @@ pub(super) async fn add_foreign_kind(table: &crate::Table, name: &str, kind: &st
#[cfg(test)]
mod tests {
#[test]
fn output_arrow_type_grammar_matches_the_shared_golden() {
let golden: serde_json::Value = serde_json::from_str(include_str!(
"../../tests/fixtures/first_class_functions/v1/arrow_types.json"
))
.unwrap();
let valid = golden["valid"].as_array().unwrap().iter();
for case in valid.chain(golden["server_only"].as_array().unwrap()) {
let raw = case["arrow_type"].as_str().unwrap();
let parsed = super::parse_output_arrow_type(raw)
.unwrap_or_else(|error| panic!("{raw}: {error}"));
assert_eq!(
serde_json::to_value(&parsed).unwrap(),
case["json"],
"{raw}"
);
}
for raw in golden["invalid"].as_array().unwrap() {
let raw = raw.as_str().unwrap();
assert!(
super::parse_output_arrow_type(raw).is_err(),
"{raw:?} should be rejected"
);
}
}
use arrow_array::record_batch;
use arrow_schema::DataType;
use futures::TryStreamExt;
rust/lancedb/src/table/create_index.rs
+38
View File
@@ -12,6 +12,7 @@ use arrow_schema::{DataType, Field};
use lance::index::DatasetIndexExt;
use lance::index::vector::VectorIndexParams;
use lance::index::vector::utils::infer_vector_dim;
use lance_arrow::json::is_json_field;
use lance_index::IndexType;
use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
use lance_index::vector::bq::RQBuildParams;
@@ -219,6 +220,14 @@ impl NativeTable {
)))
}
Index::Bitmap(_) => {
if is_json_field(field) {
return Err(Error::Schema {
message: format!(
"A BITMAP index cannot be created on the whole-document lance.json field `{}`. Create a JSON-path scalar index for structured equality or range predicates, or use FTS for document search",
field.name()
),
});
}
Self::validate_index_type(field, "Bitmap", supported_bitmap_data_type)?;
Ok(Box::new(ScalarIndexParams::for_builtin(
BuiltinIndexType::Bitmap,
@@ -1465,6 +1474,35 @@ mod tests {
assert_eq!(stats.distance_type, None);
}
#[tokio::test]
async fn test_create_bitmap_index_rejects_lance_json() {
let conn = connect("memory://").execute().await.unwrap();
let schema = Arc::new(Schema::new(vec![lance_arrow::json::json_field(
"metadata", true,
)]));
let table = conn
.create_empty_table("json_bitmap", schema)
.execute()
.await
.unwrap();
let err = table
.create_index(&["metadata"], Index::Bitmap(Default::default()))
.execute()
.await
.expect_err("a whole-document lance.json field must not support a bitmap index");
let message = err.to_string();
assert!(
message.contains("lance.json"),
"unexpected error: {message}"
);
assert!(
message.contains("JSON-path scalar index"),
"unexpected error: {message}"
);
assert!(message.contains("FTS"), "unexpected error: {message}");
}
#[tokio::test]
async fn test_create_label_list_index() {
let conn = connect("memory://").execute().await.unwrap();
rust/lancedb/src/table/merge.rs
+70 -1
View File
@@ -321,7 +321,8 @@ pub(crate) async fn execute_merge_insert(
mod tests {
use arrow_array::builder::FixedSizeBinaryBuilder;
use arrow_array::{
Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray, UInt64Array,
FixedSizeListArray, Int32Array, NullArray, RecordBatch, RecordBatchIterator,
RecordBatchReader, StringArray, UInt32Array, UInt64Array,
};
use arrow_schema::{DataType, Field, Schema};
use std::sync::Arc;
@@ -529,6 +530,74 @@ mod tests {
assert_eq!(result.num_deleted_rows, 5);
assert_eq!(table.count_rows(None).await.unwrap(), 5);
}
#[tokio::test]
async fn test_merge_insert_fixed_size_list_above_u32_child_count() {
// Arrow's FixedSizeList take kernel uses u32 child indices. Previously,
// delete-by-source materialized the target payload in a full outer join,
// causing the final list below to overflow those indices and panic.
// A Null child keeps this boundary test small in memory.
const LIST_SIZE: i32 = 65_536;
const ROW_COUNT: usize = (u32::MAX as usize / LIST_SIZE as usize) + 1;
const BATCH_SIZE: usize = 8_192;
let item = Arc::new(Field::new("item", DataType::Null, true));
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::UInt32, false),
Field::new(
"vector",
DataType::FixedSizeList(item.clone(), LIST_SIZE),
false,
),
]));
let batch = |start: usize, len: usize| {
RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(UInt32Array::from_iter_values(
start as u32..(start + len) as u32,
)),
Arc::new(FixedSizeListArray::new(
item.clone(),
LIST_SIZE,
Arc::new(NullArray::new(len * LIST_SIZE as usize)),
None,
)),
],
)
.unwrap()
};
let target_batches = (0..ROW_COUNT)
.step_by(BATCH_SIZE)
.map(|start| {
let len = (ROW_COUNT - start).min(BATCH_SIZE);
Ok(batch(start, len))
})
.collect::<Vec<_>>();
let target_data: Box<dyn RecordBatchReader + Send> =
Box::new(RecordBatchIterator::new(target_batches, schema.clone()));
let conn = connect("memory://").execute().await.unwrap();
let table = conn
.create_table("fixed_size_list_overflow", target_data)
.execute()
.await
.unwrap();
let source = batch(ROW_COUNT - 1, 1);
let mut merge = table.merge_insert(&["id"]);
merge
.when_matched_update_all(None)
.when_not_matched_by_source_delete(None);
let result = merge
.execute(Box::new(RecordBatchIterator::new([Ok(source)], schema)))
.await
.unwrap();
assert_eq!(result.num_updated_rows, 1);
assert_eq!(result.num_deleted_rows, (ROW_COUNT - 1) as u64);
assert_eq!(table.count_rows(None).await.unwrap(), 1);
}
}
#[cfg(test)]
rust/lancedb/src/table/merge/lsm.rs
+7
View File
@@ -104,6 +104,13 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
.into(),
});
}
if crate::materialized_view::materialized_view_kind(&dataset.schema().metadata)?.is_some() {
return Err(Error::NotSupported {
message: "an LSM write spec cannot be installed on a materialized view: \
rows in un-compacted tiers are invisible to refresh"
.into(),
});
}
let mut builder = dataset.initialize_mem_wal();
let writer_config_defaults = match spec {
LsmWriteSpec::Bucket {
rust/lancedb/src/table/query.rs
+3
View File
@@ -70,6 +70,9 @@ async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> R
.contains(&NamespaceClientPushdownOperation::QueryTable)
&& table.namespace_client.is_some()
&& table.dataset.current_branch().is_none()
// NsQueryTableRequest has no version field, so a pushed-down query would
// read latest and ignore the pin.
&& table.dataset.time_travel_version().is_none()
&& !requires_local_namespace_execution(query))
{
return Ok(false);

Some files were not shown because too many files have changed in this diff Show More