mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-11 07:42:26 +00:00
Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3a11f4e97c | ||
|
|
000e3b506b | ||
|
|
1b950188c3 | ||
|
|
6cc77b573c | ||
|
|
1f1d03f306 | ||
|
|
45cd053478 | ||
|
|
68749ecfa3 |
+1
-1
@@ -1,5 +1,5 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "0.38.0-beta.3"
|
||||
current_version = "0.38.0-beta.6"
|
||||
parse = """(?x)
|
||||
(?P<major>0|[1-9]\\d*)\\.
|
||||
(?P<minor>0|[1-9]\\d*)\\.
|
||||
|
||||
Generated
+49
-45
@@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.5",
|
||||
@@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -4888,8 +4888,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4911,7 +4911,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4925,7 +4925,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -4934,8 +4934,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -4945,8 +4945,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4983,8 +4983,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5013,8 +5013,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5031,8 +5031,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5041,8 +5041,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5075,8 +5075,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5107,8 +5107,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5172,8 +5172,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index-core"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5195,8 +5195,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5222,7 +5222,11 @@ dependencies = [
|
||||
"pin-project",
|
||||
"prost",
|
||||
"rand 0.9.5",
|
||||
"reqsign-core",
|
||||
"reqsign-file-read-tokio",
|
||||
"reqsign-google",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -5232,8 +5236,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5247,8 +5251,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5260,8 +5264,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -5314,8 +5318,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5329,8 +5333,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5370,8 +5374,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5384,8 +5388,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "11.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"frostem",
|
||||
"icu_segmenter",
|
||||
@@ -5398,7 +5402,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb"
|
||||
version = "0.38.0-beta.3"
|
||||
version = "0.38.0-beta.5"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"anyhow",
|
||||
@@ -5486,7 +5490,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-nodejs"
|
||||
version = "0.38.0-beta.3"
|
||||
version = "0.38.0-beta.5"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5511,7 +5515,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-python"
|
||||
version = "0.38.0-beta.3"
|
||||
version = "0.38.0-beta.5"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
|
||||
+14
-14
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lancedb = { path = "rust/lancedb", default-features = false }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
|
||||
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
|
||||
<dependency>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-core</artifactId>
|
||||
<version>0.38.0-beta.3</version>
|
||||
<version>0.38.0-beta.6</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
|
||||
@@ -169,6 +169,45 @@ Creates a new empty Table
|
||||
|
||||
***
|
||||
|
||||
### createMaterializedView()
|
||||
|
||||
```ts
|
||||
abstract createMaterializedView(
|
||||
name,
|
||||
source,
|
||||
options?): Promise<MaterializedView>
|
||||
```
|
||||
|
||||
Define a materialized view named `name` over the table `source`.
|
||||
|
||||
The view is created empty, with the query recorded in its schema
|
||||
metadata; `view.refresh()` computes the rows. The view is a normal
|
||||
table: it can be queried, indexed and searched, and it appears in
|
||||
`tableNames`. The source table must have stable row ids (create it with
|
||||
the `newTableEnableStableRowIds` storage option); they keep the view's
|
||||
provenance valid across source compactions and cannot be enabled after
|
||||
a table exists. Local databases only.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **name**: `string`
|
||||
|
||||
* **source**: `string`
|
||||
|
||||
* **options?**
|
||||
|
||||
* **options.limit?**: `number`
|
||||
|
||||
* **options.select?**: [`MaterializedViewSelect`](../type-aliases/MaterializedViewSelect.md)
|
||||
|
||||
* **options.where?**: `string`
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`MaterializedView`](MaterializedView.md)>
|
||||
|
||||
***
|
||||
|
||||
### createNamespace()
|
||||
|
||||
```ts
|
||||
@@ -499,6 +538,22 @@ List server-side jobs across the database's tables.
|
||||
|
||||
***
|
||||
|
||||
### listMaterializedViews()
|
||||
|
||||
```ts
|
||||
abstract listMaterializedViews(): Promise<string[]>
|
||||
```
|
||||
|
||||
The names of the materialized views in this database.
|
||||
|
||||
Found by reading every table's schema, so this costs an open per table.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`string`[]>
|
||||
|
||||
***
|
||||
|
||||
### listNamespaces()
|
||||
|
||||
```ts
|
||||
@@ -529,6 +584,26 @@ Child namespace names and
|
||||
|
||||
***
|
||||
|
||||
### openMaterializedView()
|
||||
|
||||
```ts
|
||||
abstract openMaterializedView(name): Promise<MaterializedView>
|
||||
```
|
||||
|
||||
Open the materialized view named `name`.
|
||||
|
||||
Rejects a table that exists but is not a materialized view.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **name**: `string`
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`MaterializedView`](MaterializedView.md)>
|
||||
|
||||
***
|
||||
|
||||
### openTable()
|
||||
|
||||
```ts
|
||||
@@ -538,18 +613,13 @@ abstract openTable(
|
||||
options?): Promise<Table>
|
||||
```
|
||||
|
||||
Open a table in the database.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **name**: `string`
|
||||
The name of the table
|
||||
|
||||
* **namespacePath?**: `string`[]
|
||||
The namespace path of the table (defaults to root namespace)
|
||||
|
||||
* **options?**: `Partial`<[`OpenTableOptions`](../interfaces/OpenTableOptions.md)>
|
||||
Additional options
|
||||
|
||||
#### Returns
|
||||
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / MaterializedView
|
||||
|
||||
# Class: MaterializedView
|
||||
|
||||
A handle on a materialized view: its table plus its definition.
|
||||
|
||||
Obtained from [Connection#createMaterializedView](Connection.md#creatematerializedview) or
|
||||
[Connection#openMaterializedView](Connection.md#openmaterializedview). The view is a normal table --
|
||||
queries, indexes and search all apply through [MaterializedView#table](MaterializedView.md#table)
|
||||
-- whose contents are maintained by [MaterializedView#refresh](MaterializedView.md#refresh).
|
||||
|
||||
## Constructors
|
||||
|
||||
### new MaterializedView()
|
||||
|
||||
```ts
|
||||
new MaterializedView(table): MaterializedView
|
||||
```
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **table**: [`Table`](Table.md)
|
||||
|
||||
#### Returns
|
||||
|
||||
[`MaterializedView`](MaterializedView.md)
|
||||
|
||||
## Accessors
|
||||
|
||||
### name
|
||||
|
||||
```ts
|
||||
get name(): string
|
||||
```
|
||||
|
||||
#### Returns
|
||||
|
||||
`string`
|
||||
|
||||
## Methods
|
||||
|
||||
### definition()
|
||||
|
||||
```ts
|
||||
definition(): Promise<MaterializedViewDefinition>
|
||||
```
|
||||
|
||||
The query that defines the view, read from its stored schema.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`MaterializedViewDefinition`](../interfaces/MaterializedViewDefinition.md)>
|
||||
|
||||
***
|
||||
|
||||
### refresh()
|
||||
|
||||
```ts
|
||||
refresh(options?): Promise<RefreshMaterializedViewResult>
|
||||
```
|
||||
|
||||
Recompute the view from its source.
|
||||
|
||||
The refresh is incremental when the source's changes can be reconciled
|
||||
into the view -- rows added, changed or removed since the last one --
|
||||
and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
|
||||
refreshes to that source version instead of the latest.
|
||||
|
||||
Concurrent refreshes of one view do not duplicate its rows. Two that
|
||||
plan the same source rows conflict on commit, and the loser throws
|
||||
rather than writing them a second time.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **options?**
|
||||
|
||||
* **options.full?**: `boolean`
|
||||
|
||||
* **options.sourceVersion?**: `number`
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`RefreshMaterializedViewResult`](../interfaces/RefreshMaterializedViewResult.md)>
|
||||
|
||||
***
|
||||
|
||||
### table()
|
||||
|
||||
```ts
|
||||
table(): Table
|
||||
```
|
||||
|
||||
The view, as the table it is.
|
||||
|
||||
#### Returns
|
||||
|
||||
[`Table`](Table.md)
|
||||
@@ -28,6 +28,7 @@
|
||||
- [Job](classes/Job.md)
|
||||
- [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
|
||||
- [MatchQuery](classes/MatchQuery.md)
|
||||
- [MaterializedView](classes/MaterializedView.md)
|
||||
- [MergeInsertBuilder](classes/MergeInsertBuilder.md)
|
||||
- [MultiMatchQuery](classes/MultiMatchQuery.md)
|
||||
- [NativeJsHeaderProvider](classes/NativeJsHeaderProvider.md)
|
||||
@@ -101,6 +102,7 @@
|
||||
- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
|
||||
- [LsmStats](interfaces/LsmStats.md)
|
||||
- [LsmWriteSpec](interfaces/LsmWriteSpec.md)
|
||||
- [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md)
|
||||
- [MemtableStats](interfaces/MemtableStats.md)
|
||||
- [MergeResult](interfaces/MergeResult.md)
|
||||
- [NativeOAuthConfig](interfaces/NativeOAuthConfig.md)
|
||||
@@ -110,6 +112,7 @@
|
||||
- [OptimizeStats](interfaces/OptimizeStats.md)
|
||||
- [QueryExecutionOptions](interfaces/QueryExecutionOptions.md)
|
||||
- [RefreshColumnResult](interfaces/RefreshColumnResult.md)
|
||||
- [RefreshMaterializedViewResult](interfaces/RefreshMaterializedViewResult.md)
|
||||
- [RemovalStats](interfaces/RemovalStats.md)
|
||||
- [RenameTableOptions](interfaces/RenameTableOptions.md)
|
||||
- [RestNamespaceConfig](interfaces/RestNamespaceConfig.md)
|
||||
@@ -142,6 +145,7 @@
|
||||
- [FieldLike](type-aliases/FieldLike.md)
|
||||
- [IntoSql](type-aliases/IntoSql.md)
|
||||
- [IntoVector](type-aliases/IntoVector.md)
|
||||
- [MaterializedViewSelect](type-aliases/MaterializedViewSelect.md)
|
||||
- [MultiVector](type-aliases/MultiVector.md)
|
||||
- [RecordBatchLike](type-aliases/RecordBatchLike.md)
|
||||
- [SchemaLike](type-aliases/SchemaLike.md)
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / MaterializedViewDefinition
|
||||
|
||||
# Interface: MaterializedViewDefinition
|
||||
|
||||
The query that defines a materialized view.
|
||||
|
||||
## Properties
|
||||
|
||||
### filter?
|
||||
|
||||
```ts
|
||||
optional filter: string;
|
||||
```
|
||||
|
||||
SQL predicate selecting the source rows the view holds.
|
||||
|
||||
***
|
||||
|
||||
### inputs
|
||||
|
||||
```ts
|
||||
inputs: string[];
|
||||
```
|
||||
|
||||
Source columns the projections and filter read.
|
||||
|
||||
***
|
||||
|
||||
### limit?
|
||||
|
||||
```ts
|
||||
optional limit: number;
|
||||
```
|
||||
|
||||
Cap on the number of rows the view holds.
|
||||
|
||||
***
|
||||
|
||||
### projections
|
||||
|
||||
```ts
|
||||
projections: [string, string][];
|
||||
```
|
||||
|
||||
`[output column, SQL expression]` pairs, in view schema order.
|
||||
|
||||
***
|
||||
|
||||
### sourceTable
|
||||
|
||||
```ts
|
||||
sourceTable: string;
|
||||
```
|
||||
|
||||
Name of the source table, in the same database as the view.
|
||||
@@ -0,0 +1,41 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / RefreshMaterializedViewResult
|
||||
|
||||
# Interface: RefreshMaterializedViewResult
|
||||
|
||||
## Properties
|
||||
|
||||
### mode
|
||||
|
||||
```ts
|
||||
mode: string;
|
||||
```
|
||||
|
||||
How the view was brought up to date: "rebuild", "incremental" or "no_op".
|
||||
|
||||
***
|
||||
|
||||
### rowsWritten
|
||||
|
||||
```ts
|
||||
rowsWritten: number;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### sourceVersion
|
||||
|
||||
```ts
|
||||
sourceVersion: number;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### version
|
||||
|
||||
```ts
|
||||
version: number;
|
||||
```
|
||||
@@ -0,0 +1,14 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / MaterializedViewSelect
|
||||
|
||||
# Type Alias: MaterializedViewSelect
|
||||
|
||||
```ts
|
||||
type MaterializedViewSelect: (string | [string, string])[] | Record<string, string>;
|
||||
```
|
||||
|
||||
The view's columns: column names, `[alias, SQL expression]` pairs, or a
|
||||
record of the same. A bare name projects itself.
|
||||
@@ -8,7 +8,7 @@
|
||||
<parent>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.38.0-beta.3</version>
|
||||
<version>0.38.0-beta.6</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
|
||||
+2
-2
@@ -6,7 +6,7 @@
|
||||
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.38.0-beta.3</version>
|
||||
<version>0.38.0-beta.6</version>
|
||||
<packaging>pom</packaging>
|
||||
<name>${project.artifactId}</name>
|
||||
<description>LanceDB Java SDK Parent POM</description>
|
||||
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>11.0.0-beta.19</lance-core.version>
|
||||
<lance-core.version>11.0.0-beta.22</lance-core.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[package]
|
||||
name = "lancedb-nodejs"
|
||||
edition.workspace = true
|
||||
version = "0.38.0-beta.3"
|
||||
version = "0.38.0-beta.6"
|
||||
publish = false
|
||||
license.workspace = true
|
||||
description.workspace = true
|
||||
|
||||
@@ -487,4 +487,52 @@ describe("embedding functions", () => {
|
||||
expect(stringSchema3).toEqual(stringExpectedSchema);
|
||||
},
|
||||
);
|
||||
test("parses one function writing several vector columns", async () => {
|
||||
class MockEmbeddingFunction extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 3;
|
||||
}
|
||||
embeddingDataType(): Float {
|
||||
return new Float32();
|
||||
}
|
||||
async computeQueryEmbeddings(_data: string) {
|
||||
return [1, 2, 3];
|
||||
}
|
||||
async computeSourceEmbeddings(data: string[]) {
|
||||
return Array.from({ length: data.length }).fill([
|
||||
1, 2, 3,
|
||||
]) as number[][];
|
||||
}
|
||||
}
|
||||
const registry = getRegistry();
|
||||
registry.register("multi_output_mock")(MockEmbeddingFunction);
|
||||
|
||||
// A materialized view can project one source vector column under two
|
||||
// names, so a table's configuration names the same function twice.
|
||||
const parsed = await registry.parseFunctions(
|
||||
new Map([
|
||||
[
|
||||
"embedding_functions",
|
||||
JSON.stringify([
|
||||
{
|
||||
name: "multi_output_mock",
|
||||
sourceColumn: "text",
|
||||
vectorColumn: "vector_a",
|
||||
model: {},
|
||||
},
|
||||
{
|
||||
name: "multi_output_mock",
|
||||
sourceColumn: "text",
|
||||
vectorColumn: "vector_b",
|
||||
model: {},
|
||||
},
|
||||
]),
|
||||
],
|
||||
]),
|
||||
);
|
||||
|
||||
expect(
|
||||
[...parsed.values()].map(({ vectorColumn }) => vectorColumn).sort(),
|
||||
).toEqual(["vector_a", "vector_b"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import * as tmp from "tmp";
|
||||
|
||||
import { Connection, connect } from "../lancedb";
|
||||
import {
|
||||
DEFINITION_META_KEY,
|
||||
definitionFromMetadata,
|
||||
} from "../lancedb/materialized_view";
|
||||
|
||||
describe("materialized views", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
let db: Connection;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
db = await connect(tmpDir.name);
|
||||
await db.createTable(
|
||||
"people",
|
||||
[
|
||||
{ name: "ada", age: 36 },
|
||||
{ name: "kid", age: 7 },
|
||||
{ name: "grace", age: 85 },
|
||||
],
|
||||
{ storageOptions: { newTableEnableStableRowIds: "true" } },
|
||||
);
|
||||
});
|
||||
afterEach(() => tmpDir.removeCallback());
|
||||
|
||||
it("rejects a stored limit a number cannot carry", () => {
|
||||
const big = new Map([
|
||||
[
|
||||
DEFINITION_META_KEY,
|
||||
'{"kind":"select","source_table":"people","limit":9007199254740993}',
|
||||
],
|
||||
]);
|
||||
expect(() => definitionFromMetadata(big, "v")).toThrow(
|
||||
/too large to represent exactly/,
|
||||
);
|
||||
|
||||
const safe = new Map([
|
||||
[
|
||||
DEFINITION_META_KEY,
|
||||
'{"kind":"select","source_table":"people","limit":42}',
|
||||
],
|
||||
]);
|
||||
expect(definitionFromMetadata(safe, "v").limit).toBe(42);
|
||||
});
|
||||
|
||||
it("creates, refreshes and queries a view", async () => {
|
||||
const view = await db.createMaterializedView("adults", "people", {
|
||||
select: ["name", ["shout", "upper(name)"]],
|
||||
where: "age >= 18",
|
||||
});
|
||||
expect(view.name).toBe("adults");
|
||||
expect(await view.table().countRows()).toBe(0);
|
||||
|
||||
const result = await view.refresh();
|
||||
expect(result.mode).toBe("rebuild");
|
||||
expect(Number(result.rowsWritten)).toBe(2);
|
||||
|
||||
const rows = await view.table().query().toArray();
|
||||
expect(rows.map((r) => r.shout).sort()).toEqual(["ADA", "GRACE"]);
|
||||
});
|
||||
|
||||
it("round-trips the definition", async () => {
|
||||
await db.createMaterializedView("adults", "people", {
|
||||
where: "age >= 18",
|
||||
});
|
||||
const view = await db.openMaterializedView("adults");
|
||||
const definition = await view.definition();
|
||||
expect(definition.sourceTable).toBe("people");
|
||||
expect(definition.filter).toBe("age >= 18");
|
||||
expect(definition.projections).toEqual([
|
||||
["name", "`name`"],
|
||||
["age", "`age`"],
|
||||
]);
|
||||
expect(definition.inputs).toEqual(["age", "name"]);
|
||||
});
|
||||
|
||||
it("refreshes incrementally after an append", async () => {
|
||||
const view = await db.createMaterializedView("copy", "people");
|
||||
await view.refresh();
|
||||
|
||||
const people = await db.openTable("people");
|
||||
await people.add([{ name: "alan", age: 41 }]);
|
||||
const result = await view.refresh();
|
||||
expect(result.mode).toBe("incremental");
|
||||
expect(Number(result.rowsWritten)).toBe(1);
|
||||
expect(await view.table().countRows()).toBe(4);
|
||||
|
||||
expect((await view.refresh()).mode).toBe("no_op");
|
||||
});
|
||||
|
||||
it("lists views and rejects non-views", async () => {
|
||||
await db.createMaterializedView("adults", "people", {
|
||||
where: "age >= 18",
|
||||
});
|
||||
expect(await db.listMaterializedViews()).toEqual(["adults"]);
|
||||
await expect(db.openMaterializedView("people")).rejects.toThrow(
|
||||
"not a materialized view",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an invalid expression at create time", async () => {
|
||||
await expect(
|
||||
db.createMaterializedView("bad", "people", {
|
||||
select: [["x", "missing + 1"]],
|
||||
}),
|
||||
).rejects.toThrow("missing");
|
||||
});
|
||||
|
||||
it("rejects invalid numeric options before creating anything", async () => {
|
||||
for (const limit of [-5, 1.5, Infinity, NaN]) {
|
||||
await expect(
|
||||
db.createMaterializedView("bad", "people", { limit }),
|
||||
).rejects.toThrow("non-negative integer");
|
||||
}
|
||||
expect(await db.listMaterializedViews()).toEqual([]);
|
||||
|
||||
const view = await db.createMaterializedView("copy", "people");
|
||||
for (const sourceVersion of [-1, 1.5, Infinity, NaN]) {
|
||||
await expect(view.refresh({ sourceVersion })).rejects.toThrow(
|
||||
"non-negative integer",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("quotes bare select names", async () => {
|
||||
await db.createTable("odd_names", [{ "order item": "widget" }], {
|
||||
storageOptions: { newTableEnableStableRowIds: "true" },
|
||||
});
|
||||
const view = await db.createMaterializedView("quoted", "odd_names", {
|
||||
select: ["order item"],
|
||||
});
|
||||
const result = await view.refresh();
|
||||
expect(Number(result.rowsWritten)).toBe(1);
|
||||
});
|
||||
|
||||
it("requires stable row ids on the source", async () => {
|
||||
await db.createTable("plain", [{ x: 1 }]);
|
||||
await expect(db.createMaterializedView("v", "plain")).rejects.toThrow(
|
||||
"stable row ids",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -75,6 +75,25 @@ async function withMockDatabase(
|
||||
}
|
||||
|
||||
describe("remote connection", () => {
|
||||
it("refuses materialized views before issuing any request", async () => {
|
||||
const paths: string[] = [];
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
paths.push(req.url ?? "");
|
||||
res.writeHead(404).end();
|
||||
},
|
||||
async (db) => {
|
||||
await expect(db.openMaterializedView("secret_table")).rejects.toThrow(
|
||||
/only on local databases/,
|
||||
);
|
||||
await expect(db.listMaterializedViews()).rejects.toThrow(
|
||||
/only on local databases/,
|
||||
);
|
||||
expect(paths).toEqual([]);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("should accept partial connection options", async () => {
|
||||
await connect("db://test", {
|
||||
apiKey: "fake",
|
||||
|
||||
@@ -2953,7 +2953,7 @@ describe("column name options", () => {
|
||||
.limit(10)
|
||||
.toArray();
|
||||
expect(results2.length).toBe(10);
|
||||
});
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
describe("when creating an empty table", () => {
|
||||
|
||||
@@ -16,6 +16,12 @@ import {
|
||||
makeEmptyTable,
|
||||
} from "./arrow";
|
||||
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
|
||||
import {
|
||||
MaterializedView,
|
||||
MaterializedViewSelect,
|
||||
normalizeSelect,
|
||||
validateNonNegativeInteger,
|
||||
} from "./materialized_view";
|
||||
import { Connection as LanceDbConnection } from "./native";
|
||||
import type {
|
||||
CreateNamespaceResponse,
|
||||
@@ -247,6 +253,41 @@ export abstract class Connection {
|
||||
* @param {string[]} namespacePath - The namespace path of the table (defaults to root namespace)
|
||||
* @param {Partial<OpenTableOptions>} options - Additional options
|
||||
*/
|
||||
/**
|
||||
* Define a materialized view named `name` over the table `source`.
|
||||
*
|
||||
* The view is created empty, with the query recorded in its schema
|
||||
* metadata; `view.refresh()` computes the rows. The view is a normal
|
||||
* table: it can be queried, indexed and searched, and it appears in
|
||||
* `tableNames`. The source table must have stable row ids (create it with
|
||||
* the `newTableEnableStableRowIds` storage option); they keep the view's
|
||||
* provenance valid across source compactions and cannot be enabled after
|
||||
* a table exists. Local databases only.
|
||||
*/
|
||||
abstract createMaterializedView(
|
||||
name: string,
|
||||
source: string,
|
||||
options?: {
|
||||
select?: MaterializedViewSelect;
|
||||
where?: string;
|
||||
limit?: number;
|
||||
},
|
||||
): Promise<MaterializedView>;
|
||||
|
||||
/**
|
||||
* Open the materialized view named `name`.
|
||||
*
|
||||
* Rejects a table that exists but is not a materialized view.
|
||||
*/
|
||||
abstract openMaterializedView(name: string): Promise<MaterializedView>;
|
||||
|
||||
/**
|
||||
* The names of the materialized views in this database.
|
||||
*
|
||||
* Found by reading every table's schema, so this costs an open per table.
|
||||
*/
|
||||
abstract listMaterializedViews(): Promise<string[]>;
|
||||
|
||||
abstract openTable(
|
||||
name: string,
|
||||
namespacePath?: string[],
|
||||
@@ -531,6 +572,35 @@ export class LocalConnection extends Connection {
|
||||
);
|
||||
}
|
||||
|
||||
async createMaterializedView(
|
||||
name: string,
|
||||
source: string,
|
||||
options?: {
|
||||
select?: MaterializedViewSelect;
|
||||
where?: string;
|
||||
limit?: number;
|
||||
},
|
||||
): Promise<MaterializedView> {
|
||||
validateNonNegativeInteger(options?.limit, "limit");
|
||||
const innerTable = await this.inner.createMaterializedView(
|
||||
name,
|
||||
source,
|
||||
normalizeSelect(options?.select),
|
||||
options?.where,
|
||||
options?.limit,
|
||||
);
|
||||
return new MaterializedView(new LocalTable(innerTable));
|
||||
}
|
||||
|
||||
async openMaterializedView(name: string): Promise<MaterializedView> {
|
||||
const innerTable = await this.inner.openMaterializedView(name);
|
||||
return new MaterializedView(new LocalTable(innerTable));
|
||||
}
|
||||
|
||||
async listMaterializedViews(): Promise<string[]> {
|
||||
return await this.inner.listMaterializedViews();
|
||||
}
|
||||
|
||||
async openTable(
|
||||
name: string,
|
||||
namespacePath?: string[],
|
||||
|
||||
@@ -21,6 +21,11 @@ import type { BaseTokenizer } from "./indices";
|
||||
import type { FtsToken } from "./table";
|
||||
|
||||
// Re-export native header provider for use with connectWithHeaderProvider
|
||||
export {
|
||||
MaterializedView,
|
||||
MaterializedViewDefinition,
|
||||
MaterializedViewSelect,
|
||||
} from "./materialized_view";
|
||||
export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
|
||||
|
||||
// OpenTelemetry metrics bridge. Only the high-level entry point is public; the
|
||||
@@ -51,6 +56,7 @@ export {
|
||||
AddResult,
|
||||
AddColumnsResult,
|
||||
RefreshColumnResult,
|
||||
RefreshMaterializedViewResult,
|
||||
AlterColumnsResult,
|
||||
UpdateFieldMetadataResult,
|
||||
DeleteResult,
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import { RefreshMaterializedViewResult } from "./native";
|
||||
import { Table } from "./table";
|
||||
|
||||
/** Schema metadata key holding a materialized view's definition. */
|
||||
export const DEFINITION_META_KEY = "mv.definition";
|
||||
|
||||
/** The query that defines a materialized view. */
|
||||
export interface MaterializedViewDefinition {
|
||||
/** Name of the source table, in the same database as the view. */
|
||||
sourceTable: string;
|
||||
/** `[output column, SQL expression]` pairs, in view schema order. */
|
||||
projections: [string, string][];
|
||||
/** SQL predicate selecting the source rows the view holds. */
|
||||
filter?: string;
|
||||
/** Cap on the number of rows the view holds. */
|
||||
limit?: number;
|
||||
/** Source columns the projections and filter read. */
|
||||
inputs: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* The view's columns: column names, `[alias, SQL expression]` pairs, or a
|
||||
* record of the same. A bare name projects itself.
|
||||
*/
|
||||
export type MaterializedViewSelect =
|
||||
| (string | [string, string])[]
|
||||
| Record<string, string>;
|
||||
|
||||
/**
|
||||
* @internal Reject a numeric option N-API would otherwise silently coerce:
|
||||
* `Infinity` reaches Rust as 0, `1.5` as 1.
|
||||
*/
|
||||
export function validateNonNegativeInteger(
|
||||
value: number | undefined,
|
||||
name: string,
|
||||
): void {
|
||||
if (value !== undefined && !(Number.isSafeInteger(value) && value >= 0)) {
|
||||
throw new Error(`${name} must be a non-negative integer`);
|
||||
}
|
||||
}
|
||||
|
||||
/** @internal Quote a column name as a Lance SQL identifier (backticks). */
|
||||
function quoteIdentifier(name: string): string {
|
||||
return "`" + name.replace(/`/g, "``") + "`";
|
||||
}
|
||||
|
||||
/**
|
||||
* @internal Normalize a select argument into `[alias, expression]` pairs.
|
||||
* A bare name projects itself and is quoted, so any valid column name works;
|
||||
* pair and record entries are kept verbatim because their right side is an
|
||||
* expression.
|
||||
*/
|
||||
export function normalizeSelect(
|
||||
select?: MaterializedViewSelect,
|
||||
): [string, string][] | undefined {
|
||||
if (select === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
if (Array.isArray(select)) {
|
||||
return select.map((item) =>
|
||||
typeof item === "string" ? [item, quoteIdentifier(item)] : item,
|
||||
);
|
||||
}
|
||||
return Object.entries(select);
|
||||
}
|
||||
|
||||
/** @internal Parse a definition off a table's stored schema metadata. */
|
||||
export function definitionFromMetadata(
|
||||
metadata: Map<string, string>,
|
||||
name: string,
|
||||
): MaterializedViewDefinition {
|
||||
const raw = metadata.get(DEFINITION_META_KEY);
|
||||
if (raw === undefined) {
|
||||
throw new Error(`Table '${name}' is not a materialized view`);
|
||||
}
|
||||
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
|
||||
const value: any = JSON.parse(raw);
|
||||
if (value.kind !== "select") {
|
||||
throw new Error(
|
||||
`materialized view '${name}' is defined by '${value.kind}', which this ` +
|
||||
"version of lancedb cannot refresh",
|
||||
);
|
||||
}
|
||||
const limit = value.limit ?? undefined;
|
||||
// JSON.parse rounds integers past 2^53; every exact u64 parses to a safe
|
||||
// integer and every rounded one does not, so this rejects precisely the
|
||||
// values a number cannot carry.
|
||||
if (limit !== undefined && !Number.isSafeInteger(limit)) {
|
||||
throw new Error(
|
||||
`materialized view '${name}' has a stored limit too large to represent exactly`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
sourceTable: value.source_table,
|
||||
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
|
||||
projections: (value.projections ?? []).map((p: any) => [
|
||||
p.output,
|
||||
p.expression,
|
||||
]),
|
||||
filter: value.filter ?? undefined,
|
||||
limit,
|
||||
inputs: value.inputs ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A handle on a materialized view: its table plus its definition.
|
||||
*
|
||||
* Obtained from {@link Connection#createMaterializedView} or
|
||||
* {@link Connection#openMaterializedView}. The view is a normal table --
|
||||
* queries, indexes and search all apply through {@link MaterializedView#table}
|
||||
* -- whose contents are maintained by {@link MaterializedView#refresh}.
|
||||
*/
|
||||
export class MaterializedView {
|
||||
private readonly inner: Table;
|
||||
|
||||
constructor(table: Table) {
|
||||
this.inner = table;
|
||||
}
|
||||
|
||||
get name(): string {
|
||||
return this.inner.name;
|
||||
}
|
||||
|
||||
/** The view, as the table it is. */
|
||||
table(): Table {
|
||||
return this.inner;
|
||||
}
|
||||
|
||||
/** The query that defines the view, read from its stored schema. */
|
||||
async definition(): Promise<MaterializedViewDefinition> {
|
||||
const schema = await this.inner.schema();
|
||||
return definitionFromMetadata(schema.metadata, this.name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Recompute the view from its source.
|
||||
*
|
||||
* The refresh is incremental when the source's changes can be reconciled
|
||||
* into the view -- rows added, changed or removed since the last one --
|
||||
* and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
|
||||
* refreshes to that source version instead of the latest.
|
||||
*
|
||||
* Concurrent refreshes of one view do not duplicate its rows. Two that
|
||||
* plan the same source rows conflict on commit, and the loser throws
|
||||
* rather than writing them a second time.
|
||||
*/
|
||||
async refresh(options?: {
|
||||
full?: boolean;
|
||||
sourceVersion?: number;
|
||||
}): Promise<RefreshMaterializedViewResult> {
|
||||
validateNonNegativeInteger(options?.sourceVersion, "sourceVersion");
|
||||
return await this.inner.refreshMaterializedView(
|
||||
options?.full,
|
||||
options?.sourceVersion,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -35,6 +35,7 @@ import {
|
||||
Branches as NativeBranches,
|
||||
OptimizeStats,
|
||||
RefreshColumnResult,
|
||||
RefreshMaterializedViewResult,
|
||||
TableStatistics,
|
||||
Tags,
|
||||
UpdateFieldMetadataResult,
|
||||
@@ -602,6 +603,18 @@ export abstract class Table {
|
||||
*/
|
||||
abstract refreshColumnAsync(column: string): Promise<Job>;
|
||||
|
||||
/**
|
||||
* Recompute this table's contents from its materialized-view definition.
|
||||
*
|
||||
* Plumbing for {@link MaterializedView.refresh}, which is the way to call
|
||||
* it: rejects tables that carry no view definition. Local tables only.
|
||||
* @ignore
|
||||
*/
|
||||
abstract refreshMaterializedView(
|
||||
full?: boolean,
|
||||
sourceVersion?: number,
|
||||
): Promise<RefreshMaterializedViewResult>;
|
||||
|
||||
/**
|
||||
* Alter the name or nullability of columns.
|
||||
* @param {ColumnAlteration[]} columnAlterations One or more alterations to
|
||||
@@ -1264,6 +1277,13 @@ export class LocalTable extends Table {
|
||||
return await this.inner.refreshColumnAsync(column);
|
||||
}
|
||||
|
||||
async refreshMaterializedView(
|
||||
full?: boolean,
|
||||
sourceVersion?: number,
|
||||
): Promise<RefreshMaterializedViewResult> {
|
||||
return await this.inner.refreshMaterializedView(full, sourceVersion);
|
||||
}
|
||||
|
||||
async alterColumns(
|
||||
columnAlterations: ColumnAlteration[],
|
||||
): Promise<AlterColumnsResult> {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-darwin-arm64",
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.6",
|
||||
"os": ["darwin"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.darwin-arm64.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.6",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-musl",
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.6",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-gnu",
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.6",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-musl",
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.6",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.6",
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-x64-msvc",
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.6",
|
||||
"os": ["win32"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.win32-x64-msvc.node",
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb",
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.5",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@lancedb/lancedb",
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.5",
|
||||
"cpu": [
|
||||
"x64",
|
||||
"arm64"
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@
|
||||
"ann"
|
||||
],
|
||||
"private": false,
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.6",
|
||||
"main": "dist/index.js",
|
||||
"exports": {
|
||||
".": "./dist/index.js",
|
||||
|
||||
@@ -266,6 +266,58 @@ impl Connection {
|
||||
Ok(Table::new(tbl))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn create_materialized_view(
|
||||
&self,
|
||||
name: String,
|
||||
source: String,
|
||||
projections: Option<Vec<Vec<String>>>,
|
||||
filter: Option<String>,
|
||||
limit: Option<i64>,
|
||||
) -> napi::Result<Table> {
|
||||
let mut builder = self.get_inner()?.create_materialized_view(name, source);
|
||||
if let Some(projections) = projections {
|
||||
let mut pairs = Vec::with_capacity(projections.len());
|
||||
for pair in projections {
|
||||
let [output, expression]: [String; 2] = pair.try_into().map_err(|_| {
|
||||
napi::Error::from_reason("each projection must be an [output, expression] pair")
|
||||
})?;
|
||||
pairs.push((output, expression));
|
||||
}
|
||||
builder = builder.select(pairs);
|
||||
}
|
||||
if let Some(filter) = filter {
|
||||
builder = builder.only_if(filter);
|
||||
}
|
||||
if let Some(limit) = limit {
|
||||
let limit = u64::try_from(limit)
|
||||
.map_err(|_| napi::Error::from_reason("limit must be a non-negative integer"))?;
|
||||
builder = builder.limit(limit);
|
||||
}
|
||||
let view = builder.execute().await.default_error()?;
|
||||
Ok(Table::new(view.table().clone()))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn open_materialized_view(&self, name: String) -> napi::Result<Table> {
|
||||
let view = self
|
||||
.get_inner()?
|
||||
.open_materialized_view(&name)
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(Table::new(view.table().clone()))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn list_materialized_views(&self) -> napi::Result<Vec<String>> {
|
||||
let views = self
|
||||
.get_inner()?
|
||||
.list_materialized_views()
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(views.into_iter().map(|v| v.name).collect())
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn open_table(
|
||||
&self,
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
// The materialized-view refresh future deepens the type graph past the
|
||||
// default trait-recursion depth; same raise as the core crate applies.
|
||||
#![recursion_limit = "256"]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use env_logger::Env;
|
||||
|
||||
@@ -381,6 +381,26 @@ impl Table {
|
||||
Ok(crate::job::Job::new(job))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn refresh_materialized_view(
|
||||
&self,
|
||||
full: Option<bool>,
|
||||
source_version: Option<i64>,
|
||||
) -> napi::Result<RefreshMaterializedViewResult> {
|
||||
let view = lancedb::MaterializedView::from_table(self.inner_ref()?.clone())
|
||||
.await
|
||||
.default_error()?;
|
||||
let mut builder = view.refresh().full(full.unwrap_or(false));
|
||||
if let Some(version) = source_version {
|
||||
let version = u64::try_from(version).map_err(|_| {
|
||||
napi::Error::from_reason("sourceVersion must be a non-negative integer")
|
||||
})?;
|
||||
builder = builder.source_version(version);
|
||||
}
|
||||
let result = builder.execute().await.default_error()?;
|
||||
Ok(result.into())
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn add_columns_with_schema(
|
||||
&self,
|
||||
@@ -1387,6 +1407,31 @@ pub struct RefreshColumnResult {
|
||||
pub version: i64,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct RefreshMaterializedViewResult {
|
||||
/// How the view was brought up to date: "rebuild", "incremental" or "no_op".
|
||||
pub mode: String,
|
||||
pub rows_written: i64,
|
||||
pub source_version: i64,
|
||||
pub version: i64,
|
||||
}
|
||||
|
||||
impl From<lancedb::RefreshMaterializedViewResult> for RefreshMaterializedViewResult {
|
||||
fn from(value: lancedb::RefreshMaterializedViewResult) -> Self {
|
||||
let mode = match value.mode {
|
||||
lancedb::RefreshMode::Rebuild => "rebuild",
|
||||
lancedb::RefreshMode::Incremental => "incremental",
|
||||
lancedb::RefreshMode::NoOp => "no_op",
|
||||
};
|
||||
Self {
|
||||
mode: mode.to_string(),
|
||||
rows_written: value.rows_written as i64,
|
||||
source_version: value.source_version as i64,
|
||||
version: value.version as i64,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<lancedb::table::RefreshColumnResult> for RefreshColumnResult {
|
||||
fn from(value: lancedb::table::RefreshColumnResult) -> Self {
|
||||
Self {
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb-python"
|
||||
version = "0.38.0-beta.3"
|
||||
version = "0.38.0-beta.6"
|
||||
publish = false
|
||||
edition.workspace = true
|
||||
description = "Python bindings for LanceDB"
|
||||
|
||||
@@ -61,7 +61,7 @@ class StreamingDataset(IterableDataset):
|
||||
|
||||
Internally ``__iter__`` runs a two-stage pipeline:
|
||||
|
||||
- **Stage 1 (I/O)**: one thread pool with ``num_splits * prefetch_batches``
|
||||
- **Stage 1 (I/O)**: one thread pool with ``num_splits * io_queue_depth``
|
||||
workers fetches raw ``RecordBatch`` objects from LanceDB in parallel
|
||||
across all splits and places them in a per-split raw-batch queue.
|
||||
- **Stage 2 (transform)**: a second thread pool with
|
||||
@@ -104,11 +104,11 @@ class StreamingDataset(IterableDataset):
|
||||
call. Larger values amortise per-request overhead (critical on object
|
||||
storage) at the cost of higher memory usage per split buffer. Defaults
|
||||
to ``DEFAULT_READ_BATCH_SIZE`` (64).
|
||||
prefetch_batches:
|
||||
io_queue_depth:
|
||||
Number of I/O batches to keep in flight per split. Higher values
|
||||
overlap storage latency with transform and training compute at the cost
|
||||
of more memory and threads. Defaults to ``DEFAULT_PREFETCH_BATCHES``
|
||||
(4).
|
||||
of more memory and threads. Must be greater than zero. Defaults to
|
||||
``DEFAULT_PREFETCH_BATCHES`` (4).
|
||||
columns:
|
||||
Optional list of column names to read. When set, only those columns
|
||||
are fetched from storage; all others are omitted. ``None`` (the
|
||||
@@ -175,6 +175,16 @@ class StreamingDataset(IterableDataset):
|
||||
Prefer the ``filter`` parameter when bad rows can be expressed as a
|
||||
SQL predicate (e.g. ``"col IS NOT NULL"``) — filtering happens before
|
||||
splits are built, so every guarantee is fully preserved.
|
||||
transform_queue_depth:
|
||||
Number of transform-result batches to buffer per split in the
|
||||
post-transform queue before backpressure is applied to the transform
|
||||
stage. When the combined count of in-flight transform futures and
|
||||
already-buffered rows for a split reaches
|
||||
``transform_queue_depth * read_batch_size``, no new transforms are
|
||||
submitted for that split until the consumer catches up. Useful for
|
||||
capping peak memory when the consumer (e.g. a GPU training step) is
|
||||
slower than the transform stage. Must be greater than zero.
|
||||
``None`` (the default) imposes no limit.
|
||||
worker_info_override:
|
||||
If set, used in place of ``torch.utils.data.get_worker_info()`` to
|
||||
determine the DataLoader worker assignment. Intended for unit tests
|
||||
@@ -194,17 +204,26 @@ class StreamingDataset(IterableDataset):
|
||||
rank: int = 0,
|
||||
world_size: int = 1,
|
||||
read_batch_size: int = DEFAULT_READ_BATCH_SIZE,
|
||||
prefetch_batches: int = DEFAULT_PREFETCH_BATCHES,
|
||||
io_queue_depth: int = DEFAULT_PREFETCH_BATCHES,
|
||||
columns: Optional[list[str]] = None,
|
||||
shuffle_clump_size: Optional[int] = None,
|
||||
filter: Optional[str] = None,
|
||||
transform: Optional[Callable] = None,
|
||||
transform_parallelism: Optional[int] = None,
|
||||
on_transform_error: Union[str, Callable[[Exception], bool]] = "raise",
|
||||
transform_queue_depth: Optional[int] = None,
|
||||
connection_factory: Optional[Callable[[str], Any]] = None,
|
||||
worker_info_override=None,
|
||||
# Deprecated; use io_queue_depth instead.
|
||||
prefetch_batches: Optional[int] = None,
|
||||
):
|
||||
super().__init__()
|
||||
if prefetch_batches is not None:
|
||||
logger.warning(
|
||||
"prefetch_batches is deprecated and will be removed in a future "
|
||||
"version; use io_queue_depth instead"
|
||||
)
|
||||
io_queue_depth = prefetch_batches
|
||||
if num_splits is None:
|
||||
num_splits = world_size
|
||||
if shuffle_seed is None:
|
||||
@@ -214,6 +233,8 @@ class StreamingDataset(IterableDataset):
|
||||
f"num_splits ({num_splits}) must be divisible by "
|
||||
f"world_size ({world_size})"
|
||||
)
|
||||
if io_queue_depth <= 0:
|
||||
raise ValueError("io_queue_depth must be greater than 0")
|
||||
if transform_parallelism is not None and transform_parallelism <= 0:
|
||||
raise ValueError("transform_parallelism must be greater than 0")
|
||||
if on_transform_error not in ("raise", "skip", "warn") and not callable(
|
||||
@@ -223,6 +244,8 @@ class StreamingDataset(IterableDataset):
|
||||
"on_transform_error must be 'raise', 'skip', 'warn', or a "
|
||||
f"callable, got {on_transform_error!r}"
|
||||
)
|
||||
if transform_queue_depth is not None and transform_queue_depth <= 0:
|
||||
raise ValueError("transform_queue_depth must be greater than 0")
|
||||
|
||||
self._table = table
|
||||
self._num_splits = num_splits
|
||||
@@ -232,13 +255,14 @@ class StreamingDataset(IterableDataset):
|
||||
self._rank = rank
|
||||
self._world_size = world_size
|
||||
self._read_batch_size = read_batch_size
|
||||
self._prefetch_batches = prefetch_batches
|
||||
self._io_queue_depth = io_queue_depth
|
||||
self._columns = columns
|
||||
self._shuffle_clump_size = shuffle_clump_size
|
||||
self._filter = filter
|
||||
self._transform = transform
|
||||
self._transform_parallelism = transform_parallelism
|
||||
self._on_transform_error = on_transform_error
|
||||
self._transform_queue_depth = transform_queue_depth
|
||||
self._connection_factory = connection_factory
|
||||
self._worker_info_override = worker_info_override
|
||||
|
||||
@@ -365,7 +389,7 @@ class StreamingDataset(IterableDataset):
|
||||
pos_consumed = list(initial_positions)
|
||||
|
||||
batch_size = self._read_batch_size
|
||||
max_prefetch = self._prefetch_batches
|
||||
io_queue_depth = self._io_queue_depth
|
||||
transform_workers = (
|
||||
self._transform_parallelism
|
||||
if self._transform_parallelism is not None
|
||||
@@ -374,6 +398,13 @@ class StreamingDataset(IterableDataset):
|
||||
final_transform = (
|
||||
self._transform if self._transform is not None else Transforms.arrow2python
|
||||
)
|
||||
# None means no limit; otherwise cap rows per split to
|
||||
# transform_queue_depth batches worth (including in-flight transforms).
|
||||
max_cooked_rows = (
|
||||
self._transform_queue_depth * batch_size
|
||||
if self._transform_queue_depth is not None
|
||||
else None
|
||||
)
|
||||
|
||||
# Per-split pipeline state. Batches are paired with the absolute
|
||||
# permutation position of their first row so that skipped rows can be
|
||||
@@ -409,7 +440,9 @@ class StreamingDataset(IterableDataset):
|
||||
io_pending[i].append((abs_start, io_pool.submit(_io_call, perm_i, indices)))
|
||||
|
||||
def _fill_io(i: int) -> None:
|
||||
while len(io_pending[i]) < max_prefetch and fetch_head[i] < split_sizes[i]:
|
||||
while (
|
||||
len(io_pending[i]) < io_queue_depth and fetch_head[i] < split_sizes[i]
|
||||
):
|
||||
_submit_io(i)
|
||||
|
||||
def _drain_io(i: int) -> None:
|
||||
@@ -487,7 +520,19 @@ class StreamingDataset(IterableDataset):
|
||||
|
||||
def _try_submit_tx(i: int) -> None:
|
||||
"""Submit transforms for raw_batches[i] up to available capacity."""
|
||||
while raw_batches[i] and tx_semaphore.acquire(blocking=False):
|
||||
while raw_batches[i]:
|
||||
# Backpressure: only submit a new transform when there is room
|
||||
# for a full batch in the post-transform queue. Checking for
|
||||
# a full batch prevents submitting a transform that would
|
||||
# overflow the limit mid-batch (e.g. 990 rows queued with a
|
||||
# capacity of 1000 and a batch_size of 128 must wait until
|
||||
# 128 rows have been consumed, not just 1).
|
||||
if max_cooked_rows is not None:
|
||||
in_pipeline = len(cooked[i]) + len(tx_pending[i]) * batch_size
|
||||
if in_pipeline + batch_size > max_cooked_rows:
|
||||
break
|
||||
if not tx_semaphore.acquire(blocking=False):
|
||||
break
|
||||
abs_start, batch = raw_batches[i].popleft()
|
||||
tx_pending[i].append(tx_pool.submit(_tx_call_guarded, abs_start, batch))
|
||||
|
||||
@@ -531,7 +576,7 @@ class StreamingDataset(IterableDataset):
|
||||
|
||||
# ── Main loop ─────────────────────────────────────────────────────────
|
||||
|
||||
with ThreadPoolExecutor(max_workers=n * max_prefetch) as io_pool:
|
||||
with ThreadPoolExecutor(max_workers=n * io_queue_depth) as io_pool:
|
||||
with ThreadPoolExecutor(max_workers=transform_workers) as tx_pool:
|
||||
self._raw_batches_ref = raw_batches
|
||||
self._cooked_ref = cooked
|
||||
|
||||
@@ -1374,6 +1374,188 @@ def test_transform_parallelism_must_be_positive(lance_table, transform_paralleli
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Backpressure / transform_queue_depth tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("transform_queue_depth", [0, -1])
|
||||
def test_transform_queue_depth_must_be_positive(lance_table, transform_queue_depth):
|
||||
"""transform_queue_depth=0 or negative must raise ValueError."""
|
||||
with pytest.raises(
|
||||
ValueError, match="transform_queue_depth must be greater than 0"
|
||||
):
|
||||
StreamingDataset(
|
||||
lance_table,
|
||||
num_splits=NUM_SPLITS,
|
||||
transform_queue_depth=transform_queue_depth,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("transform_queue_depth", [1, 2, 4])
|
||||
def test_transform_queue_depth_correctness(lance_table, transform_queue_depth):
|
||||
"""With backpressure enabled, every row is still yielded exactly once."""
|
||||
ds = StreamingDataset(
|
||||
lance_table,
|
||||
num_splits=NUM_SPLITS,
|
||||
shuffle_seed=SHUFFLE_SEED,
|
||||
transform_queue_depth=transform_queue_depth,
|
||||
read_batch_size=8,
|
||||
)
|
||||
items = list(ds)
|
||||
assert sorted(item["id"] for item in items) == list(range(NUM_ROWS))
|
||||
|
||||
|
||||
def test_transform_queue_depth_matches_no_backpressure(lance_table):
|
||||
"""With backpressure enabled the same samples are produced as without it."""
|
||||
ds_unlimited = StreamingDataset(
|
||||
lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED
|
||||
)
|
||||
ds_limited = StreamingDataset(
|
||||
lance_table,
|
||||
num_splits=NUM_SPLITS,
|
||||
shuffle_seed=SHUFFLE_SEED,
|
||||
transform_queue_depth=1,
|
||||
)
|
||||
assert [item["id"] for item in ds_unlimited] == [
|
||||
item["id"] for item in ds_limited
|
||||
], "transform_queue_depth must not affect the sample ordering or set"
|
||||
|
||||
|
||||
def test_transform_queue_depth_bounds_cooked_rows(lance_table):
|
||||
"""prefetch_queue_depth stays within transform_queue_depth * read_batch_size
|
||||
per split when observed from the main thread during iteration."""
|
||||
n_splits = 4
|
||||
batch_size = 8
|
||||
cooked_depth = 2
|
||||
# max cooked rows across all 4 splits: 4 * 2 * 8 = 64
|
||||
max_allowed = n_splits * cooked_depth * batch_size
|
||||
|
||||
ds = StreamingDataset(
|
||||
lance_table,
|
||||
num_splits=n_splits,
|
||||
shuffle_seed=SHUFFLE_SEED,
|
||||
transform_queue_depth=cooked_depth,
|
||||
read_batch_size=batch_size,
|
||||
transform_parallelism=1,
|
||||
world_size=1,
|
||||
)
|
||||
|
||||
peak = 0
|
||||
for _ in ds:
|
||||
depth = ds.prefetch_queue_depth
|
||||
if depth > peak:
|
||||
peak = depth
|
||||
|
||||
# The main thread observes depth *after* popping a row, so the peak is at
|
||||
# most max_allowed (one row already popped from the split just served).
|
||||
assert peak <= max_allowed, (
|
||||
f"prefetch_queue_depth peaked at {peak}, expected <= {max_allowed}"
|
||||
)
|
||||
|
||||
|
||||
def test_transform_queue_depth_does_not_admit_at_capacity_minus_one(tmp_path):
|
||||
"""Admission requires a full read_batch_size of free space, not just one slot.
|
||||
|
||||
The test intercepts ThreadPoolExecutor.submit to make I/O calls execute
|
||||
synchronously on the main thread. This ensures all raw batches land in
|
||||
raw_batches (via _drain_io) before _try_submit_tx evaluates the admission
|
||||
predicate for the first time. Without this, the I/O future for batch N+1
|
||||
might still be in io_pending at the capacity-minus-one transition, leaving
|
||||
raw_batches empty and causing _try_submit_tx to skip the admission check
|
||||
entirely — so both the correct and the broken predicate produce depth=0
|
||||
observations and the test cannot distinguish them.
|
||||
|
||||
With all raw batches pre-loaded in raw_batches the 4→3 cooked transition
|
||||
(consuming one row from a full cooked queue) always triggers _try_submit_tx
|
||||
against a non-empty raw_batches.
|
||||
|
||||
With transform_queue_depth=1 and batch_size=4, max_cooked_rows=4.
|
||||
A transform may only be submitted when in_pipeline + batch_size <= 4, i.e.
|
||||
when in_pipeline == 0 (cooked is completely empty). Under the old broken
|
||||
predicate (in_pipeline >= max_cooked_rows) the second transform would be
|
||||
admitted with cooked containing batch_size-1 rows still unconsumed.
|
||||
"""
|
||||
import concurrent.futures as cf
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from unittest.mock import patch
|
||||
|
||||
db = lancedb.connect(tmp_path)
|
||||
batch_size = 4
|
||||
# Four full batches → four transform submissions to observe.
|
||||
table = db.create_table("t", pa.table({"id": list(range(batch_size * 4))}))
|
||||
|
||||
cooked_at_submit: list[int] = []
|
||||
|
||||
original_submit = ThreadPoolExecutor.submit
|
||||
|
||||
def tracking_submit(self, fn, *args, **kwargs):
|
||||
name = getattr(fn, "__name__", "")
|
||||
if name == "_io_call":
|
||||
# Run I/O synchronously on the calling (main) thread and return an
|
||||
# already-completed Future. _drain_io checks fut.done(), so a
|
||||
# completed Future is moved to raw_batches immediately on the next
|
||||
# _advance call — making raw-batch readiness deterministic at the
|
||||
# capacity-minus-one transition instead of depending on I/O thread
|
||||
# scheduling.
|
||||
fut = cf.Future()
|
||||
try:
|
||||
fut.set_result(fn(*args, **kwargs))
|
||||
except Exception as exc:
|
||||
fut.set_exception(exc)
|
||||
return fut
|
||||
if name == "_tx_call_guarded":
|
||||
# Capture cooked depth synchronously on the main thread before the
|
||||
# transform worker can drain the queue.
|
||||
ref = ds._cooked_ref
|
||||
cooked_at_submit.append(len(ref[0]) if ref is not None else -1)
|
||||
return original_submit(self, fn, *args, **kwargs)
|
||||
|
||||
with patch.object(ThreadPoolExecutor, "submit", tracking_submit):
|
||||
ds = StreamingDataset(
|
||||
table,
|
||||
num_splits=1,
|
||||
shuffle_seed=42,
|
||||
read_batch_size=batch_size,
|
||||
transform_queue_depth=1,
|
||||
transform_parallelism=1,
|
||||
)
|
||||
list(ds)
|
||||
|
||||
assert len(cooked_at_submit) == 4, (
|
||||
f"Expected 4 transform submissions (one per batch), got {len(cooked_at_submit)}"
|
||||
)
|
||||
# With full-batch backpressure each transform is only admitted when the
|
||||
# cooked queue is completely empty (depth == 0). The old broken predicate
|
||||
# would admit at depth == batch_size - 1 == 3.
|
||||
assert all(depth == 0 for depth in cooked_at_submit), (
|
||||
"Transform admitted with non-empty cooked queue; full-batch backpressure "
|
||||
"requires in_pipeline + batch_size <= max_cooked_rows before admission. "
|
||||
f"Cooked depths at each submission: {cooked_at_submit}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Deprecated parameter name tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_prefetch_batches_deprecated_warns(lance_table, caplog):
|
||||
"""prefetch_batches logs a deprecation warning and behaves like io_queue_depth."""
|
||||
with caplog.at_level(logging.WARNING, logger="lancedb.streaming"):
|
||||
ds = StreamingDataset(
|
||||
lance_table,
|
||||
num_splits=NUM_SPLITS,
|
||||
shuffle_seed=SHUFFLE_SEED,
|
||||
prefetch_batches=2,
|
||||
)
|
||||
messages = [r.message for r in caplog.records if r.levelno >= logging.WARNING]
|
||||
assert any("deprecated" in m.lower() and "io_queue_depth" in m for m in messages), (
|
||||
f"Expected deprecation warning mentioning io_queue_depth; got: {messages}"
|
||||
)
|
||||
assert sorted(item["id"] for item in ds) == list(range(NUM_ROWS))
|
||||
|
||||
|
||||
def test_filter_limits_rows(tmp_path):
|
||||
"""A filter expression is applied to the permutation so only matching rows
|
||||
are yielded. IDs 0..59 pass ``id < 60``; the other 60 are excluded."""
|
||||
@@ -1954,7 +2136,7 @@ def test_doc_example_basic(tmp_path):
|
||||
|
||||
|
||||
def test_doc_example_prefetch_params(tmp_path):
|
||||
"""doc: Prefetching — read_batch_size and prefetch_batches still cover all rows."""
|
||||
"""doc: Prefetching — read_batch_size and io_queue_depth still cover all rows."""
|
||||
db = lancedb.connect(tmp_path)
|
||||
table = db.create_table("t", pa.table({"id": list(range(NUM_ROWS))}))
|
||||
|
||||
@@ -1963,7 +2145,7 @@ def test_doc_example_prefetch_params(tmp_path):
|
||||
num_splits=NUM_SPLITS,
|
||||
shuffle_seed=SHUFFLE_SEED,
|
||||
read_batch_size=8,
|
||||
prefetch_batches=2,
|
||||
io_queue_depth=2,
|
||||
)
|
||||
assert sorted(s["id"] for s in ds) == list(range(NUM_ROWS))
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb"
|
||||
version = "0.38.0-beta.3"
|
||||
version = "0.38.0-beta.6"
|
||||
edition.workspace = true
|
||||
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
||||
license.workspace = true
|
||||
|
||||
Reference in New Issue
Block a user