mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-28 00:48:40 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 10dac899e5 | |||
| 10c8894fe8 |
+1
-1
@@ -1,5 +1,5 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "0.38.0-beta.11"
|
||||
current_version = "0.38.0-beta.3"
|
||||
parse = """(?x)
|
||||
(?P<major>0|[1-9]\\d*)\\.
|
||||
(?P<minor>0|[1-9]\\d*)\\.
|
||||
|
||||
Generated
+45
-49
@@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.5",
|
||||
@@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -4888,8 +4888,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4911,7 +4911,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4925,7 +4925,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -4934,8 +4934,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -4945,8 +4945,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4983,8 +4983,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5013,8 +5013,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5031,8 +5031,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5041,8 +5041,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5075,8 +5075,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5107,8 +5107,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5172,8 +5172,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index-core"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5195,8 +5195,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5222,11 +5222,7 @@ dependencies = [
|
||||
"pin-project",
|
||||
"prost",
|
||||
"rand 0.9.5",
|
||||
"reqsign-core",
|
||||
"reqsign-file-read-tokio",
|
||||
"reqsign-google",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -5236,8 +5232,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5251,8 +5247,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5264,8 +5260,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -5318,8 +5314,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5333,8 +5329,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5374,8 +5370,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5388,8 +5384,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
|
||||
dependencies = [
|
||||
"frostem",
|
||||
"icu_segmenter",
|
||||
@@ -5402,7 +5398,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb"
|
||||
version = "0.38.0-beta.11"
|
||||
version = "0.38.0-beta.3"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"anyhow",
|
||||
@@ -5490,7 +5486,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-nodejs"
|
||||
version = "0.38.0-beta.11"
|
||||
version = "0.38.0-beta.3"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5515,7 +5511,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-python"
|
||||
version = "0.38.0-beta.11"
|
||||
version = "0.38.0-beta.3"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
|
||||
+14
-14
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lancedb = { path = "rust/lancedb", default-features = false }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
|
||||
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
|
||||
<dependency>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-core</artifactId>
|
||||
<version>0.38.0-beta.11</version>
|
||||
<version>0.38.0-beta.3</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
|
||||
@@ -1,518 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / AutoQuery
|
||||
|
||||
# Class: AutoQuery
|
||||
|
||||
A builder for automatic string searches.
|
||||
|
||||
Automatic search determines whether to use full-text or vector search from
|
||||
the table revision selected for each execution. This builder exposes the
|
||||
common operations supported by both query families.
|
||||
|
||||
## Extends
|
||||
|
||||
- `StandardQueryBase`<`NativeQuery` \| `NativeVectorQuery`>
|
||||
|
||||
## Properties
|
||||
|
||||
### inner
|
||||
|
||||
```ts
|
||||
protected inner: Query | VectorQuery | Promise<Query | VectorQuery>;
|
||||
```
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.inner`
|
||||
|
||||
## Methods
|
||||
|
||||
### analyzePlan()
|
||||
|
||||
```ts
|
||||
analyzePlan(distributedMetrics?): Promise<string>
|
||||
```
|
||||
|
||||
Executes the query and returns the physical query plan annotated with runtime metrics.
|
||||
|
||||
This is useful for debugging and performance analysis, as it shows how the query was executed
|
||||
and includes metrics such as elapsed time, rows processed, and I/O statistics.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **distributedMetrics?**: [`AnalyzePlanDistributedMetrics`](../type-aliases/AnalyzePlanDistributedMetrics.md)
|
||||
How distributed worker metrics are displayed for remote query plans.
|
||||
Defaults to `"aggregate"`.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`string`>
|
||||
|
||||
A query execution plan with runtime metrics for each step.
|
||||
|
||||
#### Example
|
||||
|
||||
```ts
|
||||
import * as lancedb from "@lancedb/lancedb"
|
||||
|
||||
const db = await lancedb.connect("./.lancedb");
|
||||
const table = await db.createTable("my_table", [
|
||||
{ vector: [1.1, 0.9], id: "1" },
|
||||
]);
|
||||
|
||||
const plan = await table.query().nearestTo([0.5, 0.2]).analyzePlan();
|
||||
|
||||
Example output (with runtime metrics inlined):
|
||||
AnalyzeExec verbose=true, metrics=[]
|
||||
ProjectionExec: expr=[id@3 as id, vector@0 as vector, _distance@2 as _distance], metrics=[output_rows=1, elapsed_compute=3.292µs]
|
||||
Take: columns="vector, _rowid, _distance, (id)", metrics=[output_rows=1, elapsed_compute=66.001µs, batches_processed=1, bytes_read=8, iops=1, requests=1]
|
||||
CoalesceBatchesExec: target_batch_size=1024, metrics=[output_rows=1, elapsed_compute=3.333µs]
|
||||
GlobalLimitExec: skip=0, fetch=10, metrics=[output_rows=1, elapsed_compute=167ns]
|
||||
FilterExec: _distance@2 IS NOT NULL, metrics=[output_rows=1, elapsed_compute=8.542µs]
|
||||
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=63.25µs, row_replacements=1]
|
||||
KNNVectorDistance: metric=l2, metrics=[output_rows=1, elapsed_compute=114.333µs, output_batches=1]
|
||||
LanceScan: uri=/path/to/data, projection=[vector], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1, elapsed_compute=103.626µs, bytes_read=549, iops=2, requests=2]
|
||||
```
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.analyzePlan`
|
||||
|
||||
***
|
||||
|
||||
### execute()
|
||||
|
||||
```ts
|
||||
protected execute(options?): AsyncGenerator<RecordBatch<any>, void, unknown>
|
||||
```
|
||||
|
||||
Execute the query and return the results as an
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **options?**: `Partial`<[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)>
|
||||
|
||||
#### Returns
|
||||
|
||||
`AsyncGenerator`<`RecordBatch`<`any`>, `void`, `unknown`>
|
||||
|
||||
#### See
|
||||
|
||||
- AsyncIterator
|
||||
of
|
||||
- RecordBatch.
|
||||
|
||||
By default, LanceDb will use many threads to calculate results and, when
|
||||
the result set is large, multiple batches will be processed at one time.
|
||||
This readahead is limited however and backpressure will be applied if this
|
||||
stream is consumed slowly (this constrains the maximum memory used by a
|
||||
single query)
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.execute`
|
||||
|
||||
***
|
||||
|
||||
### explainPlan()
|
||||
|
||||
```ts
|
||||
explainPlan(verbose): Promise<string>
|
||||
```
|
||||
|
||||
Generates an explanation of the query execution plan.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **verbose**: `boolean` = `false`
|
||||
If true, provides a more detailed explanation. Defaults to false.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`string`>
|
||||
|
||||
A Promise that resolves to a string containing the query execution plan explanation.
|
||||
|
||||
#### Example
|
||||
|
||||
```ts
|
||||
import * as lancedb from "@lancedb/lancedb"
|
||||
const db = await lancedb.connect("./.lancedb");
|
||||
const table = await db.createTable("my_table", [
|
||||
{ vector: [1.1, 0.9], id: "1" },
|
||||
]);
|
||||
const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
|
||||
```
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.explainPlan`
|
||||
|
||||
***
|
||||
|
||||
### fastSearch()
|
||||
|
||||
```ts
|
||||
fastSearch(): this
|
||||
```
|
||||
|
||||
Skip searching un-indexed data. This can make search faster, but will miss
|
||||
any data that is not yet indexed.
|
||||
|
||||
Use [Table#optimize](Table.md#optimize) to index all un-indexed data.
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.fastSearch`
|
||||
|
||||
***
|
||||
|
||||
### ~~filter()~~
|
||||
|
||||
```ts
|
||||
filter(predicate): this
|
||||
```
|
||||
|
||||
A filter statement to be applied to this query.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **predicate**: `string`
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
#### See
|
||||
|
||||
where
|
||||
|
||||
#### Deprecated
|
||||
|
||||
Use `where` instead
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.filter`
|
||||
|
||||
***
|
||||
|
||||
### fullTextSearch()
|
||||
|
||||
```ts
|
||||
fullTextSearch(query, options?): this
|
||||
```
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **query**: `string` \| [`FullTextQuery`](../interfaces/FullTextQuery.md)
|
||||
|
||||
* **options?**: `Partial`<[`FullTextSearchOptions`](../interfaces/FullTextSearchOptions.md)>
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.fullTextSearch`
|
||||
|
||||
***
|
||||
|
||||
### limit()
|
||||
|
||||
```ts
|
||||
limit(limit): this
|
||||
```
|
||||
|
||||
Set the maximum number of results to return.
|
||||
|
||||
By default, a plain search has no limit. If this method is not
|
||||
called then every valid row from the table will be returned.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **limit**: `number`
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.limit`
|
||||
|
||||
***
|
||||
|
||||
### offset()
|
||||
|
||||
```ts
|
||||
offset(offset): this
|
||||
```
|
||||
|
||||
Set the number of rows to skip before returning results.
|
||||
|
||||
This is useful for pagination.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **offset**: `number`
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.offset`
|
||||
|
||||
***
|
||||
|
||||
### orderBy()
|
||||
|
||||
```ts
|
||||
orderBy(ordering): this
|
||||
```
|
||||
|
||||
Sort the results by the specified column(s).
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **ordering**: [`ColumnOrdering`](../interfaces/ColumnOrdering.md) \| [`ColumnOrdering`](../interfaces/ColumnOrdering.md)[]
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
This query builder.
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.orderBy`
|
||||
|
||||
***
|
||||
|
||||
### outputSchema()
|
||||
|
||||
```ts
|
||||
outputSchema(): Promise<Schema<any>>
|
||||
```
|
||||
|
||||
Returns the schema of the output that will be returned by this query.
|
||||
|
||||
This can be used to inspect the types and names of the columns that will be
|
||||
returned by the query before executing it.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`Schema`<`any`>>
|
||||
|
||||
An Arrow Schema describing the output columns.
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.outputSchema`
|
||||
|
||||
***
|
||||
|
||||
### select()
|
||||
|
||||
```ts
|
||||
select(columns): this
|
||||
```
|
||||
|
||||
Return only the specified columns.
|
||||
|
||||
By default a query will return all columns from the table. However, this can have
|
||||
a very significant impact on latency. LanceDb stores data in a columnar fashion. This
|
||||
means we can finely tune our I/O to select exactly the columns we need.
|
||||
|
||||
As a best practice you should always limit queries to the columns that you need. If you
|
||||
pass in an array of column names then only those columns will be returned.
|
||||
|
||||
You can also use this method to create new "dynamic" columns based on your existing columns.
|
||||
For example, you may not care about "a" or "b" but instead simply want "a + b". This is often
|
||||
seen in the SELECT clause of an SQL query (e.g. `SELECT a+b FROM my_table`).
|
||||
|
||||
To create dynamic columns you can pass in a Map<string, string>. A column will be returned
|
||||
for each entry in the map. The key provides the name of the column. The value is
|
||||
an SQL string used to specify how the column is calculated.
|
||||
|
||||
For example, an SQL query might state `SELECT a + b AS combined, c`. The equivalent
|
||||
input to this method would be:
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **columns**: `string` \| `string`[] \| `Record`<`string`, `string`> \| `Map`<`string`, `string`>
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
#### Example
|
||||
|
||||
```ts
|
||||
new Map([["combined", "a + b"], ["c", "c"]])
|
||||
|
||||
Columns will always be returned in the order given, even if that order is different than
|
||||
the order used when adding the data.
|
||||
|
||||
Note that you can pass in a `Record<string, string>` (e.g. an object literal). This method
|
||||
uses `Object.entries` which should preserve the insertion order of the object. However,
|
||||
object insertion order is easy to get wrong and `Map` is more foolproof.
|
||||
```
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.select`
|
||||
|
||||
***
|
||||
|
||||
### toArray()
|
||||
|
||||
```ts
|
||||
toArray(options?): Promise<any[]>
|
||||
```
|
||||
|
||||
Collect the results as an array of objects.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **options?**: `Partial`<[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)>
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`any`[]>
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.toArray`
|
||||
|
||||
***
|
||||
|
||||
### toArrow()
|
||||
|
||||
```ts
|
||||
toArrow(options?): Promise<Table<any>>
|
||||
```
|
||||
|
||||
Collect the results as an Arrow
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **options?**: `Partial`<[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)>
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`Table`<`any`>>
|
||||
|
||||
#### See
|
||||
|
||||
ArrowTable.
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.toArrow`
|
||||
|
||||
***
|
||||
|
||||
### useLsm()
|
||||
|
||||
```ts
|
||||
useLsm(enable): this
|
||||
```
|
||||
|
||||
Control MemWAL read routing for this query.
|
||||
|
||||
By default (unset), when the table carries a MemWAL write spec (see
|
||||
[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so
|
||||
they also return data written via the `mergeInsert` LSM path that has not yet
|
||||
been compacted into the base table (the active/frozen in-memory memtables and
|
||||
the flushed generations), deduplicated by primary key; a table without a spec
|
||||
reads the base table.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **enable**: `boolean`
|
||||
`true` forces the LSM scanner and errors if the table has no
|
||||
MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
|
||||
even when a spec is present.
|
||||
Note: the LSM scanner does not support every query shape (e.g. reranking,
|
||||
hybrid search, `orderBy`). On a MemWAL table those shapes error unless
|
||||
`useLsm(false)` is set, because a base-only read would silently exclude
|
||||
un-compacted MemWAL data.
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.useLsm`
|
||||
|
||||
***
|
||||
|
||||
### where()
|
||||
|
||||
```ts
|
||||
where(predicate): this
|
||||
```
|
||||
|
||||
A filter statement to be applied to this query.
|
||||
|
||||
The filter should be supplied as an SQL query string. For example:
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **predicate**: `string`
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
#### Example
|
||||
|
||||
```ts
|
||||
x > 10
|
||||
y > 0 AND y < 100
|
||||
x > 5 OR y = 'test'
|
||||
|
||||
Filtering performance can often be improved by creating a scalar index
|
||||
on the filter column(s).
|
||||
|
||||
Calling this multiple times combines the filters with a logical AND rather
|
||||
than replacing the previous filter.
|
||||
```
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.where`
|
||||
|
||||
***
|
||||
|
||||
### withRowId()
|
||||
|
||||
```ts
|
||||
withRowId(): this
|
||||
```
|
||||
|
||||
Whether to return the row id in the results.
|
||||
|
||||
This column can be used to match results between different queries. For
|
||||
example, to match results from a full text search and a vector search in
|
||||
order to perform hybrid search.
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.withRowId`
|
||||
@@ -37,31 +37,6 @@ latest and stays writable.
|
||||
|
||||
***
|
||||
|
||||
### cherryPick()
|
||||
|
||||
```ts
|
||||
cherryPick(fromBranch, dryRun): Promise<CherryPickResult>
|
||||
```
|
||||
|
||||
Cherry-pick a branch onto main.
|
||||
|
||||
Set `dryRun` to `true` to preview. A failed cherry-pick resolves
|
||||
with `status: "failed"` instead of throwing.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **fromBranch**: `string`
|
||||
Branch to cherry-pick from.
|
||||
|
||||
* **dryRun**: `boolean` = `false`
|
||||
When true, only preview. Defaults to false.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`CherryPickResult`](../interfaces/CherryPickResult.md)>
|
||||
|
||||
***
|
||||
|
||||
### create()
|
||||
|
||||
```ts
|
||||
@@ -137,3 +112,28 @@ List all branches, mapping name to branch metadata.
|
||||
#### Returns
|
||||
|
||||
`Promise`<`Record`<`string`, [`BranchContents`](BranchContents.md)>>
|
||||
|
||||
***
|
||||
|
||||
### merge()
|
||||
|
||||
```ts
|
||||
merge(fromBranch, dryRun): Promise<MergeBranchResult>
|
||||
```
|
||||
|
||||
Merge a branch into main.
|
||||
|
||||
Set `dryRun` to `true` to preview the merge. A rejected merge resolves
|
||||
with `status: "rejected"` instead of throwing.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **fromBranch**: `string`
|
||||
Branch to merge from.
|
||||
|
||||
* **dryRun**: `boolean` = `false`
|
||||
When true, only preview the merge. Defaults to false.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`MergeBranchResult`](../interfaces/MergeBranchResult.md)>
|
||||
|
||||
@@ -169,45 +169,6 @@ Creates a new empty Table
|
||||
|
||||
***
|
||||
|
||||
### createMaterializedView()
|
||||
|
||||
```ts
|
||||
abstract createMaterializedView(
|
||||
name,
|
||||
source,
|
||||
options?): Promise<MaterializedView>
|
||||
```
|
||||
|
||||
Define a materialized view named `name` over the table `source`.
|
||||
|
||||
The view is created empty, with the query recorded in its schema
|
||||
metadata; `view.refresh()` computes the rows. The view is a normal
|
||||
table: it can be queried, indexed and searched, and it appears in
|
||||
`tableNames`. The source table must have stable row ids (create it with
|
||||
the `newTableEnableStableRowIds` storage option); they keep the view's
|
||||
provenance valid across source compactions and cannot be enabled after
|
||||
a table exists. Local databases only.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **name**: `string`
|
||||
|
||||
* **source**: `string`
|
||||
|
||||
* **options?**
|
||||
|
||||
* **options.limit?**: `number`
|
||||
|
||||
* **options.select?**: [`MaterializedViewSelect`](../type-aliases/MaterializedViewSelect.md)
|
||||
|
||||
* **options.where?**: `string`
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`MaterializedView`](MaterializedView.md)>
|
||||
|
||||
***
|
||||
|
||||
### createNamespace()
|
||||
|
||||
```ts
|
||||
@@ -538,22 +499,6 @@ List server-side jobs across the database's tables.
|
||||
|
||||
***
|
||||
|
||||
### listMaterializedViews()
|
||||
|
||||
```ts
|
||||
abstract listMaterializedViews(): Promise<string[]>
|
||||
```
|
||||
|
||||
The names of the materialized views in this database.
|
||||
|
||||
Found by reading every table's schema, so this costs an open per table.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`string`[]>
|
||||
|
||||
***
|
||||
|
||||
### listNamespaces()
|
||||
|
||||
```ts
|
||||
@@ -584,90 +529,6 @@ Child namespace names and
|
||||
|
||||
***
|
||||
|
||||
### listTables()
|
||||
|
||||
#### listTables(options)
|
||||
|
||||
```ts
|
||||
abstract listTables(options?): Promise<ListTablesResponse>
|
||||
```
|
||||
|
||||
List a page of the tables in this database.
|
||||
|
||||
To retrieve the tables after the page, pass the `pageToken` the response
|
||||
carries back in. A page can be shorter than `limit` without being the last
|
||||
one, so walk until a response carries no page token:
|
||||
|
||||
```ts
|
||||
const names = [];
|
||||
let pageToken = undefined;
|
||||
do {
|
||||
const page = await conn.listTables({ pageToken, limit: 100 });
|
||||
names.push(...page.tables);
|
||||
pageToken = page.pageToken;
|
||||
} while (pageToken);
|
||||
```
|
||||
|
||||
##### Parameters
|
||||
|
||||
* **options?**: `Partial`<[`ListTablesOptions`](../interfaces/ListTablesOptions.md)>
|
||||
Pagination options
|
||||
(`pageToken`, `limit`).
|
||||
|
||||
##### Returns
|
||||
|
||||
`Promise`<[`ListTablesResponse`](../interfaces/ListTablesResponse.md)>
|
||||
|
||||
A page of table names and an
|
||||
optional token for the tables after it.
|
||||
|
||||
#### listTables(namespacePath, options)
|
||||
|
||||
```ts
|
||||
abstract listTables(namespacePath?, options?): Promise<ListTablesResponse>
|
||||
```
|
||||
|
||||
List a page of the tables in this database.
|
||||
|
||||
##### Parameters
|
||||
|
||||
* **namespacePath?**: `string`[]
|
||||
The namespace path to list tables from
|
||||
(defaults to root namespace)
|
||||
|
||||
* **options?**: `Partial`<[`ListTablesOptions`](../interfaces/ListTablesOptions.md)>
|
||||
Pagination options
|
||||
(`pageToken`, `limit`).
|
||||
|
||||
##### Returns
|
||||
|
||||
`Promise`<[`ListTablesResponse`](../interfaces/ListTablesResponse.md)>
|
||||
|
||||
A page of table names and an
|
||||
optional token for the tables after it.
|
||||
|
||||
***
|
||||
|
||||
### openMaterializedView()
|
||||
|
||||
```ts
|
||||
abstract openMaterializedView(name): Promise<MaterializedView>
|
||||
```
|
||||
|
||||
Open the materialized view named `name`.
|
||||
|
||||
Rejects a table that exists but is not a materialized view.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **name**: `string`
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`MaterializedView`](MaterializedView.md)>
|
||||
|
||||
***
|
||||
|
||||
### openTable()
|
||||
|
||||
```ts
|
||||
@@ -677,13 +538,18 @@ abstract openTable(
|
||||
options?): Promise<Table>
|
||||
```
|
||||
|
||||
Open a table in the database.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **name**: `string`
|
||||
The name of the table
|
||||
|
||||
* **namespacePath?**: `string`[]
|
||||
The namespace path of the table (defaults to root namespace)
|
||||
|
||||
* **options?**: `Partial`<[`OpenTableOptions`](../interfaces/OpenTableOptions.md)>
|
||||
Additional options
|
||||
|
||||
#### Returns
|
||||
|
||||
@@ -724,7 +590,7 @@ a "not supported" error.
|
||||
|
||||
***
|
||||
|
||||
### ~~tableNames()~~
|
||||
### tableNames()
|
||||
|
||||
#### tableNames(options)
|
||||
|
||||
@@ -746,10 +612,6 @@ Tables will be returned in lexicographical order.
|
||||
|
||||
`Promise`<`string`[]>
|
||||
|
||||
##### Deprecated
|
||||
|
||||
Use [Connection.listTables](Connection.md#listtables) instead.
|
||||
|
||||
#### tableNames(namespacePath, options)
|
||||
|
||||
```ts
|
||||
@@ -772,7 +634,3 @@ Tables will be returned in lexicographical order.
|
||||
##### Returns
|
||||
|
||||
`Promise`<`string`[]>
|
||||
|
||||
##### Deprecated
|
||||
|
||||
Use [Connection.listTables](Connection.md#listtables) instead.
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / MaterializedView
|
||||
|
||||
# Class: MaterializedView
|
||||
|
||||
A handle on a materialized view: its table plus its definition.
|
||||
|
||||
Obtained from [Connection#createMaterializedView](Connection.md#creatematerializedview) or
|
||||
[Connection#openMaterializedView](Connection.md#openmaterializedview). The view is a normal table --
|
||||
queries, indexes and search all apply through [MaterializedView#table](MaterializedView.md#table)
|
||||
-- whose contents are maintained by [MaterializedView#refresh](MaterializedView.md#refresh).
|
||||
|
||||
## Constructors
|
||||
|
||||
### new MaterializedView()
|
||||
|
||||
```ts
|
||||
new MaterializedView(table): MaterializedView
|
||||
```
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **table**: [`Table`](Table.md)
|
||||
|
||||
#### Returns
|
||||
|
||||
[`MaterializedView`](MaterializedView.md)
|
||||
|
||||
## Accessors
|
||||
|
||||
### name
|
||||
|
||||
```ts
|
||||
get name(): string
|
||||
```
|
||||
|
||||
#### Returns
|
||||
|
||||
`string`
|
||||
|
||||
## Methods
|
||||
|
||||
### definition()
|
||||
|
||||
```ts
|
||||
definition(): Promise<MaterializedViewDefinition>
|
||||
```
|
||||
|
||||
The query that defines the view, read from its stored schema.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`MaterializedViewDefinition`](../interfaces/MaterializedViewDefinition.md)>
|
||||
|
||||
***
|
||||
|
||||
### refresh()
|
||||
|
||||
```ts
|
||||
refresh(options?): Promise<RefreshMaterializedViewResult>
|
||||
```
|
||||
|
||||
Recompute the view from its source.
|
||||
|
||||
The refresh is incremental when the source's changes can be reconciled
|
||||
into the view -- rows added, changed or removed since the last one --
|
||||
and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
|
||||
refreshes to that source version instead of the latest.
|
||||
|
||||
Concurrent refreshes of one view do not duplicate its rows. Two that
|
||||
plan the same source rows conflict on commit, and the loser throws
|
||||
rather than writing them a second time.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **options?**
|
||||
|
||||
* **options.full?**: `boolean`
|
||||
|
||||
* **options.sourceVersion?**: `number`
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<[`RefreshMaterializedViewResult`](../interfaces/RefreshMaterializedViewResult.md)>
|
||||
|
||||
***
|
||||
|
||||
### table()
|
||||
|
||||
```ts
|
||||
table(): Table
|
||||
```
|
||||
|
||||
The view, as the table it is.
|
||||
|
||||
#### Returns
|
||||
|
||||
[`Table`](Table.md)
|
||||
@@ -942,7 +942,7 @@ Get the schema of the table.
|
||||
abstract search(
|
||||
query,
|
||||
queryType?,
|
||||
ftsColumns?): Query | VectorQuery | AutoQuery
|
||||
ftsColumns?): Query | VectorQuery
|
||||
```
|
||||
|
||||
Create a search query to find the nearest neighbors
|
||||
@@ -964,7 +964,7 @@ of the given query
|
||||
|
||||
#### Returns
|
||||
|
||||
[`Query`](Query.md) \| [`VectorQuery`](VectorQuery.md) \| [`AutoQuery`](AutoQuery.md)
|
||||
[`Query`](Query.md) \| [`VectorQuery`](VectorQuery.md)
|
||||
|
||||
***
|
||||
|
||||
@@ -1292,18 +1292,6 @@ abstract updateFieldMetadata(updates): Promise<UpdateFieldMetadataResult>
|
||||
|
||||
Update per-field (column) metadata.
|
||||
|
||||
The following keys are treated specially, by convention, and should be
|
||||
used when appropriate:
|
||||
|
||||
- `lancedb:description`: for a human-readable description of a field.
|
||||
- `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
|
||||
names the tag category; e.g. `lancedb:tag:model: "clip"`.
|
||||
- `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
|
||||
`feature_v2` might be in the same logical column.
|
||||
- `lancedb:status`: for status options (`production`, `candidate`,
|
||||
`deprecated`, `archived`) to designate the current life cycle state of
|
||||
this column.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **updates**: [`FieldMetadataUpdate`](../interfaces/FieldMetadataUpdate.md)[]
|
||||
|
||||
+3
-10
@@ -18,7 +18,6 @@
|
||||
|
||||
## Classes
|
||||
|
||||
- [AutoQuery](classes/AutoQuery.md)
|
||||
- [BooleanQuery](classes/BooleanQuery.md)
|
||||
- [BoostQuery](classes/BoostQuery.md)
|
||||
- [BranchContents](classes/BranchContents.md)
|
||||
@@ -29,7 +28,6 @@
|
||||
- [Job](classes/Job.md)
|
||||
- [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
|
||||
- [MatchQuery](classes/MatchQuery.md)
|
||||
- [MaterializedView](classes/MaterializedView.md)
|
||||
- [MergeInsertBuilder](classes/MergeInsertBuilder.md)
|
||||
- [MultiMatchQuery](classes/MultiMatchQuery.md)
|
||||
- [NativeJsHeaderProvider](classes/NativeJsHeaderProvider.md)
|
||||
@@ -61,9 +59,6 @@
|
||||
- [BranchIndexSummary](interfaces/BranchIndexSummary.md)
|
||||
- [BranchRowCountSummary](interfaces/BranchRowCountSummary.md)
|
||||
- [BucketStats](interfaces/BucketStats.md)
|
||||
- [CherryPickError](interfaces/CherryPickError.md)
|
||||
- [CherryPickPreview](interfaces/CherryPickPreview.md)
|
||||
- [CherryPickResult](interfaces/CherryPickResult.md)
|
||||
- [ClientConfig](interfaces/ClientConfig.md)
|
||||
- [ColumnAlteration](interfaces/ColumnAlteration.md)
|
||||
- [ColumnOrdering](interfaces/ColumnOrdering.md)
|
||||
@@ -101,12 +96,12 @@
|
||||
- [JobInfo](interfaces/JobInfo.md)
|
||||
- [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
|
||||
- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
|
||||
- [ListTablesOptions](interfaces/ListTablesOptions.md)
|
||||
- [ListTablesResponse](interfaces/ListTablesResponse.md)
|
||||
- [LsmStats](interfaces/LsmStats.md)
|
||||
- [LsmWriteSpec](interfaces/LsmWriteSpec.md)
|
||||
- [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md)
|
||||
- [MemtableStats](interfaces/MemtableStats.md)
|
||||
- [MergeBlocker](interfaces/MergeBlocker.md)
|
||||
- [MergeBranchResult](interfaces/MergeBranchResult.md)
|
||||
- [MergePreview](interfaces/MergePreview.md)
|
||||
- [MergeResult](interfaces/MergeResult.md)
|
||||
- [NativeOAuthConfig](interfaces/NativeOAuthConfig.md)
|
||||
- [OAuthConfig](interfaces/OAuthConfig.md)
|
||||
@@ -115,7 +110,6 @@
|
||||
- [OptimizeStats](interfaces/OptimizeStats.md)
|
||||
- [QueryExecutionOptions](interfaces/QueryExecutionOptions.md)
|
||||
- [RefreshColumnResult](interfaces/RefreshColumnResult.md)
|
||||
- [RefreshMaterializedViewResult](interfaces/RefreshMaterializedViewResult.md)
|
||||
- [RemovalStats](interfaces/RemovalStats.md)
|
||||
- [RenameTableOptions](interfaces/RenameTableOptions.md)
|
||||
- [RestNamespaceConfig](interfaces/RestNamespaceConfig.md)
|
||||
@@ -148,7 +142,6 @@
|
||||
- [FieldLike](type-aliases/FieldLike.md)
|
||||
- [IntoSql](type-aliases/IntoSql.md)
|
||||
- [IntoVector](type-aliases/IntoVector.md)
|
||||
- [MaterializedViewSelect](type-aliases/MaterializedViewSelect.md)
|
||||
- [MultiVector](type-aliases/MultiVector.md)
|
||||
- [RecordBatchLike](type-aliases/RecordBatchLike.md)
|
||||
- [SchemaLike](type-aliases/SchemaLike.md)
|
||||
|
||||
@@ -50,14 +50,6 @@ changedColumns: BranchColumnChange[];
|
||||
|
||||
***
|
||||
|
||||
### errors
|
||||
|
||||
```ts
|
||||
errors: CherryPickError[];
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### fromBranch
|
||||
|
||||
```ts
|
||||
@@ -74,6 +66,22 @@ mainVersion: number;
|
||||
|
||||
***
|
||||
|
||||
### mergeBlockers
|
||||
|
||||
```ts
|
||||
mergeBlockers: MergeBlocker[];
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### mergeable
|
||||
|
||||
```ts
|
||||
mergeable: boolean;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### parentVersion
|
||||
|
||||
```ts
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / CherryPickPreview
|
||||
|
||||
# Interface: CherryPickPreview
|
||||
|
||||
Changes that would be, or were, promoted by a cherry-pick.
|
||||
|
||||
## Properties
|
||||
|
||||
### promotedColumns
|
||||
|
||||
```ts
|
||||
promotedColumns: string[];
|
||||
```
|
||||
@@ -17,8 +17,7 @@ metadata: Record<string, null | string>;
|
||||
```
|
||||
|
||||
Metadata key/value pairs. Merged into the field's existing metadata by
|
||||
default; a value of `null` deletes that key. See
|
||||
[Table.updateFieldMetadata](../classes/Table.md#updatefieldmetadata) for the conventional `lancedb:*` keys.
|
||||
default; a value of `null` deletes that key.
|
||||
|
||||
***
|
||||
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / ListTablesOptions
|
||||
|
||||
# Interface: ListTablesOptions
|
||||
|
||||
## Properties
|
||||
|
||||
### limit?
|
||||
|
||||
```ts
|
||||
optional limit: number;
|
||||
```
|
||||
|
||||
An upper bound on how many tables to return.
|
||||
|
||||
A page may hold fewer than this and still not be the last one, so keep
|
||||
going while the response carries a page token rather than while pages are
|
||||
full.
|
||||
|
||||
***
|
||||
|
||||
### pageToken?
|
||||
|
||||
```ts
|
||||
optional pageToken: string;
|
||||
```
|
||||
|
||||
Token from a previous response, to resume listing where it left off.
|
||||
|
||||
The token is opaque: it carries whatever the database needs to resume, and
|
||||
callers should not construct or interpret one.
|
||||
@@ -1,23 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / ListTablesResponse
|
||||
|
||||
# Interface: ListTablesResponse
|
||||
|
||||
## Properties
|
||||
|
||||
### pageToken?
|
||||
|
||||
```ts
|
||||
optional pageToken: string;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### tables
|
||||
|
||||
```ts
|
||||
tables: string[];
|
||||
```
|
||||
@@ -1,59 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / MaterializedViewDefinition
|
||||
|
||||
# Interface: MaterializedViewDefinition
|
||||
|
||||
The query that defines a materialized view.
|
||||
|
||||
## Properties
|
||||
|
||||
### filter?
|
||||
|
||||
```ts
|
||||
optional filter: string;
|
||||
```
|
||||
|
||||
SQL predicate selecting the source rows the view holds.
|
||||
|
||||
***
|
||||
|
||||
### inputs
|
||||
|
||||
```ts
|
||||
inputs: string[];
|
||||
```
|
||||
|
||||
Source columns the projections and filter read.
|
||||
|
||||
***
|
||||
|
||||
### limit?
|
||||
|
||||
```ts
|
||||
optional limit: number;
|
||||
```
|
||||
|
||||
Cap on the number of rows the view holds.
|
||||
|
||||
***
|
||||
|
||||
### projections
|
||||
|
||||
```ts
|
||||
projections: [string, string][];
|
||||
```
|
||||
|
||||
`[output column, SQL expression]` pairs, in view schema order.
|
||||
|
||||
***
|
||||
|
||||
### sourceTable
|
||||
|
||||
```ts
|
||||
sourceTable: string;
|
||||
```
|
||||
|
||||
Name of the source table, in the same database as the view.
|
||||
@@ -2,11 +2,11 @@
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / CherryPickError
|
||||
[@lancedb/lancedb](../globals.md) / MergeBlocker
|
||||
|
||||
# Interface: CherryPickError
|
||||
# Interface: MergeBlocker
|
||||
|
||||
A reason why a cherry-pick cannot currently land.
|
||||
A reason why a branch cannot currently be merged.
|
||||
|
||||
## Properties
|
||||
|
||||
+6
-6
@@ -2,11 +2,11 @@
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / CherryPickResult
|
||||
[@lancedb/lancedb](../globals.md) / MergeBranchResult
|
||||
|
||||
# Interface: CherryPickResult
|
||||
# Interface: MergeBranchResult
|
||||
|
||||
Result of previewing or attempting a cherry-pick.
|
||||
Result of previewing or attempting a branch merge.
|
||||
|
||||
## Properties
|
||||
|
||||
@@ -29,7 +29,7 @@ optional mainVersionAfter: number;
|
||||
### preview
|
||||
|
||||
```ts
|
||||
preview: CherryPickPreview;
|
||||
preview: MergePreview;
|
||||
```
|
||||
|
||||
***
|
||||
@@ -38,9 +38,9 @@ preview: CherryPickPreview;
|
||||
|
||||
```ts
|
||||
status:
|
||||
| "failed"
|
||||
| "unknown"
|
||||
| "rejected"
|
||||
| "ready"
|
||||
| "notImplemented"
|
||||
| "cherryPicked";
|
||||
| "merged";
|
||||
```
|
||||
@@ -0,0 +1,17 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / MergePreview
|
||||
|
||||
# Interface: MergePreview
|
||||
|
||||
Changes that would be, or were, promoted by a branch merge.
|
||||
|
||||
## Properties
|
||||
|
||||
### promotedColumns
|
||||
|
||||
```ts
|
||||
promotedColumns: string[];
|
||||
```
|
||||
@@ -1,41 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / RefreshMaterializedViewResult
|
||||
|
||||
# Interface: RefreshMaterializedViewResult
|
||||
|
||||
## Properties
|
||||
|
||||
### mode
|
||||
|
||||
```ts
|
||||
mode: string;
|
||||
```
|
||||
|
||||
How the view was brought up to date: "rebuild", "incremental" or "no_op".
|
||||
|
||||
***
|
||||
|
||||
### rowsWritten
|
||||
|
||||
```ts
|
||||
rowsWritten: number;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### sourceVersion
|
||||
|
||||
```ts
|
||||
sourceVersion: number;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
### version
|
||||
|
||||
```ts
|
||||
version: number;
|
||||
```
|
||||
@@ -4,16 +4,11 @@
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / TableNamesOptions
|
||||
|
||||
# Interface: ~~TableNamesOptions~~
|
||||
|
||||
## Deprecated
|
||||
|
||||
Use [ListTablesOptions](ListTablesOptions.md) with [Connection.listTables](../classes/Connection.md#listtables)
|
||||
instead.
|
||||
# Interface: TableNamesOptions
|
||||
|
||||
## Properties
|
||||
|
||||
### ~~limit?~~
|
||||
### limit?
|
||||
|
||||
```ts
|
||||
optional limit: number;
|
||||
@@ -23,7 +18,7 @@ An optional limit to the number of results to return.
|
||||
|
||||
***
|
||||
|
||||
### ~~startAfter?~~
|
||||
### startAfter?
|
||||
|
||||
```ts
|
||||
optional startAfter: string;
|
||||
|
||||
@@ -10,12 +10,16 @@
|
||||
function getRegistry(): EmbeddingFunctionRegistry
|
||||
```
|
||||
|
||||
Get the global embedding function registry.
|
||||
|
||||
LanceDB built-in providers are initialized when this public API is first
|
||||
used, so importing the root package does not change automatic search
|
||||
selection for tables without embedding metadata.
|
||||
Utility function to get the global instance of the registry
|
||||
|
||||
## Returns
|
||||
|
||||
[`EmbeddingFunctionRegistry`](../classes/EmbeddingFunctionRegistry.md)
|
||||
|
||||
`EmbeddingFunctionRegistry` The global instance of the registry
|
||||
|
||||
## Example
|
||||
|
||||
```ts
|
||||
const registry = getRegistry();
|
||||
const openai = registry.get("openai").create();
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / MaterializedViewSelect
|
||||
|
||||
# Type Alias: MaterializedViewSelect
|
||||
|
||||
```ts
|
||||
type MaterializedViewSelect: (string | [string, string])[] | Record<string, string>;
|
||||
```
|
||||
|
||||
The view's columns: column names, `[alias, SQL expression]` pairs, or a
|
||||
record of the same. A bare name projects itself.
|
||||
@@ -102,12 +102,6 @@ listing a storage directory.
|
||||
|
||||
::: lancedb.job.AsyncJob
|
||||
|
||||
## Materialized Views (Synchronous)
|
||||
|
||||
::: lancedb.materialized_view.MaterializedView
|
||||
|
||||
::: lancedb.materialized_view.MaterializedViewDefinition
|
||||
|
||||
## Expressions
|
||||
|
||||
Type-safe expression builder for filters and projections. Use these instead
|
||||
@@ -159,8 +153,6 @@ and combined with [BooleanQuery][lancedb.query.BooleanQuery].
|
||||
|
||||
::: lancedb.query.FullTextOperator
|
||||
|
||||
::: lancedb.query.DocumentGranularity
|
||||
|
||||
::: lancedb.query.Occur
|
||||
|
||||
## Embeddings
|
||||
@@ -223,14 +215,10 @@ tokens = list(
|
||||
Blob columns store large binary values out of line so they can be read lazily
|
||||
instead of being materialized with the rest of the row.
|
||||
|
||||
`lancedb.BlobType` is `lance.blob.BlobType` when pylance is installed. Without
|
||||
pylance, LanceDB uses a matching `lance.blob.v2` extension type so blob columns
|
||||
still work. Queries return descriptors. Call
|
||||
[`fetch_blob_files`][lancedb.table.Table.fetch_blob_files] for lazy reads or
|
||||
[`fetch_blobs`][lancedb.table.Table.fetch_blobs] for eager bytes.
|
||||
|
||||
::: lancedb.blob
|
||||
|
||||
::: lancedb.BlobType
|
||||
|
||||
::: lancedb._blob.BlobFile
|
||||
options:
|
||||
show_root_full_path: false
|
||||
@@ -267,8 +255,6 @@ still work. Queries return descriptors. Call
|
||||
|
||||
::: lancedb.streaming.StreamingDataset
|
||||
|
||||
::: lancedb.streaming.StreamingDataLoader
|
||||
|
||||
::: lancedb.permutation.permutation_builder
|
||||
|
||||
::: lancedb.permutation.PermutationBuilder
|
||||
@@ -309,10 +295,6 @@ Table hold your actual data as a collection of records / rows.
|
||||
|
||||
::: lancedb.table.AsyncBranches
|
||||
|
||||
## Materialized Views (Asynchronous)
|
||||
|
||||
::: lancedb.materialized_view.AsyncMaterializedView
|
||||
|
||||
## Indices (Asynchronous)
|
||||
|
||||
Indices can be created on a table to speed up queries. This section
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
<parent>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.38.0-beta.11</version>
|
||||
<version>0.38.0-beta.3</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
|
||||
+2
-2
@@ -6,7 +6,7 @@
|
||||
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.38.0-beta.11</version>
|
||||
<version>0.38.0-beta.3</version>
|
||||
<packaging>pom</packaging>
|
||||
<name>${project.artifactId}</name>
|
||||
<description>LanceDB Java SDK Parent POM</description>
|
||||
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>12.0.0-beta.2</lance-core.version>
|
||||
<lance-core.version>11.0.0-beta.18</lance-core.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[package]
|
||||
name = "lancedb-nodejs"
|
||||
edition.workspace = true
|
||||
version = "0.38.0-beta.11"
|
||||
version = "0.38.0-beta.3"
|
||||
publish = false
|
||||
license.workspace = true
|
||||
description.workspace = true
|
||||
|
||||
@@ -1,16 +1,11 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
import * as fs from "node:fs";
|
||||
import * as vm from "node:vm";
|
||||
import * as arrow15 from "apache-arrow-15";
|
||||
import * as arrow16 from "apache-arrow-16";
|
||||
import * as arrow17 from "apache-arrow-17";
|
||||
import * as arrow18 from "apache-arrow-18";
|
||||
|
||||
import {
|
||||
Field as CurrentField,
|
||||
LargeBinary as CurrentLargeBinary,
|
||||
Schema as CurrentSchema,
|
||||
Vector as CurrentVector,
|
||||
convertToTable,
|
||||
tableFromIPC as currentTableFromIPC,
|
||||
@@ -41,59 +36,6 @@ function sampleRecords(): Array<Record<string, any>> {
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
it("serializes an Arrow Table created in another JavaScript realm", async () => {
|
||||
const context = vm.createContext({
|
||||
TextDecoder,
|
||||
TextEncoder,
|
||||
console,
|
||||
setTimeout,
|
||||
clearTimeout,
|
||||
});
|
||||
vm.runInContext(
|
||||
fs.readFileSync(
|
||||
require.resolve("apache-arrow-15/Arrow.es2015.min"),
|
||||
"utf8",
|
||||
),
|
||||
context,
|
||||
);
|
||||
const foreignTable: unknown = vm.runInContext(
|
||||
"Arrow.tableFromArrays({ id: new Int32Array([1, 2, 3]), text: ['foo', 'bar', 'baz'] })",
|
||||
context,
|
||||
);
|
||||
|
||||
const foreignMetadata = (
|
||||
foreignTable as { schema: { metadata: Map<string, string> } }
|
||||
).schema.metadata;
|
||||
expect(foreignMetadata).not.toBeInstanceOf(Map);
|
||||
|
||||
const buf = await fromDataToBuffer(
|
||||
foreignTable as Parameters<typeof fromDataToBuffer>[0],
|
||||
);
|
||||
const actual = currentTableFromIPC(buf);
|
||||
|
||||
expect(actual.numRows).toBe(3);
|
||||
expect(actual.getChild("id")?.toJSON()).toEqual([1, 2, 3]);
|
||||
expect(actual.getChild("text")?.toJSON()).toEqual(["foo", "bar", "baz"]);
|
||||
});
|
||||
|
||||
it("preserves field metadata from a provided schema", async function () {
|
||||
const jsonMetadata = new Map([["ARROW:extension:name", "lance.json"]]);
|
||||
const schema = new CurrentSchema([
|
||||
new CurrentField("meta", new CurrentLargeBinary(), true, jsonMetadata),
|
||||
]);
|
||||
|
||||
const table = makeArrowTable(
|
||||
[{ meta: Buffer.from(JSON.stringify({ source: "test" })) }],
|
||||
{ schema },
|
||||
);
|
||||
|
||||
expect(table.schema.fields[0].metadata).toEqual(jsonMetadata);
|
||||
|
||||
const roundTripped = currentTableFromIPC(await fromTableToBuffer(table));
|
||||
expect(roundTripped.schema.fields[0].metadata).toEqual(jsonMetadata);
|
||||
});
|
||||
|
||||
describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
"Arrow",
|
||||
(
|
||||
@@ -573,137 +515,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
);
|
||||
});
|
||||
|
||||
it("will allow matching inferred types across records", function () {
|
||||
expect(() =>
|
||||
makeArrowTable([{ value: 1 }, { value: 2 }]),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("will reject mismatched inferred types across records", function () {
|
||||
expect(() => makeArrowTable([{ value: 1 }, { value: "two" }])).toThrow(
|
||||
"Failed to infer schema for data. Previously inferred type Float64 but found Utf8 for field value at row 1. Consider providing an explicit schema.",
|
||||
);
|
||||
});
|
||||
|
||||
it("will ignore generated dictionary IDs when comparing inferred types", function () {
|
||||
const table = makeArrowTable([{ str: "a" }, { str: "b" }], {
|
||||
dictionaryEncodeStrings: true,
|
||||
});
|
||||
|
||||
expect(table.getChild("str")?.toJSON()).toEqual(["a", "b"]);
|
||||
});
|
||||
|
||||
it("will preserve null values without treating them as type mismatches", function () {
|
||||
for (const records of [
|
||||
[{ vector: [1, 2, 3] }, { vector: null }],
|
||||
[{ vector: null }, { vector: [1, 2, 3] }],
|
||||
]) {
|
||||
const table = makeArrowTable(records);
|
||||
|
||||
expect(table.numRows).toBe(2);
|
||||
expect(table.getChild("vector")?.nullCount).toBe(1);
|
||||
}
|
||||
});
|
||||
|
||||
it("will preserve empty variable-size lists", function () {
|
||||
for (const records of [
|
||||
[{ items: [1] }, { items: [] }],
|
||||
[{ items: [] }, { items: [1] }],
|
||||
]) {
|
||||
const table = makeArrowTable(records);
|
||||
expect(
|
||||
table
|
||||
.getChild("items")
|
||||
?.toJSON()
|
||||
.map((value) => value.toJSON()),
|
||||
).toEqual(records.map((record) => record.items));
|
||||
}
|
||||
});
|
||||
|
||||
it("will propagate deferred evidence through nested lists", function () {
|
||||
for (const records of [
|
||||
[{ items: [1] }, { items: [null] }],
|
||||
[{ items: [null] }, { items: [1] }],
|
||||
[{ items: [null, 1] }, { items: [2, null] }],
|
||||
]) {
|
||||
const table = makeArrowTable(records);
|
||||
expect(
|
||||
table
|
||||
.getChild("items")
|
||||
?.toJSON()
|
||||
.map((value) => value.toJSON()),
|
||||
).toEqual(records.map((record) => record.items));
|
||||
}
|
||||
|
||||
const nestedRecords = [{ items: [[1]] }, { items: [[null]] }];
|
||||
const nestedTable = makeArrowTable(nestedRecords);
|
||||
expect(
|
||||
nestedTable
|
||||
.getChild("items")
|
||||
?.toJSON()
|
||||
.map((value) =>
|
||||
value
|
||||
.toJSON()
|
||||
.map((nestedValue: { toJSON: () => unknown[] }) =>
|
||||
nestedValue.toJSON(),
|
||||
),
|
||||
),
|
||||
).toEqual(nestedRecords.map((record) => record.items));
|
||||
});
|
||||
|
||||
it("will reject incompatible deferred evidence within a list", function () {
|
||||
for (const items of [
|
||||
[[], 1],
|
||||
[1, []],
|
||||
[[null], 1],
|
||||
[1, [null]],
|
||||
]) {
|
||||
expect(() => makeArrowTable([{ items }])).toThrow(
|
||||
"Failed to infer data type for field items at row 0.",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("will reject empty fixed-size lists", function () {
|
||||
expect(() =>
|
||||
makeArrowTable([{ vector: [1, 2, 3] }, { vector: [] }]),
|
||||
).toThrow(
|
||||
"Failed to infer schema for data. Previously inferred type FixedSizeList[3]<Float32> but found List[0] for field vector at row 1.",
|
||||
);
|
||||
});
|
||||
|
||||
it("will reject inferred leaf and branch shape changes", function () {
|
||||
expect(() =>
|
||||
makeArrowTable([{ value: 1 }, { value: { nested: 2 } }]),
|
||||
).toThrow(
|
||||
"Failed to infer schema for data. Previously inferred type Float64 but found Struct for field value at row 1.",
|
||||
);
|
||||
expect(() =>
|
||||
makeArrowTable([{ value: { nested: 1 } }, { value: 2 }]),
|
||||
).toThrow(
|
||||
"Failed to infer schema for data. Previously inferred type Struct but found Float64 for field value at row 1.",
|
||||
);
|
||||
});
|
||||
|
||||
it("will allow null values around inferred struct values", function () {
|
||||
for (const { records, nullIndex } of [
|
||||
{
|
||||
records: [{ value: null }, { value: { nested: 2 } }],
|
||||
nullIndex: 0,
|
||||
},
|
||||
{
|
||||
records: [{ value: { nested: 1 } }, { value: null }],
|
||||
nullIndex: 1,
|
||||
},
|
||||
]) {
|
||||
const table = makeArrowTable(records);
|
||||
const values = table.getChild("value");
|
||||
|
||||
expect(values?.nullCount).toBe(1);
|
||||
expect(values?.get(nullIndex)).toBeNull();
|
||||
}
|
||||
});
|
||||
|
||||
it("will allow a schema to be provided", async function () {
|
||||
await checkTableCreation(
|
||||
async (records, _, schema) =>
|
||||
|
||||
@@ -4,13 +4,7 @@
|
||||
import { readdirSync } from "fs";
|
||||
import { Field, Float64, Schema } from "apache-arrow";
|
||||
import * as tmp from "tmp";
|
||||
import {
|
||||
Connection,
|
||||
ListTablesResponse,
|
||||
Table,
|
||||
connect,
|
||||
connectNamespace,
|
||||
} from "../lancedb";
|
||||
import { Connection, Table, connect, connectNamespace } from "../lancedb";
|
||||
import { LocalTable } from "../lancedb/table";
|
||||
|
||||
describe("when connecting", () => {
|
||||
@@ -53,7 +47,6 @@ describe("given a connection", () => {
|
||||
await db.close();
|
||||
expect(db.isOpen()).toBe(false);
|
||||
await expect(db.tableNames()).rejects.toThrow("Connection is closed");
|
||||
await expect(db.listTables()).rejects.toThrow("Connection is closed");
|
||||
await expect(db.renameTable("a", "b")).rejects.toThrow(
|
||||
"Connection is closed",
|
||||
);
|
||||
@@ -136,66 +129,6 @@ describe("given a connection", () => {
|
||||
expect(tables).toEqual(["b", "c"]);
|
||||
});
|
||||
|
||||
it("should respect limit and page token when listing tables", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
|
||||
await db.createTable("b", [{ id: 1 }]);
|
||||
await db.createTable("a", [{ id: 1 }]);
|
||||
await db.createTable("c", [{ id: 1 }]);
|
||||
|
||||
const all = await db.listTables();
|
||||
expect(all.tables).toEqual(["a", "b", "c"]);
|
||||
expect(all.pageToken).toBeUndefined();
|
||||
|
||||
const first = await db.listTables({ limit: 1 });
|
||||
expect(first.tables).toEqual(["a"]);
|
||||
expect(first.pageToken).toBeDefined();
|
||||
|
||||
const second = await db.listTables({
|
||||
limit: 1,
|
||||
pageToken: first.pageToken,
|
||||
});
|
||||
expect(second.tables).toEqual(["b"]);
|
||||
});
|
||||
|
||||
it("should visit every table exactly once when walking pages", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
|
||||
const created = ["a", "b", "c", "d", "e"];
|
||||
for (const name of created) {
|
||||
await db.createTable(name, [{ id: 1 }]);
|
||||
}
|
||||
|
||||
const seen: string[] = [];
|
||||
let pageToken: string | undefined = undefined;
|
||||
do {
|
||||
const page: ListTablesResponse = await db.listTables({
|
||||
limit: 2,
|
||||
pageToken,
|
||||
});
|
||||
seen.push(...page.tables);
|
||||
pageToken = page.pageToken;
|
||||
} while (pageToken);
|
||||
|
||||
expect(seen).toEqual(created);
|
||||
});
|
||||
|
||||
it("should list tables in a namespace", async () => {
|
||||
const db = await connect(tmpDir.name, {
|
||||
// biome-ignore lint/style/useNamingConvention: opaque backend property key, must match Rust
|
||||
namespaceClientProperties: { manifest_enabled: "true" },
|
||||
});
|
||||
await db.createNamespace(["child"]);
|
||||
await db.createTable("nested", [{ id: 1 }], ["child"]);
|
||||
|
||||
await expect(db.listTables(["child"])).resolves.toEqual(
|
||||
expect.objectContaining({ tables: ["nested"] }),
|
||||
);
|
||||
await expect(db.listTables()).resolves.toEqual(
|
||||
expect.objectContaining({ tables: [] }),
|
||||
);
|
||||
});
|
||||
|
||||
it("should create tables in v2 mode", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const data = [...Array(10000).keys()].map((i) => ({ id: i }));
|
||||
|
||||
@@ -187,58 +187,6 @@ describe("embedding functions", () => {
|
||||
const vector0 = JSON.parse(JSON.stringify(arr[0].vector));
|
||||
expect(vector0).toEqual([1, 2, 3]);
|
||||
});
|
||||
it("should append multiple Python embeddings with the same alias", async () => {
|
||||
@register("python-mock")
|
||||
// biome-ignore lint/correctness/noUnusedVariables: the decorator registers this class
|
||||
class MockEmbeddingFunction extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 3;
|
||||
}
|
||||
embeddingDataType(): Float {
|
||||
return new Float32();
|
||||
}
|
||||
async computeQueryEmbeddings(_data: string) {
|
||||
return [1, 2, 3];
|
||||
}
|
||||
async computeSourceEmbeddings(data: string[]) {
|
||||
return data.map((value) =>
|
||||
value === "hello world" ? [1, 2, 3] : [4, 5, 6],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const metadata = new Map([
|
||||
[
|
||||
"embedding_functions",
|
||||
'[{"source_column":"text1","vector_column":"vector1","name":"python-mock","model":{}},{"source_column":"text2","vector_column":"vector2","name":"python-mock","model":{}}]',
|
||||
],
|
||||
]);
|
||||
const schema = new Schema(
|
||||
[
|
||||
new Field("text1", new Utf8(), true),
|
||||
new Field("text2", new Utf8(), true),
|
||||
new Field(
|
||||
"vector1",
|
||||
new FixedSizeList(3, new Field("item", new Float32(), true)),
|
||||
true,
|
||||
),
|
||||
new Field(
|
||||
"vector2",
|
||||
new FixedSizeList(3, new Field("item", new Float32(), true)),
|
||||
true,
|
||||
),
|
||||
],
|
||||
metadata,
|
||||
);
|
||||
|
||||
const db = await connect(tmpDir.name);
|
||||
const table = await db.createEmptyTable("test", schema);
|
||||
await table.add([{ text1: "hello world", text2: "goodbye world" }]);
|
||||
|
||||
const rows = await table.query().toArray();
|
||||
expect(JSON.parse(JSON.stringify(rows[0].vector1))).toEqual([1, 2, 3]);
|
||||
expect(JSON.parse(JSON.stringify(rows[0].vector2))).toEqual([4, 5, 6]);
|
||||
});
|
||||
|
||||
it("should append generated vectors to a non-nullable schema", async () => {
|
||||
@register("non_nullable_schema_test")
|
||||
@@ -539,52 +487,4 @@ describe("embedding functions", () => {
|
||||
expect(stringSchema3).toEqual(stringExpectedSchema);
|
||||
},
|
||||
);
|
||||
test("parses one function writing several vector columns", async () => {
|
||||
class MockEmbeddingFunction extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 3;
|
||||
}
|
||||
embeddingDataType(): Float {
|
||||
return new Float32();
|
||||
}
|
||||
async computeQueryEmbeddings(_data: string) {
|
||||
return [1, 2, 3];
|
||||
}
|
||||
async computeSourceEmbeddings(data: string[]) {
|
||||
return Array.from({ length: data.length }).fill([
|
||||
1, 2, 3,
|
||||
]) as number[][];
|
||||
}
|
||||
}
|
||||
const registry = getRegistry();
|
||||
registry.register("multi_output_mock")(MockEmbeddingFunction);
|
||||
|
||||
// A materialized view can project one source vector column under two
|
||||
// names, so a table's configuration names the same function twice.
|
||||
const parsed = await registry.parseFunctions(
|
||||
new Map([
|
||||
[
|
||||
"embedding_functions",
|
||||
JSON.stringify([
|
||||
{
|
||||
name: "multi_output_mock",
|
||||
sourceColumn: "text",
|
||||
vectorColumn: "vector_a",
|
||||
model: {},
|
||||
},
|
||||
{
|
||||
name: "multi_output_mock",
|
||||
sourceColumn: "text",
|
||||
vectorColumn: "vector_b",
|
||||
model: {},
|
||||
},
|
||||
]),
|
||||
],
|
||||
]),
|
||||
);
|
||||
|
||||
expect(
|
||||
[...parsed.values()].map(({ vectorColumn }) => vectorColumn).sort(),
|
||||
).toEqual(["vector_a", "vector_b"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import type { OpenAIEmbeddingFunction } from "../lancedb/embedding/openai";
|
||||
import type { EmbeddingFunctionRegistry } from "../lancedb/embedding/registry";
|
||||
|
||||
type EmbeddingModule = typeof import("../lancedb/embedding");
|
||||
type OpenAIModule = typeof import("../lancedb/embedding/openai");
|
||||
type RegistryModule = typeof import("../lancedb/embedding/registry");
|
||||
|
||||
describe("embedding function registry", () => {
|
||||
const registries: EmbeddingFunctionRegistry[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const registry of registries) {
|
||||
registry.reset();
|
||||
}
|
||||
registries.length = 0;
|
||||
});
|
||||
|
||||
it("defers built-in providers until the public registry API is used", () => {
|
||||
jest.isolateModules(() => {
|
||||
const embedding = require("../lancedb/embedding") as EmbeddingModule;
|
||||
const { getRegistry: getInternalRegistry } =
|
||||
require("../lancedb/embedding/registry") as RegistryModule;
|
||||
const registry = getInternalRegistry();
|
||||
registries.push(registry);
|
||||
|
||||
expect(registry.length()).toBe(0);
|
||||
expect(embedding.getRegistry()).toBe(registry);
|
||||
expect(registry.get("openai")).toBeDefined();
|
||||
expect(registry.get("huggingface")).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves automatic FTS search in a fresh process", () => {
|
||||
execFileSync(
|
||||
process.execPath,
|
||||
[resolve(__dirname, "fixtures", "auto_fts_search.cjs")],
|
||||
{ stdio: "pipe" },
|
||||
);
|
||||
});
|
||||
|
||||
it("shares registrations across duplicated provider module graphs", () => {
|
||||
let registeringRegistry: EmbeddingFunctionRegistry | undefined;
|
||||
let latestOpenAIConstructor: typeof OpenAIEmbeddingFunction | undefined;
|
||||
|
||||
jest.isolateModules(() => {
|
||||
require("../lancedb/embedding/openai");
|
||||
const { getRegistry } =
|
||||
require("../lancedb/embedding/registry") as RegistryModule;
|
||||
registeringRegistry = getRegistry();
|
||||
registries.push(registeringRegistry);
|
||||
expect(registeringRegistry.get("openai")).toBeDefined();
|
||||
});
|
||||
|
||||
expect(() => {
|
||||
jest.isolateModules(() => {
|
||||
const { OpenAIEmbeddingFunction } =
|
||||
require("../lancedb/embedding/openai") as OpenAIModule;
|
||||
latestOpenAIConstructor = OpenAIEmbeddingFunction;
|
||||
const { getRegistry } =
|
||||
require("../lancedb/embedding/registry") as RegistryModule;
|
||||
registries.push(getRegistry());
|
||||
});
|
||||
}).not.toThrow();
|
||||
|
||||
const previousApiKey = process.env.OPENAI_API_KEY;
|
||||
process.env.OPENAI_API_KEY = "test";
|
||||
try {
|
||||
const latestOpenAI = registeringRegistry!
|
||||
.get<OpenAIEmbeddingFunction>("openai")!
|
||||
.create();
|
||||
expect(latestOpenAI).toBeInstanceOf(latestOpenAIConstructor!);
|
||||
} finally {
|
||||
if (previousApiKey === undefined) {
|
||||
delete process.env.OPENAI_API_KEY;
|
||||
} else {
|
||||
process.env.OPENAI_API_KEY = previousApiKey;
|
||||
}
|
||||
}
|
||||
|
||||
jest.isolateModules(() => {
|
||||
const { getRegistry } =
|
||||
require("../lancedb/embedding") as EmbeddingModule;
|
||||
const publicRegistry = getRegistry();
|
||||
registries.push(publicRegistry);
|
||||
expect(publicRegistry).toBe(registeringRegistry);
|
||||
expect(publicRegistry.get("openai")).toBeDefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,33 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
const assert = require("node:assert/strict");
|
||||
const tmp = require("tmp");
|
||||
const { connect, embedding, Index } = require("../../dist");
|
||||
const { getRegistry } = require("../../dist/embedding/registry");
|
||||
|
||||
async function main() {
|
||||
assert.equal(typeof embedding.getRegistry, "function");
|
||||
assert.equal(getRegistry().length(), 0);
|
||||
assert.equal(embedding.getRegistry(), getRegistry());
|
||||
assert.equal(getRegistry().length(), 2);
|
||||
|
||||
const dir = tmp.dirSync({ unsafeCleanup: true });
|
||||
let db;
|
||||
try {
|
||||
db = await connect(dir.name);
|
||||
const table = await db.createTable("docs", [{ text: "hello world" }]);
|
||||
await table.createIndex("text", { config: Index.fts() });
|
||||
|
||||
const rows = await table.search("hello").toArray();
|
||||
assert.equal(rows[0].text, "hello world");
|
||||
} finally {
|
||||
db?.close();
|
||||
dir.removeCallback();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -1,147 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import * as tmp from "tmp";
|
||||
|
||||
import { Connection, connect } from "../lancedb";
|
||||
import {
|
||||
DEFINITION_META_KEY,
|
||||
definitionFromMetadata,
|
||||
} from "../lancedb/materialized_view";
|
||||
|
||||
describe("materialized views", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
let db: Connection;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
db = await connect(tmpDir.name);
|
||||
await db.createTable(
|
||||
"people",
|
||||
[
|
||||
{ name: "ada", age: 36 },
|
||||
{ name: "kid", age: 7 },
|
||||
{ name: "grace", age: 85 },
|
||||
],
|
||||
{ storageOptions: { newTableEnableStableRowIds: "true" } },
|
||||
);
|
||||
});
|
||||
afterEach(() => tmpDir.removeCallback());
|
||||
|
||||
it("rejects a stored limit a number cannot carry", () => {
|
||||
const big = new Map([
|
||||
[
|
||||
DEFINITION_META_KEY,
|
||||
'{"kind":"select","source_table":"people","limit":9007199254740993}',
|
||||
],
|
||||
]);
|
||||
expect(() => definitionFromMetadata(big, "v")).toThrow(
|
||||
/too large to represent exactly/,
|
||||
);
|
||||
|
||||
const safe = new Map([
|
||||
[
|
||||
DEFINITION_META_KEY,
|
||||
'{"kind":"select","source_table":"people","limit":42}',
|
||||
],
|
||||
]);
|
||||
expect(definitionFromMetadata(safe, "v").limit).toBe(42);
|
||||
});
|
||||
|
||||
it("creates, refreshes and queries a view", async () => {
|
||||
const view = await db.createMaterializedView("adults", "people", {
|
||||
select: ["name", ["shout", "upper(name)"]],
|
||||
where: "age >= 18",
|
||||
});
|
||||
expect(view.name).toBe("adults");
|
||||
expect(await view.table().countRows()).toBe(0);
|
||||
|
||||
const result = await view.refresh();
|
||||
expect(result.mode).toBe("rebuild");
|
||||
expect(Number(result.rowsWritten)).toBe(2);
|
||||
|
||||
const rows = await view.table().query().toArray();
|
||||
expect(rows.map((r) => r.shout).sort()).toEqual(["ADA", "GRACE"]);
|
||||
});
|
||||
|
||||
it("round-trips the definition", async () => {
|
||||
await db.createMaterializedView("adults", "people", {
|
||||
where: "age >= 18",
|
||||
});
|
||||
const view = await db.openMaterializedView("adults");
|
||||
const definition = await view.definition();
|
||||
expect(definition.sourceTable).toBe("people");
|
||||
expect(definition.filter).toBe("age >= 18");
|
||||
expect(definition.projections).toEqual([
|
||||
["name", "`name`"],
|
||||
["age", "`age`"],
|
||||
]);
|
||||
expect(definition.inputs).toEqual(["age", "name"]);
|
||||
});
|
||||
|
||||
it("refreshes incrementally after an append", async () => {
|
||||
const view = await db.createMaterializedView("copy", "people");
|
||||
await view.refresh();
|
||||
|
||||
const people = await db.openTable("people");
|
||||
await people.add([{ name: "alan", age: 41 }]);
|
||||
const result = await view.refresh();
|
||||
expect(result.mode).toBe("incremental");
|
||||
expect(Number(result.rowsWritten)).toBe(1);
|
||||
expect(await view.table().countRows()).toBe(4);
|
||||
|
||||
expect((await view.refresh()).mode).toBe("no_op");
|
||||
});
|
||||
|
||||
it("lists views and rejects non-views", async () => {
|
||||
await db.createMaterializedView("adults", "people", {
|
||||
where: "age >= 18",
|
||||
});
|
||||
expect(await db.listMaterializedViews()).toEqual(["adults"]);
|
||||
await expect(db.openMaterializedView("people")).rejects.toThrow(
|
||||
"not a materialized view",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an invalid expression at create time", async () => {
|
||||
await expect(
|
||||
db.createMaterializedView("bad", "people", {
|
||||
select: [["x", "missing + 1"]],
|
||||
}),
|
||||
).rejects.toThrow("missing");
|
||||
});
|
||||
|
||||
it("rejects invalid numeric options before creating anything", async () => {
|
||||
for (const limit of [-5, 1.5, Infinity, NaN]) {
|
||||
await expect(
|
||||
db.createMaterializedView("bad", "people", { limit }),
|
||||
).rejects.toThrow("non-negative integer");
|
||||
}
|
||||
expect(await db.listMaterializedViews()).toEqual([]);
|
||||
|
||||
const view = await db.createMaterializedView("copy", "people");
|
||||
for (const sourceVersion of [-1, 1.5, Infinity, NaN]) {
|
||||
await expect(view.refresh({ sourceVersion })).rejects.toThrow(
|
||||
"non-negative integer",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("quotes bare select names", async () => {
|
||||
await db.createTable("odd_names", [{ "order item": "widget" }], {
|
||||
storageOptions: { newTableEnableStableRowIds: "true" },
|
||||
});
|
||||
const view = await db.createMaterializedView("quoted", "odd_names", {
|
||||
select: ["order item"],
|
||||
});
|
||||
const result = await view.refresh();
|
||||
expect(Number(result.rowsWritten)).toBe(1);
|
||||
});
|
||||
|
||||
it("requires stable row ids on the source", async () => {
|
||||
await db.createTable("plain", [{ x: 1 }]);
|
||||
await expect(db.createMaterializedView("v", "plain")).rejects.toThrow(
|
||||
"stable row ids",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -75,25 +75,6 @@ async function withMockDatabase(
|
||||
}
|
||||
|
||||
describe("remote connection", () => {
|
||||
it("refuses materialized views before issuing any request", async () => {
|
||||
const paths: string[] = [];
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
paths.push(req.url ?? "");
|
||||
res.writeHead(404).end();
|
||||
},
|
||||
async (db) => {
|
||||
await expect(db.openMaterializedView("secret_table")).rejects.toThrow(
|
||||
/only on local databases/,
|
||||
);
|
||||
await expect(db.listMaterializedViews()).rejects.toThrow(
|
||||
/only on local databases/,
|
||||
);
|
||||
expect(paths).toEqual([]);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("should accept partial connection options", async () => {
|
||||
await connect("db://test", {
|
||||
apiKey: "fake",
|
||||
@@ -330,7 +311,7 @@ describe("remote connection", () => {
|
||||
expect(createIndexBody?.["custom_stop_words"]).toEqual(["the"]);
|
||||
});
|
||||
|
||||
it("diffs and cherry-picks remote branches", async () => {
|
||||
it("diffs and merges remote branches", async () => {
|
||||
const sampleDiff = {
|
||||
fromBranch: "exp",
|
||||
parentVersion: 1,
|
||||
@@ -352,9 +333,10 @@ describe("remote connection", () => {
|
||||
changedColumns: [],
|
||||
addedIndexes: [],
|
||||
removedIndexes: [],
|
||||
errors: [],
|
||||
mergeable: true,
|
||||
mergeBlockers: [],
|
||||
};
|
||||
const cherryPickBodies: Record<string, unknown>[] = [];
|
||||
const mergeBodies: Record<string, unknown>[] = [];
|
||||
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
@@ -384,16 +366,17 @@ describe("remote connection", () => {
|
||||
.end(JSON.stringify(sampleDiff));
|
||||
return;
|
||||
}
|
||||
if (path.endsWith("/branches/cherry_pick/")) {
|
||||
cherryPickBodies.push(body);
|
||||
if (path.endsWith("/branches/merge/")) {
|
||||
mergeBodies.push(body);
|
||||
const dryRun = body["dry_run"] === true;
|
||||
const response = {
|
||||
status: dryRun ? "ready" : "failed",
|
||||
status: dryRun ? "ready" : "rejected",
|
||||
diff: dryRun
|
||||
? sampleDiff
|
||||
: {
|
||||
...sampleDiff,
|
||||
errors: [
|
||||
mergeable: false,
|
||||
mergeBlockers: [
|
||||
{ code: "baseMoved", message: "main has advanced" },
|
||||
],
|
||||
},
|
||||
@@ -415,19 +398,19 @@ describe("remote connection", () => {
|
||||
|
||||
await expect(branches.diff("exp")).resolves.toEqual(sampleDiff);
|
||||
|
||||
const failed = await branches.cherryPick("exp");
|
||||
expect(failed.status).toBe("failed");
|
||||
expect(failed.diff.errors).toEqual([
|
||||
const rejected = await branches.merge("exp");
|
||||
expect(rejected.status).toBe("rejected");
|
||||
expect(rejected.diff.mergeBlockers).toEqual([
|
||||
{ code: "baseMoved", message: "main has advanced" },
|
||||
]);
|
||||
|
||||
const preview = await branches.cherryPick("exp", true);
|
||||
const preview = await branches.merge("exp", true);
|
||||
expect(preview.status).toBe("ready");
|
||||
expect(preview.preview.promotedColumns).toEqual(["tag"]);
|
||||
},
|
||||
);
|
||||
|
||||
expect(cherryPickBodies).toEqual([
|
||||
expect(mergeBodies).toEqual([
|
||||
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
|
||||
{ from_branch: "exp", dry_run: false },
|
||||
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
|
||||
|
||||
@@ -11,13 +11,10 @@ import * as arrow17 from "apache-arrow-17";
|
||||
import * as arrow18 from "apache-arrow-18";
|
||||
|
||||
import {
|
||||
AutoQuery,
|
||||
Connection,
|
||||
MatchQuery,
|
||||
PhraseQuery,
|
||||
Query,
|
||||
Table,
|
||||
VectorQuery,
|
||||
connect,
|
||||
tokenize,
|
||||
} from "../lancedb";
|
||||
@@ -685,56 +682,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
},
|
||||
);
|
||||
|
||||
// https://github.com/lancedb/lancedb/issues/1963
|
||||
it("should query documents with LangChain PDF metadata", async () => {
|
||||
const tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
try {
|
||||
const db = await connect(tmpDir.name);
|
||||
const documents = [
|
||||
{
|
||||
text: "first page",
|
||||
vector: [1, 0],
|
||||
source: "first.pdf",
|
||||
loc: { pageNumber: 1, lines: { from: 1, to: 12 } },
|
||||
pdf: {
|
||||
version: "1.10.100",
|
||||
info: {
|
||||
format: "PDF 1.7",
|
||||
producer: "pdf.js",
|
||||
creator: "Writer",
|
||||
},
|
||||
totalPages: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
text: "second page",
|
||||
vector: [0, 1],
|
||||
source: "second.pdf",
|
||||
loc: { pageNumber: 2, lines: { from: 13, to: 24 } },
|
||||
pdf: {
|
||||
version: "1.10.100",
|
||||
info: {
|
||||
format: "PDF 1.7",
|
||||
producer: "pdf.js",
|
||||
creator: "Writer",
|
||||
},
|
||||
totalPages: 2,
|
||||
},
|
||||
},
|
||||
];
|
||||
const documentsTable = await db.createTable("documents", documents);
|
||||
|
||||
const results = await documentsTable.query().toArray();
|
||||
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results[0].source).toBe("first.pdf");
|
||||
expect(results[0].pdf.info.producer).toBe("pdf.js");
|
||||
expect(results[1].loc.pageNumber).toBe(2);
|
||||
} finally {
|
||||
tmpDir.removeCallback();
|
||||
}
|
||||
});
|
||||
|
||||
describe("merge insert", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
let table: Table;
|
||||
@@ -1830,194 +1777,6 @@ describe("Read consistency interval", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("automatic search schema consistency", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
|
||||
class SchemaRefreshEmbedding extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 2;
|
||||
}
|
||||
|
||||
embeddingDataType() {
|
||||
return new Float32();
|
||||
}
|
||||
|
||||
async computeSourceEmbeddings(data: string[]) {
|
||||
return data.map((value) => [value.length, 1]);
|
||||
}
|
||||
|
||||
async computeQueryEmbeddings(value: string) {
|
||||
return [value.length, 1];
|
||||
}
|
||||
}
|
||||
|
||||
function embeddingSchema() {
|
||||
const func = new SchemaRefreshEmbedding();
|
||||
return LanceSchema({
|
||||
text: func.sourceField(new Utf8()),
|
||||
vector: func.vectorField(),
|
||||
});
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
getRegistry().reset();
|
||||
register("schema-refresh")(SchemaRefreshEmbedding);
|
||||
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
getRegistry().reset();
|
||||
tmpDir.removeCallback();
|
||||
});
|
||||
|
||||
it("uses the schema refreshed from another connection", async () => {
|
||||
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
|
||||
try {
|
||||
const stale = await first.createTable("docs", [{ text: "before" }], {
|
||||
schema: embeddingSchema(),
|
||||
});
|
||||
const replacement = await second.createTable(
|
||||
"docs",
|
||||
[{ text: "after hello" }],
|
||||
{ mode: "overwrite" },
|
||||
);
|
||||
await replacement.createIndex("text", { config: Index.fts() });
|
||||
|
||||
const search = stale.search("hello");
|
||||
expect(search).toBeInstanceOf(AutoQuery);
|
||||
expect(search).not.toBeInstanceOf(Query);
|
||||
expect(search).not.toBeInstanceOf(VectorQuery);
|
||||
expect("nprobes" in search).toBe(false);
|
||||
|
||||
const rows = await search.toArray();
|
||||
expect(rows[0].text).toBe("after hello");
|
||||
expect((await stale.schema()).metadata.has("embedding_functions")).toBe(
|
||||
false,
|
||||
);
|
||||
} finally {
|
||||
first.close();
|
||||
second.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("tracks embedding metadata across checkout and restore", async () => {
|
||||
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
|
||||
try {
|
||||
await first.createTable("docs", [{ text: "before" }], {
|
||||
schema: embeddingSchema(),
|
||||
});
|
||||
const table = await second.createTable(
|
||||
"docs",
|
||||
[{ text: "after hello" }],
|
||||
{ mode: "overwrite" },
|
||||
);
|
||||
await table.createIndex("text", { config: Index.fts() });
|
||||
|
||||
await table.checkout(1);
|
||||
expect((await table.search("before").toArray())[0].text).toBe("before");
|
||||
|
||||
await table.checkoutLatest();
|
||||
expect((await table.search("hello").toArray())[0].text).toBe(
|
||||
"after hello",
|
||||
);
|
||||
|
||||
await table.checkout(1);
|
||||
await table.restore();
|
||||
expect((await table.search("before").toArray())[0].text).toBe("before");
|
||||
} finally {
|
||||
first.close();
|
||||
second.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("pins automatic search while computing an embedding", async () => {
|
||||
let markStarted!: () => void;
|
||||
let releaseEmbedding!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
markStarted = resolve;
|
||||
});
|
||||
const released = new Promise<void>((resolve) => {
|
||||
releaseEmbedding = resolve;
|
||||
});
|
||||
|
||||
class BlockingEmbedding extends SchemaRefreshEmbedding {
|
||||
async computeQueryEmbeddings(value: string) {
|
||||
markStarted();
|
||||
await released;
|
||||
return [value.length, 1];
|
||||
}
|
||||
}
|
||||
|
||||
register("schema-refresh-blocking")(BlockingEmbedding);
|
||||
const func = new BlockingEmbedding();
|
||||
const schema = LanceSchema({
|
||||
text: func.sourceField(new Utf8()),
|
||||
vector: func.vectorField(),
|
||||
});
|
||||
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
|
||||
try {
|
||||
const table = await first.createTable(
|
||||
"docs",
|
||||
[{ text: "hello before" }],
|
||||
{ schema },
|
||||
);
|
||||
const pending = table.search("hello").toArray();
|
||||
await started;
|
||||
|
||||
const replacement = await second.createTable(
|
||||
"docs",
|
||||
[{ text: "hello after" }],
|
||||
{ mode: "overwrite" },
|
||||
);
|
||||
await replacement.createIndex("text", { config: Index.fts() });
|
||||
releaseEmbedding();
|
||||
|
||||
expect((await pending)[0].text).toBe("hello before");
|
||||
} finally {
|
||||
releaseEmbedding();
|
||||
first.close();
|
||||
second.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("refreshes a reused automatic search for every execution", async () => {
|
||||
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
|
||||
try {
|
||||
const table = await first.createTable("docs", [
|
||||
{ text: "hello before", marker: "before" },
|
||||
]);
|
||||
await table.createIndex("text", { config: Index.fts() });
|
||||
const search = table.search("hello").select(["text"]);
|
||||
|
||||
const before = (await search.toArray())[0];
|
||||
expect(before.text).toBe("hello before");
|
||||
expect(before.marker).toBeUndefined();
|
||||
|
||||
const replacement = await second.createTable(
|
||||
"docs",
|
||||
[{ text: "hello after", marker: "after" }],
|
||||
{ mode: "overwrite" },
|
||||
);
|
||||
await replacement.createIndex("text", { config: Index.fts() });
|
||||
|
||||
const after = (await search.toArray())[0];
|
||||
expect(after.text).toBe("hello after");
|
||||
expect(after.marker).toBeUndefined();
|
||||
} finally {
|
||||
first.close();
|
||||
second.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("schema evolution", function () {
|
||||
let tmpDir: tmp.DirResult;
|
||||
beforeEach(() => {
|
||||
@@ -2585,24 +2344,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
);
|
||||
});
|
||||
|
||||
test("full text search if only an unrelated embedding function is registered", async () => {
|
||||
register("unused")(
|
||||
class extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 3;
|
||||
}
|
||||
embeddingDataType() {
|
||||
return new Float32();
|
||||
}
|
||||
async computeQueryEmbeddings(_data: string) {
|
||||
return [1, 2, 3];
|
||||
}
|
||||
async computeSourceEmbeddings(data: string[]) {
|
||||
return data.map(() => [1, 2, 3]);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
test("full text search if no embedding function provided", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const data = [
|
||||
{ text: "hello world", vector: [0.1, 0.2, 0.3] },
|
||||
@@ -2624,306 +2366,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
expect(results2[0].text).toBe(data[1].text);
|
||||
});
|
||||
|
||||
test("auto search stays consistent with the active revision", async () => {
|
||||
let initCalls = 0;
|
||||
let queryCalls = 0;
|
||||
let markStarted!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
markStarted = resolve;
|
||||
});
|
||||
let releaseEmbedding!: () => void;
|
||||
const embeddingReleased = new Promise<void>((resolve) => {
|
||||
releaseEmbedding = resolve;
|
||||
});
|
||||
|
||||
@register("refresh-test")
|
||||
class TestEmbedding extends EmbeddingFunction<string> {
|
||||
async init() {
|
||||
initCalls += 1;
|
||||
}
|
||||
ndims() {
|
||||
return 1;
|
||||
}
|
||||
embeddingDataType() {
|
||||
return new arrow.Float32();
|
||||
}
|
||||
async computeQueryEmbeddings(value: string) {
|
||||
queryCalls += 1;
|
||||
if (value === "blocked") {
|
||||
markStarted();
|
||||
await embeddingReleased;
|
||||
}
|
||||
return value === "greetings" ? [0.1] : [0.2];
|
||||
}
|
||||
async computeSourceEmbeddings(values: string[]) {
|
||||
return values.map((value) =>
|
||||
value === "hello world" ? [0.1] : [0.2],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const writer = await connect(tmpDir.name);
|
||||
await writer.createTable("test", [{ text: "plain", vector: [0.0] }]);
|
||||
const reader = await connect(tmpDir.name, {
|
||||
readConsistencyInterval: 0,
|
||||
});
|
||||
const tracked = await reader.openTable("test");
|
||||
type SnapshotCountingNative = {
|
||||
querySnapshot: () => Promise<unknown>;
|
||||
};
|
||||
const native = (tracked as unknown as { inner: SnapshotCountingNative })
|
||||
.inner;
|
||||
const querySnapshot = native.querySnapshot.bind(native);
|
||||
let snapshotCalls = 0;
|
||||
native.querySnapshot = async () => {
|
||||
snapshotCalls += 1;
|
||||
return await querySnapshot();
|
||||
};
|
||||
const autoQuery = tracked.search("greetings").select(["text"]).limit(1);
|
||||
|
||||
const func = new TestEmbedding();
|
||||
const schema = LanceSchema({
|
||||
text: func.sourceField(new arrow.Utf8()),
|
||||
vector: func.vectorField(),
|
||||
});
|
||||
const data = [{ text: "hello world" }, { text: "goodbye world" }];
|
||||
await writer.createTable("test", data, { mode: "overwrite", schema });
|
||||
const baselineInitCalls = initCalls;
|
||||
|
||||
expect(
|
||||
(await tracked.schema()).metadata.get("embedding_functions"),
|
||||
).toBeDefined();
|
||||
const results = await autoQuery.toArray();
|
||||
expect(results[0].text).toBe(data[0].text);
|
||||
expect(initCalls).toBe(baselineInitCalls + 1);
|
||||
expect(queryCalls).toBe(1);
|
||||
expect(snapshotCalls).toBe(1);
|
||||
|
||||
const repeatedResults = await autoQuery.toArray();
|
||||
expect(repeatedResults[0].text).toBe(data[0].text);
|
||||
expect(initCalls).toBe(baselineInitCalls + 1);
|
||||
expect(queryCalls).toBe(1);
|
||||
expect(snapshotCalls).toBe(2);
|
||||
|
||||
const pending = tracked
|
||||
.search("blocked")
|
||||
.select(["text"])
|
||||
.limit(1)
|
||||
.toArray();
|
||||
await started;
|
||||
|
||||
const ftsData = [
|
||||
{ text: "greetings from full text", vector: [0.0] },
|
||||
{ text: "blocked from full text", vector: [0.0] },
|
||||
];
|
||||
const ftsTable = await writer.createTable("test", ftsData, {
|
||||
mode: "overwrite",
|
||||
});
|
||||
await ftsTable.createIndex("text", { config: Index.fts() });
|
||||
releaseEmbedding();
|
||||
|
||||
const pendingResults = await pending;
|
||||
expect(pendingResults[0].text).toBe(data[1].text);
|
||||
|
||||
expect(
|
||||
(await tracked.schema()).metadata.get("embedding_functions"),
|
||||
).toBeUndefined();
|
||||
const ftsResults = await autoQuery.toArray();
|
||||
expect(ftsResults[0].text).toBe(ftsData[0].text);
|
||||
});
|
||||
|
||||
test("auto search keeps newer preparation during a revision race", async () => {
|
||||
let aCalls = 0;
|
||||
let bCalls = 0;
|
||||
let markAStarted!: () => void;
|
||||
const aStarted = new Promise<void>((resolve) => {
|
||||
markAStarted = resolve;
|
||||
});
|
||||
let releaseA!: () => void;
|
||||
const aReleased = new Promise<void>((resolve) => {
|
||||
releaseA = resolve;
|
||||
});
|
||||
let markBStarted!: () => void;
|
||||
const bStarted = new Promise<void>((resolve) => {
|
||||
markBStarted = resolve;
|
||||
});
|
||||
let releaseB!: () => void;
|
||||
const bReleased = new Promise<void>((resolve) => {
|
||||
releaseB = resolve;
|
||||
});
|
||||
|
||||
@register("race-a")
|
||||
class EmbeddingA extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 1;
|
||||
}
|
||||
embeddingDataType() {
|
||||
return new arrow.Float32();
|
||||
}
|
||||
async computeQueryEmbeddings() {
|
||||
aCalls += 1;
|
||||
markAStarted();
|
||||
await aReleased;
|
||||
return [0.1];
|
||||
}
|
||||
async computeSourceEmbeddings(values: string[]) {
|
||||
return values.map(() => [0.1]);
|
||||
}
|
||||
}
|
||||
|
||||
@register("race-b")
|
||||
class EmbeddingB extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 1;
|
||||
}
|
||||
embeddingDataType() {
|
||||
return new arrow.Float32();
|
||||
}
|
||||
async computeQueryEmbeddings() {
|
||||
bCalls += 1;
|
||||
markBStarted();
|
||||
await bReleased;
|
||||
return [0.2];
|
||||
}
|
||||
async computeSourceEmbeddings(values: string[]) {
|
||||
return values.map(() => [0.2]);
|
||||
}
|
||||
}
|
||||
|
||||
const writer = await connect(tmpDir.name);
|
||||
const embeddingA = new EmbeddingA();
|
||||
const schemaA = LanceSchema({
|
||||
text: embeddingA.sourceField(new arrow.Utf8()),
|
||||
vector: embeddingA.vectorField(),
|
||||
});
|
||||
await writer.createTable("race", [{ text: "revision a" }], {
|
||||
schema: schemaA,
|
||||
});
|
||||
const reader = await connect(tmpDir.name, {
|
||||
readConsistencyInterval: 0,
|
||||
});
|
||||
const tracked = await reader.openTable("race");
|
||||
const query = tracked.search("query");
|
||||
|
||||
const first = query.toArray();
|
||||
await aStarted;
|
||||
|
||||
const embeddingB = new EmbeddingB();
|
||||
const schemaB = LanceSchema({
|
||||
text: embeddingB.sourceField(new arrow.Utf8()),
|
||||
vector: embeddingB.vectorField(),
|
||||
});
|
||||
await writer.createTable("race", [{ text: "revision b" }], {
|
||||
mode: "overwrite",
|
||||
schema: schemaB,
|
||||
});
|
||||
const second = query.toArray();
|
||||
await bStarted;
|
||||
|
||||
releaseA();
|
||||
releaseB();
|
||||
await Promise.all([first, second]);
|
||||
expect(aCalls).toBe(1);
|
||||
expect(bCalls).toBe(1);
|
||||
});
|
||||
|
||||
test("stale FTS routing keeps newer vector preparation", async () => {
|
||||
let vectorCalls = 0;
|
||||
let markVectorStarted!: () => void;
|
||||
const vectorStarted = new Promise<void>((resolve) => {
|
||||
markVectorStarted = resolve;
|
||||
});
|
||||
let releaseVector!: () => void;
|
||||
const vectorReleased = new Promise<void>((resolve) => {
|
||||
releaseVector = resolve;
|
||||
});
|
||||
|
||||
@register("stale-fts-race")
|
||||
class RaceEmbedding extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 1;
|
||||
}
|
||||
embeddingDataType() {
|
||||
return new arrow.Float32();
|
||||
}
|
||||
async computeQueryEmbeddings() {
|
||||
vectorCalls += 1;
|
||||
markVectorStarted();
|
||||
await vectorReleased;
|
||||
return [0.1];
|
||||
}
|
||||
async computeSourceEmbeddings(values: string[]) {
|
||||
return values.map(() => [0.1]);
|
||||
}
|
||||
}
|
||||
|
||||
const writer = await connect(tmpDir.name);
|
||||
const ftsTable = await writer.createTable("stale_fts", [
|
||||
{ text: "hello", vector: [0.0] },
|
||||
]);
|
||||
await ftsTable.createIndex("text", { config: Index.fts() });
|
||||
|
||||
const reader = await connect(tmpDir.name, {
|
||||
readConsistencyInterval: 0,
|
||||
});
|
||||
const tracked = await reader.openTable("stale_fts");
|
||||
type Snapshot = {
|
||||
schema: () => Promise<Buffer>;
|
||||
};
|
||||
type NativeWithSnapshot = {
|
||||
querySnapshot: () => Promise<Snapshot>;
|
||||
};
|
||||
const native = (tracked as unknown as { inner: NativeWithSnapshot })
|
||||
.inner;
|
||||
const querySnapshot = native.querySnapshot.bind(native);
|
||||
let snapshotCalls = 0;
|
||||
let markStaleSchemaStarted!: () => void;
|
||||
const staleSchemaStarted = new Promise<void>((resolve) => {
|
||||
markStaleSchemaStarted = resolve;
|
||||
});
|
||||
let releaseStaleSchema!: () => void;
|
||||
const staleSchemaReleased = new Promise<void>((resolve) => {
|
||||
releaseStaleSchema = resolve;
|
||||
});
|
||||
native.querySnapshot = async () => {
|
||||
const snapshot = await querySnapshot();
|
||||
snapshotCalls += 1;
|
||||
if (snapshotCalls === 1) {
|
||||
const schema = snapshot.schema.bind(snapshot);
|
||||
snapshot.schema = async () => {
|
||||
markStaleSchemaStarted();
|
||||
await staleSchemaReleased;
|
||||
return await schema();
|
||||
};
|
||||
}
|
||||
return snapshot;
|
||||
};
|
||||
|
||||
const query = tracked.search("hello");
|
||||
const staleFtsExecution = query.toArray();
|
||||
await staleSchemaStarted;
|
||||
|
||||
const embedding = new RaceEmbedding();
|
||||
const vectorSchema = LanceSchema({
|
||||
text: embedding.sourceField(new arrow.Utf8()),
|
||||
vector: embedding.vectorField(),
|
||||
});
|
||||
await writer.createTable("stale_fts", [{ text: "hello" }], {
|
||||
mode: "overwrite",
|
||||
schema: vectorSchema,
|
||||
});
|
||||
|
||||
const vectorExecution = query.toArray();
|
||||
await vectorStarted;
|
||||
releaseStaleSchema();
|
||||
await staleFtsExecution;
|
||||
releaseVector();
|
||||
await vectorExecution;
|
||||
|
||||
await query.toArray();
|
||||
expect(vectorCalls).toBe(1);
|
||||
});
|
||||
|
||||
test("tokenizes FTS queries by column or index name", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const data = [
|
||||
@@ -3474,30 +2916,6 @@ describe("column name options", () => {
|
||||
expect(results[1].query_index).toBe(1);
|
||||
});
|
||||
|
||||
test("observes promised additional vectors while the query is pending", async () => {
|
||||
const initialVector = new Promise<number[]>(() => undefined);
|
||||
const query = table.query().nearestTo(initialVector);
|
||||
const unhandled: unknown[] = [];
|
||||
const onUnhandled = (reason: unknown) => unhandled.push(reason);
|
||||
process.on("unhandledRejection", onUnhandled);
|
||||
|
||||
try {
|
||||
query.addQueryVector(Promise.reject(new Error("extra vector failed")));
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(unhandled).toEqual([]);
|
||||
|
||||
const rejectedQuery = table
|
||||
.query()
|
||||
.nearestTo([0.1, 0.2])
|
||||
.addQueryVector(Promise.reject(new Error("consumed vector failed")));
|
||||
await expect(rejectedQuery.toArray()).rejects.toThrow(
|
||||
"consumed vector failed",
|
||||
);
|
||||
} finally {
|
||||
process.off("unhandledRejection", onUnhandled);
|
||||
}
|
||||
});
|
||||
|
||||
test("index and search multivectors", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const data = [];
|
||||
@@ -3535,7 +2953,7 @@ describe("column name options", () => {
|
||||
.limit(10)
|
||||
.toArray();
|
||||
expect(results2.length).toBe(10);
|
||||
}, 30_000);
|
||||
});
|
||||
});
|
||||
|
||||
describe("when creating an empty table", () => {
|
||||
@@ -3561,27 +2979,6 @@ describe("when creating an empty table", () => {
|
||||
expect((actualSchema.fields[1].type as Float64).precision).toBe(2);
|
||||
});
|
||||
|
||||
it("can add and query JSON data", async () => {
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int32(), true),
|
||||
new Field(
|
||||
"meta",
|
||||
new Utf8(),
|
||||
true,
|
||||
new Map([["ARROW:extension:name", "arrow.json"]]),
|
||||
),
|
||||
]);
|
||||
const table = await con.createEmptyTable("json", schema);
|
||||
const meta = JSON.stringify({ x: 1 });
|
||||
|
||||
await table.add([{ id: 1, meta }]);
|
||||
|
||||
const rows = await table.query().toArray();
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].id).toBe(1);
|
||||
expect(rows[0].meta).toBe(meta);
|
||||
});
|
||||
|
||||
it("can create an empty table from schema that specifies field types by name", async () => {
|
||||
const schemaLike = {
|
||||
fields: [
|
||||
|
||||
@@ -170,7 +170,7 @@ test("basic table examples", async () => {
|
||||
// --8<-- [end:create_index]
|
||||
|
||||
// --8<-- [start:delete_rows]
|
||||
await tbl.delete("item = 'fizz'");
|
||||
await tbl.delete('item = "fizz"');
|
||||
// --8<-- [end:delete_rows]
|
||||
|
||||
// --8<-- [start:drop_table]
|
||||
|
||||
+309
-35
@@ -5,6 +5,7 @@ import {
|
||||
Data as ArrowData,
|
||||
Table as ArrowTable,
|
||||
Binary,
|
||||
Bool,
|
||||
BufferType,
|
||||
DataType,
|
||||
DateUnit,
|
||||
@@ -17,7 +18,12 @@ import {
|
||||
FixedSizeList,
|
||||
Float,
|
||||
Float32,
|
||||
Float64,
|
||||
Int,
|
||||
Int8,
|
||||
Int16,
|
||||
Int32,
|
||||
Int64,
|
||||
LargeBinary,
|
||||
List,
|
||||
Null,
|
||||
@@ -30,16 +36,17 @@ import {
|
||||
Struct,
|
||||
Timestamp,
|
||||
Type,
|
||||
Uint8,
|
||||
Uint16,
|
||||
Uint32,
|
||||
Utf8,
|
||||
Vector,
|
||||
makeVector as arrowMakeVector,
|
||||
util as arrowUtil,
|
||||
vectorFromArray as badVectorFromArray,
|
||||
makeBuilder,
|
||||
makeData,
|
||||
} from "apache-arrow";
|
||||
import { Buffers } from "apache-arrow/data";
|
||||
import { typedArrayToArrowType } from "./arrow_type";
|
||||
import { type EmbeddingFunction } from "./embedding/embedding_function";
|
||||
import {
|
||||
EmbeddingFunctionConfig,
|
||||
@@ -52,7 +59,14 @@ import {
|
||||
sanitizeTable,
|
||||
sanitizeType,
|
||||
} from "./sanitize";
|
||||
import { inferSchema } from "./schema";
|
||||
|
||||
/**
|
||||
* Check if a field name indicates a vector column.
|
||||
*/
|
||||
function nameSuggestsVectorColumn(fieldName: string): boolean {
|
||||
const nameLower = fieldName.toLowerCase();
|
||||
return nameLower.includes("vector") || nameLower.includes("embedding");
|
||||
}
|
||||
|
||||
export * from "apache-arrow";
|
||||
export type SchemaLike =
|
||||
@@ -72,7 +86,8 @@ export type FieldLike =
|
||||
};
|
||||
|
||||
export type DataLike =
|
||||
| import("apache-arrow").Data
|
||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||
| import("apache-arrow").Data<Struct<any>>
|
||||
| {
|
||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||
type: any;
|
||||
@@ -81,7 +96,6 @@ export type DataLike =
|
||||
stride: number;
|
||||
nullable: boolean;
|
||||
children: DataLike[];
|
||||
dictionary?: { data: readonly DataLike[] };
|
||||
get nullCount(): number;
|
||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||
values: Buffers<any>[BufferType.DATA];
|
||||
@@ -445,6 +459,110 @@ export function makeArrowTable(
|
||||
return new ArrowTable(inferredSchema, finalColumns);
|
||||
}
|
||||
|
||||
function inferSchema(
|
||||
data: Array<Record<string, unknown>>,
|
||||
schema: Schema | undefined,
|
||||
opts: MakeArrowTableOptions,
|
||||
): Schema {
|
||||
// We will collect all fields we see in the data.
|
||||
const pathTree = new PathTree<DataType>();
|
||||
|
||||
for (const [rowI, row] of data.entries()) {
|
||||
for (const [path, value] of rowPathsAndValues(row)) {
|
||||
if (!pathTree.has(path)) {
|
||||
// First time seeing this field.
|
||||
if (schema !== undefined) {
|
||||
const field = getFieldForPath(schema, path);
|
||||
if (field === undefined) {
|
||||
throw new Error(
|
||||
`Found field not in schema: ${path.join(".")} at row ${rowI}`,
|
||||
);
|
||||
} else {
|
||||
pathTree.set(path, field.type);
|
||||
}
|
||||
} else {
|
||||
const inferredType = inferType(value, path, opts);
|
||||
if (inferredType === undefined) {
|
||||
throw new Error(`Failed to infer data type for field ${path.join(
|
||||
".",
|
||||
)} at row ${rowI}. \
|
||||
Consider providing an explicit schema.`);
|
||||
}
|
||||
pathTree.set(path, inferredType);
|
||||
}
|
||||
} else if (schema === undefined) {
|
||||
const currentType = pathTree.get(path);
|
||||
const newType = inferType(value, path, opts);
|
||||
if (currentType !== newType) {
|
||||
new Error(`Failed to infer schema for data. Previously inferred type \
|
||||
${currentType} but found ${newType} at row ${rowI}. Consider \
|
||||
providing an explicit schema.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (schema === undefined) {
|
||||
function fieldsFromPathTree(pathTree: PathTree<DataType>): Field[] {
|
||||
const fields = [];
|
||||
for (const [name, value] of pathTree.map.entries()) {
|
||||
if (value instanceof PathTree) {
|
||||
const children = fieldsFromPathTree(value);
|
||||
fields.push(new Field(name, new Struct(children), true));
|
||||
} else {
|
||||
fields.push(new Field(name, value, true));
|
||||
}
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
const fields = fieldsFromPathTree(pathTree);
|
||||
return new Schema(fields);
|
||||
} else {
|
||||
function takeMatchingFields(
|
||||
fields: Field[],
|
||||
pathTree: PathTree<DataType>,
|
||||
): Field[] {
|
||||
const outFields = [];
|
||||
for (const field of fields) {
|
||||
if (pathTree.map.has(field.name)) {
|
||||
const value = pathTree.get([field.name]);
|
||||
if (value instanceof PathTree) {
|
||||
const struct = field.type as Struct;
|
||||
const children = takeMatchingFields(struct.children, value);
|
||||
outFields.push(
|
||||
new Field(field.name, new Struct(children), field.nullable),
|
||||
);
|
||||
} else {
|
||||
outFields.push(
|
||||
new Field(field.name, value as DataType, field.nullable),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return outFields;
|
||||
}
|
||||
const fields = takeMatchingFields(schema.fields, pathTree);
|
||||
return new Schema(fields);
|
||||
}
|
||||
}
|
||||
|
||||
function* rowPathsAndValues(
|
||||
row: Record<string, unknown>,
|
||||
basePath: string[] = [],
|
||||
): Generator<[string[], unknown]> {
|
||||
for (const [key, value] of Object.entries(row)) {
|
||||
if (isObject(value)) {
|
||||
yield* rowPathsAndValues(value, [...basePath, key]);
|
||||
} else {
|
||||
// Skip undefined values - they should be treated the same as missing fields
|
||||
// for embedding function purposes
|
||||
if (value !== undefined) {
|
||||
yield [[...basePath, key], value];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function isObject(value: unknown): value is Record<string, unknown> {
|
||||
return (
|
||||
typeof value === "object" &&
|
||||
@@ -459,19 +577,146 @@ function isObject(value: unknown): value is Record<string, unknown> {
|
||||
);
|
||||
}
|
||||
|
||||
function valueAtPath(datum: Record<string, unknown>, path: string[]): unknown {
|
||||
let current: unknown = datum;
|
||||
function getFieldForPath(schema: Schema, path: string[]): Field | undefined {
|
||||
let current: Field | Schema = schema;
|
||||
for (const key of path) {
|
||||
if (current == null) {
|
||||
return null;
|
||||
}
|
||||
if (isObject(current) && (Object.hasOwn(current, key) || key in current)) {
|
||||
current = current[key];
|
||||
if (current instanceof Schema) {
|
||||
const field: Field | undefined = current.fields.find(
|
||||
(f) => f.name === key,
|
||||
);
|
||||
if (field === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
current = field;
|
||||
} else if (current instanceof Field && DataType.isStruct(current.type)) {
|
||||
const struct: Struct = current.type;
|
||||
const field = struct.children.find((f) => f.name === key);
|
||||
if (field === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
current = field;
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
return current;
|
||||
if (current instanceof Field) {
|
||||
return current;
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to infer which Arrow type to use for a given value.
|
||||
*
|
||||
* May return undefined if the type cannot be inferred.
|
||||
*/
|
||||
function inferType(
|
||||
value: unknown,
|
||||
path: string[],
|
||||
opts: MakeArrowTableOptions,
|
||||
): DataType | undefined {
|
||||
if (typeof value === "bigint") {
|
||||
return new Int64();
|
||||
} else if (typeof value === "number") {
|
||||
// Even if it's an integer, it's safer to assume Float64. Users can
|
||||
// always provide an explicit schema or use BigInt if they mean integer.
|
||||
return new Float64();
|
||||
} else if (typeof value === "string") {
|
||||
if (opts.dictionaryEncodeStrings) {
|
||||
return new Dictionary(new Utf8(), new Int32());
|
||||
} else {
|
||||
return new Utf8();
|
||||
}
|
||||
} else if (typeof value === "boolean") {
|
||||
return new Bool();
|
||||
} else if (value instanceof Buffer) {
|
||||
return new Binary();
|
||||
} else if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
|
||||
const info = typedArrayToArrowType(value);
|
||||
if (info !== undefined) {
|
||||
const child = new Field("item", info.elementType, true);
|
||||
return new FixedSizeList(info.length, child);
|
||||
}
|
||||
return undefined;
|
||||
} else if (Array.isArray(value)) {
|
||||
if (value.length === 0) {
|
||||
return undefined; // Without any values we can't infer the type
|
||||
}
|
||||
if (path.length === 1 && Object.hasOwn(opts.vectorColumns, path[0])) {
|
||||
const floatType = sanitizeType(opts.vectorColumns[path[0]].type);
|
||||
return new FixedSizeList(
|
||||
value.length,
|
||||
new Field("item", floatType, true),
|
||||
);
|
||||
}
|
||||
const valueType = inferType(value[0], path, opts);
|
||||
if (valueType === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
// Try to automatically detect embedding columns.
|
||||
if (nameSuggestsVectorColumn(path[path.length - 1])) {
|
||||
// Check if value is a Uint8Array for integer vector type determination
|
||||
if (value instanceof Uint8Array) {
|
||||
// For integer vectors, we default to Uint8 (matching Python implementation)
|
||||
const child = new Field("item", new Uint8(), true);
|
||||
return new FixedSizeList(value.length, child);
|
||||
} else {
|
||||
// For float vectors, we default to Float32
|
||||
const child = new Field("item", new Float32(), true);
|
||||
return new FixedSizeList(value.length, child);
|
||||
}
|
||||
} else {
|
||||
const child = new Field("item", valueType, true);
|
||||
return new List(child);
|
||||
}
|
||||
} else {
|
||||
// TODO: timestamp
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
class PathTree<V> {
|
||||
map: Map<string, V | PathTree<V>>;
|
||||
|
||||
constructor(entries?: [string[], V][]) {
|
||||
this.map = new Map();
|
||||
if (entries !== undefined) {
|
||||
for (const [path, value] of entries) {
|
||||
this.set(path, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
has(path: string[]): boolean {
|
||||
let ref: PathTree<V> = this;
|
||||
for (const part of path) {
|
||||
if (!(ref instanceof PathTree) || !ref.map.has(part)) {
|
||||
return false;
|
||||
}
|
||||
ref = ref.map.get(part) as PathTree<V>;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
get(path: string[]): V | undefined {
|
||||
let ref: PathTree<V> = this;
|
||||
for (const part of path) {
|
||||
if (!(ref instanceof PathTree) || !ref.map.has(part)) {
|
||||
return undefined;
|
||||
}
|
||||
ref = ref.map.get(part) as PathTree<V>;
|
||||
}
|
||||
return ref as V;
|
||||
}
|
||||
set(path: string[], value: V): void {
|
||||
let ref: PathTree<V> = this;
|
||||
for (const part of path.slice(0, path.length - 1)) {
|
||||
if (!ref.map.has(part)) {
|
||||
ref.map.set(part, new PathTree<V>());
|
||||
}
|
||||
ref = ref.map.get(part) as PathTree<V>;
|
||||
}
|
||||
ref.map.set(path[path.length - 1], value);
|
||||
}
|
||||
}
|
||||
|
||||
function transposeData(
|
||||
@@ -479,26 +724,37 @@ function transposeData(
|
||||
field: Field,
|
||||
path: string[] = [],
|
||||
): Vector {
|
||||
const valuesPath = [...path, field.name];
|
||||
const values = data.map((datum) => valueAtPath(datum, valuesPath));
|
||||
if (field.type instanceof Struct) {
|
||||
const childFields = field.type.children;
|
||||
const fullPath = [...path, field.name];
|
||||
const childVectors = childFields.map((child) => {
|
||||
return transposeData(data, child, valuesPath);
|
||||
return transposeData(data, child, fullPath);
|
||||
});
|
||||
const nullCount = values.filter((value) => value === null).length;
|
||||
const structData = makeData({
|
||||
type: field.type,
|
||||
length: values.length,
|
||||
nullCount,
|
||||
nullBitmap:
|
||||
nullCount > 0
|
||||
? arrowUtil.packBools(values.map((value) => value !== null))
|
||||
: undefined,
|
||||
children: childVectors as unknown as ArrowData<DataType>[],
|
||||
});
|
||||
return arrowMakeVector(structData);
|
||||
} else {
|
||||
const valuesPath = [...path, field.name];
|
||||
const values = data.map((datum) => {
|
||||
let current: unknown = datum;
|
||||
for (const key of valuesPath) {
|
||||
if (current == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (
|
||||
isObject(current) &&
|
||||
(Object.hasOwn(current, key) || key in current)
|
||||
) {
|
||||
current = current[key];
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return current;
|
||||
});
|
||||
return makeVector(values, field.type, undefined, field.nullable);
|
||||
}
|
||||
}
|
||||
@@ -541,6 +797,32 @@ function makeListVector(lists: unknown[][]): Vector<unknown> {
|
||||
return listBuilder.finish().toVector();
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a JS TypedArray instance to the corresponding Arrow element DataType
|
||||
* and its length. Returns undefined if the value is not a recognized TypedArray.
|
||||
*/
|
||||
function typedArrayToArrowType(
|
||||
value: ArrayBufferView,
|
||||
): { elementType: DataType; length: number } | undefined {
|
||||
if (value instanceof Float32Array)
|
||||
return { elementType: new Float32(), length: value.length };
|
||||
if (value instanceof Float64Array)
|
||||
return { elementType: new Float64(), length: value.length };
|
||||
if (value instanceof Uint8Array)
|
||||
return { elementType: new Uint8(), length: value.length };
|
||||
if (value instanceof Uint16Array)
|
||||
return { elementType: new Uint16(), length: value.length };
|
||||
if (value instanceof Uint32Array)
|
||||
return { elementType: new Uint32(), length: value.length };
|
||||
if (value instanceof Int8Array)
|
||||
return { elementType: new Int8(), length: value.length };
|
||||
if (value instanceof Int16Array)
|
||||
return { elementType: new Int16(), length: value.length };
|
||||
if (value instanceof Int32Array)
|
||||
return { elementType: new Int32(), length: value.length };
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Helper function to convert an Array of JS values to an Arrow Vector */
|
||||
function makeVector(
|
||||
values: unknown[],
|
||||
@@ -1180,12 +1462,8 @@ export function ensureNestedFieldsExist(
|
||||
completeRow[field.name] = row[field.name];
|
||||
}
|
||||
} else {
|
||||
// Keep a missing struct valid while filling each of its children with
|
||||
// null. This is distinct from an explicitly null struct value.
|
||||
completeRow[field.name] =
|
||||
field.type.constructor.name === "Struct"
|
||||
? ensureStructFieldsExist({}, field.type as Struct)
|
||||
: null;
|
||||
// Field is missing from the data - set to null
|
||||
completeRow[field.name] = null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1220,12 +1498,8 @@ function ensureStructFieldsExist(
|
||||
completeStruct[childField.name] = data[childField.name];
|
||||
}
|
||||
} else {
|
||||
// Keep a missing struct valid while filling each of its children with
|
||||
// null. This is distinct from an explicitly null struct value.
|
||||
completeStruct[childField.name] =
|
||||
childField.type.constructor.name === "Struct"
|
||||
? ensureStructFieldsExist({}, childField.type as Struct)
|
||||
: null;
|
||||
// Field is missing - set to null
|
||||
completeStruct[childField.name] = null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import {
|
||||
type DataType,
|
||||
Float32,
|
||||
Float64,
|
||||
Int8,
|
||||
Int16,
|
||||
Int32,
|
||||
Uint8,
|
||||
Uint16,
|
||||
Uint32,
|
||||
} from "apache-arrow";
|
||||
|
||||
/**
|
||||
* Map a JS TypedArray instance to the corresponding Arrow element type and
|
||||
* length. Returns undefined when the view is not a supported TypedArray.
|
||||
*/
|
||||
export function typedArrayToArrowType(
|
||||
value: ArrayBufferView,
|
||||
): { elementType: DataType; length: number } | undefined {
|
||||
if (value instanceof Float32Array)
|
||||
return { elementType: new Float32(), length: value.length };
|
||||
if (value instanceof Float64Array)
|
||||
return { elementType: new Float64(), length: value.length };
|
||||
if (value instanceof Uint8Array)
|
||||
return { elementType: new Uint8(), length: value.length };
|
||||
if (value instanceof Uint16Array)
|
||||
return { elementType: new Uint16(), length: value.length };
|
||||
if (value instanceof Uint32Array)
|
||||
return { elementType: new Uint32(), length: value.length };
|
||||
if (value instanceof Int8Array)
|
||||
return { elementType: new Int8(), length: value.length };
|
||||
if (value instanceof Int16Array)
|
||||
return { elementType: new Int16(), length: value.length };
|
||||
if (value instanceof Int32Array)
|
||||
return { elementType: new Int32(), length: value.length };
|
||||
return undefined;
|
||||
}
|
||||
@@ -16,12 +16,6 @@ import {
|
||||
makeEmptyTable,
|
||||
} from "./arrow";
|
||||
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
|
||||
import {
|
||||
MaterializedView,
|
||||
MaterializedViewSelect,
|
||||
normalizeSelect,
|
||||
validateNonNegativeInteger,
|
||||
} from "./materialized_view";
|
||||
import { Connection as LanceDbConnection } from "./native";
|
||||
import type {
|
||||
CreateNamespaceResponse,
|
||||
@@ -31,14 +25,12 @@ import type {
|
||||
JobDescription,
|
||||
JobInfo,
|
||||
ListNamespacesResponse,
|
||||
ListTablesResponse,
|
||||
} from "./native";
|
||||
export type {
|
||||
CreateNamespaceResponse,
|
||||
DescribeNamespaceResponse,
|
||||
DropNamespaceResponse,
|
||||
ListNamespacesResponse,
|
||||
ListTablesResponse,
|
||||
};
|
||||
import { sanitizeTable } from "./sanitize";
|
||||
import { LocalTable, Table } from "./table";
|
||||
@@ -136,10 +128,6 @@ export interface OpenTableOptions {
|
||||
indexCacheSize?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Use {@link ListTablesOptions} with {@link Connection.listTables}
|
||||
* instead.
|
||||
*/
|
||||
export interface TableNamesOptions {
|
||||
/**
|
||||
* If present, only return names that come lexicographically after the
|
||||
@@ -153,24 +141,6 @@ export interface TableNamesOptions {
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export interface ListTablesOptions {
|
||||
/**
|
||||
* Token from a previous response, to resume listing where it left off.
|
||||
*
|
||||
* The token is opaque: it carries whatever the database needs to resume, and
|
||||
* callers should not construct or interpret one.
|
||||
*/
|
||||
pageToken?: string;
|
||||
/**
|
||||
* An upper bound on how many tables to return.
|
||||
*
|
||||
* A page may hold fewer than this and still not be the last one, so keep
|
||||
* going while the response carries a page token rather than while pages are
|
||||
* full.
|
||||
*/
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export interface ListNamespacesOptions {
|
||||
/** Token from a previous response for pagination. */
|
||||
pageToken?: string;
|
||||
@@ -255,7 +225,6 @@ export abstract class Connection {
|
||||
* @param {Partial<TableNamesOptions>} options - options to control the
|
||||
* paging / start point (backwards compatibility)
|
||||
*
|
||||
* @deprecated Use {@link Connection.listTables} instead.
|
||||
*/
|
||||
abstract tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
|
||||
/**
|
||||
@@ -266,94 +235,18 @@ export abstract class Connection {
|
||||
* @param {Partial<TableNamesOptions>} options - options to control the
|
||||
* paging / start point
|
||||
*
|
||||
* @deprecated Use {@link Connection.listTables} instead.
|
||||
*/
|
||||
abstract tableNames(
|
||||
namespacePath?: string[],
|
||||
options?: Partial<TableNamesOptions>,
|
||||
): Promise<string[]>;
|
||||
|
||||
/**
|
||||
* List a page of the tables in this database.
|
||||
*
|
||||
* To retrieve the tables after the page, pass the `pageToken` the response
|
||||
* carries back in. A page can be shorter than `limit` without being the last
|
||||
* one, so walk until a response carries no page token:
|
||||
*
|
||||
* ```ts
|
||||
* const names = [];
|
||||
* let pageToken = undefined;
|
||||
* do {
|
||||
* const page = await conn.listTables({ pageToken, limit: 100 });
|
||||
* names.push(...page.tables);
|
||||
* pageToken = page.pageToken;
|
||||
* } while (pageToken);
|
||||
* ```
|
||||
*
|
||||
* @param {Partial<ListTablesOptions>} options - Pagination options
|
||||
* (`pageToken`, `limit`).
|
||||
* @returns {Promise<ListTablesResponse>} A page of table names and an
|
||||
* optional token for the tables after it.
|
||||
*/
|
||||
abstract listTables(
|
||||
options?: Partial<ListTablesOptions>,
|
||||
): Promise<ListTablesResponse>;
|
||||
/**
|
||||
* List a page of the tables in this database.
|
||||
*
|
||||
* @param {string[]} namespacePath - The namespace path to list tables from
|
||||
* (defaults to root namespace)
|
||||
* @param {Partial<ListTablesOptions>} options - Pagination options
|
||||
* (`pageToken`, `limit`).
|
||||
* @returns {Promise<ListTablesResponse>} A page of table names and an
|
||||
* optional token for the tables after it.
|
||||
*/
|
||||
abstract listTables(
|
||||
namespacePath?: string[],
|
||||
options?: Partial<ListTablesOptions>,
|
||||
): Promise<ListTablesResponse>;
|
||||
|
||||
/**
|
||||
* Open a table in the database.
|
||||
* @param {string} name - The name of the table
|
||||
* @param {string[]} namespacePath - The namespace path of the table (defaults to root namespace)
|
||||
* @param {Partial<OpenTableOptions>} options - Additional options
|
||||
*/
|
||||
/**
|
||||
* Define a materialized view named `name` over the table `source`.
|
||||
*
|
||||
* The view is created empty, with the query recorded in its schema
|
||||
* metadata; `view.refresh()` computes the rows. The view is a normal
|
||||
* table: it can be queried, indexed and searched, and it appears in
|
||||
* `tableNames`. The source table must have stable row ids (create it with
|
||||
* the `newTableEnableStableRowIds` storage option); they keep the view's
|
||||
* provenance valid across source compactions and cannot be enabled after
|
||||
* a table exists. Local databases only.
|
||||
*/
|
||||
abstract createMaterializedView(
|
||||
name: string,
|
||||
source: string,
|
||||
options?: {
|
||||
select?: MaterializedViewSelect;
|
||||
where?: string;
|
||||
limit?: number;
|
||||
},
|
||||
): Promise<MaterializedView>;
|
||||
|
||||
/**
|
||||
* Open the materialized view named `name`.
|
||||
*
|
||||
* Rejects a table that exists but is not a materialized view.
|
||||
*/
|
||||
abstract openMaterializedView(name: string): Promise<MaterializedView>;
|
||||
|
||||
/**
|
||||
* The names of the materialized views in this database.
|
||||
*
|
||||
* Found by reading every table's schema, so this costs an open per table.
|
||||
*/
|
||||
abstract listMaterializedViews(): Promise<string[]>;
|
||||
|
||||
abstract openTable(
|
||||
name: string,
|
||||
namespacePath?: string[],
|
||||
@@ -638,54 +531,6 @@ export class LocalConnection extends Connection {
|
||||
);
|
||||
}
|
||||
|
||||
async createMaterializedView(
|
||||
name: string,
|
||||
source: string,
|
||||
options?: {
|
||||
select?: MaterializedViewSelect;
|
||||
where?: string;
|
||||
limit?: number;
|
||||
},
|
||||
): Promise<MaterializedView> {
|
||||
validateNonNegativeInteger(options?.limit, "limit");
|
||||
const innerTable = await this.inner.createMaterializedView(
|
||||
name,
|
||||
source,
|
||||
normalizeSelect(options?.select),
|
||||
options?.where,
|
||||
options?.limit,
|
||||
);
|
||||
return new MaterializedView(new LocalTable(innerTable));
|
||||
}
|
||||
|
||||
async openMaterializedView(name: string): Promise<MaterializedView> {
|
||||
const innerTable = await this.inner.openMaterializedView(name);
|
||||
return new MaterializedView(new LocalTable(innerTable));
|
||||
}
|
||||
|
||||
async listMaterializedViews(): Promise<string[]> {
|
||||
return await this.inner.listMaterializedViews();
|
||||
}
|
||||
|
||||
async listTables(
|
||||
namespacePathOrOptions?: string[] | Partial<ListTablesOptions>,
|
||||
options?: Partial<ListTablesOptions>,
|
||||
): Promise<ListTablesResponse> {
|
||||
// Detect if first argument is namespacePath array or options object
|
||||
const namespacePath = Array.isArray(namespacePathOrOptions)
|
||||
? namespacePathOrOptions
|
||||
: undefined;
|
||||
const listTablesOptions = Array.isArray(namespacePathOrOptions)
|
||||
? options
|
||||
: namespacePathOrOptions;
|
||||
|
||||
return this.inner.listTables(
|
||||
namespacePath ?? [],
|
||||
listTablesOptions?.pageToken,
|
||||
listTablesOptions?.limit,
|
||||
);
|
||||
}
|
||||
|
||||
async openTable(
|
||||
name: string,
|
||||
namespacePath?: string[],
|
||||
|
||||
@@ -4,15 +4,7 @@
|
||||
import { Field, Schema } from "../arrow";
|
||||
import { sanitizeType } from "../sanitize";
|
||||
import { EmbeddingFunction } from "./embedding_function";
|
||||
import {
|
||||
EmbeddingFunctionConfig,
|
||||
EmbeddingFunctionRegistry,
|
||||
getRegistry as getGlobalRegistry,
|
||||
registerBuiltIn,
|
||||
} from "./registry";
|
||||
|
||||
type OpenAIModule = typeof import("./openai");
|
||||
type TransformersModule = typeof import("./transformers");
|
||||
import { EmbeddingFunctionConfig, getRegistry } from "./registry";
|
||||
|
||||
export {
|
||||
FieldOptions,
|
||||
@@ -22,39 +14,7 @@ export {
|
||||
EmbeddingFunctionConstructor,
|
||||
} from "./embedding_function";
|
||||
|
||||
export {
|
||||
EmbeddingFunctionRegistry,
|
||||
parseEmbeddingMetadata,
|
||||
register,
|
||||
} from "./registry";
|
||||
export type {
|
||||
CreateReturnType,
|
||||
EmbeddingFunctionConfig,
|
||||
EmbeddingFunctionCreate,
|
||||
EmbeddingMetadataEntry,
|
||||
ResolvedEmbeddingFunctionConfig,
|
||||
} from "./registry";
|
||||
|
||||
function initializeBuiltInProviders() {
|
||||
const { OpenAIEmbeddingFunction } = require("./openai") as OpenAIModule;
|
||||
const { TransformersEmbeddingFunction } =
|
||||
require("./transformers") as TransformersModule;
|
||||
|
||||
registerBuiltIn("openai", OpenAIEmbeddingFunction);
|
||||
registerBuiltIn("huggingface", TransformersEmbeddingFunction);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the global embedding function registry.
|
||||
*
|
||||
* LanceDB built-in providers are initialized when this public API is first
|
||||
* used, so importing the root package does not change automatic search
|
||||
* selection for tables without embedding metadata.
|
||||
*/
|
||||
export function getRegistry(): EmbeddingFunctionRegistry {
|
||||
initializeBuiltInProviders();
|
||||
return getGlobalRegistry();
|
||||
}
|
||||
export * from "./registry";
|
||||
|
||||
/**
|
||||
* Create a schema with embedding functions.
|
||||
|
||||
@@ -5,13 +5,14 @@ import type OpenAI from "openai";
|
||||
import type { EmbeddingCreateParams } from "openai/resources/index";
|
||||
import { Float, Float32 } from "../arrow";
|
||||
import { EmbeddingFunction } from "./embedding_function";
|
||||
import { registerBuiltIn } from "./registry";
|
||||
import { register } from "./registry";
|
||||
|
||||
export type OpenAIOptions = {
|
||||
apiKey: string;
|
||||
model: EmbeddingCreateParams["model"];
|
||||
};
|
||||
|
||||
@register("openai")
|
||||
export class OpenAIEmbeddingFunction extends EmbeddingFunction<
|
||||
string,
|
||||
Partial<OpenAIOptions>
|
||||
@@ -99,5 +100,3 @@ export class OpenAIEmbeddingFunction extends EmbeddingFunction<
|
||||
return response.data[0].embedding;
|
||||
}
|
||||
}
|
||||
|
||||
registerBuiltIn("openai", OpenAIEmbeddingFunction);
|
||||
|
||||
@@ -7,10 +7,6 @@ import {
|
||||
} from "./embedding_function";
|
||||
import "reflect-metadata";
|
||||
|
||||
const builtInFunctionsKey = Symbol.for(
|
||||
"@lancedb/lancedb::embedding-built-in-functions::v1",
|
||||
);
|
||||
|
||||
export type CreateReturnType<T> = T extends { init: () => Promise<void> }
|
||||
? Promise<T>
|
||||
: T;
|
||||
@@ -63,15 +59,6 @@ export class EmbeddingFunctionRegistry {
|
||||
};
|
||||
}
|
||||
|
||||
/** @ignore */
|
||||
setBuiltIn<
|
||||
T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
|
||||
>(name: string, ctor: T): T {
|
||||
this.#functions.set(name, ctor);
|
||||
Reflect.defineMetadata("lancedb::embedding::name", name, ctor);
|
||||
return ctor;
|
||||
}
|
||||
|
||||
get<T extends EmbeddingFunction<unknown>>(
|
||||
name: string,
|
||||
): EmbeddingFunctionCreate<T> | undefined;
|
||||
@@ -109,7 +96,6 @@ export class EmbeddingFunctionRegistry {
|
||||
*/
|
||||
reset(this: EmbeddingFunctionRegistry) {
|
||||
this.#functions.clear();
|
||||
getBuiltInFunctions(this).clear();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -197,56 +183,12 @@ export class EmbeddingFunctionRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
function getBuiltInFunctions(registry: EmbeddingFunctionRegistry): Set<string> {
|
||||
const registryWithBuiltIns = registry as EmbeddingFunctionRegistry & {
|
||||
[key: symbol]: Set<string> | undefined;
|
||||
};
|
||||
let builtInFunctions = registryWithBuiltIns[builtInFunctionsKey];
|
||||
if (builtInFunctions === undefined) {
|
||||
builtInFunctions = new Set<string>();
|
||||
registryWithBuiltIns[builtInFunctionsKey] = builtInFunctions;
|
||||
}
|
||||
return builtInFunctions;
|
||||
}
|
||||
|
||||
// Server bundlers can load the side-effect embedding entry points and the public
|
||||
// embedding API from separate module graphs. Keep their registry shared.
|
||||
const registryKey = Symbol.for(
|
||||
"@lancedb/lancedb::embedding-function-registry::v1",
|
||||
);
|
||||
const registryGlobal = globalThis as typeof globalThis & {
|
||||
[key: symbol]: EmbeddingFunctionRegistry | undefined;
|
||||
};
|
||||
|
||||
function getGlobalRegistry(): EmbeddingFunctionRegistry {
|
||||
const existingRegistry = registryGlobal[registryKey];
|
||||
if (existingRegistry !== undefined) {
|
||||
return existingRegistry;
|
||||
}
|
||||
const registry = new EmbeddingFunctionRegistry();
|
||||
registryGlobal[registryKey] = registry;
|
||||
return registry;
|
||||
}
|
||||
|
||||
const _REGISTRY = getGlobalRegistry();
|
||||
const _REGISTRY = new EmbeddingFunctionRegistry();
|
||||
|
||||
export function register(name?: string) {
|
||||
return _REGISTRY.register(name);
|
||||
}
|
||||
|
||||
/** @ignore */
|
||||
export function registerBuiltIn<
|
||||
T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
|
||||
>(name: string, ctor: T): T {
|
||||
const builtInFunctions = getBuiltInFunctions(_REGISTRY);
|
||||
if (builtInFunctions.has(name)) {
|
||||
return _REGISTRY.setBuiltIn(name, ctor);
|
||||
}
|
||||
_REGISTRY.register(name)(ctor);
|
||||
builtInFunctions.add(name);
|
||||
return ctor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility function to get the global instance of the registry
|
||||
* @returns `EmbeddingFunctionRegistry` The global instance of the registry
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import { Float, Float32 } from "../arrow";
|
||||
import { EmbeddingFunction } from "./embedding_function";
|
||||
import { registerBuiltIn } from "./registry";
|
||||
import { register } from "./registry";
|
||||
|
||||
export type XenovaTransformerOptions = {
|
||||
/** The wasm compatible model to use */
|
||||
@@ -31,6 +31,7 @@ export type XenovaTransformerOptions = {
|
||||
};
|
||||
};
|
||||
|
||||
@register("huggingface")
|
||||
export class TransformersEmbeddingFunction extends EmbeddingFunction<
|
||||
string,
|
||||
Partial<XenovaTransformerOptions>
|
||||
@@ -157,8 +158,6 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
|
||||
}
|
||||
}
|
||||
|
||||
registerBuiltIn("huggingface", TransformersEmbeddingFunction);
|
||||
|
||||
const tensorDiv = (
|
||||
src: import("@huggingface/transformers").Tensor,
|
||||
divBy: number,
|
||||
|
||||
+3
-12
@@ -21,11 +21,6 @@ import type { BaseTokenizer } from "./indices";
|
||||
import type { FtsToken } from "./table";
|
||||
|
||||
// Re-export native header provider for use with connectWithHeaderProvider
|
||||
export {
|
||||
MaterializedView,
|
||||
MaterializedViewDefinition,
|
||||
MaterializedViewSelect,
|
||||
} from "./materialized_view";
|
||||
export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
|
||||
|
||||
// OpenTelemetry metrics bridge. Only the high-level entry point is public; the
|
||||
@@ -56,7 +51,6 @@ export {
|
||||
AddResult,
|
||||
AddColumnsResult,
|
||||
RefreshColumnResult,
|
||||
RefreshMaterializedViewResult,
|
||||
AlterColumnsResult,
|
||||
UpdateFieldMetadataResult,
|
||||
DeleteResult,
|
||||
@@ -81,13 +75,11 @@ export {
|
||||
Connection,
|
||||
CreateTableOptions,
|
||||
TableNamesOptions,
|
||||
ListTablesOptions,
|
||||
OpenTableOptions,
|
||||
ListNamespacesOptions,
|
||||
CreateNamespaceOptions,
|
||||
DropNamespaceOptions,
|
||||
ListNamespacesResponse,
|
||||
ListTablesResponse,
|
||||
CreateNamespaceResponse,
|
||||
DropNamespaceResponse,
|
||||
DescribeNamespaceResponse,
|
||||
@@ -103,7 +95,6 @@ export {
|
||||
} from "./native.js";
|
||||
|
||||
export {
|
||||
AutoQuery,
|
||||
ExecutableQuery,
|
||||
Query,
|
||||
QueryBase,
|
||||
@@ -144,10 +135,10 @@ export {
|
||||
BranchColumnChange,
|
||||
BranchIndexSummary,
|
||||
BranchRowCountSummary,
|
||||
CherryPickError,
|
||||
MergeBlocker,
|
||||
BranchDiff,
|
||||
CherryPickPreview,
|
||||
CherryPickResult,
|
||||
MergePreview,
|
||||
MergeBranchResult,
|
||||
AddDataOptions,
|
||||
UpdateOptions,
|
||||
OptimizeOptions,
|
||||
|
||||
@@ -1,161 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import { RefreshMaterializedViewResult } from "./native";
|
||||
import { Table } from "./table";
|
||||
|
||||
/** Schema metadata key holding a materialized view's definition. */
|
||||
export const DEFINITION_META_KEY = "mv.definition";
|
||||
|
||||
/** The query that defines a materialized view. */
|
||||
export interface MaterializedViewDefinition {
|
||||
/** Name of the source table, in the same database as the view. */
|
||||
sourceTable: string;
|
||||
/** `[output column, SQL expression]` pairs, in view schema order. */
|
||||
projections: [string, string][];
|
||||
/** SQL predicate selecting the source rows the view holds. */
|
||||
filter?: string;
|
||||
/** Cap on the number of rows the view holds. */
|
||||
limit?: number;
|
||||
/** Source columns the projections and filter read. */
|
||||
inputs: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* The view's columns: column names, `[alias, SQL expression]` pairs, or a
|
||||
* record of the same. A bare name projects itself.
|
||||
*/
|
||||
export type MaterializedViewSelect =
|
||||
| (string | [string, string])[]
|
||||
| Record<string, string>;
|
||||
|
||||
/**
|
||||
* @internal Reject a numeric option N-API would otherwise silently coerce:
|
||||
* `Infinity` reaches Rust as 0, `1.5` as 1.
|
||||
*/
|
||||
export function validateNonNegativeInteger(
|
||||
value: number | undefined,
|
||||
name: string,
|
||||
): void {
|
||||
if (value !== undefined && !(Number.isSafeInteger(value) && value >= 0)) {
|
||||
throw new Error(`${name} must be a non-negative integer`);
|
||||
}
|
||||
}
|
||||
|
||||
/** @internal Quote a column name as a Lance SQL identifier (backticks). */
|
||||
function quoteIdentifier(name: string): string {
|
||||
return "`" + name.replace(/`/g, "``") + "`";
|
||||
}
|
||||
|
||||
/**
|
||||
* @internal Normalize a select argument into `[alias, expression]` pairs.
|
||||
* A bare name projects itself and is quoted, so any valid column name works;
|
||||
* pair and record entries are kept verbatim because their right side is an
|
||||
* expression.
|
||||
*/
|
||||
export function normalizeSelect(
|
||||
select?: MaterializedViewSelect,
|
||||
): [string, string][] | undefined {
|
||||
if (select === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
if (Array.isArray(select)) {
|
||||
return select.map((item) =>
|
||||
typeof item === "string" ? [item, quoteIdentifier(item)] : item,
|
||||
);
|
||||
}
|
||||
return Object.entries(select);
|
||||
}
|
||||
|
||||
/** @internal Parse a definition off a table's stored schema metadata. */
|
||||
export function definitionFromMetadata(
|
||||
metadata: Map<string, string>,
|
||||
name: string,
|
||||
): MaterializedViewDefinition {
|
||||
const raw = metadata.get(DEFINITION_META_KEY);
|
||||
if (raw === undefined) {
|
||||
throw new Error(`Table '${name}' is not a materialized view`);
|
||||
}
|
||||
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
|
||||
const value: any = JSON.parse(raw);
|
||||
if (value.kind !== "select") {
|
||||
throw new Error(
|
||||
`materialized view '${name}' is defined by '${value.kind}', which this ` +
|
||||
"version of lancedb cannot refresh",
|
||||
);
|
||||
}
|
||||
const limit = value.limit ?? undefined;
|
||||
// JSON.parse rounds integers past 2^53; every exact u64 parses to a safe
|
||||
// integer and every rounded one does not, so this rejects precisely the
|
||||
// values a number cannot carry.
|
||||
if (limit !== undefined && !Number.isSafeInteger(limit)) {
|
||||
throw new Error(
|
||||
`materialized view '${name}' has a stored limit too large to represent exactly`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
sourceTable: value.source_table,
|
||||
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
|
||||
projections: (value.projections ?? []).map((p: any) => [
|
||||
p.output,
|
||||
p.expression,
|
||||
]),
|
||||
filter: value.filter ?? undefined,
|
||||
limit,
|
||||
inputs: value.inputs ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A handle on a materialized view: its table plus its definition.
|
||||
*
|
||||
* Obtained from {@link Connection#createMaterializedView} or
|
||||
* {@link Connection#openMaterializedView}. The view is a normal table --
|
||||
* queries, indexes and search all apply through {@link MaterializedView#table}
|
||||
* -- whose contents are maintained by {@link MaterializedView#refresh}.
|
||||
*/
|
||||
export class MaterializedView {
|
||||
private readonly inner: Table;
|
||||
|
||||
constructor(table: Table) {
|
||||
this.inner = table;
|
||||
}
|
||||
|
||||
get name(): string {
|
||||
return this.inner.name;
|
||||
}
|
||||
|
||||
/** The view, as the table it is. */
|
||||
table(): Table {
|
||||
return this.inner;
|
||||
}
|
||||
|
||||
/** The query that defines the view, read from its stored schema. */
|
||||
async definition(): Promise<MaterializedViewDefinition> {
|
||||
const schema = await this.inner.schema();
|
||||
return definitionFromMetadata(schema.metadata, this.name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Recompute the view from its source.
|
||||
*
|
||||
* The refresh is incremental when the source's changes can be reconciled
|
||||
* into the view -- rows added, changed or removed since the last one --
|
||||
* and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
|
||||
* refreshes to that source version instead of the latest.
|
||||
*
|
||||
* Concurrent refreshes of one view do not duplicate its rows. Two that
|
||||
* plan the same source rows conflict on commit, and the loser throws
|
||||
* rather than writing them a second time.
|
||||
*/
|
||||
async refresh(options?: {
|
||||
full?: boolean;
|
||||
sourceVersion?: number;
|
||||
}): Promise<RefreshMaterializedViewResult> {
|
||||
validateNonNegativeInteger(options?.sourceVersion, "sourceVersion");
|
||||
return await this.inner.refreshMaterializedView(
|
||||
options?.full,
|
||||
options?.sourceVersion,
|
||||
);
|
||||
}
|
||||
}
|
||||
+106
-205
@@ -100,29 +100,6 @@ export interface FullTextSearchOptions {
|
||||
columns?: string | string[];
|
||||
}
|
||||
|
||||
function nearestToNative(
|
||||
inner: NativeQuery,
|
||||
vector: Awaited<IntoVector>,
|
||||
): NativeVectorQuery {
|
||||
const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector);
|
||||
if (raw) {
|
||||
return inner.nearestToRaw(raw.data, raw.dtype);
|
||||
}
|
||||
return inner.nearestTo(Float32Array.from(vector as number[]));
|
||||
}
|
||||
|
||||
function addQueryVectorToNative(
|
||||
inner: NativeVectorQuery,
|
||||
vector: Awaited<IntoVector>,
|
||||
) {
|
||||
const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector);
|
||||
if (raw) {
|
||||
inner.addQueryVectorRaw(raw.data, raw.dtype);
|
||||
} else {
|
||||
inner.addQueryVector(Float32Array.from(vector as number[]));
|
||||
}
|
||||
}
|
||||
|
||||
/** Common methods supported by all query types
|
||||
*
|
||||
* @see {@link Query}
|
||||
@@ -134,15 +111,13 @@ export class QueryBase<
|
||||
NativeQueryType extends NativeQuery | NativeVectorQuery | NativeTakeQuery,
|
||||
> implements AsyncIterable<RecordBatch>
|
||||
{
|
||||
protected inner!: NativeQueryType | Promise<NativeQueryType>;
|
||||
|
||||
/**
|
||||
* @hidden
|
||||
*/
|
||||
protected constructor(inner?: NativeQueryType | Promise<NativeQueryType>) {
|
||||
if (inner !== undefined) {
|
||||
this.inner = inner;
|
||||
}
|
||||
protected constructor(
|
||||
protected inner: NativeQueryType | Promise<NativeQueryType>,
|
||||
) {
|
||||
// intentionally empty
|
||||
}
|
||||
|
||||
// call a function on the inner (either a promise or the actual object)
|
||||
@@ -160,15 +135,6 @@ export class QueryBase<
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the native query used by the next terminal operation.
|
||||
*
|
||||
* @hidden
|
||||
*/
|
||||
protected async getInner(): Promise<NativeQueryType> {
|
||||
return this.inner;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return only the specified columns.
|
||||
*
|
||||
@@ -241,11 +207,16 @@ export class QueryBase<
|
||||
/**
|
||||
* @hidden
|
||||
*/
|
||||
protected async nativeExecute(
|
||||
protected nativeExecute(
|
||||
options?: Partial<QueryExecutionOptions>,
|
||||
): Promise<NativeBatchIterator> {
|
||||
const inner = await this.getInner();
|
||||
return inner.execute(options?.maxBatchLength, options?.timeoutMs);
|
||||
if (this.inner instanceof Promise) {
|
||||
return this.inner.then((inner) =>
|
||||
inner.execute(options?.maxBatchLength, options?.timeoutMs),
|
||||
);
|
||||
} else {
|
||||
return this.inner.execute(options?.maxBatchLength, options?.timeoutMs);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -274,7 +245,12 @@ export class QueryBase<
|
||||
/** Collect the results as an Arrow @see {@link ArrowTable}. */
|
||||
async toArrow(options?: Partial<QueryExecutionOptions>): Promise<ArrowTable> {
|
||||
const batches = [];
|
||||
const inner = await this.getInner();
|
||||
let inner;
|
||||
if (this.inner instanceof Promise) {
|
||||
inner = await this.inner;
|
||||
} else {
|
||||
inner = this.inner;
|
||||
}
|
||||
for await (const batch of new RecordBatchIterable(inner, options)) {
|
||||
batches.push(batch);
|
||||
}
|
||||
@@ -303,8 +279,11 @@ export class QueryBase<
|
||||
* @returns A Promise that resolves to a string containing the query execution plan explanation.
|
||||
*/
|
||||
async explainPlan(verbose = false): Promise<string> {
|
||||
const inner = await this.getInner();
|
||||
return inner.explainPlan(verbose);
|
||||
if (this.inner instanceof Promise) {
|
||||
return this.inner.then((inner) => inner.explainPlan(verbose));
|
||||
} else {
|
||||
return this.inner.explainPlan(verbose);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -342,8 +321,13 @@ export class QueryBase<
|
||||
distributedMetrics?: AnalyzePlanDistributedMetrics,
|
||||
): Promise<string> {
|
||||
const distributedMetricsMode = distributedMetrics ?? "aggregate";
|
||||
const inner = await this.getInner();
|
||||
return inner.analyzePlan(distributedMetricsMode);
|
||||
if (this.inner instanceof Promise) {
|
||||
return this.inner.then((inner) =>
|
||||
inner.analyzePlan(distributedMetricsMode),
|
||||
);
|
||||
} else {
|
||||
return this.inner.analyzePlan(distributedMetricsMode);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -355,8 +339,12 @@ export class QueryBase<
|
||||
* @returns An Arrow Schema describing the output columns.
|
||||
*/
|
||||
async outputSchema(): Promise<import("./arrow").Schema> {
|
||||
const inner = await this.getInner();
|
||||
const schemaBuffer = await inner.outputSchema();
|
||||
let schemaBuffer: Buffer;
|
||||
if (this.inner instanceof Promise) {
|
||||
schemaBuffer = await this.inner.then((inner) => inner.outputSchema());
|
||||
} else {
|
||||
schemaBuffer = await this.inner.outputSchema();
|
||||
}
|
||||
const schema = tableFromIPC(schemaBuffer).schema;
|
||||
return schema;
|
||||
}
|
||||
@@ -368,7 +356,7 @@ export class StandardQueryBase<
|
||||
extends QueryBase<NativeQueryType>
|
||||
implements ExecutableQuery
|
||||
{
|
||||
constructor(inner?: NativeQueryType | Promise<NativeQueryType>) {
|
||||
constructor(inner: NativeQueryType | Promise<NativeQueryType>) {
|
||||
super(inner);
|
||||
}
|
||||
|
||||
@@ -522,13 +510,6 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
super(inner);
|
||||
}
|
||||
|
||||
/**
|
||||
* @hidden
|
||||
*/
|
||||
protected doVectorCall(fn: (inner: NativeVectorQuery) => void) {
|
||||
super.doCall(fn);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the number of partitions to search (probe)
|
||||
*
|
||||
@@ -556,7 +537,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* the minimum and maximum to the same value.
|
||||
*/
|
||||
nprobes(nprobes: number): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.nprobes(nprobes));
|
||||
super.doCall((inner) => inner.nprobes(nprobes));
|
||||
|
||||
return this;
|
||||
}
|
||||
@@ -570,7 +551,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* but will also increase latency.
|
||||
*/
|
||||
minimumNprobes(minimumNprobes: number): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.minimumNprobes(minimumNprobes));
|
||||
super.doCall((inner) => inner.minimumNprobes(minimumNprobes));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -584,7 +565,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* potential false negatives.
|
||||
*/
|
||||
maximumNprobes(maximumNprobes: number): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.maximumNprobes(maximumNprobes));
|
||||
super.doCall((inner) => inner.maximumNprobes(maximumNprobes));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -597,7 +578,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* `undefined` means no lower or upper bound.
|
||||
*/
|
||||
distanceRange(lowerBound?: number, upperBound?: number): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.distanceRange(lowerBound, upperBound));
|
||||
super.doCall((inner) => inner.distanceRange(lowerBound, upperBound));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -611,7 +592,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* also increase the latency of your query. The default value is 1.5*limit.
|
||||
*/
|
||||
ef(ef: number): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.ef(ef));
|
||||
super.doCall((inner) => inner.ef(ef));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -625,7 +606,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* whose data type is a fixed-size-list of floats.
|
||||
*/
|
||||
column(column: string): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.column(column));
|
||||
super.doCall((inner) => inner.column(column));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -646,7 +627,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
distanceType(
|
||||
distanceType: Required<IvfPqOptions>["distanceType"],
|
||||
): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.distanceType(distanceType));
|
||||
super.doCall((inner) => inner.distanceType(distanceType));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -680,7 +661,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* distance between the query vector and the actual uncompressed vector.
|
||||
*/
|
||||
refineFactor(refineFactor: number): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.refineFactor(refineFactor));
|
||||
super.doCall((inner) => inner.refineFactor(refineFactor));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -705,7 +686,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* factor can often help restore some of the results lost by post filtering.
|
||||
*/
|
||||
postfilter(): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.postfilter());
|
||||
super.doCall((inner) => inner.postfilter());
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -719,7 +700,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* calculate your recall to select an appropriate value for nprobes.
|
||||
*/
|
||||
bypassVectorIndex(): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.bypassVectorIndex());
|
||||
super.doCall((inner) => inner.bypassVectorIndex());
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -727,39 +708,43 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* Add a query vector to the search
|
||||
*
|
||||
* This method can be called multiple times to add multiple query vectors
|
||||
* to the search. A column called `query_index` will be added to indicate the index
|
||||
* of the query vector that produced the result. Flat searches share one table scan
|
||||
* across the query vectors, avoiding the scan and memory amplification of running
|
||||
* multiple queries concurrently. Indexed searches may still perform per-vector
|
||||
* index work.
|
||||
* to the search. If multiple query vectors are added, then they will be searched
|
||||
* in parallel, and the results will be concatenated. A column called `query_index`
|
||||
* will be added to indicate the index of the query vector that produced the result.
|
||||
*
|
||||
* Performance wise, this is equivalent to running multiple queries concurrently.
|
||||
*/
|
||||
addQueryVector(vector: IntoVector): VectorQuery {
|
||||
if (vector instanceof Promise) {
|
||||
// Observe the promise as soon as it is accepted. The existing native
|
||||
// query may still be pending, and delaying observation until it resolves
|
||||
// can otherwise surface a fast rejection as unhandled.
|
||||
const settledVector = vector.then(
|
||||
(value) => ({ status: "fulfilled" as const, value }),
|
||||
(reason) => ({ status: "rejected" as const, reason }),
|
||||
);
|
||||
const res = (async () => {
|
||||
const inner = await this.getInner();
|
||||
const outcome = await settledVector;
|
||||
if (outcome.status === "rejected") {
|
||||
throw outcome.reason;
|
||||
try {
|
||||
const v = await vector;
|
||||
// biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
|
||||
const value: any = this.addQueryVector(v);
|
||||
const inner = value.inner as
|
||||
| NativeVectorQuery
|
||||
| Promise<NativeVectorQuery>;
|
||||
return inner;
|
||||
} catch (e) {
|
||||
return Promise.reject(e);
|
||||
}
|
||||
addQueryVectorToNative(inner, outcome.value);
|
||||
return inner;
|
||||
})();
|
||||
return new VectorQuery(res);
|
||||
} else {
|
||||
this.doVectorCall((inner) => addQueryVectorToNative(inner, vector));
|
||||
super.doCall((inner) => {
|
||||
const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector);
|
||||
if (raw) {
|
||||
inner.addQueryVectorRaw(raw.data, raw.dtype);
|
||||
} else {
|
||||
inner.addQueryVector(Float32Array.from(vector as number[]));
|
||||
}
|
||||
});
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
rerank(reranker: Reranker): VectorQuery {
|
||||
this.doVectorCall((inner) =>
|
||||
super.doCall((inner) =>
|
||||
inner.rerank(async (args) => {
|
||||
const vecResults = await fromBufferToRecordBatch(args.vecResults);
|
||||
const ftsResults = await fromBufferToRecordBatch(args.ftsResults);
|
||||
@@ -778,71 +763,6 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a string query whose vector/FTS routing is resolved against the active
|
||||
* table schema when the query executes.
|
||||
*
|
||||
* @hidden
|
||||
*/
|
||||
export function createAutoQuery(
|
||||
table: NativeTable,
|
||||
query: string,
|
||||
columns: string[] | null,
|
||||
getVector: (metadata: string) => Promise<Awaited<IntoVector>>,
|
||||
): AutoQuery {
|
||||
type RouteSnapshot = {
|
||||
table: NativeTable;
|
||||
embeddingMetadata: string | undefined;
|
||||
};
|
||||
type CachedPreparation = {
|
||||
metadata: string;
|
||||
vector: Promise<Awaited<IntoVector>>;
|
||||
};
|
||||
|
||||
let cachedPreparation: CachedPreparation | undefined;
|
||||
|
||||
const snapshotRoute = async (): Promise<RouteSnapshot> => {
|
||||
const snapshot = await table.querySnapshot();
|
||||
const schema = tableFromIPC(await snapshot.schema()).schema;
|
||||
return {
|
||||
table: snapshot,
|
||||
embeddingMetadata: schema.metadata.get("embedding_functions"),
|
||||
};
|
||||
};
|
||||
|
||||
const createInner = async (): Promise<NativeQuery | NativeVectorQuery> => {
|
||||
const route = await snapshotRoute();
|
||||
if (route.embeddingMetadata === undefined) {
|
||||
const inner = route.table.query();
|
||||
inner.fullTextSearch({ query, columns });
|
||||
return inner;
|
||||
}
|
||||
|
||||
const metadata = route.embeddingMetadata;
|
||||
if (cachedPreparation?.metadata !== metadata) {
|
||||
cachedPreparation = {
|
||||
metadata,
|
||||
vector: Promise.resolve().then(() => getVector(metadata)),
|
||||
};
|
||||
}
|
||||
|
||||
const preparation = cachedPreparation;
|
||||
let vector: Awaited<IntoVector>;
|
||||
try {
|
||||
vector = await preparation.vector;
|
||||
} catch (error) {
|
||||
if (cachedPreparation === preparation) {
|
||||
cachedPreparation = undefined;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
return nearestToNative(route.table.query(), vector);
|
||||
};
|
||||
|
||||
return new AutoQuery(createInner);
|
||||
}
|
||||
|
||||
/**
|
||||
* A query that returns a subset of the rows in the table.
|
||||
*
|
||||
@@ -868,51 +788,6 @@ export class TakeQuery extends QueryBase<NativeTakeQuery> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A builder for automatic string searches.
|
||||
*
|
||||
* Automatic search determines whether to use full-text or vector search from
|
||||
* the table revision selected for each execution. This builder exposes the
|
||||
* common operations supported by both query families.
|
||||
*
|
||||
* @hideconstructor
|
||||
*/
|
||||
export class AutoQuery extends StandardQueryBase<
|
||||
NativeQuery | NativeVectorQuery
|
||||
> {
|
||||
private readonly calls: Array<
|
||||
(inner: NativeQuery | NativeVectorQuery) => void
|
||||
> = [];
|
||||
|
||||
/** @hidden */
|
||||
constructor(
|
||||
private readonly createInner: () => Promise<
|
||||
NativeQuery | NativeVectorQuery
|
||||
>,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
/** @hidden */
|
||||
protected override doCall(
|
||||
fn: (inner: NativeQuery | NativeVectorQuery) => void,
|
||||
) {
|
||||
this.calls.push(fn);
|
||||
}
|
||||
|
||||
/** @hidden */
|
||||
protected override async getInner(): Promise<
|
||||
NativeQuery | NativeVectorQuery
|
||||
> {
|
||||
const calls = [...this.calls];
|
||||
const inner = await this.createInner();
|
||||
for (const call of calls) {
|
||||
call(inner);
|
||||
}
|
||||
return inner;
|
||||
}
|
||||
}
|
||||
|
||||
/** A builder for LanceDB queries.
|
||||
*
|
||||
* @see {@link Table#query}, {@link Table#search}
|
||||
@@ -965,19 +840,45 @@ export class Query extends StandardQueryBase<NativeQuery> {
|
||||
* a default `limit` of 10 will be used. @see {@link Query#limit}
|
||||
*/
|
||||
nearestTo(vector: IntoVector): VectorQuery {
|
||||
const inner = this.inner;
|
||||
if (inner instanceof Promise) {
|
||||
const nativeQuery = inner.then(async (resolvedInner) =>
|
||||
nearestToNative(resolvedInner, await vector),
|
||||
);
|
||||
const callNearestTo = (
|
||||
inner: NativeQuery,
|
||||
resolved: Float32Array | Float64Array | Uint8Array | number[],
|
||||
): NativeVectorQuery => {
|
||||
const raw = Array.isArray(resolved)
|
||||
? null
|
||||
: extractVectorBuffer(resolved);
|
||||
if (raw) {
|
||||
return inner.nearestToRaw(raw.data, raw.dtype);
|
||||
}
|
||||
return inner.nearestTo(Float32Array.from(resolved as number[]));
|
||||
};
|
||||
|
||||
if (this.inner instanceof Promise) {
|
||||
const nativeQuery = this.inner.then(async (inner) => {
|
||||
const resolved = vector instanceof Promise ? await vector : vector;
|
||||
return callNearestTo(inner, resolved);
|
||||
});
|
||||
return new VectorQuery(nativeQuery);
|
||||
}
|
||||
if (vector instanceof Promise) {
|
||||
return new VectorQuery(
|
||||
vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)),
|
||||
);
|
||||
const res = (async () => {
|
||||
try {
|
||||
const v = await vector;
|
||||
// biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
|
||||
const value: any = this.nearestTo(v);
|
||||
const inner = value.inner as
|
||||
| NativeVectorQuery
|
||||
| Promise<NativeVectorQuery>;
|
||||
return inner;
|
||||
} catch (e) {
|
||||
return Promise.reject(e);
|
||||
}
|
||||
})();
|
||||
return new VectorQuery(res);
|
||||
} else {
|
||||
const vectorQuery = callNearestTo(this.inner, vector);
|
||||
return new VectorQuery(vectorQuery);
|
||||
}
|
||||
return new VectorQuery(nearestToNative(inner, vector));
|
||||
}
|
||||
|
||||
nearestToText(query: string | FullTextQuery, columns?: string[]): Query {
|
||||
|
||||
@@ -94,24 +94,17 @@ export function sanitizeMetadata(
|
||||
if (metadataLike === undefined || metadataLike === null) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
let entries: IterableIterator<[unknown, unknown]>;
|
||||
try {
|
||||
entries = Map.prototype.entries.call(metadataLike);
|
||||
} catch {
|
||||
if (!(metadataLike instanceof Map)) {
|
||||
throw Error("Expected metadata, if present, to be a Map<string, string>");
|
||||
}
|
||||
|
||||
const metadata = new Map<string, string>();
|
||||
for (const [key, value] of entries) {
|
||||
if (typeof key !== "string" || typeof value !== "string") {
|
||||
for (const item of metadataLike) {
|
||||
if (typeof item[0] !== "string" || typeof item[1] !== "string") {
|
||||
throw Error(
|
||||
"Expected metadata, if present, to be a Map<string, string> but it had non-string keys or values",
|
||||
);
|
||||
}
|
||||
metadata.set(key, value);
|
||||
}
|
||||
return metadata;
|
||||
return metadataLike as Map<string, string>;
|
||||
}
|
||||
|
||||
export function sanitizeInt(typeLike: object) {
|
||||
|
||||
@@ -1,567 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import {
|
||||
Binary,
|
||||
Bool,
|
||||
DataType,
|
||||
Dictionary,
|
||||
Field,
|
||||
FixedSizeList,
|
||||
Float32,
|
||||
Float64,
|
||||
Int32,
|
||||
Int64,
|
||||
List,
|
||||
Schema,
|
||||
Struct,
|
||||
Utf8,
|
||||
util as arrowUtil,
|
||||
} from "apache-arrow";
|
||||
import { typedArrayToArrowType } from "./arrow_type";
|
||||
import { sanitizeType } from "./sanitize";
|
||||
|
||||
type InferenceOptions = {
|
||||
dictionaryEncodeStrings: boolean;
|
||||
vectorColumns: Record<string, { type: unknown }>;
|
||||
};
|
||||
|
||||
/**
|
||||
* Infer the Arrow schema represented by a set of records.
|
||||
*
|
||||
* This is the intentionally small interface to schema inference. The stateful
|
||||
* details of combining partial type evidence are encapsulated below so callers
|
||||
* only need to provide records, an optional schema, and inference options.
|
||||
*/
|
||||
export function inferSchema(
|
||||
data: Array<Record<string, unknown>>,
|
||||
schema: Schema | undefined,
|
||||
options: InferenceOptions,
|
||||
): Schema {
|
||||
return new SchemaInferrer(schema, options).infer(data);
|
||||
}
|
||||
|
||||
class SchemaInferrer {
|
||||
private readonly fields = new FieldTree();
|
||||
|
||||
constructor(
|
||||
private readonly providedSchema: Schema | undefined,
|
||||
private readonly options: InferenceOptions,
|
||||
) {}
|
||||
|
||||
infer(data: Array<Record<string, unknown>>): Schema {
|
||||
for (const [row, record] of data.entries()) {
|
||||
for (const [path, value] of recordPathsAndValues(record)) {
|
||||
this.observe(path, value, row);
|
||||
}
|
||||
}
|
||||
|
||||
return this.providedSchema === undefined
|
||||
? new Schema(fieldsFromTree(this.fields))
|
||||
: new Schema(matchingFields(this.providedSchema.fields, this.fields));
|
||||
}
|
||||
|
||||
private observe(path: string[], value: unknown, row: number): void {
|
||||
const current = this.fields.get(path);
|
||||
if (current === undefined) {
|
||||
this.addField(path, value, row);
|
||||
} else if (this.providedSchema === undefined) {
|
||||
this.updateInferredField(path, value, row, current);
|
||||
}
|
||||
}
|
||||
|
||||
private addField(path: string[], value: unknown, row: number): void {
|
||||
if (this.providedSchema !== undefined) {
|
||||
this.addSchemaField(this.providedSchema, path, row);
|
||||
return;
|
||||
}
|
||||
|
||||
const evidence =
|
||||
this.inferType(value, path) ?? DeferredTypeEvidence.from(value, row);
|
||||
if (evidence === undefined) {
|
||||
throw typeInferenceError(path, row);
|
||||
}
|
||||
|
||||
const conflict = this.fields.set(
|
||||
path,
|
||||
evidence,
|
||||
(existing) =>
|
||||
existing instanceof DeferredTypeEvidence && existing.isOnlyNulls(),
|
||||
);
|
||||
if (conflict !== undefined) {
|
||||
throw branchConflictError(conflict, row, "Struct");
|
||||
}
|
||||
}
|
||||
|
||||
private addSchemaField(schema: Schema, path: string[], row: number): void {
|
||||
const field = fieldAtPath(schema, path);
|
||||
if (field === undefined) {
|
||||
throw new Error(
|
||||
`Found field not in schema: ${path.join(".")} at row ${row}`,
|
||||
);
|
||||
}
|
||||
|
||||
const conflict = this.fields.set(path, field.type);
|
||||
if (conflict !== undefined) {
|
||||
throw branchConflictError(conflict, row, "Struct");
|
||||
}
|
||||
}
|
||||
|
||||
private updateInferredField(
|
||||
path: string[],
|
||||
value: unknown,
|
||||
row: number,
|
||||
current: FieldNode,
|
||||
): void {
|
||||
const newType = this.inferType(value, path);
|
||||
const deferred = DeferredTypeEvidence.from(value, row);
|
||||
|
||||
if (current instanceof FieldTree) {
|
||||
if (deferred?.isOnlyNulls()) {
|
||||
return;
|
||||
}
|
||||
throw schemaInferenceError(
|
||||
path,
|
||||
row,
|
||||
"Struct",
|
||||
describeEvidence(newType ?? deferred),
|
||||
);
|
||||
}
|
||||
|
||||
if (current instanceof DeferredTypeEvidence) {
|
||||
this.resolveDeferredField(path, row, current, newType, deferred);
|
||||
return;
|
||||
}
|
||||
|
||||
if (newType !== undefined) {
|
||||
if (!inferredTypesEqual(current, newType)) {
|
||||
throw schemaInferenceError(
|
||||
path,
|
||||
row,
|
||||
describeEvidence(current),
|
||||
describeEvidence(newType),
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (deferred === undefined || !deferred.matches(current)) {
|
||||
throw schemaInferenceError(
|
||||
path,
|
||||
row,
|
||||
describeEvidence(current),
|
||||
describeEvidence(deferred),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private resolveDeferredField(
|
||||
path: string[],
|
||||
row: number,
|
||||
current: DeferredTypeEvidence,
|
||||
newType: DataType | undefined,
|
||||
deferred: DeferredTypeEvidence | undefined,
|
||||
): void {
|
||||
if (newType !== undefined) {
|
||||
if (!current.matches(newType)) {
|
||||
throw schemaInferenceError(
|
||||
path,
|
||||
row,
|
||||
current.describe(),
|
||||
describeEvidence(newType),
|
||||
);
|
||||
}
|
||||
this.fields.set(path, newType);
|
||||
return;
|
||||
}
|
||||
|
||||
if (deferred !== undefined) {
|
||||
this.fields.set(path, current.merge(deferred));
|
||||
return;
|
||||
}
|
||||
|
||||
throw schemaInferenceError(
|
||||
path,
|
||||
row,
|
||||
current.describe(),
|
||||
describeEvidence(newType),
|
||||
);
|
||||
}
|
||||
|
||||
private inferType(value: unknown, path: string[]): DataType | undefined {
|
||||
if (typeof value === "bigint") {
|
||||
return new Int64();
|
||||
}
|
||||
if (typeof value === "number") {
|
||||
return new Float64();
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
return this.options.dictionaryEncodeStrings
|
||||
? new Dictionary(new Utf8(), new Int32())
|
||||
: new Utf8();
|
||||
}
|
||||
if (typeof value === "boolean") {
|
||||
return new Bool();
|
||||
}
|
||||
if (value instanceof Buffer) {
|
||||
return new Binary();
|
||||
}
|
||||
if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
|
||||
const typedArray = typedArrayToArrowType(value);
|
||||
return typedArray === undefined
|
||||
? undefined
|
||||
: new FixedSizeList(
|
||||
typedArray.length,
|
||||
new Field("item", typedArray.elementType, true),
|
||||
);
|
||||
}
|
||||
if (!Array.isArray(value) || value.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const configuredVector =
|
||||
path.length === 1 ? this.options.vectorColumns[path[0]] : undefined;
|
||||
if (configuredVector !== undefined) {
|
||||
return new FixedSizeList(
|
||||
value.length,
|
||||
new Field("item", sanitizeType(configuredVector.type), true),
|
||||
);
|
||||
}
|
||||
|
||||
const itemType = this.inferArrayItemType(value, path);
|
||||
if (itemType === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return nameSuggestsVectorColumn(path[path.length - 1])
|
||||
? new FixedSizeList(value.length, new Field("item", new Float32(), true))
|
||||
: new List(new Field("item", itemType, true));
|
||||
}
|
||||
|
||||
private inferArrayItemType(
|
||||
values: unknown[],
|
||||
path: string[],
|
||||
): DataType | undefined {
|
||||
let itemType: DataType | undefined;
|
||||
const deferredItems: unknown[] = [];
|
||||
|
||||
for (const value of values) {
|
||||
const candidate = this.inferType(value, path);
|
||||
if (candidate === undefined) {
|
||||
if (!isDeferredValue(value)) {
|
||||
return undefined;
|
||||
}
|
||||
deferredItems.push(value);
|
||||
} else if (itemType === undefined) {
|
||||
itemType = candidate;
|
||||
} else if (!inferredTypesEqual(itemType, candidate)) {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
if (itemType === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
return deferredItems.every((value) =>
|
||||
deferredValueMatchesType(value, itemType),
|
||||
)
|
||||
? itemType
|
||||
: undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Nulls and empty/all-null lists that do not determine a type by themselves. */
|
||||
class DeferredTypeEvidence {
|
||||
private constructor(
|
||||
private readonly values: Array<{ value: unknown; row: number }>,
|
||||
) {}
|
||||
|
||||
static from(value: unknown, row: number): DeferredTypeEvidence | undefined {
|
||||
return isDeferredValue(value)
|
||||
? new DeferredTypeEvidence([{ value, row }])
|
||||
: undefined;
|
||||
}
|
||||
|
||||
isOnlyNulls(): boolean {
|
||||
return this.values.every(({ value }) => value == null);
|
||||
}
|
||||
|
||||
matches(type: DataType): boolean {
|
||||
return this.values.every(({ value }) =>
|
||||
deferredValueMatchesType(value, type),
|
||||
);
|
||||
}
|
||||
|
||||
merge(other: DeferredTypeEvidence): DeferredTypeEvidence {
|
||||
return new DeferredTypeEvidence([...this.values, ...other.values]);
|
||||
}
|
||||
|
||||
describe(): string {
|
||||
const list = this.values.find(({ value }) => Array.isArray(value));
|
||||
return list === undefined
|
||||
? "null"
|
||||
: `List[${(list.value as unknown[]).length}]`;
|
||||
}
|
||||
|
||||
firstRow(): number {
|
||||
return this.values[0].row;
|
||||
}
|
||||
}
|
||||
|
||||
type FieldNode = DataType | DeferredTypeEvidence | FieldTree;
|
||||
type LeafNode = Exclude<FieldNode, FieldTree>;
|
||||
type FieldConflict = { path: string[]; value: FieldNode };
|
||||
|
||||
/** Nested field state, kept separate from Arrow's eventual Struct types. */
|
||||
class FieldTree {
|
||||
private readonly children = new Map<string, FieldNode>();
|
||||
|
||||
get(path: string[]): FieldNode | undefined {
|
||||
let current: FieldNode = this;
|
||||
for (const part of path) {
|
||||
if (!(current instanceof FieldTree)) {
|
||||
return undefined;
|
||||
}
|
||||
const child = current.children.get(part);
|
||||
if (child === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
current = child;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
set(
|
||||
path: string[],
|
||||
value: LeafNode,
|
||||
canReplaceLeaf: (value: LeafNode) => boolean = () => false,
|
||||
): FieldConflict | undefined {
|
||||
let branch: FieldTree = this;
|
||||
for (const [index, part] of path.slice(0, -1).entries()) {
|
||||
const child = branch.children.get(part);
|
||||
if (child === undefined || (isLeaf(child) && canReplaceLeaf(child))) {
|
||||
const nextBranch = new FieldTree();
|
||||
branch.children.set(part, nextBranch);
|
||||
branch = nextBranch;
|
||||
} else if (child instanceof FieldTree) {
|
||||
branch = child;
|
||||
} else {
|
||||
return { path: path.slice(0, index + 1), value: child };
|
||||
}
|
||||
}
|
||||
|
||||
const name = path[path.length - 1];
|
||||
const current = branch.children.get(name);
|
||||
if (current instanceof FieldTree) {
|
||||
return { path, value: current };
|
||||
}
|
||||
branch.children.set(name, value);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
entries(): IterableIterator<[string, FieldNode]> {
|
||||
return this.children.entries();
|
||||
}
|
||||
|
||||
has(name: string): boolean {
|
||||
return this.children.has(name);
|
||||
}
|
||||
}
|
||||
|
||||
function isLeaf(value: FieldNode): value is LeafNode {
|
||||
return !(value instanceof FieldTree);
|
||||
}
|
||||
|
||||
function fieldsFromTree(tree: FieldTree, path: string[] = []): Field[] {
|
||||
const fields: Field[] = [];
|
||||
for (const [name, value] of tree.entries()) {
|
||||
if (value instanceof FieldTree) {
|
||||
fields.push(
|
||||
new Field(
|
||||
name,
|
||||
new Struct(fieldsFromTree(value, [...path, name])),
|
||||
true,
|
||||
),
|
||||
);
|
||||
} else if (value instanceof DeferredTypeEvidence) {
|
||||
throw typeInferenceError([...path, name], value.firstRow());
|
||||
} else {
|
||||
fields.push(new Field(name, value, true));
|
||||
}
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
function matchingFields(fields: Field[], tree: FieldTree): Field[] {
|
||||
const matches: Field[] = [];
|
||||
for (const field of fields) {
|
||||
if (!tree.has(field.name)) {
|
||||
continue;
|
||||
}
|
||||
const value = tree.get([field.name]);
|
||||
if (value instanceof FieldTree) {
|
||||
const struct = field.type as Struct;
|
||||
matches.push(
|
||||
new Field(
|
||||
field.name,
|
||||
new Struct(matchingFields(struct.children, value)),
|
||||
field.nullable,
|
||||
field.metadata,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
matches.push(field);
|
||||
}
|
||||
}
|
||||
return matches;
|
||||
}
|
||||
|
||||
function* recordPathsAndValues(
|
||||
record: Record<string, unknown>,
|
||||
path: string[] = [],
|
||||
): Generator<[string[], unknown]> {
|
||||
for (const [name, value] of Object.entries(record)) {
|
||||
if (isRecord(value)) {
|
||||
yield* recordPathsAndValues(value, [...path, name]);
|
||||
} else if (value !== undefined) {
|
||||
yield [[...path, name], value];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return (
|
||||
typeof value === "object" &&
|
||||
value !== null &&
|
||||
!Array.isArray(value) &&
|
||||
!(value instanceof RegExp) &&
|
||||
!(value instanceof Date) &&
|
||||
!(value instanceof Set) &&
|
||||
!(value instanceof Map) &&
|
||||
!(value instanceof Buffer) &&
|
||||
!ArrayBuffer.isView(value)
|
||||
);
|
||||
}
|
||||
|
||||
function fieldAtPath(schema: Schema, path: string[]): Field | undefined {
|
||||
let fields = schema.fields;
|
||||
let field: Field | undefined;
|
||||
for (const [index, name] of path.entries()) {
|
||||
field = fields.find((candidate) => candidate.name === name);
|
||||
if (field === undefined || index === path.length - 1) {
|
||||
return field;
|
||||
}
|
||||
if (!DataType.isStruct(field.type)) {
|
||||
return undefined;
|
||||
}
|
||||
fields = field.type.children;
|
||||
}
|
||||
return field;
|
||||
}
|
||||
|
||||
function isDeferredValue(value: unknown): boolean {
|
||||
return (
|
||||
value == null || (Array.isArray(value) && value.every(isDeferredValue))
|
||||
);
|
||||
}
|
||||
|
||||
function deferredValueMatchesType(value: unknown, type: DataType): boolean {
|
||||
if (value == null) {
|
||||
return true;
|
||||
}
|
||||
if (!Array.isArray(value)) {
|
||||
return false;
|
||||
}
|
||||
if (DataType.isList(type)) {
|
||||
return value.every((item) =>
|
||||
deferredValueMatchesType(item, type.valueType),
|
||||
);
|
||||
}
|
||||
if (DataType.isFixedSizeList(type)) {
|
||||
return (
|
||||
value.length === type.listSize &&
|
||||
value.every((item) => deferredValueMatchesType(item, type.valueType))
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function inferredTypesEqual(current: DataType, candidate: DataType): boolean {
|
||||
if (DataType.isDictionary(current)) {
|
||||
return (
|
||||
DataType.isDictionary(candidate) &&
|
||||
current.isOrdered === candidate.isOrdered &&
|
||||
inferredTypesEqual(current.indices, candidate.indices) &&
|
||||
inferredTypesEqual(current.dictionary, candidate.dictionary)
|
||||
);
|
||||
}
|
||||
if (DataType.isList(current)) {
|
||||
return (
|
||||
DataType.isList(candidate) &&
|
||||
current.valueField.name === candidate.valueField.name &&
|
||||
current.valueField.nullable === candidate.valueField.nullable &&
|
||||
inferredTypesEqual(current.valueType, candidate.valueType)
|
||||
);
|
||||
}
|
||||
if (DataType.isFixedSizeList(current)) {
|
||||
return (
|
||||
DataType.isFixedSizeList(candidate) &&
|
||||
current.listSize === candidate.listSize &&
|
||||
current.valueField.name === candidate.valueField.name &&
|
||||
current.valueField.nullable === candidate.valueField.nullable &&
|
||||
inferredTypesEqual(current.valueType, candidate.valueType)
|
||||
);
|
||||
}
|
||||
return arrowUtil.compareTypes(current, candidate);
|
||||
}
|
||||
|
||||
function describeEvidence(
|
||||
evidence: DataType | DeferredTypeEvidence | undefined,
|
||||
): string {
|
||||
if (evidence === undefined) {
|
||||
return "an unsupported value";
|
||||
}
|
||||
return evidence instanceof DeferredTypeEvidence
|
||||
? evidence.describe()
|
||||
: evidence.toString();
|
||||
}
|
||||
|
||||
function branchConflictError(
|
||||
conflict: FieldConflict,
|
||||
row: number,
|
||||
candidate: string,
|
||||
): Error {
|
||||
return schemaInferenceError(
|
||||
conflict.path,
|
||||
row,
|
||||
conflict.value instanceof FieldTree
|
||||
? "Struct"
|
||||
: describeEvidence(conflict.value),
|
||||
candidate,
|
||||
);
|
||||
}
|
||||
|
||||
function schemaInferenceError(
|
||||
path: string[],
|
||||
row: number,
|
||||
currentType: string,
|
||||
newType: string,
|
||||
): Error {
|
||||
return new Error(
|
||||
`Failed to infer schema for data. Previously inferred type ${currentType} ` +
|
||||
`but found ${newType} for field ${path.join(".")} at row ${row}. ` +
|
||||
"Consider providing an explicit schema.",
|
||||
);
|
||||
}
|
||||
|
||||
function typeInferenceError(path: string[], row: number): Error {
|
||||
return new Error(
|
||||
`Failed to infer data type for field ${path.join(".")} at row ${row}. ` +
|
||||
"Consider providing an explicit schema.",
|
||||
);
|
||||
}
|
||||
|
||||
function nameSuggestsVectorColumn(name: string): boolean {
|
||||
const normalized = name.toLowerCase();
|
||||
return normalized.includes("vector") || normalized.includes("embedding");
|
||||
}
|
||||
+34
-83
@@ -35,7 +35,6 @@ import {
|
||||
Branches as NativeBranches,
|
||||
OptimizeStats,
|
||||
RefreshColumnResult,
|
||||
RefreshMaterializedViewResult,
|
||||
TableStatistics,
|
||||
Tags,
|
||||
UpdateFieldMetadataResult,
|
||||
@@ -43,12 +42,10 @@ import {
|
||||
Table as _NativeTable,
|
||||
} from "./native";
|
||||
import {
|
||||
AutoQuery,
|
||||
FullTextQuery,
|
||||
Query,
|
||||
TakeQuery,
|
||||
VectorQuery,
|
||||
createAutoQuery,
|
||||
instanceOfFullTextQuery,
|
||||
} from "./query";
|
||||
import { sanitizeType } from "./sanitize";
|
||||
@@ -525,7 +522,7 @@ export abstract class Table {
|
||||
query: string | IntoVector | MultiVector | FullTextQuery,
|
||||
queryType?: string,
|
||||
ftsColumns?: string | string[],
|
||||
): VectorQuery | Query | AutoQuery;
|
||||
): VectorQuery | Query;
|
||||
/**
|
||||
* Search the table with a given query vector.
|
||||
*
|
||||
@@ -605,18 +602,6 @@ export abstract class Table {
|
||||
*/
|
||||
abstract refreshColumnAsync(column: string): Promise<Job>;
|
||||
|
||||
/**
|
||||
* Recompute this table's contents from its materialized-view definition.
|
||||
*
|
||||
* Plumbing for {@link MaterializedView.refresh}, which is the way to call
|
||||
* it: rejects tables that carry no view definition. Local tables only.
|
||||
* @ignore
|
||||
*/
|
||||
abstract refreshMaterializedView(
|
||||
full?: boolean,
|
||||
sourceVersion?: number,
|
||||
): Promise<RefreshMaterializedViewResult>;
|
||||
|
||||
/**
|
||||
* Alter the name or nullability of columns.
|
||||
* @param {ColumnAlteration[]} columnAlterations One or more alterations to
|
||||
@@ -630,18 +615,6 @@ export abstract class Table {
|
||||
|
||||
/**
|
||||
* Update per-field (column) metadata.
|
||||
*
|
||||
* The following keys are treated specially, by convention, and should be
|
||||
* used when appropriate:
|
||||
*
|
||||
* - `lancedb:description`: for a human-readable description of a field.
|
||||
* - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
|
||||
* names the tag category; e.g. `lancedb:tag:model: "clip"`.
|
||||
* - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
|
||||
* `feature_v2` might be in the same logical column.
|
||||
* - `lancedb:status`: for status options (`production`, `candidate`,
|
||||
* `deprecated`, `archived`) to designate the current life cycle state of
|
||||
* this column.
|
||||
* @param {FieldMetadataUpdate[]} updates One or more per-field updates. Each
|
||||
* update's metadata is merged into the field's existing metadata by default;
|
||||
* a value of `null` deletes that key, and `replace: true` swaps the whole map.
|
||||
@@ -989,11 +962,10 @@ export class LocalTable extends Table {
|
||||
return this.inner.display();
|
||||
}
|
||||
|
||||
private async getEmbeddingFunctions(
|
||||
inner: _NativeTable = this.inner,
|
||||
): Promise<Map<string, EmbeddingFunctionConfig>> {
|
||||
const schemaBuf = await inner.schema();
|
||||
const schema = tableFromIPC(schemaBuf).schema;
|
||||
private async getEmbeddingFunctions(): Promise<
|
||||
Map<string, EmbeddingFunctionConfig>
|
||||
> {
|
||||
const schema = await this.schema();
|
||||
const registry = getRegistry();
|
||||
return registry.parseFunctions(schema.metadata);
|
||||
}
|
||||
@@ -1175,7 +1147,7 @@ export class LocalTable extends Table {
|
||||
query: string | IntoVector | MultiVector | FullTextQuery,
|
||||
queryType: string = "auto",
|
||||
ftsColumns?: string | string[],
|
||||
): VectorQuery | Query | AutoQuery {
|
||||
): VectorQuery | Query {
|
||||
if (typeof query !== "string" && !instanceOfFullTextQuery(query)) {
|
||||
if (queryType === "fts") {
|
||||
throw new Error("Cannot perform full text search on a vector query");
|
||||
@@ -1190,28 +1162,14 @@ export class LocalTable extends Table {
|
||||
});
|
||||
}
|
||||
|
||||
if (queryType === "auto") {
|
||||
if (instanceOfFullTextQuery(query)) {
|
||||
return this.query().fullTextSearch(query, {
|
||||
columns: ftsColumns,
|
||||
});
|
||||
}
|
||||
|
||||
const columns =
|
||||
typeof ftsColumns === "string" ? [ftsColumns] : (ftsColumns ?? null);
|
||||
return createAutoQuery(this.inner, query, columns, async (metadata) => {
|
||||
const functions = await getRegistry().parseFunctions(
|
||||
new Map([["embedding_functions", metadata]]),
|
||||
);
|
||||
// TODO: Support multiple embedding functions
|
||||
const embeddingFunc: EmbeddingFunctionConfig | undefined = functions
|
||||
.values()
|
||||
.next().value;
|
||||
// The route only calls this callback when embedding metadata exists.
|
||||
// parseFunctions either yields a provider or reports malformed metadata.
|
||||
if (!embeddingFunc)
|
||||
throw new Error("Invalid embedding function metadata");
|
||||
return await embeddingFunc.function.computeQueryEmbeddings(query);
|
||||
// The query type is auto or vector
|
||||
// fall back to full text search if no embedding functions are defined and the query is a string
|
||||
if (
|
||||
queryType === "auto" &&
|
||||
(getRegistry().length() === 0 || instanceOfFullTextQuery(query))
|
||||
) {
|
||||
return this.query().fullTextSearch(query, {
|
||||
columns: ftsColumns,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1306,13 +1264,6 @@ export class LocalTable extends Table {
|
||||
return await this.inner.refreshColumnAsync(column);
|
||||
}
|
||||
|
||||
async refreshMaterializedView(
|
||||
full?: boolean,
|
||||
sourceVersion?: number,
|
||||
): Promise<RefreshMaterializedViewResult> {
|
||||
return await this.inner.refreshMaterializedView(full, sourceVersion);
|
||||
}
|
||||
|
||||
async alterColumns(
|
||||
columnAlterations: ColumnAlteration[],
|
||||
): Promise<AlterColumnsResult> {
|
||||
@@ -1567,8 +1518,7 @@ export interface FieldMetadataUpdate {
|
||||
path: string;
|
||||
/**
|
||||
* Metadata key/value pairs. Merged into the field's existing metadata by
|
||||
* default; a value of `null` deletes that key. See
|
||||
* {@link Table.updateFieldMetadata} for the conventional `lancedb:*` keys.
|
||||
* default; a value of `null` deletes that key.
|
||||
*/
|
||||
metadata: Record<string, string | null>;
|
||||
/** If true, replace the field's entire metadata map instead of merging. */
|
||||
@@ -1607,8 +1557,8 @@ export interface BranchRowCountSummary {
|
||||
deltaAvailable: boolean;
|
||||
}
|
||||
|
||||
/** A reason why a cherry-pick cannot currently land. */
|
||||
export interface CherryPickError {
|
||||
/** A reason why a branch cannot currently be merged. */
|
||||
export interface MergeBlocker {
|
||||
code: string;
|
||||
message: string;
|
||||
}
|
||||
@@ -1628,19 +1578,20 @@ export interface BranchDiff {
|
||||
changedColumns: BranchColumnChange[];
|
||||
addedIndexes: BranchIndexSummary[];
|
||||
removedIndexes: BranchIndexSummary[];
|
||||
errors: CherryPickError[];
|
||||
mergeable: boolean;
|
||||
mergeBlockers: MergeBlocker[];
|
||||
}
|
||||
|
||||
/** Changes that would be, or were, promoted by a cherry-pick. */
|
||||
export interface CherryPickPreview {
|
||||
/** Changes that would be, or were, promoted by a branch merge. */
|
||||
export interface MergePreview {
|
||||
promotedColumns: string[];
|
||||
}
|
||||
|
||||
/** Result of previewing or attempting a cherry-pick. */
|
||||
export interface CherryPickResult {
|
||||
status: "ready" | "failed" | "notImplemented" | "cherryPicked" | "unknown";
|
||||
/** Result of previewing or attempting a branch merge. */
|
||||
export interface MergeBranchResult {
|
||||
status: "ready" | "rejected" | "notImplemented" | "merged" | "unknown";
|
||||
diff: BranchDiff;
|
||||
preview: CherryPickPreview;
|
||||
preview: MergePreview;
|
||||
mainVersionAfter?: number;
|
||||
}
|
||||
|
||||
@@ -1703,21 +1654,21 @@ export class Branches {
|
||||
}
|
||||
|
||||
/**
|
||||
* Cherry-pick a branch onto main.
|
||||
* Merge a branch into main.
|
||||
*
|
||||
* Set `dryRun` to `true` to preview. A failed cherry-pick resolves
|
||||
* with `status: "failed"` instead of throwing.
|
||||
* Set `dryRun` to `true` to preview the merge. A rejected merge resolves
|
||||
* with `status: "rejected"` instead of throwing.
|
||||
*
|
||||
* @param fromBranch Branch to cherry-pick from.
|
||||
* @param dryRun When true, only preview. Defaults to false.
|
||||
* @param fromBranch Branch to merge from.
|
||||
* @param dryRun When true, only preview the merge. Defaults to false.
|
||||
*/
|
||||
async cherryPick(
|
||||
async merge(
|
||||
fromBranch: string,
|
||||
dryRun: boolean = false,
|
||||
): Promise<CherryPickResult> {
|
||||
return (await this.#inner.cherryPick(
|
||||
): Promise<MergeBranchResult> {
|
||||
return (await this.#inner.merge(
|
||||
fromBranch,
|
||||
dryRun,
|
||||
)) as unknown as CherryPickResult;
|
||||
)) as unknown as MergeBranchResult;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-darwin-arm64",
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.3",
|
||||
"os": ["darwin"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.darwin-arm64.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.3",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-musl",
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.3",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-gnu",
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.3",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-musl",
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.3",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.3",
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-x64-msvc",
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.3",
|
||||
"os": ["win32"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.win32-x64-msvc.node",
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb",
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.3",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@lancedb/lancedb",
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.3",
|
||||
"cpu": [
|
||||
"x64",
|
||||
"arm64"
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@
|
||||
"ann"
|
||||
],
|
||||
"private": false,
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.3",
|
||||
"main": "dist/index.js",
|
||||
"exports": {
|
||||
".": "./dist/index.js",
|
||||
|
||||
@@ -17,7 +17,6 @@ use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection, conne
|
||||
|
||||
use lance_namespace::models::{
|
||||
CreateNamespaceRequest, DescribeNamespaceRequest, DropNamespaceRequest, ListNamespacesRequest,
|
||||
ListTablesRequest,
|
||||
};
|
||||
use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
|
||||
|
||||
@@ -37,12 +36,6 @@ pub struct ListNamespacesResponse {
|
||||
pub page_token: Option<String>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct ListTablesResponse {
|
||||
pub tables: Vec<String>,
|
||||
pub page_token: Option<String>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct CreateNamespaceResponse {
|
||||
pub properties: Option<HashMap<String, String>>,
|
||||
@@ -213,33 +206,6 @@ impl Connection {
|
||||
op.execute().await.default_error()
|
||||
}
|
||||
|
||||
/// List a page of tables in the database.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn list_tables(
|
||||
&self,
|
||||
namespace_path: Option<Vec<String>>,
|
||||
page_token: Option<String>,
|
||||
limit: Option<u32>,
|
||||
) -> napi::Result<ListTablesResponse> {
|
||||
let request = ListTablesRequest {
|
||||
// The root namespace is an empty path, not an absent one: a namespace-backed
|
||||
// database rejects a request that names no namespace.
|
||||
id: Some(namespace_path.unwrap_or_default()),
|
||||
page_token,
|
||||
limit: limit.map(|limit| i32::try_from(limit).unwrap_or(i32::MAX)),
|
||||
..Default::default()
|
||||
};
|
||||
let response = self
|
||||
.get_inner()?
|
||||
.list_tables(request)
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(ListTablesResponse {
|
||||
tables: response.tables,
|
||||
page_token: response.page_token,
|
||||
})
|
||||
}
|
||||
|
||||
/// Create table from a Apache Arrow IPC (file) buffer.
|
||||
///
|
||||
/// Parameters:
|
||||
@@ -300,58 +266,6 @@ impl Connection {
|
||||
Ok(Table::new(tbl))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn create_materialized_view(
|
||||
&self,
|
||||
name: String,
|
||||
source: String,
|
||||
projections: Option<Vec<Vec<String>>>,
|
||||
filter: Option<String>,
|
||||
limit: Option<i64>,
|
||||
) -> napi::Result<Table> {
|
||||
let mut builder = self.get_inner()?.create_materialized_view(name, source);
|
||||
if let Some(projections) = projections {
|
||||
let mut pairs = Vec::with_capacity(projections.len());
|
||||
for pair in projections {
|
||||
let [output, expression]: [String; 2] = pair.try_into().map_err(|_| {
|
||||
napi::Error::from_reason("each projection must be an [output, expression] pair")
|
||||
})?;
|
||||
pairs.push((output, expression));
|
||||
}
|
||||
builder = builder.select(pairs);
|
||||
}
|
||||
if let Some(filter) = filter {
|
||||
builder = builder.only_if(filter);
|
||||
}
|
||||
if let Some(limit) = limit {
|
||||
let limit = u64::try_from(limit)
|
||||
.map_err(|_| napi::Error::from_reason("limit must be a non-negative integer"))?;
|
||||
builder = builder.limit(limit);
|
||||
}
|
||||
let view = builder.execute().await.default_error()?;
|
||||
Ok(Table::new(view.table().clone()))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn open_materialized_view(&self, name: String) -> napi::Result<Table> {
|
||||
let view = self
|
||||
.get_inner()?
|
||||
.open_materialized_view(&name)
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(Table::new(view.table().clone()))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn list_materialized_views(&self) -> napi::Result<Vec<String>> {
|
||||
let views = self
|
||||
.get_inner()?
|
||||
.list_materialized_views()
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(views.into_iter().map(|v| v.name).collect())
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn open_table(
|
||||
&self,
|
||||
|
||||
+2
-5
@@ -14,12 +14,9 @@ pub struct Job {
|
||||
}
|
||||
|
||||
impl Job {
|
||||
pub(crate) fn new<T>(inner: lancedb::Job<T>) -> Self
|
||||
where
|
||||
T: Clone + Send + Sync + 'static,
|
||||
{
|
||||
pub(crate) fn new(inner: lancedb::Job) -> Self {
|
||||
Self {
|
||||
inner: Arc::new(inner.map(|_| ())),
|
||||
inner: Arc::new(inner),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
// The materialized-view refresh future deepens the type graph past the
|
||||
// default trait-recursion depth; same raise as the core crate applies.
|
||||
#![recursion_limit = "256"]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use env_logger::Env;
|
||||
|
||||
+3
-61
@@ -278,13 +278,6 @@ impl Table {
|
||||
Ok(Query::new(self.inner_ref()?.query()))
|
||||
}
|
||||
|
||||
/// Return a read-only table handle pinned to the current query revision.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn query_snapshot(&self) -> napi::Result<Self> {
|
||||
let snapshot = self.inner_ref()?.query_snapshot().await.default_error()?;
|
||||
Ok(Self::new(snapshot))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub fn take_offsets(&self, offsets: Vec<i64>) -> napi::Result<TakeQuery> {
|
||||
Ok(TakeQuery::new(
|
||||
@@ -388,26 +381,6 @@ impl Table {
|
||||
Ok(crate::job::Job::new(job))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn refresh_materialized_view(
|
||||
&self,
|
||||
full: Option<bool>,
|
||||
source_version: Option<i64>,
|
||||
) -> napi::Result<RefreshMaterializedViewResult> {
|
||||
let view = lancedb::MaterializedView::from_table(self.inner_ref()?.clone())
|
||||
.await
|
||||
.default_error()?;
|
||||
let mut builder = view.refresh().full(full.unwrap_or(false));
|
||||
if let Some(version) = source_version {
|
||||
let version = u64::try_from(version).map_err(|_| {
|
||||
napi::Error::from_reason("sourceVersion must be a non-negative integer")
|
||||
})?;
|
||||
builder = builder.source_version(version);
|
||||
}
|
||||
let result = builder.execute().await.default_error()?;
|
||||
Ok(result.into())
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn add_columns_with_schema(
|
||||
&self,
|
||||
@@ -561,12 +534,6 @@ impl Table {
|
||||
.default_error()
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn checkout_current(&self) -> napi::Result<Self> {
|
||||
let table = self.inner_ref()?.checkout_current().await.default_error()?;
|
||||
Ok(Self::new(table))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn checkout(&self, version: i64) -> napi::Result<()> {
|
||||
self.inner_ref()?
|
||||
@@ -1420,31 +1387,6 @@ pub struct RefreshColumnResult {
|
||||
pub version: i64,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct RefreshMaterializedViewResult {
|
||||
/// How the view was brought up to date: "rebuild", "incremental" or "no_op".
|
||||
pub mode: String,
|
||||
pub rows_written: i64,
|
||||
pub source_version: i64,
|
||||
pub version: i64,
|
||||
}
|
||||
|
||||
impl From<lancedb::RefreshMaterializedViewResult> for RefreshMaterializedViewResult {
|
||||
fn from(value: lancedb::RefreshMaterializedViewResult) -> Self {
|
||||
let mode = match value.mode {
|
||||
lancedb::RefreshMode::Rebuild => "rebuild",
|
||||
lancedb::RefreshMode::Incremental => "incremental",
|
||||
lancedb::RefreshMode::NoOp => "no_op",
|
||||
};
|
||||
Self {
|
||||
mode: mode.to_string(),
|
||||
rows_written: value.rows_written as i64,
|
||||
source_version: value.source_version as i64,
|
||||
version: value.version as i64,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<lancedb::table::RefreshColumnResult> for RefreshColumnResult {
|
||||
fn from(value: lancedb::table::RefreshColumnResult) -> Self {
|
||||
Self {
|
||||
@@ -1663,18 +1605,18 @@ impl Branches {
|
||||
}
|
||||
|
||||
#[napi(ts_return_type = "Promise<Record<string, unknown>>")]
|
||||
pub async fn cherry_pick(
|
||||
pub async fn merge(
|
||||
&self,
|
||||
from_branch: String,
|
||||
dry_run: Option<bool>,
|
||||
) -> napi::Result<serde_json::Value> {
|
||||
let result = self
|
||||
.inner
|
||||
.cherry_pick(&from_branch, dry_run.unwrap_or(false))
|
||||
.merge_branch(&from_branch, dry_run.unwrap_or(false))
|
||||
.await
|
||||
.default_error()?;
|
||||
serde_json::to_value(result).map_err(|err| {
|
||||
napi::Error::from_reason(format!("failed to serialize cherry-pick result: {err}"))
|
||||
napi::Error::from_reason(format!("failed to serialize branch merge result: {err}"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+6
-5
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb-python"
|
||||
version = "0.38.0-beta.11"
|
||||
version = "0.38.0-beta.3"
|
||||
publish = false
|
||||
edition.workspace = true
|
||||
description = "Python bindings for LanceDB"
|
||||
@@ -26,9 +26,7 @@ lance-namespace-impls.workspace = true
|
||||
lance-io.workspace = true
|
||||
env_logger.workspace = true
|
||||
log.workspace = true
|
||||
# Maturin enables extension-module mode for Python builds. Keeping it out of
|
||||
# Cargo features lets Rust unit tests link against libpython.
|
||||
pyo3 = { version = "0.28", features = ["abi3-py310", "chrono"] }
|
||||
pyo3 = { version = "0.28", features = ["extension-module", "abi3-py310", "chrono"] }
|
||||
chrono.workspace = true
|
||||
pyo3-async-runtimes = { version = "0.28", features = [
|
||||
"attributes",
|
||||
@@ -43,7 +41,10 @@ tokio.workspace = true
|
||||
libc = "0.2"
|
||||
|
||||
[build-dependencies]
|
||||
pyo3-build-config = { version = "0.28", features = ["abi3-py310"] }
|
||||
pyo3-build-config = { version = "0.28", features = [
|
||||
"extension-module",
|
||||
"abi3-py310",
|
||||
] }
|
||||
|
||||
[features]
|
||||
default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/cos", "lancedb/goosefs", "lancedb/metrics-otel"]
|
||||
|
||||
@@ -38,25 +38,6 @@ Stable releases are created about every 2 weeks. For the latest features and bug
|
||||
pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb
|
||||
```
|
||||
|
||||
### Threading in CPU-limited containers
|
||||
|
||||
LanceDB uses separate pools for compute work and storage I/O. On a container with
|
||||
two visible CPUs, current releases intentionally use one compute worker by default;
|
||||
no manual configuration is needed. If every query logs an I/O core reservation
|
||||
warning on a two-CPU container, upgrade from LanceDB 0.21.1 or earlier.
|
||||
|
||||
The two commonly tuned environment variables control different resources:
|
||||
|
||||
- `LANCE_CPU_THREADS` overrides the number of compute workers. One worker is the
|
||||
appropriate setting for a two-CPU container when an explicit override is needed.
|
||||
- `LANCE_IO_THREADS` controls concurrent storage operations, not reserved CPU
|
||||
cores. Its default can be greater than the number of CPUs because I/O workers
|
||||
spend much of their time waiting for storage.
|
||||
|
||||
Keep the defaults unless measurements show that the workload benefits from an
|
||||
override. See the [Lance threading model](https://lance.org/guide/performance/#threading-model)
|
||||
for the current defaults and tuning guidance.
|
||||
|
||||
## Usage
|
||||
|
||||
### Basic Example
|
||||
|
||||
@@ -101,12 +101,9 @@ azure = ["adlfs>=2024.2.0"]
|
||||
[tool.maturin]
|
||||
python-source = "python"
|
||||
module-name = "lancedb._lancedb"
|
||||
# uv installs the project as an editable package before `uv run`, so keep that
|
||||
# bootstrap build consistent with `maturin develop`.
|
||||
editable-profile = "dev"
|
||||
|
||||
[build-system]
|
||||
requires = ["maturin>=1.10"]
|
||||
requires = ["maturin>=1.4"]
|
||||
build-backend = "maturin"
|
||||
|
||||
[tool.ruff.lint]
|
||||
|
||||
@@ -6,7 +6,7 @@ import importlib.metadata
|
||||
import os
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import timedelta
|
||||
from typing import Dict, Optional, Union, Any, List, Iterable, TYPE_CHECKING
|
||||
from typing import Dict, Optional, Union, Any, List, Iterable
|
||||
|
||||
__version__ = importlib.metadata.version("lancedb")
|
||||
|
||||
@@ -20,7 +20,7 @@ from .db import AsyncConnection, DBConnection, LanceDBConnection
|
||||
from .remote import ClientConfig
|
||||
from .remote.db import RemoteDBConnection
|
||||
from .expr import Expr, col, lit, func
|
||||
from .schema import blob, vector
|
||||
from .schema import blob, vector, BlobType
|
||||
from .job import AsyncJob, Job
|
||||
from .functions import (
|
||||
FunctionArtifactRequest as FunctionArtifactRequest,
|
||||
@@ -29,15 +29,9 @@ from .functions import (
|
||||
FunctionRegistrationRequest as FunctionRegistrationRequest,
|
||||
FunctionVersion as FunctionVersion,
|
||||
PythonRuntimeSpec as PythonRuntimeSpec,
|
||||
RefreshColumnResult as RefreshColumnResult,
|
||||
UdfDefinition as UdfDefinition,
|
||||
udf as udf,
|
||||
)
|
||||
from .materialized_view import (
|
||||
AsyncMaterializedView,
|
||||
MaterializedView,
|
||||
MaterializedViewDefinition,
|
||||
)
|
||||
from .table import AsyncTable, Table
|
||||
from .types import BaseTokenizerType
|
||||
from ._lancedb import Session
|
||||
@@ -49,19 +43,6 @@ from .namespace import (
|
||||
)
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from lance.blob import BlobType as BlobType
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
if name == "BlobType":
|
||||
from .schema import BlobType
|
||||
|
||||
globals()["BlobType"] = BlobType
|
||||
return BlobType
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
def _check_s3_bucket_with_dots(
|
||||
uri: str, storage_options: Optional[Dict[str, str]]
|
||||
) -> None:
|
||||
@@ -192,18 +173,6 @@ def connect(
|
||||
... },
|
||||
... )
|
||||
|
||||
For Azure Blob Storage, credentials can be passed directly without setting
|
||||
environment variables:
|
||||
|
||||
>>> azure_storage_options = {
|
||||
... "account_name": "some-account",
|
||||
... "account_key": "some-key",
|
||||
... }
|
||||
>>> db = lancedb.connect( # doctest: +SKIP
|
||||
... "az://my-container/my-database",
|
||||
... storage_options=azure_storage_options,
|
||||
... )
|
||||
|
||||
For tests and temporary data, use an in-memory database:
|
||||
|
||||
>>> db = lancedb.connect("memory://")
|
||||
@@ -490,10 +459,6 @@ async def connect_async(
|
||||
--------
|
||||
|
||||
>>> import lancedb
|
||||
>>> azure_storage_options = {
|
||||
... "account_name": "some-account",
|
||||
... "account_key": "some-key",
|
||||
... }
|
||||
>>> async def doctest_example():
|
||||
... # For a local directory, provide a path to the database
|
||||
... db = await lancedb.connect_async("~/.lancedb")
|
||||
@@ -501,11 +466,6 @@ async def connect_async(
|
||||
... db = await lancedb.connect_async("s3://my-bucket/lancedb",
|
||||
... storage_options={
|
||||
... "aws_access_key_id": "***"})
|
||||
... # Azure credentials can also be passed directly
|
||||
... db = await lancedb.connect_async(
|
||||
... "az://my-container/my-database",
|
||||
... storage_options=azure_storage_options,
|
||||
... )
|
||||
... # For tests and temporary data, use an in-memory database
|
||||
... db = await lancedb.connect_async("memory://")
|
||||
... # Connect to LanceDB cloud
|
||||
@@ -546,9 +506,6 @@ async def connect_async(
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AsyncMaterializedView",
|
||||
"MaterializedView",
|
||||
"MaterializedViewDefinition",
|
||||
"connect",
|
||||
"connect_async",
|
||||
"tokenize",
|
||||
|
||||
@@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Optional, Union
|
||||
import pyarrow as pa
|
||||
|
||||
from .expr import Expr
|
||||
from .schema import row_addressable_blob_v2_paths
|
||||
from .schema import blob_v2_column_paths
|
||||
from .types import BlobMode, QueryProjection, QueryProjectionSpec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -119,7 +119,7 @@ def blob_v2_projection_sources(
|
||||
schema: pa.Schema,
|
||||
projection: QueryProjection,
|
||||
) -> dict[str, str]:
|
||||
blob_columns = row_addressable_blob_v2_paths(schema)
|
||||
blob_columns = blob_v2_column_paths(schema)
|
||||
if not blob_columns:
|
||||
return {}
|
||||
columns = set(blob_columns)
|
||||
@@ -140,9 +140,7 @@ def v2_projection_needs_row_id(
|
||||
) -> bool:
|
||||
if with_row_id:
|
||||
return False
|
||||
return projection_includes_blob_column(
|
||||
projection, row_addressable_blob_v2_paths(schema)
|
||||
)
|
||||
return projection_includes_blob_column(projection, blob_v2_column_paths(schema))
|
||||
|
||||
|
||||
def blob_auto_row_id_for_scan(
|
||||
@@ -272,8 +270,7 @@ def _iter_projection_pairs(
|
||||
if isinstance(expr, str):
|
||||
yield name, expr
|
||||
elif isinstance(expr, Expr):
|
||||
source = expr._column_name()
|
||||
yield name, source if source is not None else expr.to_sql()
|
||||
yield name, expr.to_sql()
|
||||
return
|
||||
for column in projection:
|
||||
if isinstance(column, str):
|
||||
@@ -283,8 +280,7 @@ def _iter_projection_pairs(
|
||||
if isinstance(expr, str):
|
||||
yield name, expr
|
||||
elif isinstance(expr, Expr):
|
||||
source = expr._column_name()
|
||||
yield name, source if source is not None else expr.to_sql()
|
||||
yield name, expr.to_sql()
|
||||
|
||||
|
||||
def _set_blob_column(tbl: pa.Table, output_name: str, blobs: pa.Array) -> pa.Table:
|
||||
|
||||
@@ -87,7 +87,6 @@ class PyExpr:
|
||||
def contains(self, substr: "PyExpr") -> "PyExpr": ...
|
||||
def isin(self, values: List["PyExpr"]) -> "PyExpr": ...
|
||||
def cast(self, data_type: pa.DataType) -> "PyExpr": ...
|
||||
def column_name(self) -> Optional[str]: ...
|
||||
def to_sql(self) -> str: ...
|
||||
|
||||
def expr_col(name: str) -> PyExpr: ...
|
||||
@@ -148,7 +147,7 @@ class Connection(object):
|
||||
limit: Optional[int],
|
||||
) -> list[str]: ... # Deprecated: Use list_tables instead
|
||||
def job(self, job_id: str) -> Job: ...
|
||||
async def create_function_async(self, request_json: str) -> Job: ...
|
||||
async def create_function_async(self, request_json: str) -> FunctionJob: ...
|
||||
async def get_function(self, name: str, version: str) -> str: ...
|
||||
async def list_jobs(self) -> List[JobInfo]: ...
|
||||
async def get_job(self, job_id: str) -> Optional[JobDescription]: ...
|
||||
@@ -198,15 +197,6 @@ class Connection(object):
|
||||
cur_namespace_path: Optional[List[str]] = None,
|
||||
new_namespace_path: Optional[List[str]] = None,
|
||||
) -> None: ...
|
||||
async def create_materialized_view(
|
||||
self,
|
||||
name: str,
|
||||
source: str,
|
||||
projections: Optional[List[Tuple[str, str]]] = None,
|
||||
filter: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
) -> Table: ...
|
||||
async def list_materialized_views(self) -> List[str]: ...
|
||||
async def drop_table(
|
||||
self, name: str, namespace_path: Optional[List[str]] = None
|
||||
) -> None: ...
|
||||
@@ -235,7 +225,14 @@ class Job:
|
||||
@property
|
||||
def id(self) -> Optional[str]: ...
|
||||
async def status(self) -> str: ...
|
||||
async def wait(self) -> Optional[str]: ...
|
||||
async def wait(self) -> None: ...
|
||||
async def cancel(self) -> None: ...
|
||||
|
||||
class FunctionJob:
|
||||
@property
|
||||
def id(self) -> Optional[str]: ...
|
||||
async def status(self) -> str: ...
|
||||
async def wait(self) -> str: ...
|
||||
async def cancel(self) -> None: ...
|
||||
|
||||
class JobInfo:
|
||||
@@ -284,7 +281,6 @@ class Table:
|
||||
mode: Literal["append", "overwrite"],
|
||||
progress: Optional[Any] = None,
|
||||
write_parallelism: Optional[int] = None,
|
||||
allow_external_blob_outside_bases: bool = False,
|
||||
) -> AddResult: ...
|
||||
async def update(
|
||||
self, updates: Dict[str, str], where: Optional[str]
|
||||
@@ -359,9 +355,6 @@ class Table:
|
||||
) -> AddColumnsResult: ...
|
||||
async def refresh_column(self, column: str) -> RefreshColumnResult: ...
|
||||
async def refresh_column_async(self, column: str) -> Job: ...
|
||||
async def refresh_materialized_view(
|
||||
self, full: bool = False, source_version: Optional[int] = None
|
||||
) -> RefreshMaterializedViewResult: ...
|
||||
async def add_columns_with_schema(self, schema: pa.Schema) -> AddColumnsResult: ...
|
||||
async def alter_columns(
|
||||
self, columns: list[dict[str, Any]]
|
||||
@@ -427,7 +420,7 @@ class Branches:
|
||||
async def checkout(self, name: str, version: Optional[int] = None) -> Table: ...
|
||||
async def delete(self, name: str) -> None: ...
|
||||
async def diff(self, from_branch: str) -> Dict[str, Any]: ...
|
||||
async def cherry_pick(
|
||||
async def merge(
|
||||
self, from_branch: str, dry_run: bool = False
|
||||
) -> Dict[str, Any]: ...
|
||||
|
||||
@@ -609,7 +602,6 @@ class PyQueryRequest:
|
||||
filter: Optional[Union[str, bytes]]
|
||||
full_text_search: Optional[FullTextQuery]
|
||||
select: Optional[Union[str, List[str]]]
|
||||
select_source_columns: Optional[Dict[str, str]]
|
||||
fast_search: Optional[bool]
|
||||
with_row_id: Optional[bool]
|
||||
use_lsm: Optional[bool]
|
||||
@@ -712,12 +704,6 @@ class RefreshColumnResult:
|
||||
rows_filled: int
|
||||
version: int
|
||||
|
||||
class RefreshMaterializedViewResult:
|
||||
mode: str
|
||||
rows_written: int
|
||||
source_version: int
|
||||
version: int
|
||||
|
||||
class AlterColumnsResult:
|
||||
version: int
|
||||
|
||||
|
||||
+9
-206
@@ -16,7 +16,6 @@ from typing import (
|
||||
Iterable,
|
||||
List,
|
||||
Literal,
|
||||
Mapping,
|
||||
Optional,
|
||||
Union,
|
||||
)
|
||||
@@ -47,13 +46,7 @@ from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
|
||||
from . import __version__
|
||||
from ._lancedb import connect as lancedb_connect # type: ignore
|
||||
from .functions import FunctionVersion, UdfDefinition
|
||||
from .job import AsyncJob, Job, _typed_job
|
||||
from .materialized_view import (
|
||||
AsyncMaterializedView,
|
||||
MaterializedView,
|
||||
SelectArg,
|
||||
normalize_select,
|
||||
)
|
||||
from .job import AsyncJob, Job, _function_job
|
||||
from .table import (
|
||||
AsyncTable,
|
||||
LanceTable,
|
||||
@@ -517,70 +510,6 @@ class DBConnection(EnforceOverrides):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def create_materialized_view(
|
||||
self,
|
||||
name: str,
|
||||
source: str,
|
||||
*,
|
||||
select: SelectArg = None,
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
) -> MaterializedView:
|
||||
"""Define a materialized view named ``name`` over the table ``source``.
|
||||
|
||||
The view is created empty, with the query recorded in its schema
|
||||
metadata; ``view.refresh()`` computes the rows. The view is a normal
|
||||
table: it can be queried, indexed and searched, and it appears in
|
||||
``table_names``. Local databases only.
|
||||
|
||||
The source table must have stable row ids (create it with the
|
||||
``new_table_enable_stable_row_ids`` storage option): they keep the
|
||||
view's provenance valid across source compactions, and cannot be
|
||||
enabled after a table exists.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name: str
|
||||
The name of the view.
|
||||
source: str
|
||||
The name of the source table, in this database.
|
||||
select: list or dict, optional
|
||||
The view's columns: column names, ``(alias, SQL expression)``
|
||||
pairs, or a dict of the same. Omitting it selects every source
|
||||
column, expanded against the source schema at creation time.
|
||||
where: str, optional
|
||||
SQL predicate; only matching source rows appear in the view.
|
||||
limit: int, optional
|
||||
Cap the view at this many rows, in materialization order.
|
||||
|
||||
Returns
|
||||
-------
|
||||
MaterializedView
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
"materialized views are not supported on this connection type"
|
||||
)
|
||||
|
||||
def open_materialized_view(self, name: str) -> MaterializedView:
|
||||
"""Open the materialized view named ``name``.
|
||||
|
||||
Raises ``ValueError`` if the table exists but is not a materialized
|
||||
view.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
"materialized views are not supported on this connection type"
|
||||
)
|
||||
|
||||
def list_materialized_views(self) -> List[str]:
|
||||
"""The names of the materialized views in this database.
|
||||
|
||||
Found by reading every table's schema, so this costs an open per
|
||||
table.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
"materialized views are not supported on this connection type"
|
||||
)
|
||||
|
||||
def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
|
||||
"""Drop a table from the database.
|
||||
|
||||
@@ -688,35 +617,17 @@ class DBConnection(EnforceOverrides):
|
||||
"""
|
||||
raise NotImplementedError("serialize is not supported for this connection type")
|
||||
|
||||
def create_function(
|
||||
self,
|
||||
definition: UdfDefinition,
|
||||
*,
|
||||
secrets: Optional[Mapping[str, str]] = None,
|
||||
) -> FunctionVersion:
|
||||
def create_function(self, definition: UdfDefinition) -> FunctionVersion:
|
||||
"""Register a scalar Python UDF and wait for its immutable version.
|
||||
|
||||
``secrets`` must contain exactly the names declared by
|
||||
``@udf(secrets=[...])``. Values are sent in the create request and
|
||||
stored server-side in the private execution artifact; returned
|
||||
Function and Job metadata contain only the declared names.
|
||||
This is the blocking counterpart of :meth:`create_function_async`.
|
||||
Local connections raise ``NotImplementedError``.
|
||||
"""
|
||||
return self.create_function_async(definition, secrets=secrets).wait()
|
||||
return self.create_function_async(definition).wait()
|
||||
|
||||
def create_function_async(
|
||||
self,
|
||||
definition: UdfDefinition,
|
||||
*,
|
||||
secrets: Optional[Mapping[str, str]] = None,
|
||||
) -> Job[FunctionVersion]:
|
||||
def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
|
||||
"""Register a scalar Python UDF through the remote Function catalog.
|
||||
|
||||
``secrets`` must contain exactly the names declared by
|
||||
``@udf(secrets=[...])``. Values are sent in the create request and
|
||||
stored server-side in the private execution artifact; returned
|
||||
Function and Job metadata contain only the declared names.
|
||||
Submission returns a typed job. The immutable Function version becomes
|
||||
available only when :meth:`Job.wait` succeeds. Local connections raise
|
||||
``NotImplementedError``.
|
||||
@@ -1225,58 +1136,6 @@ class LanceDBConnection(DBConnection):
|
||||
tbl.checkout(version)
|
||||
return tbl
|
||||
|
||||
@override
|
||||
def create_materialized_view(
|
||||
self,
|
||||
name: str,
|
||||
source: str,
|
||||
*,
|
||||
select: SelectArg = None,
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
) -> MaterializedView:
|
||||
"""Define a materialized view named ``name`` over the table ``source``.
|
||||
See
|
||||
[DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view].
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> import lancedb
|
||||
>>> db = lancedb.connect(
|
||||
... "./.lancedb",
|
||||
... storage_options={"new_table_enable_stable_row_ids": "true"},
|
||||
... )
|
||||
>>> data = [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]
|
||||
>>> table = db.create_table("people", data)
|
||||
>>> view = db.create_materialized_view(
|
||||
... "adults",
|
||||
... "people",
|
||||
... select=["name", ("shout", "upper(name)")],
|
||||
... where="age >= 18",
|
||||
... )
|
||||
>>> result = view.refresh()
|
||||
>>> result.rows_written
|
||||
1
|
||||
"""
|
||||
LOOP.run(
|
||||
self._conn.create_materialized_view(
|
||||
name, source, select=select, where=where, limit=limit
|
||||
)
|
||||
)
|
||||
return MaterializedView(self.open_table(name))
|
||||
|
||||
@override
|
||||
def open_materialized_view(self, name: str) -> MaterializedView:
|
||||
"""Open the materialized view named ``name``."""
|
||||
view = MaterializedView(self.open_table(name))
|
||||
view.definition
|
||||
return view
|
||||
|
||||
@override
|
||||
def list_materialized_views(self) -> List[str]:
|
||||
"""The names of the materialized views in this database."""
|
||||
return LOOP.run(self._conn.list_materialized_views())
|
||||
|
||||
def clone_table(
|
||||
self,
|
||||
target_table_name: str,
|
||||
@@ -1424,13 +1283,8 @@ class LanceDBConnection(DBConnection):
|
||||
return Job(self._conn.job(job_id))
|
||||
|
||||
@override
|
||||
def create_function_async(
|
||||
self,
|
||||
definition: UdfDefinition,
|
||||
*,
|
||||
secrets: Optional[Mapping[str, str]] = None,
|
||||
) -> Job[FunctionVersion]:
|
||||
job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets))
|
||||
def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
|
||||
job = LOOP.run(self._conn.create_function_async(definition))
|
||||
return Job(job)
|
||||
|
||||
@override
|
||||
@@ -2052,50 +1906,6 @@ class AsyncConnection(object):
|
||||
await tbl.checkout(version)
|
||||
return tbl
|
||||
|
||||
async def create_materialized_view(
|
||||
self,
|
||||
name: str,
|
||||
source: str,
|
||||
*,
|
||||
select: SelectArg = None,
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
) -> AsyncMaterializedView:
|
||||
"""Define a materialized view named ``name`` over the table ``source``.
|
||||
See
|
||||
[DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view].
|
||||
"""
|
||||
inner = await self._inner.create_materialized_view(
|
||||
name,
|
||||
source,
|
||||
projections=normalize_select(select),
|
||||
filter=where,
|
||||
limit=limit,
|
||||
)
|
||||
return AsyncMaterializedView(AsyncTable(inner))
|
||||
|
||||
async def open_materialized_view(self, name: str) -> AsyncMaterializedView:
|
||||
"""Open the materialized view named ``name``.
|
||||
|
||||
Raises ``ValueError`` if the table exists but is not a materialized
|
||||
view.
|
||||
"""
|
||||
if self.uri.startswith("db://"):
|
||||
raise NotImplementedError(
|
||||
"materialized views are supported only on local databases"
|
||||
)
|
||||
view = AsyncMaterializedView(await self.open_table(name))
|
||||
await view.definition()
|
||||
return view
|
||||
|
||||
async def list_materialized_views(self) -> List[str]:
|
||||
"""The names of the materialized views in this database.
|
||||
|
||||
Found by reading every table's schema, so this costs an open per
|
||||
table.
|
||||
"""
|
||||
return await self._inner.list_materialized_views()
|
||||
|
||||
async def clone_table(
|
||||
self,
|
||||
target_table_name: str,
|
||||
@@ -2249,26 +2059,19 @@ class AsyncConnection(object):
|
||||
return AsyncJob(self._inner.job(job_id))
|
||||
|
||||
async def create_function_async(
|
||||
self,
|
||||
definition: UdfDefinition,
|
||||
*,
|
||||
secrets: Optional[Mapping[str, str]] = None,
|
||||
self, definition: UdfDefinition
|
||||
) -> AsyncJob[FunctionVersion]:
|
||||
"""Register a scalar Python UDF through the remote Function catalog.
|
||||
|
||||
``secrets`` must contain exactly the names declared by
|
||||
``@udf(secrets=[...])``. Values are sent in the create request and
|
||||
stored server-side in the private execution artifact; returned
|
||||
Function and Job metadata contain only the declared names.
|
||||
The returned typed job resolves to the immutable Function version.
|
||||
Local connections raise ``NotImplementedError``.
|
||||
"""
|
||||
if not isinstance(definition, UdfDefinition):
|
||||
raise TypeError("create_function_async requires a @udf definition")
|
||||
inner = await self._inner.create_function_async(
|
||||
definition._submission_json(secrets)
|
||||
definition.registration_request.to_canonical_json()
|
||||
)
|
||||
return _typed_job(inner, FunctionVersion.from_json)
|
||||
return _function_job(inner)
|
||||
|
||||
async def get_function(self, name: str, *, version: str) -> FunctionVersion:
|
||||
"""Open one exact immutable Function version from the remote catalog."""
|
||||
|
||||
@@ -249,10 +249,6 @@ class Expr:
|
||||
|
||||
# ── utilities ────────────────────────────────────────────────────────────
|
||||
|
||||
def _column_name(self) -> str | None:
|
||||
"""Return the source name when this is a bare column expression."""
|
||||
return self._inner.column_name()
|
||||
|
||||
def to_sql(self) -> str:
|
||||
"""Render the expression as a SQL string (useful for debugging)."""
|
||||
return self._inner.to_sql()
|
||||
@@ -316,7 +312,7 @@ def func(name: str, *args: ExprLike) -> Expr:
|
||||
--------
|
||||
>>> from lancedb.expr import col, func
|
||||
>>> func("lower", col("name"))
|
||||
Expr(lower(`name`))
|
||||
Expr(lower(name))
|
||||
"""
|
||||
inner_args = [_coerce(a)._inner for a in args]
|
||||
return Expr(expr_func(name, inner_args))
|
||||
|
||||
@@ -1,30 +1,26 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
"""Canonical Function values exchanged with LanceDB Enterprise services.
|
||||
"""Canonical values exchanged with LanceDB Enterprise Function services.
|
||||
|
||||
These immutable models contain client/wire state only. Catalog persistence,
|
||||
environment bake, secret resolution, and execution are owned by Sophon.
|
||||
``RefreshColumnResult`` is also the backend-neutral result of a local
|
||||
expression-backed refresh job.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import builtins
|
||||
import base64
|
||||
import functools
|
||||
import hashlib
|
||||
import importlib
|
||||
import inspect
|
||||
import symtable
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
import textwrap
|
||||
import types
|
||||
import uuid
|
||||
from collections.abc import Mapping
|
||||
from datetime import date, datetime
|
||||
from typing import (
|
||||
@@ -222,7 +218,6 @@ class PythonEnvironmentSpec(_RemoteValue):
|
||||
|
||||
kind: str
|
||||
packages: tuple[str, ...] = ()
|
||||
channels: tuple[str, ...] = ()
|
||||
path: Optional[str] = None
|
||||
modules: tuple[str, ...] = ()
|
||||
image: Optional[str] = None
|
||||
@@ -276,7 +271,7 @@ class FunctionVersion(_RemoteValue):
|
||||
|
||||
Every input must be a direct [lancedb.col][lancedb.expr.col]
|
||||
reference. The returned application is immutable and retains a
|
||||
named-struct output as one binding, so every row's sibling values
|
||||
named-struct output as one sibling group, so every row's sibling values
|
||||
come from one logical Function evaluation. Map result fields to table
|
||||
columns with
|
||||
[FunctionApplication.rename][lancedb.functions.FunctionApplication.rename],
|
||||
@@ -326,14 +321,15 @@ class FunctionVersion(_RemoteValue):
|
||||
function=FunctionVersionRef(name=self.name, version=self.version),
|
||||
inputs=tuple(bindings),
|
||||
output=self.signature.output,
|
||||
group_id=f"fg_{uuid.uuid4().hex}",
|
||||
)
|
||||
|
||||
|
||||
class FunctionRegistrationRequest(_RemoteValue):
|
||||
"""Stable remote registration envelope produced by :func:`udf`.
|
||||
|
||||
Only secret names are represented. Secret values are supplied separately
|
||||
when the definition is submitted and are not part of this durable value.
|
||||
Only secret names are represented. Secret values are resolved inside the
|
||||
remote service and have no client request field.
|
||||
"""
|
||||
|
||||
name: str
|
||||
@@ -369,7 +365,7 @@ class ApplicationInput(_OpenRemoteValue):
|
||||
class FunctionApplication(_OpenRemoteValue):
|
||||
"""Immutable pre-declaration application of an exact Function version.
|
||||
|
||||
A named-struct output remains one application through table
|
||||
A named-struct output remains one grouped application through table
|
||||
declaration and execution.
|
||||
[FunctionApplication.rename][lancedb.functions.FunctionApplication.rename]
|
||||
records the result-field to table-column mapping without splitting sibling
|
||||
@@ -379,6 +375,7 @@ class FunctionApplication(_OpenRemoteValue):
|
||||
function: FunctionVersionRef
|
||||
inputs: tuple[ApplicationInput, ...]
|
||||
output: FunctionOutput
|
||||
group_id: str
|
||||
columns: Mapping[str, str] = Field(default_factory=dict)
|
||||
|
||||
def _known_dict(self) -> dict[str, Any]:
|
||||
@@ -450,10 +447,12 @@ class OutputMapping(_RemoteValue):
|
||||
|
||||
|
||||
class FunctionBinding(_RemoteValue):
|
||||
"""Immutable Function binding persisted by the Enterprise table service."""
|
||||
"""Immutable grouped binding persisted by the Enterprise table service."""
|
||||
|
||||
binding_id: str
|
||||
revision: _UInt64
|
||||
function: FunctionVersionRef
|
||||
group_id: str
|
||||
inputs: tuple[InputBinding, ...]
|
||||
outputs: tuple[OutputMapping, ...]
|
||||
input_schema: Optional[Mapping[str, Any]] = None
|
||||
@@ -461,11 +460,7 @@ class FunctionBinding(_RemoteValue):
|
||||
|
||||
|
||||
class RefreshColumnResult(_RemoteValue):
|
||||
"""Terminal result of an expression-backed or Function-backed refresh Job.
|
||||
|
||||
Local jobs produce this value in process. LanceDB Cloud and Enterprise
|
||||
decode the same value from the durable server-job terminal payload.
|
||||
"""
|
||||
"""Terminal result of a remote Function-column refresh Job."""
|
||||
|
||||
rows_assigned: _UInt64
|
||||
rows_failed: _UInt64
|
||||
@@ -486,80 +481,61 @@ class RefreshColumnResult(_RemoteValue):
|
||||
|
||||
_FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$")
|
||||
_SECRET_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
# Keep this byte limit aligned with Sophon's MAX_FUNCTION_SECRET_VALUE_BYTES.
|
||||
_MAX_FUNCTION_SECRET_VALUE_BYTES = 64 * 1024
|
||||
_MAX_FUNCTION_SECRET_VALUES_BYTES = 512 * 1024
|
||||
|
||||
|
||||
def _validate_secret_value(name: str, value: Any) -> str:
|
||||
"""Validate one secret value before building the create request."""
|
||||
if not isinstance(value, str):
|
||||
raise TypeError(f"Function secret {name!r} value must be a string")
|
||||
if not value:
|
||||
raise ValueError(f"Function secret {name!r} value must be non-empty")
|
||||
if "\0" in value:
|
||||
raise ValueError(f"Function secret {name!r} value must not contain NUL")
|
||||
value_bytes = len(value.encode("utf-8"))
|
||||
if value_bytes > _MAX_FUNCTION_SECRET_VALUE_BYTES:
|
||||
raise ValueError(
|
||||
f"Function secret {name!r} value exceeds the "
|
||||
f"{_MAX_FUNCTION_SECRET_VALUE_BYTES}-byte limit"
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
_GRAMMAR_PRIMITIVES = (
|
||||
(pa.bool_(), "bool"),
|
||||
(pa.int8(), "int8"),
|
||||
(pa.int16(), "int16"),
|
||||
(pa.int32(), "int32"),
|
||||
(pa.int64(), "int64"),
|
||||
(pa.uint8(), "uint8"),
|
||||
(pa.uint16(), "uint16"),
|
||||
(pa.uint32(), "uint32"),
|
||||
(pa.uint64(), "uint64"),
|
||||
(pa.float16(), "float16"),
|
||||
(pa.float32(), "float32"),
|
||||
(pa.float64(), "float64"),
|
||||
(pa.string(), "utf8"),
|
||||
(pa.binary(), "binary"),
|
||||
(pa.date32(), "date32"),
|
||||
(pa.date64(), "date64"),
|
||||
)
|
||||
|
||||
|
||||
def _canonical_arrow_type(data_type: pa.DataType) -> str:
|
||||
"""The server's V1 Function type grammar. Anything outside it is rejected
|
||||
here rather than at registration."""
|
||||
for candidate, name in _GRAMMAR_PRIMITIVES:
|
||||
primitive_types = (
|
||||
(pa.bool_(), "bool"),
|
||||
(pa.int8(), "int8"),
|
||||
(pa.int16(), "int16"),
|
||||
(pa.int32(), "int32"),
|
||||
(pa.int64(), "int64"),
|
||||
(pa.uint8(), "uint8"),
|
||||
(pa.uint16(), "uint16"),
|
||||
(pa.uint32(), "uint32"),
|
||||
(pa.uint64(), "uint64"),
|
||||
(pa.float16(), "float16"),
|
||||
(pa.float32(), "float32"),
|
||||
(pa.float64(), "float64"),
|
||||
(pa.string(), "utf8"),
|
||||
(pa.large_utf8(), "large_utf8"),
|
||||
(pa.binary(), "binary"),
|
||||
(pa.large_binary(), "large_binary"),
|
||||
(pa.date32(), "date32"),
|
||||
(pa.date64(), "date64"),
|
||||
)
|
||||
for candidate, name in primitive_types:
|
||||
if data_type == candidate:
|
||||
return name
|
||||
if pa.types.is_list(data_type) or pa.types.is_large_list(data_type):
|
||||
prefix = "list" if pa.types.is_list(data_type) else "large_list"
|
||||
return f"{prefix}<{_canonical_list_item(data_type)}>"
|
||||
if pa.types.is_fixed_size_list(data_type) and data_type.list_size > 0:
|
||||
if pa.types.is_fixed_size_binary(data_type):
|
||||
return f"fixed_size_binary[{data_type.byte_width}]"
|
||||
if pa.types.is_list(data_type):
|
||||
return f"list<{_canonical_arrow_type(data_type.value_type)}>"
|
||||
if pa.types.is_large_list(data_type):
|
||||
return f"large_list<{_canonical_arrow_type(data_type.value_type)}>"
|
||||
if pa.types.is_fixed_size_list(data_type):
|
||||
return (
|
||||
f"fixed_size_list<{_canonical_list_item(data_type)}, {data_type.list_size}>"
|
||||
f"fixed_size_list<{_canonical_arrow_type(data_type.value_type)}>"
|
||||
f"[{data_type.list_size}]"
|
||||
)
|
||||
if pa.types.is_struct(data_type):
|
||||
fields = ",".join(
|
||||
f"{field.name}:{_canonical_arrow_type(field.type)}" for field in data_type
|
||||
)
|
||||
return f"struct<{fields}>"
|
||||
if pa.types.is_timestamp(data_type):
|
||||
timezone = f",tz={data_type.tz}" if data_type.tz is not None else ""
|
||||
return f"timestamp[{data_type.unit}{timezone}]"
|
||||
if pa.types.is_time32(data_type) or pa.types.is_time64(data_type):
|
||||
return f"time[{data_type.unit}]"
|
||||
if pa.types.is_duration(data_type):
|
||||
return f"duration[{data_type.unit}]"
|
||||
if pa.types.is_decimal(data_type):
|
||||
bit_width = data_type.bit_width
|
||||
return f"decimal{bit_width}({data_type.precision},{data_type.scale})"
|
||||
raise TypeError(f"unsupported Arrow type for Function signature: {data_type}")
|
||||
|
||||
|
||||
def _canonical_list_item(data_type: pa.DataType) -> str:
|
||||
"""The grammar names only the item type; it always means a non-nullable
|
||||
child called `item`, so any other child metadata cannot be represented."""
|
||||
child = data_type.value_field
|
||||
if child.name != "item" or child.nullable or child.metadata:
|
||||
raise TypeError(
|
||||
"unsupported Arrow type for Function signature: list items must be a "
|
||||
f"non-nullable field named 'item', got {child}"
|
||||
)
|
||||
return _canonical_arrow_type(child.type)
|
||||
|
||||
|
||||
def _list_of(item: pa.DataType) -> pa.DataType:
|
||||
return pa.list_(pa.field("item", item, nullable=False))
|
||||
|
||||
|
||||
def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]:
|
||||
nullable = False
|
||||
origin = get_origin(annotation)
|
||||
@@ -607,7 +583,7 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]:
|
||||
value_type, value_nullable = _annotation_type(arguments[0])
|
||||
if value_nullable:
|
||||
raise TypeError("nullable Function list elements are not supported")
|
||||
return _list_of(value_type), nullable
|
||||
return pa.list_(value_type), nullable
|
||||
raise TypeError(f"unsupported Function annotation: {annotation!r}")
|
||||
|
||||
|
||||
@@ -754,104 +730,6 @@ def _literal_source(value: Any) -> str:
|
||||
)
|
||||
|
||||
|
||||
_DYNAMIC_NAMESPACE_ACCESS = frozenset(
|
||||
{"globals", "locals", "vars", "eval", "exec", "compile", "__import__"}
|
||||
)
|
||||
# Modules that hand out namespaces (`sys.modules`, `builtins`, importers,
|
||||
# introspection). The artifact's module namespace holds only the names it was
|
||||
# packaged with, so reaching around it cannot be represented.
|
||||
_NAMESPACE_MODULES = frozenset(
|
||||
{"sys", "builtins", "importlib", "inspect", "gc", "ctypes", "types"}
|
||||
)
|
||||
|
||||
|
||||
def _namespace_acquisition(
|
||||
definition: ast.FunctionDef, references: set[str]
|
||||
) -> list[str]:
|
||||
found = set(references & _DYNAMIC_NAMESPACE_ACCESS)
|
||||
for node in ast.walk(definition):
|
||||
if isinstance(node, ast.Import):
|
||||
found.update(
|
||||
alias.name
|
||||
for alias in node.names
|
||||
if alias.name.split(".")[0] in _NAMESPACE_MODULES
|
||||
)
|
||||
elif isinstance(node, ast.ImportFrom) and node.module:
|
||||
if node.module.split(".")[0] in _NAMESPACE_MODULES:
|
||||
found.add(node.module)
|
||||
return sorted(found)
|
||||
|
||||
|
||||
def _module_references(module_source: str) -> set[str]:
|
||||
"""Names any scope in `module_source` binds or loads at module scope.
|
||||
Python's own scope analysis on the exact text that ships: free variables
|
||||
belong to an enclosing scope inside the function, and postponed
|
||||
annotations are not runtime loads."""
|
||||
|
||||
def visit(table: symtable.SymbolTable, found: set[str]) -> None:
|
||||
for symbol in table.get_symbols():
|
||||
if symbol.is_global() and (
|
||||
symbol.is_referenced() or symbol.is_declared_global()
|
||||
):
|
||||
found.add(symbol.get_name())
|
||||
for child in table.get_children():
|
||||
visit(child, found)
|
||||
|
||||
found: set[str] = set()
|
||||
for table in symtable.symtable(module_source, "<udf>", "exec").get_children():
|
||||
visit(table, found)
|
||||
return found
|
||||
|
||||
|
||||
def _global_source(name: str, value: Any) -> str:
|
||||
"""One module-level line that rebinds `name` to `value` in the artifact:
|
||||
an import for modules and importable classes/functions, a literal otherwise."""
|
||||
if isinstance(value, types.ModuleType):
|
||||
if value.__name__.split(".")[0] in _NAMESPACE_MODULES:
|
||||
raise ValueError(
|
||||
f"@udf cannot package dynamic namespace access: {value.__name__!r}"
|
||||
)
|
||||
try:
|
||||
imported = importlib.import_module(value.__name__)
|
||||
except ImportError:
|
||||
imported = None
|
||||
if imported is not value:
|
||||
raise TypeError(
|
||||
f"Function source references module {name!r} that does not import "
|
||||
f"as {value.__name__!r}"
|
||||
)
|
||||
return f"import {value.__name__} as {name}"
|
||||
module_name = getattr(value, "__module__", None)
|
||||
qualname = getattr(value, "__qualname__", None)
|
||||
if (
|
||||
isinstance(module_name, str)
|
||||
and isinstance(qualname, str)
|
||||
and module_name != "__main__"
|
||||
and "." not in qualname
|
||||
and "<" not in qualname
|
||||
):
|
||||
try:
|
||||
imported = getattr(importlib.import_module(module_name), qualname)
|
||||
except (ImportError, AttributeError):
|
||||
imported = None
|
||||
if imported is value:
|
||||
return f"from {module_name} import {qualname} as {name}"
|
||||
return f"{name} = {_literal_source(value)}"
|
||||
|
||||
|
||||
def _is_recursive_reference(function: Callable[..., Any], name: str) -> bool:
|
||||
"""`name` inside the body means the function itself unless the module has
|
||||
since bound it to something else."""
|
||||
if name != function.__name__:
|
||||
return False
|
||||
bound = function.__globals__.get(name, function)
|
||||
if bound is function:
|
||||
return True
|
||||
# The decorator's own result is the one wrapper known to call `function`
|
||||
# unchanged; any other binding may behave differently from a self-call.
|
||||
return type(bound) is UdfDefinition and bound._function is function
|
||||
|
||||
|
||||
def _package_source(function: Callable[..., Any]) -> bytes:
|
||||
if not inspect.isfunction(function) or inspect.iscoroutinefunction(function):
|
||||
raise TypeError("@udf requires a synchronous Python function")
|
||||
@@ -876,46 +754,23 @@ def _package_source(function: Callable[..., Any]) -> bytes:
|
||||
closure = inspect.getclosurevars(function)
|
||||
if closure.nonlocals:
|
||||
raise ValueError("@udf cannot package functions that capture closure values")
|
||||
function_source = ast.unparse(definition)
|
||||
module_header = "from __future__ import annotations"
|
||||
references = _module_references(f"{module_header}\n\n{function_source}\n")
|
||||
dynamic = _namespace_acquisition(definition, references)
|
||||
if dynamic:
|
||||
raise ValueError(f"@udf cannot package dynamic namespace access: {dynamic!r}")
|
||||
# Resolve every module-scope reference the way the interpreter would: the
|
||||
# function's own globals first (a module global may shadow a builtin, and
|
||||
# nested scopes are not visible to getclosurevars), then its builtins.
|
||||
# The artifact runs under the standard builtins; only the exact mapping is
|
||||
# provably equivalent (a subclass or copy can change lookups and hooks).
|
||||
if function.__builtins__ is not vars(builtins):
|
||||
raise ValueError("@udf cannot package a non-standard builtins environment")
|
||||
globals_source = []
|
||||
unresolved = []
|
||||
for name in sorted(references):
|
||||
if name == function.__name__:
|
||||
if not _is_recursive_reference(function, name):
|
||||
raise ValueError(
|
||||
f"@udf cannot package {name!r}: the module binds that name to "
|
||||
"another value, which the artifact's own definition would shadow"
|
||||
)
|
||||
continue
|
||||
if name in function.__globals__:
|
||||
globals_source.append(_global_source(name, function.__globals__[name]))
|
||||
elif hasattr(builtins, name):
|
||||
pass
|
||||
else:
|
||||
unresolved.append(name)
|
||||
if unresolved:
|
||||
if closure.unbound:
|
||||
raise ValueError(
|
||||
f"@udf source contains unresolved global names: {unresolved!r}"
|
||||
f"@udf source contains unresolved global names: {sorted(closure.unbound)!r}"
|
||||
)
|
||||
globals_source = []
|
||||
for name, value in sorted(closure.globals.items()):
|
||||
if isinstance(value, types.ModuleType):
|
||||
globals_source.append(f"import {value.__name__} as {name}")
|
||||
else:
|
||||
globals_source.append(f"{name} = {_literal_source(value)}")
|
||||
|
||||
parts = [module_header]
|
||||
function_source = ast.unparse(definition)
|
||||
parts = ["from __future__ import annotations"]
|
||||
if globals_source:
|
||||
parts.extend(["", *globals_source])
|
||||
parts.extend(["", function_source, ""])
|
||||
packaged = "\n".join(parts)
|
||||
return packaged.encode("utf-8")
|
||||
return "\n".join(parts).encode("utf-8")
|
||||
|
||||
|
||||
class UdfDefinition:
|
||||
@@ -938,25 +793,13 @@ class UdfDefinition:
|
||||
env: Mapping[str, str],
|
||||
secrets: tuple[str, ...],
|
||||
python_version: Optional[str],
|
||||
conda: tuple[str, ...] = (),
|
||||
conda_channels: tuple[str, ...] = (),
|
||||
):
|
||||
function_name = name or function.__name__
|
||||
if not _FUNCTION_NAME.fullmatch(function_name):
|
||||
raise ValueError(f"invalid Function name: {function_name!r}")
|
||||
if pip and conda:
|
||||
raise ValueError("a Function environment is pip or conda, not both")
|
||||
if conda_channels and not conda:
|
||||
raise ValueError("conda_channels requires conda packages")
|
||||
packages = tuple(sorted(set(conda if conda else pip)))
|
||||
packages = tuple(sorted(set(pip)))
|
||||
if any(not package or package != package.strip() for package in packages):
|
||||
raise ValueError("package requirements must be non-empty and trimmed")
|
||||
if conda:
|
||||
environment_spec = PythonEnvironmentSpec(
|
||||
kind="conda", packages=packages, channels=tuple(conda_channels)
|
||||
)
|
||||
else:
|
||||
environment_spec = PythonEnvironmentSpec(kind="pip", packages=packages)
|
||||
raise ValueError("pip requirements must be non-empty and trimmed")
|
||||
environment = dict(env)
|
||||
if any(
|
||||
not isinstance(key, str) or not isinstance(value, str)
|
||||
@@ -974,6 +817,7 @@ class UdfDefinition:
|
||||
raise ValueError(
|
||||
f"Function env and secret names must be disjoint: {sorted(overlap)!r}"
|
||||
)
|
||||
|
||||
signature = _infer_signature(function, input_schema, output_schema)
|
||||
source = _package_source(function)
|
||||
digest = f"sha256:{hashlib.sha256(source).hexdigest()}"
|
||||
@@ -981,7 +825,7 @@ class UdfDefinition:
|
||||
kind="python",
|
||||
python_version=python_version
|
||||
or f"{sys.version_info.major}.{sys.version_info.minor}",
|
||||
environment=environment_spec,
|
||||
environment=PythonEnvironmentSpec(kind="pip", packages=packages),
|
||||
env=environment,
|
||||
)
|
||||
self._function = function
|
||||
@@ -1008,59 +852,9 @@ class UdfDefinition:
|
||||
|
||||
@property
|
||||
def registration_request(self) -> FunctionRegistrationRequest:
|
||||
"""The immutable, value-free client model for a Function submission."""
|
||||
"""The immutable request sent by ``create_function_async``."""
|
||||
return self._request
|
||||
|
||||
def _submission_json(self, secrets: Optional[Mapping[str, str]]) -> str:
|
||||
"""Build one registration submission without retaining values on self."""
|
||||
if secrets is None:
|
||||
secret_values: Mapping[str, str] = {}
|
||||
elif not isinstance(secrets, Mapping):
|
||||
raise TypeError("Function secrets must be a mapping of names to strings")
|
||||
else:
|
||||
secret_values = secrets
|
||||
|
||||
if any(not isinstance(name, str) for name in secret_values):
|
||||
raise TypeError("Function secret names must be strings")
|
||||
expected = set(self._request.required_secrets)
|
||||
provided = set(secret_values)
|
||||
if provided != expected:
|
||||
missing = sorted(expected - provided)
|
||||
unexpected = sorted(provided - expected)
|
||||
details = []
|
||||
if missing:
|
||||
details.append(f"missing: {missing!r}")
|
||||
if unexpected:
|
||||
details.append(f"unexpected: {unexpected!r}")
|
||||
raise ValueError(
|
||||
"Function secret values must exactly match the declared secrets ("
|
||||
+ "; ".join(details)
|
||||
+ ")"
|
||||
)
|
||||
|
||||
canonical_values = {}
|
||||
total_bytes = 0
|
||||
for name in sorted(secret_values):
|
||||
value = _validate_secret_value(name, secret_values[name])
|
||||
total_bytes += len(value.encode("utf-8"))
|
||||
if total_bytes > _MAX_FUNCTION_SECRET_VALUES_BYTES:
|
||||
raise ValueError(
|
||||
"Function secret values exceed the "
|
||||
f"{_MAX_FUNCTION_SECRET_VALUES_BYTES}-byte request limit"
|
||||
)
|
||||
canonical_values[name] = value
|
||||
|
||||
submission = self._request._known_dict()
|
||||
if canonical_values:
|
||||
submission["secret_values"] = canonical_values
|
||||
return json.dumps(
|
||||
submission,
|
||||
ensure_ascii=False,
|
||||
allow_nan=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
return self._function(*args, **kwargs)
|
||||
|
||||
@@ -1080,8 +874,6 @@ def udf(
|
||||
env: Optional[Mapping[str, str]] = None,
|
||||
secrets: tuple[str, ...] | list[str] = (),
|
||||
python_version: Optional[str] = None,
|
||||
conda: tuple[str, ...] | list[str] = (),
|
||||
conda_channels: tuple[str, ...] | list[str] = (),
|
||||
) -> Callable[[Callable[..., Any]], UdfDefinition]: ...
|
||||
|
||||
|
||||
@@ -1095,8 +887,6 @@ def udf(
|
||||
env: Optional[Mapping[str, str]] = None,
|
||||
secrets: tuple[str, ...] | list[str] = (),
|
||||
python_version: Optional[str] = None,
|
||||
conda: tuple[str, ...] | list[str] = (),
|
||||
conda_channels: tuple[str, ...] | list[str] = (),
|
||||
):
|
||||
"""Prepare a scalar Python callable for remote Function registration.
|
||||
|
||||
@@ -1119,25 +909,14 @@ def udf(
|
||||
provided together with ``input_schema``.
|
||||
pip : sequence of str, optional
|
||||
Pip requirements for the remote environment.
|
||||
conda : sequence of str, optional
|
||||
Conda packages for the remote environment, instead of ``pip``.
|
||||
conda_channels : sequence of str, optional
|
||||
Conda channels in priority order; requires ``conda``.
|
||||
env : mapping of str to str, optional
|
||||
Non-secret environment variables. Use ``secrets`` for credentials.
|
||||
secrets : sequence of str, optional
|
||||
Names of secrets required by the callable. Supply their values separately
|
||||
to ``create_function`` or ``create_function_async``.
|
||||
Names of secrets resolved by the remote service. Secret values are not
|
||||
accepted by this API or included in the registration request.
|
||||
python_version : str, optional
|
||||
Remote Python major/minor version. Defaults to the client version.
|
||||
|
||||
The packaged artifact is a snapshot: the function source plus exactly
|
||||
the module-level names it references (modules as imports, importable
|
||||
classes and functions as imports, literals inline). Code that reaches the
|
||||
module namespace another way -- ``globals()``/``eval``, ``sys.modules``,
|
||||
``builtins`` -- is rejected where it can be seen and otherwise
|
||||
unsupported; closures and a non-standard ``__builtins__`` are rejected.
|
||||
|
||||
Returns
|
||||
-------
|
||||
UdfDefinition
|
||||
@@ -1154,10 +933,6 @@ def udf(
|
||||
... return value * 2
|
||||
>>> score(1.5)
|
||||
3.0
|
||||
>>> db.create_function( # doctest: +SKIP
|
||||
... score, secrets={"MODEL_TOKEN": "user-secret-value"}
|
||||
... )
|
||||
|
||||
"""
|
||||
|
||||
def decorate(target: Callable[..., Any]) -> UdfDefinition:
|
||||
@@ -1170,8 +945,6 @@ def udf(
|
||||
env={} if env is None else env,
|
||||
secrets=tuple(secrets),
|
||||
python_version=python_version,
|
||||
conda=tuple(conda),
|
||||
conda_channels=tuple(conda_channels),
|
||||
)
|
||||
|
||||
if function is None:
|
||||
|
||||
@@ -7,7 +7,6 @@ from typing import List, Literal, Optional
|
||||
from ._lancedb import (
|
||||
IndexConfig,
|
||||
)
|
||||
from .query import DocumentGranularity
|
||||
from .types import BaseTokenizerType
|
||||
|
||||
lang_mapping = {
|
||||
@@ -122,11 +121,6 @@ class FTS:
|
||||
|
||||
>>> config = FTS(block_size=256)
|
||||
|
||||
Create an index that treats each deepest-list element as one document:
|
||||
|
||||
>>> from lancedb.query import DocumentGranularity
|
||||
>>> config = FTS(document_granularity=DocumentGranularity.LIST_ELEMENT)
|
||||
|
||||
Attributes
|
||||
----------
|
||||
with_position : bool, default False
|
||||
@@ -169,20 +163,6 @@ class FTS:
|
||||
The number of documents per compressed posting block. Supported values
|
||||
are 128 and 256. A value of 256 uses the experimental FTS V3 format
|
||||
and may introduce breaking changes.
|
||||
memory_limit : int, optional
|
||||
The total memory limit in MiB for the local FTS build stage. The limit
|
||||
is divided evenly among indexing workers. This build-only setting is
|
||||
not persisted with the index and does not apply to remote tables.
|
||||
num_workers : int, optional
|
||||
The number of workers for a local FTS build. By default Lance uses
|
||||
roughly half of the available CPU cores. The effective value is
|
||||
limited by the available compute capacity. This build-only setting is
|
||||
not persisted with the index and does not apply to remote tables.
|
||||
document_granularity : DocumentGranularity, default ROW
|
||||
``ROW`` treats the selected text in one table row as one document.
|
||||
``LIST_ELEMENT`` treats each element of the deepest list on the indexed
|
||||
field path as one document and returns its physical coordinates in
|
||||
``_doc_index`` for matching queries.
|
||||
|
||||
Notes
|
||||
-----
|
||||
@@ -205,9 +185,6 @@ class FTS:
|
||||
prefix_only: bool = False
|
||||
block_size: int = 128
|
||||
custom_stop_words: Optional[List[str]] = None
|
||||
memory_limit: Optional[int] = None
|
||||
num_workers: Optional[int] = None
|
||||
document_granularity: DocumentGranularity = DocumentGranularity.ROW
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -5,11 +5,12 @@
|
||||
|
||||
import asyncio
|
||||
from datetime import timedelta
|
||||
from typing import Any, Callable, Generic, Optional, TypeVar, cast
|
||||
from typing import Any, Generic, Optional, TypeVar, cast
|
||||
|
||||
from lancedb.background_loop import LOOP
|
||||
|
||||
from . import _lancedb
|
||||
from .functions import FunctionVersion
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
@@ -17,18 +18,11 @@ T = TypeVar("T")
|
||||
class AsyncJob(Generic[T]):
|
||||
"""A handle to an operation that may still be running.
|
||||
|
||||
The operation may already be complete when the handle is created. ``T``
|
||||
is the endpoint's terminal result type; unit-result jobs resolve to
|
||||
``None``.
|
||||
The operation may already be complete when the handle is created.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
inner: Optional[Any],
|
||||
result_decoder: Optional[Callable[[Any], T]] = None,
|
||||
):
|
||||
def __init__(self, inner: Optional[Any]):
|
||||
self._inner = inner
|
||||
self._result_decoder = result_decoder
|
||||
|
||||
@property
|
||||
def id(self) -> Optional[str]:
|
||||
@@ -56,21 +50,17 @@ class AsyncJob(Generic[T]):
|
||||
async def wait(self, timeout: Optional[timedelta] = None) -> T:
|
||||
"""Wait until the operation reaches a terminal state.
|
||||
|
||||
Returns the endpoint's typed result, or ``None`` for a unit-result
|
||||
job.
|
||||
|
||||
Raises `JobFailedError` if the operation failed, `JobCancelledError`
|
||||
if it was cancelled, and `TimeoutError` if `timeout` elapses first.
|
||||
"""
|
||||
if self._inner is None:
|
||||
return cast(T, None)
|
||||
if timeout is None:
|
||||
result = await self._inner.wait()
|
||||
else:
|
||||
result = await asyncio.wait_for(self._inner.wait(), timeout.total_seconds())
|
||||
if self._result_decoder is not None:
|
||||
return self._result_decoder(result)
|
||||
return cast(T, result)
|
||||
return cast(T, await self._inner.wait())
|
||||
return cast(
|
||||
T,
|
||||
await asyncio.wait_for(self._inner.wait(), timeout.total_seconds()),
|
||||
)
|
||||
|
||||
async def cancel(self):
|
||||
"""Request cancellation. Cancelling a finished operation is a no-op."""
|
||||
@@ -80,7 +70,7 @@ class AsyncJob(Generic[T]):
|
||||
|
||||
|
||||
class Job(Generic[T]):
|
||||
"""Synchronous counterpart of `AsyncJob` with the same result type."""
|
||||
"""Synchronous counterpart of `AsyncJob`."""
|
||||
|
||||
def __init__(self, inner: Optional[AsyncJob[T]]):
|
||||
self._inner = inner
|
||||
@@ -106,9 +96,6 @@ class Job(Generic[T]):
|
||||
def wait(self, timeout: Optional[timedelta] = None) -> T:
|
||||
"""Block until the operation reaches a terminal state.
|
||||
|
||||
Returns the endpoint's typed result, or ``None`` for a unit-result
|
||||
job.
|
||||
|
||||
Raises `JobFailedError` if the operation failed, `JobCancelledError`
|
||||
if it was cancelled, and `TimeoutError` if `timeout` elapses first.
|
||||
"""
|
||||
@@ -123,8 +110,23 @@ class Job(Generic[T]):
|
||||
LOOP.run(self._inner.cancel())
|
||||
|
||||
|
||||
def _typed_job(
|
||||
inner: "_lancedb.Job", result_decoder: Callable[[str], T]
|
||||
) -> AsyncJob[T]:
|
||||
"""Bind an internal JSON-producing job to its public result model."""
|
||||
return AsyncJob(inner, result_decoder)
|
||||
class _FunctionJobAdapter:
|
||||
def __init__(self, inner: "_lancedb.FunctionJob"):
|
||||
self._inner = inner
|
||||
|
||||
@property
|
||||
def id(self) -> Optional[str]:
|
||||
return self._inner.id
|
||||
|
||||
async def status(self) -> str:
|
||||
return await self._inner.status()
|
||||
|
||||
async def wait(self) -> FunctionVersion:
|
||||
return FunctionVersion.from_json(await self._inner.wait())
|
||||
|
||||
async def cancel(self):
|
||||
await self._inner.cancel()
|
||||
|
||||
|
||||
def _function_job(inner: "_lancedb.FunctionJob") -> AsyncJob[FunctionVersion]:
|
||||
return AsyncJob(_FunctionJobAdapter(inner))
|
||||
|
||||
@@ -1,178 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
"""Materialized views: tables defined by a query over a source table and
|
||||
maintained by refresh. See ``DBConnection.create_materialized_view``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Tuple, Union
|
||||
|
||||
from .background_loop import LOOP
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import pyarrow as pa
|
||||
|
||||
from ._lancedb import RefreshMaterializedViewResult
|
||||
from .table import AsyncTable, LanceTable
|
||||
|
||||
DEFINITION_META_KEY = b"mv.definition"
|
||||
|
||||
SelectArg = Union[
|
||||
str,
|
||||
Sequence[Union[str, Tuple[str, str]]],
|
||||
Dict[str, str],
|
||||
None,
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class MaterializedViewDefinition:
|
||||
"""The query that defines a materialized view."""
|
||||
|
||||
source_table: str
|
||||
"""Name of the source table, in the same database as the view."""
|
||||
projections: List[Tuple[str, str]]
|
||||
"""``(output column, SQL expression)`` pairs, in view schema order."""
|
||||
filter: Optional[str] = None
|
||||
"""SQL predicate selecting the source rows the view holds."""
|
||||
limit: Optional[int] = None
|
||||
"""Cap on the number of rows the view holds."""
|
||||
inputs: List[str] = field(default_factory=list)
|
||||
"""Source columns the projections and filter read."""
|
||||
|
||||
|
||||
def _definition_from_schema(
|
||||
schema: "pa.Schema", name: str
|
||||
) -> MaterializedViewDefinition:
|
||||
metadata = schema.metadata or {}
|
||||
raw = metadata.get(DEFINITION_META_KEY)
|
||||
if raw is None:
|
||||
raise ValueError(f"Table '{name}' is not a materialized view")
|
||||
value = json.loads(raw)
|
||||
kind = value.get("kind")
|
||||
if kind != "select":
|
||||
raise NotImplementedError(
|
||||
f"materialized view '{name}' is defined by '{kind}', which this "
|
||||
"version of lancedb cannot refresh"
|
||||
)
|
||||
return MaterializedViewDefinition(
|
||||
source_table=value["source_table"],
|
||||
projections=[
|
||||
(p["output"], p["expression"]) for p in value.get("projections", [])
|
||||
],
|
||||
filter=value.get("filter"),
|
||||
limit=value.get("limit"),
|
||||
inputs=value.get("inputs", []),
|
||||
)
|
||||
|
||||
|
||||
def _quote_identifier(name: str) -> str:
|
||||
"""Quote a column name as a Lance SQL identifier (backticks)."""
|
||||
escaped = name.replace("`", "``")
|
||||
return f"`{escaped}`"
|
||||
|
||||
|
||||
def normalize_select(select: SelectArg) -> Optional[List[Tuple[str, str]]]:
|
||||
"""``select`` items may be a column name, an ``(alias, expression)`` pair,
|
||||
or a dict of the same. A bare name projects itself and is quoted, so any
|
||||
valid column name works; dict and pair entries are kept verbatim because
|
||||
their right side is an expression.
|
||||
|
||||
A lone string is one column, not a sequence of its characters."""
|
||||
if select is None:
|
||||
return None
|
||||
if isinstance(select, str):
|
||||
select = [select]
|
||||
if isinstance(select, dict):
|
||||
return list(select.items())
|
||||
normalized = []
|
||||
for item in select:
|
||||
if isinstance(item, str):
|
||||
normalized.append((item, _quote_identifier(item)))
|
||||
else:
|
||||
alias, expression = item
|
||||
normalized.append((alias, expression))
|
||||
return normalized
|
||||
|
||||
|
||||
class AsyncMaterializedView:
|
||||
"""A handle on a materialized view: its table plus its definition.
|
||||
|
||||
Obtained from ``AsyncConnection.create_materialized_view`` or
|
||||
``AsyncConnection.open_materialized_view``.
|
||||
"""
|
||||
|
||||
def __init__(self, table: "AsyncTable"):
|
||||
self._table = table
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"AsyncMaterializedView(name={self.name!r})"
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._table.name
|
||||
|
||||
@property
|
||||
def table(self) -> "AsyncTable":
|
||||
"""The view, as the table it is. Queries, indexes and search all
|
||||
apply; writes are not blocked, but a rebuild replaces them."""
|
||||
return self._table
|
||||
|
||||
async def definition(self) -> MaterializedViewDefinition:
|
||||
"""The query that defines the view, read from its stored schema."""
|
||||
return _definition_from_schema(await self._table.schema(), self.name)
|
||||
|
||||
async def refresh(
|
||||
self, *, full: bool = False, source_version: Optional[int] = None
|
||||
) -> "RefreshMaterializedViewResult":
|
||||
"""Recompute the view from its source.
|
||||
|
||||
The refresh is incremental when the source's changes can be
|
||||
reconciled into the view -- rows added, changed or removed since the
|
||||
last one -- and otherwise rebuilds. ``full=True`` forces a rebuild;
|
||||
``source_version`` refreshes to that source version instead of the
|
||||
latest.
|
||||
|
||||
Concurrent refreshes of one view do not duplicate its rows. Two that
|
||||
plan the same source rows conflict on commit, and the loser raises
|
||||
rather than writing them a second time.
|
||||
"""
|
||||
return await self._table._inner.refresh_materialized_view(
|
||||
full=full, source_version=source_version
|
||||
)
|
||||
|
||||
|
||||
class MaterializedView:
|
||||
"""Synchronous variant of
|
||||
[AsyncMaterializedView][lancedb.materialized_view.AsyncMaterializedView]."""
|
||||
|
||||
def __init__(self, table: "LanceTable"):
|
||||
self._table = table
|
||||
self._async = AsyncMaterializedView(table._table)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"MaterializedView(name={self.name!r})"
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._table.name
|
||||
|
||||
@property
|
||||
def table(self) -> "LanceTable":
|
||||
"""The view, as the table it is."""
|
||||
return self._table
|
||||
|
||||
@property
|
||||
def definition(self) -> MaterializedViewDefinition:
|
||||
"""The query that defines the view, read from its stored schema."""
|
||||
return _definition_from_schema(self._table.schema, self.name)
|
||||
|
||||
def refresh(
|
||||
self, *, full: bool = False, source_version: Optional[int] = None
|
||||
) -> "RefreshMaterializedViewResult":
|
||||
"""Recompute the view from its source. See
|
||||
[AsyncMaterializedView.refresh][lancedb.materialized_view.AsyncMaterializedView.refresh]."""
|
||||
return LOOP.run(self._async.refresh(full=full, source_version=source_version))
|
||||
@@ -61,11 +61,6 @@ from lance_namespace import (
|
||||
NamespaceExistsRequest,
|
||||
TableExistsRequest,
|
||||
)
|
||||
from lancedb.materialized_view import (
|
||||
AsyncMaterializedView,
|
||||
MaterializedView,
|
||||
SelectArg,
|
||||
)
|
||||
from lancedb.table import AsyncTable, LanceTable, Table
|
||||
from lancedb.util import validate_table_name
|
||||
from lancedb.common import DATA
|
||||
@@ -624,42 +619,6 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
tbl.checkout(version)
|
||||
return tbl
|
||||
|
||||
@override
|
||||
def create_materialized_view(
|
||||
self,
|
||||
name: str,
|
||||
source: str,
|
||||
*,
|
||||
select: "SelectArg" = None,
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
) -> "MaterializedView":
|
||||
"""Define a materialized view over a table in the root namespace.
|
||||
See
|
||||
[DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view].
|
||||
"""
|
||||
return MaterializedView(
|
||||
self.open_table(
|
||||
LOOP.run(
|
||||
self._inner.create_materialized_view(
|
||||
name, source, select=select, where=where, limit=limit
|
||||
)
|
||||
).name
|
||||
)
|
||||
)
|
||||
|
||||
@override
|
||||
def open_materialized_view(self, name: str) -> "MaterializedView":
|
||||
"""Open the materialized view named ``name``."""
|
||||
view = MaterializedView(self.open_table(name))
|
||||
view.definition
|
||||
return view
|
||||
|
||||
@override
|
||||
def list_materialized_views(self) -> List[str]:
|
||||
"""The names of the materialized views in the root namespace."""
|
||||
return LOOP.run(self._inner.list_materialized_views())
|
||||
|
||||
@override
|
||||
def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
|
||||
if namespace_path is None:
|
||||
@@ -1182,33 +1141,6 @@ class AsyncLanceNamespaceDBConnection:
|
||||
route_pushdown_to_rust=self._route_pushdown_to_rust,
|
||||
)
|
||||
|
||||
async def create_materialized_view(
|
||||
self,
|
||||
name: str,
|
||||
source: str,
|
||||
*,
|
||||
select: "SelectArg" = None,
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
) -> "AsyncMaterializedView":
|
||||
"""Define a materialized view over a table in the root namespace."""
|
||||
view = await self._inner.create_materialized_view(
|
||||
name, source, select=select, where=where, limit=limit
|
||||
)
|
||||
# Reopen through the namespace so the view's table carries the
|
||||
# namespace client and pushdown configuration a bare inner table lacks.
|
||||
return AsyncMaterializedView(await self.open_table(view.name))
|
||||
|
||||
async def open_materialized_view(self, name: str) -> "AsyncMaterializedView":
|
||||
"""Open the materialized view named ``name``."""
|
||||
view = AsyncMaterializedView(await self.open_table(name))
|
||||
await view.definition()
|
||||
return view
|
||||
|
||||
async def list_materialized_views(self) -> List[str]:
|
||||
"""The names of the materialized views in the root namespace."""
|
||||
return await self._inner.list_materialized_views()
|
||||
|
||||
async def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
|
||||
"""Drop a table from the namespace."""
|
||||
if namespace_path is None:
|
||||
|
||||
@@ -391,15 +391,6 @@ def _table_to_pickle_state(table: Table) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _drop_base_version(permutation_data: pa.Table) -> pa.Table:
|
||||
"""Strip the recorded base version so the reader leaves the base table unpinned."""
|
||||
metadata = dict(permutation_data.schema.metadata or {})
|
||||
if metadata.pop(b"base_version", None) is None:
|
||||
return permutation_data
|
||||
metadata.pop(b"base_branch", None)
|
||||
return permutation_data.replace_schema_metadata(metadata)
|
||||
|
||||
|
||||
def _table_from_pickle_state(state: dict[str, Any]) -> Table:
|
||||
from . import connect
|
||||
|
||||
@@ -688,15 +679,11 @@ class Permutation:
|
||||
from . import connect
|
||||
|
||||
connection_factory = state["connection_factory"]
|
||||
rebuilt_base = False
|
||||
if connection_factory is not None:
|
||||
base_table = connection_factory(state["base_table_name"])
|
||||
elif "base_table_state" in state:
|
||||
base_state = state["base_table_state"]
|
||||
rebuilt_base = base_state["kind"] == "memory"
|
||||
base_table = _table_from_pickle_state(base_state)
|
||||
base_table = _table_from_pickle_state(state["base_table_state"])
|
||||
elif "base_table_data" in state:
|
||||
rebuilt_base = True
|
||||
# In-memory base table inlined into the pickle; rebuild the same
|
||||
# way we rebuild the in-memory permutation table.
|
||||
mem_db = connect("memory://")
|
||||
@@ -714,14 +701,11 @@ class Permutation:
|
||||
)
|
||||
|
||||
permutation_table: Optional[Table] = None
|
||||
permutation_data = state["permutation_data"]
|
||||
if permutation_data is not None:
|
||||
if rebuilt_base:
|
||||
# The base table was materialized from Arrow, so it is a fresh
|
||||
# single-version dataset and the recorded pin cannot resolve on it.
|
||||
permutation_data = _drop_base_version(permutation_data)
|
||||
if state["permutation_data"] is not None:
|
||||
mem_db = connect("memory://")
|
||||
permutation_table = mem_db.create_table("permutation", permutation_data)
|
||||
permutation_table = mem_db.create_table(
|
||||
"permutation", state["permutation_data"]
|
||||
)
|
||||
|
||||
self.base_table = base_table
|
||||
self.permutation_table = permutation_table
|
||||
|
||||
@@ -167,12 +167,6 @@ def _projection_to_scanner_kwargs(columns: QueryProjection) -> Dict[str, Any]:
|
||||
return {"columns": projection}
|
||||
|
||||
|
||||
def _query_request_projection(req: "PyQueryRequest") -> QueryProjection:
|
||||
if req.select_source_columns is not None:
|
||||
return req.select_source_columns
|
||||
return req.select
|
||||
|
||||
|
||||
def _scanner_kwargs_for_query(
|
||||
query: Query,
|
||||
blob_mode: BlobMode,
|
||||
@@ -381,13 +375,6 @@ class FullTextOperator(str, Enum):
|
||||
OR = "OR"
|
||||
|
||||
|
||||
class DocumentGranularity(str, Enum):
|
||||
"""The unit treated as one full-text-search document."""
|
||||
|
||||
ROW = "row"
|
||||
LIST_ELEMENT = "list_element"
|
||||
|
||||
|
||||
class Occur(str, Enum):
|
||||
SHOULD = "SHOULD"
|
||||
MUST = "MUST"
|
||||
@@ -491,10 +478,6 @@ class MatchQuery(FullTextQuery):
|
||||
prefix_length : int, optional
|
||||
The number of beginning characters being unchanged for fuzzy matching.
|
||||
This is useful to achieve prefix matching.
|
||||
document_granularity : DocumentGranularity, optional
|
||||
Explicitly select row or deepest-list-element documents. If omitted,
|
||||
the indexed granularity is inferred. When both granularities are indexed
|
||||
for the field, this must be specified. With no index, row granularity is used.
|
||||
"""
|
||||
|
||||
query: str
|
||||
@@ -504,9 +487,6 @@ class MatchQuery(FullTextQuery):
|
||||
max_expansions: int = pydantic.Field(50, kw_only=True)
|
||||
operator: FullTextOperator = pydantic.Field(FullTextOperator.OR, kw_only=True)
|
||||
prefix_length: int = pydantic.Field(0, kw_only=True)
|
||||
document_granularity: Optional[DocumentGranularity] = pydantic.Field(
|
||||
None, kw_only=True
|
||||
)
|
||||
|
||||
def query_type(self) -> FullTextQueryType:
|
||||
return FullTextQueryType.MATCH
|
||||
@@ -523,20 +503,11 @@ class PhraseQuery(FullTextQuery):
|
||||
The query string to match against.
|
||||
column : str
|
||||
The name of the column to match against.
|
||||
slop : int, default 0
|
||||
The maximum number of intervening positions permitted in the phrase.
|
||||
document_granularity : DocumentGranularity, optional
|
||||
Explicitly select row or deepest-list-element documents. If omitted,
|
||||
the indexed granularity is inferred. When both granularities are indexed
|
||||
for the field, this must be specified. With no index, row granularity is used.
|
||||
"""
|
||||
|
||||
query: str
|
||||
column: str
|
||||
slop: int = pydantic.Field(0, kw_only=True)
|
||||
document_granularity: Optional[DocumentGranularity] = pydantic.Field(
|
||||
None, kw_only=True
|
||||
)
|
||||
|
||||
def query_type(self) -> FullTextQueryType:
|
||||
return FullTextQueryType.MATCH_PHRASE
|
||||
@@ -2805,16 +2776,15 @@ class AsyncQueryBase(object):
|
||||
|
||||
req = self._inner.to_query_request()
|
||||
schema = await self._table.schema()
|
||||
projection = _query_request_projection(req)
|
||||
self._blob_auto_row_id = blob_auto_row_id_for_scan(
|
||||
schema,
|
||||
projection,
|
||||
req.select,
|
||||
with_row_id=self._with_row_id,
|
||||
)
|
||||
if not self._blob_auto_row_id:
|
||||
self._blob_paths = ()
|
||||
return
|
||||
self._blob_paths = tuple(blob_v2_projection_sources(schema, projection).keys())
|
||||
self._blob_paths = tuple(blob_v2_projection_sources(schema, req.select).keys())
|
||||
self._inner.with_row_id()
|
||||
|
||||
def select(self, columns: Union[List[str], dict[str, str]]) -> Self:
|
||||
@@ -3408,10 +3378,9 @@ class AsyncQuery(AsyncStandardQuery):
|
||||
pass in multiple vectors. When multiple vectors are passed in, if the vector
|
||||
column is with multivector type, then the vectors will be treated as a single
|
||||
query. Or the vectors will be treated as multiple queries, this can be useful
|
||||
if you want to find the nearest vectors to multiple query vectors. Flat
|
||||
searches share one table scan across the query vectors, avoiding the scan
|
||||
and memory amplification of making multiple queries concurrently. If
|
||||
multiple vectors are passed in then
|
||||
if you want to find the nearest vectors to multiple query vectors.
|
||||
This is not expected to be faster than making multiple queries concurrently;
|
||||
it is just a convenience method. If multiple vectors are passed in then
|
||||
an additional column `query_index` will be added to the results. This column
|
||||
will contain the index of the query vector that the result is nearest to.
|
||||
"""
|
||||
@@ -3540,8 +3509,8 @@ class AsyncFTSQuery(AsyncStandardQuery):
|
||||
|
||||
Typically, a single vector is passed in as the query. However, you can also
|
||||
pass in multiple vectors. This can be useful if you want to find the nearest
|
||||
vectors to multiple query vectors. Flat searches share one table scan across
|
||||
the query vectors instead of issuing concurrent full scans.
|
||||
vectors to multiple query vectors. This is not expected to be faster than
|
||||
making multiple queries concurrently; it is just a convenience method.
|
||||
If multiple vectors are passed in then an additional column `query_index`
|
||||
will be added to the results. This column will contain the index of the
|
||||
query vector that the result is nearest to.
|
||||
@@ -3901,15 +3870,14 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
|
||||
blob_paths: tuple[str, ...] = ()
|
||||
if self._table is not None:
|
||||
schema = await self._table.schema()
|
||||
projection = _query_request_projection(req)
|
||||
blob_auto_row_id = blob_auto_row_id_for_scan(
|
||||
schema,
|
||||
projection,
|
||||
req.select,
|
||||
with_row_id=self._with_row_id,
|
||||
)
|
||||
if blob_auto_row_id:
|
||||
blob_paths = tuple(
|
||||
blob_v2_projection_sources(schema, projection).keys()
|
||||
blob_v2_projection_sources(schema, req.select).keys()
|
||||
)
|
||||
self._blob_auto_row_id = blob_auto_row_id
|
||||
self._blob_paths = blob_paths
|
||||
|
||||
@@ -7,7 +7,7 @@ import json
|
||||
import logging
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Mapping, Optional, Union
|
||||
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
|
||||
from urllib.parse import urlparse
|
||||
import warnings
|
||||
|
||||
@@ -25,7 +25,6 @@ from ..common import DATA
|
||||
from ..db import DBConnection, LOOP
|
||||
from ..functions import FunctionVersion, UdfDefinition
|
||||
from ..job import AsyncJob, Job
|
||||
from ..materialized_view import MaterializedView, SelectArg
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .._lancedb import JobDescription, JobInfo
|
||||
@@ -649,32 +648,6 @@ class RemoteDBConnection(DBConnection):
|
||||
namespace_path=namespace_path,
|
||||
)
|
||||
|
||||
@override
|
||||
def create_materialized_view(
|
||||
self,
|
||||
name: str,
|
||||
source: str,
|
||||
*,
|
||||
select: SelectArg = None,
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
) -> MaterializedView:
|
||||
raise NotImplementedError(
|
||||
"materialized views are supported only on local databases"
|
||||
)
|
||||
|
||||
@override
|
||||
def open_materialized_view(self, name: str) -> MaterializedView:
|
||||
raise NotImplementedError(
|
||||
"materialized views are supported only on local databases"
|
||||
)
|
||||
|
||||
@override
|
||||
def list_materialized_views(self) -> List[str]:
|
||||
raise NotImplementedError(
|
||||
"materialized views are supported only on local databases"
|
||||
)
|
||||
|
||||
@override
|
||||
def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
|
||||
"""Drop a table from the database.
|
||||
@@ -742,15 +715,8 @@ class RemoteDBConnection(DBConnection):
|
||||
return Job(self._conn.job(job_id))
|
||||
|
||||
@override
|
||||
def create_function_async(
|
||||
self,
|
||||
definition: UdfDefinition,
|
||||
*,
|
||||
secrets: Optional[Mapping[str, str]] = None,
|
||||
) -> Job[FunctionVersion]:
|
||||
return Job(
|
||||
LOOP.run(self._conn.create_function_async(definition, secrets=secrets))
|
||||
)
|
||||
def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
|
||||
return Job(LOOP.run(self._conn.create_function_async(definition)))
|
||||
|
||||
@override
|
||||
def get_function(self, name: str, *, version: str) -> FunctionVersion:
|
||||
|
||||
@@ -36,7 +36,6 @@ from lancedb._lancedb import (
|
||||
UpdateResult,
|
||||
)
|
||||
from lancedb.embeddings.base import EmbeddingFunctionConfig
|
||||
from lancedb.expr import Expr
|
||||
from lancedb.index import (
|
||||
FTS,
|
||||
BTree,
|
||||
@@ -50,7 +49,7 @@ from lancedb.index import (
|
||||
LabelList,
|
||||
)
|
||||
from lancedb.job import Job
|
||||
from lancedb.functions import FunctionApplication, RefreshColumnResult
|
||||
from lancedb.functions import FunctionApplication
|
||||
from lancedb.remote.db import LOOP
|
||||
from lancedb.table import IndexConfigType, KNOWN_METRICS
|
||||
import pyarrow as pa
|
||||
@@ -62,7 +61,6 @@ from lancedb.table import _normalize_progress
|
||||
|
||||
from ..query import (
|
||||
AnalyzePlanDistributedMetrics,
|
||||
DocumentGranularity,
|
||||
LanceQueryBuilder,
|
||||
LanceTakeQueryBuilder,
|
||||
LanceVectorQueryBuilder,
|
||||
@@ -351,7 +349,6 @@ class RemoteTable(Table):
|
||||
ngram_max_length: int = 3,
|
||||
prefix_only: bool = False,
|
||||
block_size: int = 128,
|
||||
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
|
||||
name: Optional[str] = None,
|
||||
):
|
||||
"""Create a full-text search index on a column.
|
||||
@@ -374,7 +371,6 @@ class RemoteTable(Table):
|
||||
ngram_max_length=ngram_max_length,
|
||||
prefix_only=prefix_only,
|
||||
block_size=block_size,
|
||||
document_granularity=document_granularity,
|
||||
)
|
||||
LOOP.run(
|
||||
self._table.create_index(
|
||||
@@ -614,7 +610,6 @@ class RemoteTable(Table):
|
||||
fill_value: float = 0.0,
|
||||
progress: Optional[Union[bool, Callable, Any]] = None,
|
||||
write_parallelism: Optional[int] = None,
|
||||
allow_external_blob_outside_bases: bool = False,
|
||||
) -> AddResult:
|
||||
"""Add more data to the [Table][lancedb.table.Table].
|
||||
|
||||
@@ -647,8 +642,6 @@ class RemoteTable(Table):
|
||||
data in flight. Defaults to an estimate based on the data size,
|
||||
capped at the number of CPU cores. Lower this if bulk ingestion is
|
||||
using too much memory.
|
||||
allow_external_blob_outside_bases: bool, default False
|
||||
Not supported on LanceDB Cloud. Setting this raises.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -665,7 +658,6 @@ class RemoteTable(Table):
|
||||
fill_value=fill_value,
|
||||
progress=progress,
|
||||
write_parallelism=write_parallelism,
|
||||
allow_external_blob_outside_bases=allow_external_blob_outside_bases,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
@@ -864,7 +856,7 @@ class RemoteTable(Table):
|
||||
|
||||
def update(
|
||||
self,
|
||||
where: Optional[Union[str, Expr]] = None,
|
||||
where: Optional[str] = None,
|
||||
values: Optional[dict] = None,
|
||||
*,
|
||||
values_sql: Optional[Dict[str, str]] = None,
|
||||
@@ -875,11 +867,9 @@ class RemoteTable(Table):
|
||||
|
||||
Parameters
|
||||
----------
|
||||
where: str or [Expr][lancedb.expr.Expr], optional
|
||||
The filter condition. Can be a SQL string or a type-safe
|
||||
[Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
|
||||
[lit][lancedb.expr.lit]. The filter must not be empty, or it will
|
||||
error.
|
||||
where: str, optional
|
||||
The SQL where clause to use when updating rows. For example, 'x = 2'
|
||||
or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
|
||||
values: dict, optional
|
||||
The values to update. The keys are the column names and the values
|
||||
are the values to set.
|
||||
@@ -982,7 +972,7 @@ class RemoteTable(Table):
|
||||
def refresh_column(self, column: str):
|
||||
return LOOP.run(self._table.refresh_column(column))
|
||||
|
||||
def refresh_column_async(self, column: str) -> Job[RefreshColumnResult]:
|
||||
def refresh_column_async(self, column: str) -> Job:
|
||||
return Job(LOOP.run(self._table.refresh_column_async(column)))
|
||||
|
||||
def alter_columns(
|
||||
|
||||
+34
-101
@@ -4,34 +4,30 @@
|
||||
|
||||
"""Schema helpers for Lance blob columns."""
|
||||
|
||||
import importlib
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pyarrow as pa
|
||||
import pyarrow.ipc
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from lance.blob import BlobType as BlobType
|
||||
|
||||
_BLOB_EXTENSION_NAME = "lance.blob.v2"
|
||||
_BLOB_V1_KEY = "lance-encoding:blob"
|
||||
_ARROW_EXT_NAME_KEY = "ARROW:extension:name"
|
||||
_BLOB_V2_STORAGE_TYPE = pa.struct(
|
||||
[
|
||||
pa.field("data", pa.large_binary(), nullable=True),
|
||||
pa.field("uri", pa.utf8(), nullable=True),
|
||||
pa.field("position", pa.uint64(), nullable=True),
|
||||
pa.field("size", pa.uint64(), nullable=True),
|
||||
]
|
||||
)
|
||||
_resolved_blob_type = None
|
||||
|
||||
|
||||
class _FallbackBlobType(pa.ExtensionType):
|
||||
"""lance.blob.v2 extension type used when pylance is not installed."""
|
||||
class BlobType(pa.ExtensionType):
|
||||
"""PyArrow extension type for a Lance blob v2 column.
|
||||
|
||||
Queries return descriptors; call :meth:`~lancedb.table.Table.fetch_blob_files`
|
||||
for lazy reads or :meth:`~lancedb.table.Table.fetch_blobs` for eager bytes.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
pa.ExtensionType.__init__(self, _BLOB_V2_STORAGE_TYPE, _BLOB_EXTENSION_NAME)
|
||||
storage_type = pa.struct(
|
||||
[
|
||||
pa.field("data", pa.large_binary(), nullable=True),
|
||||
pa.field("uri", pa.utf8(), nullable=True),
|
||||
pa.field("position", pa.uint64(), nullable=True),
|
||||
pa.field("size", pa.uint64(), nullable=True),
|
||||
]
|
||||
)
|
||||
super().__init__(storage_type, _BLOB_EXTENSION_NAME)
|
||||
|
||||
def __arrow_ext_serialize__(self) -> bytes:
|
||||
return b""
|
||||
@@ -39,16 +35,23 @@ class _FallbackBlobType(pa.ExtensionType):
|
||||
@classmethod
|
||||
def __arrow_ext_deserialize__(
|
||||
cls, storage_type: pa.DataType, serialized: bytes
|
||||
) -> "_FallbackBlobType":
|
||||
) -> "BlobType":
|
||||
return cls()
|
||||
|
||||
def __reduce__(self):
|
||||
# Ensure pickle round-trips on older pyarrow (apache/arrow#35599).
|
||||
return type(self).__arrow_ext_deserialize__, (
|
||||
self.storage_type,
|
||||
self.__arrow_ext_serialize__(),
|
||||
)
|
||||
|
||||
|
||||
try:
|
||||
pa.register_extension_type(BlobType()) # type: ignore[arg-type]
|
||||
except pa.ArrowKeyError:
|
||||
pass
|
||||
|
||||
|
||||
def _metadata_value(metadata: dict, key: str):
|
||||
return metadata.get(key.encode()) or metadata.get(key)
|
||||
|
||||
@@ -89,105 +92,43 @@ def is_blob_like_field(field: pa.Field) -> bool:
|
||||
return is_blob_v2_field(field) or _metadata_marks_legacy_blob(field.metadata or {})
|
||||
|
||||
|
||||
def _collect_blob_paths(schema: pa.Schema, is_blob) -> list[tuple[str, bool]]:
|
||||
"""Walk the schema and return (path, has_list_ancestor) for each blob field."""
|
||||
paths: list[tuple[str, bool]] = []
|
||||
def _collect_blob_paths(schema: pa.Schema, is_blob) -> list[str]:
|
||||
paths: list[str] = []
|
||||
|
||||
def walk(fields, prefix: str, has_list_ancestor: bool) -> None:
|
||||
def walk(fields, prefix: str) -> None:
|
||||
for field in fields:
|
||||
path = f"{prefix}.{field.name}" if prefix else field.name
|
||||
if is_blob(field):
|
||||
paths.append((path, has_list_ancestor))
|
||||
paths.append(path)
|
||||
elif pa.types.is_struct(field.type):
|
||||
walk(field.type, path, has_list_ancestor)
|
||||
walk(field.type, path)
|
||||
elif (
|
||||
pa.types.is_list(field.type)
|
||||
or pa.types.is_large_list(field.type)
|
||||
or pa.types.is_fixed_size_list(field.type)
|
||||
):
|
||||
walk([field.type.value_field], path, True)
|
||||
walk([field.type.value_field], path)
|
||||
|
||||
walk(schema, "", False)
|
||||
walk(schema, "")
|
||||
return paths
|
||||
|
||||
|
||||
def blob_column_paths(schema: pa.Schema) -> list[str]:
|
||||
"""Dotted paths of blob-like columns (v2 extension or legacy metadata)."""
|
||||
return [path for path, _ in _collect_blob_paths(schema, is_blob_like_field)]
|
||||
return _collect_blob_paths(schema, is_blob_like_field)
|
||||
|
||||
|
||||
def blob_v2_column_paths(schema: pa.Schema) -> list[str]:
|
||||
return [path for path, _ in _collect_blob_paths(schema, is_blob_v2_field)]
|
||||
|
||||
|
||||
def row_addressable_blob_v2_paths(schema: pa.Schema) -> list[str]:
|
||||
"""Blob v2 paths with one blob addressable by table row id.
|
||||
|
||||
``fetch_blobs`` and the descriptor row-id ride-along address one blob per
|
||||
row, so a blob inside a list container has no row-id slot and no fetch
|
||||
path. Those columns still store and query as raw descriptors.
|
||||
"""
|
||||
return [
|
||||
path
|
||||
for path, has_list_ancestor in _collect_blob_paths(schema, is_blob_v2_field)
|
||||
if not has_list_ancestor
|
||||
]
|
||||
return _collect_blob_paths(schema, is_blob_v2_field)
|
||||
|
||||
|
||||
def schema_has_blob_field(schema: pa.Schema) -> bool:
|
||||
return bool(blob_column_paths(schema))
|
||||
|
||||
|
||||
def _deserialize_registered_type(extension_type: pa.ExtensionType) -> pa.DataType:
|
||||
"""Return the type Arrow reconstructs for this extension name."""
|
||||
schema = pa.schema([pa.field("value", extension_type)])
|
||||
restored = pa.ipc.read_schema(schema.serialize())
|
||||
return restored.field("value").type
|
||||
|
||||
|
||||
def _resolve_blob_type():
|
||||
"""Return the BlobType class this process should use.
|
||||
|
||||
pylance's class when it owns the lance.blob.v2 registry entry,
|
||||
otherwise LanceDB's fallback. A different registered class is an error.
|
||||
"""
|
||||
global _resolved_blob_type
|
||||
if _resolved_blob_type is not None:
|
||||
return _resolved_blob_type
|
||||
try:
|
||||
blob_module = importlib.import_module("lance.blob")
|
||||
except ModuleNotFoundError as err:
|
||||
if err.name not in ("lance", "lance.blob"):
|
||||
raise
|
||||
else:
|
||||
blob_type = getattr(blob_module, "BlobType", None)
|
||||
if blob_type is not None:
|
||||
registered_type = _deserialize_registered_type(blob_type())
|
||||
if type(registered_type) is not blob_type:
|
||||
registered_cls = type(registered_type)
|
||||
raise ValueError(
|
||||
"lance.blob.v2 is already registered by "
|
||||
f"{registered_cls.__module__}.{registered_cls.__qualname__}"
|
||||
)
|
||||
_resolved_blob_type = blob_type
|
||||
return blob_type
|
||||
try:
|
||||
pa.register_extension_type(_FallbackBlobType()) # type: ignore[arg-type]
|
||||
except pa.ArrowKeyError as err:
|
||||
raise ValueError(
|
||||
"lance.blob.v2 is already registered by another extension class"
|
||||
) from err
|
||||
_resolved_blob_type = _FallbackBlobType
|
||||
return _resolved_blob_type
|
||||
|
||||
|
||||
def blob(name: str, nullable: bool = True) -> pa.Field:
|
||||
"""Create a Lance blob v2 column field.
|
||||
|
||||
When pylance is installed this is ``lance.blob.BlobType``.
|
||||
"""
|
||||
blob_type = _resolve_blob_type()
|
||||
return pa.field(name, blob_type(), nullable=nullable)
|
||||
"""Create a Lance blob v2 column field."""
|
||||
return pa.field(name, BlobType(), nullable=nullable)
|
||||
|
||||
|
||||
def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataType:
|
||||
@@ -214,11 +155,3 @@ def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataTyp
|
||||
... ])
|
||||
"""
|
||||
return pa.list_(value_type, dimension)
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
if name == "BlobType":
|
||||
blob_type = _resolve_blob_type()
|
||||
globals()["BlobType"] = blob_type
|
||||
return blob_type
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
+90
-1036
File diff suppressed because it is too large
Load Diff
+98
-349
@@ -40,7 +40,7 @@ from ._blob import (
|
||||
from .types import BlobMode
|
||||
from lancedb.arrow import peek_reader
|
||||
from lancedb.background_loop import LOOP, embedding_executor
|
||||
from lancedb.job import AsyncJob, Job, _typed_job
|
||||
from lancedb.job import AsyncJob, Job
|
||||
from .dependencies import (
|
||||
_check_for_hugging_face,
|
||||
_check_for_lance,
|
||||
@@ -72,10 +72,7 @@ from .index import (
|
||||
FTS,
|
||||
)
|
||||
from .expr import Expr
|
||||
from .functions import (
|
||||
FunctionApplication,
|
||||
RefreshColumnResult as RefreshColumnJobResult,
|
||||
)
|
||||
from .functions import FunctionApplication
|
||||
from .merge import LanceMergeInsertBuilder
|
||||
from .pydantic import LanceModel, model_to_dict
|
||||
from .query import (
|
||||
@@ -85,7 +82,6 @@ from .query import (
|
||||
AsyncQuery,
|
||||
AsyncTakeQuery,
|
||||
AsyncVectorQuery,
|
||||
DocumentGranularity,
|
||||
FullTextQuery,
|
||||
LanceEmptyQueryBuilder,
|
||||
LanceFtsQueryBuilder,
|
||||
@@ -104,12 +100,7 @@ from .util import (
|
||||
value_to_sql,
|
||||
)
|
||||
from .index import lang_mapping
|
||||
from .schema import (
|
||||
blob_v2_column_paths,
|
||||
is_blob_v2_field,
|
||||
row_addressable_blob_v2_paths,
|
||||
schema_has_blob_field,
|
||||
)
|
||||
from .schema import blob_v2_column_paths, schema_has_blob_field
|
||||
|
||||
|
||||
def _should_push_down_query_table(
|
||||
@@ -431,7 +422,6 @@ def _cast_to_target_schema(
|
||||
|
||||
def gen():
|
||||
for batch in reader:
|
||||
batch = _coerce_blob_write_columns(batch, reordered_schema)
|
||||
# Table but not RecordBatch has cast.
|
||||
cast_batches = (
|
||||
pa.Table.from_batches([batch]).cast(reordered_schema).to_batches()
|
||||
@@ -444,166 +434,6 @@ def _cast_to_target_schema(
|
||||
return pa.RecordBatchReader.from_batches(reordered_schema, gen())
|
||||
|
||||
|
||||
def _coerce_blob_write_columns(
|
||||
batch: pa.RecordBatch, target_schema: pa.Schema
|
||||
) -> pa.RecordBatch:
|
||||
"""Materialize blob storage structs before the stream leaves Python.
|
||||
|
||||
merge_insert requires its source reader to already match the table's
|
||||
physical schema. Unlike add and insert, it does not pass through
|
||||
LanceDB's Rust blob coercion, so preserving binary input here would
|
||||
reach Lance as binary and fail the schema check.
|
||||
"""
|
||||
columns = []
|
||||
fields = []
|
||||
changed = False
|
||||
for field, column in zip(batch.schema, batch.columns):
|
||||
target_field = target_schema.field(field.name)
|
||||
coerced = _coerce_blob_value(column, target_field)
|
||||
if coerced is not column:
|
||||
column = coerced
|
||||
field = pa.field(
|
||||
field.name,
|
||||
coerced.type,
|
||||
field.nullable,
|
||||
target_field.metadata,
|
||||
)
|
||||
changed = True
|
||||
columns.append(column)
|
||||
fields.append(field)
|
||||
if not changed:
|
||||
return batch
|
||||
return pa.RecordBatch.from_arrays(
|
||||
columns, schema=pa.schema(fields, metadata=batch.schema.metadata)
|
||||
)
|
||||
|
||||
|
||||
def _coerce_blob_value(column: pa.Array, target_field: pa.Field) -> pa.Array:
|
||||
if is_blob_v2_field(target_field) and _can_coerce_to_blob(column.type):
|
||||
return _coerce_value_to_blob(column, target_field)
|
||||
|
||||
target_type = target_field.type
|
||||
if pa.types.is_struct(target_type) and pa.types.is_struct(column.type):
|
||||
children = []
|
||||
fields = []
|
||||
changed = False
|
||||
for source_field in column.type:
|
||||
source_column = column.field(source_field.name)
|
||||
nested_target = next(
|
||||
(field for field in target_type if field.name == source_field.name),
|
||||
None,
|
||||
)
|
||||
if nested_target is None:
|
||||
children.append(source_column)
|
||||
fields.append(source_field)
|
||||
continue
|
||||
coerced = _coerce_blob_value(source_column, nested_target)
|
||||
if coerced is not source_column:
|
||||
changed = True
|
||||
child_array, child_type = _physical_array_and_type(coerced)
|
||||
children.append(child_array)
|
||||
fields.append(
|
||||
pa.field(
|
||||
source_field.name,
|
||||
child_type,
|
||||
source_field.nullable,
|
||||
nested_target.metadata,
|
||||
)
|
||||
)
|
||||
if not changed:
|
||||
return column
|
||||
return pa.StructArray.from_arrays(
|
||||
children,
|
||||
fields=fields,
|
||||
mask=column.is_null() if column.null_count else None,
|
||||
)
|
||||
|
||||
if _is_list_like(target_type) and _is_list_like(column.type):
|
||||
return _coerce_blob_list_values(column, target_type.value_field)
|
||||
|
||||
return column
|
||||
|
||||
|
||||
def _coerce_blob_list_values(
|
||||
column: pa.Array, target_value_field: pa.Field
|
||||
) -> pa.Array:
|
||||
"""Coerce blob values inside a list column, preserving offsets and nulls.
|
||||
|
||||
Works on the raw child values window instead of ``pc.list_flatten`` because
|
||||
flatten drops values spanned by null slots, which would misalign offsets.
|
||||
"""
|
||||
mask = column.is_null() if column.null_count else None
|
||||
if pa.types.is_fixed_size_list(column.type):
|
||||
list_size = column.type.list_size
|
||||
values = column.values.slice(column.offset * list_size, len(column) * list_size)
|
||||
coerced = _coerce_blob_value(values, target_value_field)
|
||||
if coerced is values:
|
||||
return column
|
||||
physical_values, _ = _physical_array_and_type(coerced)
|
||||
return pa.FixedSizeListArray.from_arrays(physical_values, list_size, mask=mask)
|
||||
offsets = column.offsets
|
||||
first_offset = offsets[0].as_py()
|
||||
values = column.values.slice(
|
||||
first_offset,
|
||||
offsets[-1].as_py() - first_offset,
|
||||
)
|
||||
coerced = _coerce_blob_value(values, target_value_field)
|
||||
if coerced is values:
|
||||
return column
|
||||
physical_values, _ = _physical_array_and_type(coerced)
|
||||
if first_offset:
|
||||
offsets = pc.subtract(offsets, pa.scalar(first_offset, offsets.type))
|
||||
if pa.types.is_large_list(column.type):
|
||||
return pa.LargeListArray.from_arrays(offsets, physical_values, mask=mask)
|
||||
return pa.ListArray.from_arrays(offsets, physical_values, mask=mask)
|
||||
|
||||
|
||||
def _coerce_value_to_blob(values: pa.Array, target_field: pa.Field) -> pa.Array:
|
||||
if pa.types.is_null(values.type):
|
||||
data = pa.nulls(len(values), type=pa.large_binary())
|
||||
elif pa.types.is_large_binary(values.type):
|
||||
data = values
|
||||
else:
|
||||
data = values.cast(pa.large_binary())
|
||||
length = len(values)
|
||||
storage_type = target_field.type
|
||||
if isinstance(storage_type, pa.ExtensionType):
|
||||
storage_type = storage_type.storage_type
|
||||
storage_fields = list(storage_type)
|
||||
children = []
|
||||
for storage_field in storage_fields:
|
||||
if storage_field.name == "data":
|
||||
children.append(data)
|
||||
else:
|
||||
children.append(pa.nulls(length, type=storage_field.type))
|
||||
storage = pa.StructArray.from_arrays(
|
||||
children,
|
||||
fields=storage_fields,
|
||||
mask=values.is_null() if values.null_count else None,
|
||||
)
|
||||
if isinstance(target_field.type, pa.ExtensionType):
|
||||
return pa.ExtensionArray.from_storage(target_field.type, storage)
|
||||
return storage
|
||||
|
||||
|
||||
def _physical_array_and_type(array: pa.Array) -> tuple[pa.Array, pa.DataType]:
|
||||
if isinstance(array.type, pa.ExtensionType):
|
||||
return array.storage, array.type.storage_type
|
||||
return array, array.type
|
||||
|
||||
|
||||
def _can_coerce_to_blob(data_type: pa.DataType) -> bool:
|
||||
return _is_binary_like(data_type) or pa.types.is_null(data_type)
|
||||
|
||||
|
||||
def _is_binary_like(data_type: pa.DataType) -> bool:
|
||||
return (
|
||||
pa.types.is_binary(data_type)
|
||||
or pa.types.is_large_binary(data_type)
|
||||
or pa.types.is_binary_view(data_type)
|
||||
)
|
||||
|
||||
|
||||
def _field_extension_name(field: pa.Field) -> Optional[str]:
|
||||
extension_name = getattr(field.type, "extension_name", None)
|
||||
if extension_name is not None:
|
||||
@@ -630,73 +460,65 @@ def _align_field_types(
|
||||
target_field = next((f for f in target_fields if f.name == field.name), None)
|
||||
if target_field is None:
|
||||
raise ValueError(f"Field '{field.name}' not found in target schema")
|
||||
new_fields.append(_align_field(field, target_field))
|
||||
# Preserve arrow.json input until it reaches Lance. LanceDB exposes stored
|
||||
# JSON columns as lance.json (JSONB-backed LargeBinary), but casting the
|
||||
# input to that storage type here merely relabels the raw JSON bytes as
|
||||
# JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
|
||||
if (
|
||||
_field_extension_name(field) == "arrow.json"
|
||||
and _field_extension_name(target_field) == "lance.json"
|
||||
):
|
||||
new_fields.append(field)
|
||||
continue
|
||||
if pa.types.is_struct(target_field.type):
|
||||
if pa.types.is_struct(field.type):
|
||||
new_type = pa.struct(
|
||||
_align_field_types(
|
||||
field.type.fields,
|
||||
target_field.type.fields,
|
||||
)
|
||||
)
|
||||
else:
|
||||
new_type = target_field.type
|
||||
elif pa.types.is_list(target_field.type):
|
||||
if _is_list_like(field.type):
|
||||
new_type = pa.list_(
|
||||
_align_field_types(
|
||||
[field.type.value_field],
|
||||
[target_field.type.value_field],
|
||||
)[0]
|
||||
)
|
||||
else:
|
||||
new_type = target_field.type
|
||||
elif pa.types.is_large_list(target_field.type):
|
||||
if _is_list_like(field.type):
|
||||
new_type = pa.large_list(
|
||||
_align_field_types(
|
||||
[field.type.value_field],
|
||||
[target_field.type.value_field],
|
||||
)[0]
|
||||
)
|
||||
else:
|
||||
new_type = target_field.type
|
||||
elif pa.types.is_fixed_size_list(target_field.type):
|
||||
if _is_list_like(field.type):
|
||||
new_type = pa.list_(
|
||||
_align_field_types(
|
||||
[field.type.value_field],
|
||||
[target_field.type.value_field],
|
||||
)[0],
|
||||
target_field.type.list_size,
|
||||
)
|
||||
else:
|
||||
new_type = target_field.type
|
||||
else:
|
||||
new_type = target_field.type
|
||||
new_fields.append(
|
||||
pa.field(field.name, new_type, field.nullable, target_field.metadata)
|
||||
)
|
||||
return new_fields
|
||||
|
||||
|
||||
def _align_list_value_field(
|
||||
value_field: pa.Field, target_value_field: pa.Field
|
||||
) -> pa.Field:
|
||||
# A list has exactly one child, so the inferred child name ("item") aligns
|
||||
# positionally and adopts the table's child name; pa.Table.cast renames it.
|
||||
return _align_field(value_field, target_value_field).with_name(
|
||||
target_value_field.name
|
||||
)
|
||||
|
||||
|
||||
def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field:
|
||||
# Preserve arrow.json input until it reaches Lance. LanceDB exposes stored
|
||||
# JSON columns as lance.json (JSONB-backed LargeBinary), but casting the
|
||||
# input to that storage type here merely relabels the raw JSON bytes as
|
||||
# JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
|
||||
if (
|
||||
_field_extension_name(field) == "arrow.json"
|
||||
and _field_extension_name(target_field) == "lance.json"
|
||||
):
|
||||
return field
|
||||
if pa.types.is_struct(target_field.type):
|
||||
if pa.types.is_struct(field.type):
|
||||
new_type = pa.struct(
|
||||
_align_field_types(
|
||||
field.type.fields,
|
||||
target_field.type.fields,
|
||||
)
|
||||
)
|
||||
else:
|
||||
new_type = target_field.type
|
||||
elif pa.types.is_list(target_field.type):
|
||||
if _is_list_like(field.type):
|
||||
new_type = pa.list_(
|
||||
_align_list_value_field(
|
||||
field.type.value_field, target_field.type.value_field
|
||||
)
|
||||
)
|
||||
else:
|
||||
new_type = target_field.type
|
||||
elif pa.types.is_large_list(target_field.type):
|
||||
if _is_list_like(field.type):
|
||||
new_type = pa.large_list(
|
||||
_align_list_value_field(
|
||||
field.type.value_field, target_field.type.value_field
|
||||
)
|
||||
)
|
||||
else:
|
||||
new_type = target_field.type
|
||||
elif pa.types.is_fixed_size_list(target_field.type):
|
||||
if _is_list_like(field.type):
|
||||
new_type = pa.list_(
|
||||
_align_list_value_field(
|
||||
field.type.value_field, target_field.type.value_field
|
||||
),
|
||||
target_field.type.list_size,
|
||||
)
|
||||
else:
|
||||
new_type = target_field.type
|
||||
else:
|
||||
new_type = target_field.type
|
||||
return pa.field(field.name, new_type, field.nullable, target_field.metadata)
|
||||
|
||||
|
||||
def _infer_subschema(
|
||||
schema: List[pa.Field],
|
||||
reference_fields: List[pa.Field],
|
||||
@@ -763,7 +585,7 @@ def sanitize_create_table(
|
||||
schema = data.schema
|
||||
else:
|
||||
if schema is not None:
|
||||
data = pa.Table.from_batches([], schema=schema)
|
||||
data = pa.Table.from_pylist([], schema)
|
||||
if schema is None:
|
||||
if data is None:
|
||||
raise ValueError("Either data or schema must be provided")
|
||||
@@ -1343,7 +1165,6 @@ class Table(ABC):
|
||||
ngram_max_length: int = 3,
|
||||
prefix_only: bool = False,
|
||||
block_size: int = 128,
|
||||
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
|
||||
wait_timeout: Optional[timedelta] = None,
|
||||
name: Optional[str] = None,
|
||||
):
|
||||
@@ -1422,11 +1243,6 @@ class Table(ABC):
|
||||
The number of documents per compressed posting block. Must be 128
|
||||
or 256. A value of 256 uses the experimental FTS V3 format and
|
||||
may introduce breaking changes.
|
||||
document_granularity: DocumentGranularity, default ROW
|
||||
``ROW`` treats the selected text in one table row as one document.
|
||||
``LIST_ELEMENT`` treats each element of the deepest list on the field
|
||||
path as one document and returns its physical coordinates in
|
||||
``_doc_index`` for matching queries.
|
||||
wait_timeout: timedelta, optional
|
||||
The timeout to wait if indexing is asynchronous.
|
||||
name: str, optional
|
||||
@@ -1450,7 +1266,6 @@ class Table(ABC):
|
||||
fill_value: float = 0.0,
|
||||
progress: Optional[Union[bool, Callable, Any]] = None,
|
||||
write_parallelism: Optional[int] = None,
|
||||
allow_external_blob_outside_bases: bool = False,
|
||||
) -> AddResult:
|
||||
"""Add more data to the [Table][lancedb.table.Table].
|
||||
|
||||
@@ -1502,10 +1317,6 @@ class Table(ABC):
|
||||
data in flight. Defaults to an estimate based on the data size,
|
||||
capped at the number of CPU cores. Lower this if bulk ingestion is
|
||||
using too much memory.
|
||||
allow_external_blob_outside_bases: bool, default False
|
||||
Store blob URIs that sit outside registered blob bases. The row
|
||||
keeps a reference, so the object has to stay readable. Local
|
||||
tables only.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -1918,7 +1729,7 @@ class Table(ABC):
|
||||
@abstractmethod
|
||||
def update(
|
||||
self,
|
||||
where: Optional[Union[str, Expr]] = None,
|
||||
where: Optional[str] = None,
|
||||
values: Optional[dict] = None,
|
||||
*,
|
||||
values_sql: Optional[Dict[str, str]] = None,
|
||||
@@ -1933,11 +1744,9 @@ class Table(ABC):
|
||||
|
||||
Parameters
|
||||
----------
|
||||
where: str or [Expr][lancedb.expr.Expr], optional
|
||||
The filter condition. Can be a SQL string or a type-safe
|
||||
[Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
|
||||
[lit][lancedb.expr.lit]. The filter must not be empty, or it will
|
||||
error.
|
||||
where: str, optional
|
||||
The SQL where clause to use when updating rows. For example, 'x = 2'
|
||||
or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
|
||||
values: dict, optional
|
||||
The values to update. The keys are the column names and the values
|
||||
are the values to set.
|
||||
@@ -1955,7 +1764,6 @@ class Table(ABC):
|
||||
Examples
|
||||
--------
|
||||
>>> import lancedb
|
||||
>>> from lancedb.expr import col
|
||||
>>> import pandas as pd
|
||||
>>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1.0, 2], [3, 4], [5, 6]]})
|
||||
>>> db = lancedb.connect("./.lancedb")
|
||||
@@ -1965,7 +1773,7 @@ class Table(ABC):
|
||||
0 1 [1.0, 2.0]
|
||||
1 2 [3.0, 4.0]
|
||||
2 3 [5.0, 6.0]
|
||||
>>> table.update(where=col("x") == 2, values={"vector": [10.0, 10]})
|
||||
>>> table.update(where="x = 2", values={"vector": [10.0, 10]})
|
||||
UpdateResult(rows_updated=1, version=2)
|
||||
>>> table.to_pandas()
|
||||
x vector
|
||||
@@ -2161,7 +1969,7 @@ class Table(ABC):
|
||||
A mapping with one ``FunctionApplication`` value keeps its scalar
|
||||
or named-struct result in the named table column. A bare
|
||||
named-struct application expands its ordered result fields as one
|
||||
atomic binding; aliases come from ``rename(columns=...)``.
|
||||
atomic sibling group; aliases come from ``rename(columns=...)``.
|
||||
Function columns are supported only on LanceDB Cloud and
|
||||
Enterprise.
|
||||
computed: Dict[str, str], optional
|
||||
@@ -2231,7 +2039,7 @@ class Table(ABC):
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def refresh_column_async(self, column: str) -> Job[RefreshColumnJobResult]:
|
||||
def refresh_column_async(self, column: str) -> Job:
|
||||
"""
|
||||
Like :meth:`refresh_column`, but returns a handle to the refresh job
|
||||
instead of blocking until it completes.
|
||||
@@ -2242,12 +2050,6 @@ class Table(ABC):
|
||||
than failing the job. On local tables the job runs in-process; on
|
||||
LanceDB Cloud and Enterprise it is the server's backfill job.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Job[RefreshColumnResult]
|
||||
A job whose successful ``wait`` returns row counts plus the source
|
||||
and published table versions.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> import lancedb
|
||||
@@ -2256,9 +2058,7 @@ class Table(ABC):
|
||||
>>> table.add_columns(computed={"doubled": "x * 2"})
|
||||
AddColumnsResult(version=2)
|
||||
>>> job = table.refresh_column_async("doubled")
|
||||
>>> result = job.wait()
|
||||
>>> result.rows_assigned
|
||||
2
|
||||
>>> job.wait()
|
||||
>>> job.status()
|
||||
'finished'
|
||||
"""
|
||||
@@ -2304,25 +2104,12 @@ class Table(ABC):
|
||||
----------
|
||||
updates : dict
|
||||
One or more dicts, each with:
|
||||
|
||||
- "path": str — dot-path to the field (e.g. "embedding" or "a.b.c").
|
||||
- "metadata": dict[str, str | None] — keys to set; a value of ``None``
|
||||
deletes that key.
|
||||
- "replace": bool, optional — replace the field's whole metadata map
|
||||
instead of merging (default False).
|
||||
|
||||
The following keys are treated specially, by convention, and should
|
||||
be used when appropriate:
|
||||
|
||||
- "lancedb:description": for a human-readable description of a field.
|
||||
- ``"lancedb:tag:<name>"`` for a user-defined key-value tag, where the
|
||||
suffix names the tag category; e.g. "lancedb:tag:model": "clip".
|
||||
- "lancedb:logical-column" for a column grouping; e.g. "feature_v1"
|
||||
and "feature_v2" might be in the same logical column.
|
||||
- "lancedb:status" for status options ("production", "candidate",
|
||||
"deprecated", "archived") to designate the current life cycle
|
||||
state of this column.
|
||||
|
||||
Returns
|
||||
-------
|
||||
UpdateFieldMetadataResult
|
||||
@@ -2872,7 +2659,7 @@ class LanceTable(Table):
|
||||
arrow_tbl = self.to_arrow()
|
||||
if blob_mode == "descriptions":
|
||||
arrow_tbl = strip_auto_row_ids(
|
||||
arrow_tbl, row_addressable_blob_v2_paths(self.schema)
|
||||
arrow_tbl, blob_v2_column_paths(self.schema)
|
||||
)
|
||||
return arrow_tbl.to_pandas(**kwargs)
|
||||
|
||||
@@ -3470,7 +3257,6 @@ class LanceTable(Table):
|
||||
ngram_max_length: int = 3,
|
||||
prefix_only: bool = False,
|
||||
block_size: int = 128,
|
||||
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
|
||||
name: Optional[str] = None,
|
||||
):
|
||||
"""Create a full-text search index on a column.
|
||||
@@ -3522,11 +3308,7 @@ class LanceTable(Table):
|
||||
tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)
|
||||
tokenizer_configs["custom_stop_words"] = custom_stop_words
|
||||
|
||||
config = FTS(
|
||||
block_size=block_size,
|
||||
document_granularity=document_granularity,
|
||||
**tokenizer_configs,
|
||||
)
|
||||
config = FTS(block_size=block_size, **tokenizer_configs)
|
||||
|
||||
try:
|
||||
LOOP.run(
|
||||
@@ -3616,7 +3398,6 @@ class LanceTable(Table):
|
||||
fill_value: float = 0.0,
|
||||
progress: Optional[Union[bool, Callable, Any]] = None,
|
||||
write_parallelism: Optional[int] = None,
|
||||
allow_external_blob_outside_bases: bool = False,
|
||||
) -> AddResult:
|
||||
"""Add data to the table.
|
||||
If vector columns are missing and the table
|
||||
@@ -3644,9 +3425,6 @@ class LanceTable(Table):
|
||||
data in flight. Defaults to an estimate based on the data size,
|
||||
capped at the number of CPU cores. Lower this if bulk ingestion is
|
||||
using too much memory.
|
||||
allow_external_blob_outside_bases: bool, default False
|
||||
Allow blob URIs outside registered bases. See :meth:`Table.add`.
|
||||
Local tables only.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -3663,7 +3441,6 @@ class LanceTable(Table):
|
||||
fill_value=fill_value,
|
||||
progress=progress,
|
||||
write_parallelism=write_parallelism,
|
||||
allow_external_blob_outside_bases=allow_external_blob_outside_bases,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
@@ -4018,7 +3795,7 @@ class LanceTable(Table):
|
||||
|
||||
def update(
|
||||
self,
|
||||
where: Optional[Union[str, Expr]] = None,
|
||||
where: Optional[str] = None,
|
||||
values: Optional[dict] = None,
|
||||
*,
|
||||
values_sql: Optional[Dict[str, str]] = None,
|
||||
@@ -4029,11 +3806,9 @@ class LanceTable(Table):
|
||||
|
||||
Parameters
|
||||
----------
|
||||
where: str or [Expr][lancedb.expr.Expr], optional
|
||||
The filter condition. Can be a SQL string or a type-safe
|
||||
[Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
|
||||
[lit][lancedb.expr.lit]. The filter must not be empty, or it will
|
||||
error.
|
||||
where: str, optional
|
||||
The SQL where clause to use when updating rows. For example, 'x = 2'
|
||||
or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
|
||||
values: dict, optional
|
||||
The values to update. The keys are the column names and the values
|
||||
are the values to set.
|
||||
@@ -4051,7 +3826,6 @@ class LanceTable(Table):
|
||||
Examples
|
||||
--------
|
||||
>>> import lancedb
|
||||
>>> from lancedb.expr import col
|
||||
>>> import pandas as pd
|
||||
>>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1.0, 2], [3, 4], [5, 6]]})
|
||||
>>> db = lancedb.connect("./.lancedb")
|
||||
@@ -4061,7 +3835,7 @@ class LanceTable(Table):
|
||||
0 1 [1.0, 2.0]
|
||||
1 2 [3.0, 4.0]
|
||||
2 3 [5.0, 6.0]
|
||||
>>> table.update(where=col("x") == 2, values={"vector": [10.0, 10]})
|
||||
>>> table.update(where="x = 2", values={"vector": [10.0, 10]})
|
||||
UpdateResult(rows_updated=1, version=2)
|
||||
>>> table.to_pandas()
|
||||
x vector
|
||||
@@ -4308,7 +4082,7 @@ class LanceTable(Table):
|
||||
[`AsyncTable.refresh_column`][lancedb.AsyncTable.refresh_column]."""
|
||||
return LOOP.run(self._table.refresh_column(column))
|
||||
|
||||
def refresh_column_async(self, column: str) -> Job[RefreshColumnJobResult]:
|
||||
def refresh_column_async(self, column: str) -> Job:
|
||||
"""Fill a computed column's unfilled rows, returning a handle to the
|
||||
refresh job. See
|
||||
[`Table.refresh_column_async`][lancedb.table.Table.refresh_column_async].
|
||||
@@ -5276,9 +5050,7 @@ class AsyncTable:
|
||||
if blob_mode == "descriptions" or not schema_has_blob_field(schema):
|
||||
arrow_tbl = await self.to_arrow()
|
||||
if blob_mode == "descriptions":
|
||||
arrow_tbl = strip_auto_row_ids(
|
||||
arrow_tbl, row_addressable_blob_v2_paths(schema)
|
||||
)
|
||||
arrow_tbl = strip_auto_row_ids(arrow_tbl, blob_v2_column_paths(schema))
|
||||
return arrow_tbl.to_pandas(**kwargs)
|
||||
|
||||
if blob_mode == "lazy" and get_uri_scheme(await self.uri()) == "memory":
|
||||
@@ -5582,7 +5354,6 @@ class AsyncTable:
|
||||
fill_value: Optional[float] = None,
|
||||
progress: Optional[Union[bool, Callable, Any]] = None,
|
||||
write_parallelism: Optional[int] = None,
|
||||
allow_external_blob_outside_bases: bool = False,
|
||||
) -> AddResult:
|
||||
"""Add more data to the [AsyncTable][lancedb.table.AsyncTable].
|
||||
|
||||
@@ -5613,9 +5384,6 @@ class AsyncTable:
|
||||
data in flight. Defaults to an estimate based on the data size,
|
||||
capped at the number of CPU cores. Lower this if bulk ingestion is
|
||||
using too much memory.
|
||||
allow_external_blob_outside_bases: bool, default False
|
||||
Allow blob URIs outside registered bases. See :meth:`Table.add`.
|
||||
Local tables only.
|
||||
|
||||
"""
|
||||
schema = await self.schema()
|
||||
@@ -5652,7 +5420,6 @@ class AsyncTable:
|
||||
mode or "append",
|
||||
progress=progress,
|
||||
write_parallelism=write_parallelism,
|
||||
allow_external_blob_outside_bases=allow_external_blob_outside_bases,
|
||||
)
|
||||
except RuntimeError as e:
|
||||
if "Cast error" in str(e):
|
||||
@@ -6177,7 +5944,7 @@ class AsyncTable:
|
||||
self,
|
||||
updates: Optional[Dict[str, Any]] = None,
|
||||
*,
|
||||
where: Optional[Union[str, Expr]] = None,
|
||||
where: Optional[str] = None,
|
||||
updates_sql: Optional[Dict[str, str]] = None,
|
||||
) -> UpdateResult:
|
||||
"""
|
||||
@@ -6192,11 +5959,9 @@ class AsyncTable:
|
||||
The updates to apply. The keys should be the name of the column to
|
||||
update. The values should be the new values to assign. This is
|
||||
required unless updates_sql is supplied.
|
||||
where: str or [Expr][lancedb.expr.Expr], optional
|
||||
The filter condition. Can be a SQL string or a type-safe
|
||||
[Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
|
||||
[lit][lancedb.expr.lit]. Only rows that satisfy this filter will
|
||||
be updated.
|
||||
where: str, optional
|
||||
An SQL filter that controls which rows are updated. For example, 'x = 2'
|
||||
or 'x IN (1, 2, 3)'. Only rows that satisfy this filter will be udpated.
|
||||
updates_sql: dict, optional
|
||||
The updates to apply, expressed as SQL expression strings. The keys should
|
||||
be column names. The values should be SQL expressions. These can be SQL
|
||||
@@ -6214,14 +5979,13 @@ class AsyncTable:
|
||||
--------
|
||||
>>> import asyncio
|
||||
>>> import lancedb
|
||||
>>> from lancedb.expr import col
|
||||
>>> import pandas as pd
|
||||
>>> async def demo_update():
|
||||
... data = pd.DataFrame({"x": [1, 2], "vector": [[1, 2], [3, 4]]})
|
||||
... db = await lancedb.connect_async("./.lancedb")
|
||||
... table = await db.create_table("my_table", data)
|
||||
... # x is [1, 2], vector is [[1, 2], [3, 4]]
|
||||
... await table.update({"vector": [10, 10]}, where=col("x") == 2)
|
||||
... await table.update({"vector": [10, 10]}, where="x = 2")
|
||||
... # x is [1, 2], vector is [[1, 2], [10, 10]]
|
||||
... await table.update(updates_sql={"x": "x + 1"})
|
||||
... # x is [2, 3], vector is [[1, 2], [10, 10]]
|
||||
@@ -6235,8 +5999,7 @@ class AsyncTable:
|
||||
if updates is not None:
|
||||
updates_sql = {k: value_to_sql(v) for k, v in updates.items()}
|
||||
|
||||
predicate = where.to_sql() if isinstance(where, Expr) else where
|
||||
return await self._inner.update(updates_sql, predicate)
|
||||
return await self._inner.update(updates_sql, where)
|
||||
|
||||
async def add_columns(
|
||||
self,
|
||||
@@ -6264,7 +6027,7 @@ class AsyncTable:
|
||||
A mapping with one ``FunctionApplication`` value keeps its scalar
|
||||
or named-struct result in the named table column. A bare
|
||||
named-struct application expands its ordered result fields as one
|
||||
atomic binding; aliases come from ``rename(columns=...)``.
|
||||
atomic sibling group; aliases come from ``rename(columns=...)``.
|
||||
Function columns are supported only on LanceDB Cloud and
|
||||
Enterprise.
|
||||
computed: Dict[str, str], optional
|
||||
@@ -6301,7 +6064,7 @@ class AsyncTable:
|
||||
isinstance(value, FunctionApplication) for value in transforms.values()
|
||||
):
|
||||
raise ValueError(
|
||||
"one add_columns call declares exactly one Function binding"
|
||||
"one add_columns call declares exactly one Function sibling group"
|
||||
)
|
||||
function_output_name, function_application = next(iter(transforms.items()))
|
||||
|
||||
@@ -6359,9 +6122,7 @@ class AsyncTable:
|
||||
"""
|
||||
return await self._inner.refresh_column(column)
|
||||
|
||||
async def refresh_column_async(
|
||||
self, column: str
|
||||
) -> AsyncJob[RefreshColumnJobResult]:
|
||||
async def refresh_column_async(self, column: str) -> AsyncJob:
|
||||
"""
|
||||
Like :meth:`refresh_column`, but returns a handle to the refresh job
|
||||
instead of blocking until it completes.
|
||||
@@ -6373,12 +6134,6 @@ class AsyncTable:
|
||||
in-process; on LanceDB Cloud and Enterprise it is the server's
|
||||
backfill job.
|
||||
|
||||
Returns
|
||||
-------
|
||||
AsyncJob[RefreshColumnResult]
|
||||
A job whose successful ``wait`` returns row counts plus the source
|
||||
and published table versions.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> import asyncio
|
||||
@@ -6388,16 +6143,12 @@ class AsyncTable:
|
||||
... table = await db.create_table("computed_job_async_demo", [{"x": 1}])
|
||||
... await table.add_columns(computed={"doubled": "x * 2"})
|
||||
... job = await table.refresh_column_async("doubled")
|
||||
... result = await job.wait()
|
||||
... assert result.rows_assigned == 1
|
||||
... await job.wait()
|
||||
... return await job.status()
|
||||
>>> asyncio.run(refresh_in_background())
|
||||
'finished'
|
||||
"""
|
||||
return _typed_job(
|
||||
await self._inner.refresh_column_async(column),
|
||||
RefreshColumnJobResult.from_json,
|
||||
)
|
||||
return AsyncJob(await self._inner.refresh_column_async(column))
|
||||
|
||||
async def alter_columns(
|
||||
self, *alterations: Iterable[dict[str, Any]]
|
||||
@@ -7050,21 +6801,21 @@ class Branches:
|
||||
"""Diff a branch against main."""
|
||||
return LOOP.run(self._table.branches.diff(from_branch))
|
||||
|
||||
def cherry_pick(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]:
|
||||
"""Cherry-pick a branch onto main, or dry-run.
|
||||
def merge(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]:
|
||||
"""Merge a branch into main, or dry-run.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
from_branch: str
|
||||
Branch to cherry-pick from.
|
||||
Branch to merge from.
|
||||
dry_run: bool, default False
|
||||
When True, only preview. When False, attempt the cherry-pick.
|
||||
When True, only preview. When False, attempt the merge.
|
||||
|
||||
Notes
|
||||
-----
|
||||
A failed cherry-pick returns ``status="failed"`` instead of raising.
|
||||
A rejected merge returns ``status="rejected"`` instead of raising.
|
||||
"""
|
||||
return LOOP.run(self._table.branches.cherry_pick(from_branch, dry_run))
|
||||
return LOOP.run(self._table.branches.merge(from_branch, dry_run))
|
||||
|
||||
def _wrap(
|
||||
self, async_table: "AsyncTable", version: Optional[int] = None
|
||||
@@ -7200,11 +6951,9 @@ class AsyncBranches:
|
||||
"""Diff a branch against main."""
|
||||
return await self._table.branches.diff(from_branch)
|
||||
|
||||
async def cherry_pick(
|
||||
self, from_branch: str, dry_run: bool = False
|
||||
) -> Dict[str, Any]:
|
||||
"""Cherry-pick a branch onto main, or dry-run.
|
||||
async def merge(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]:
|
||||
"""Merge a branch into main, or dry-run.
|
||||
|
||||
A failed cherry-pick returns ``status="failed"`` instead of raising.
|
||||
A rejected merge returns ``status="rejected"`` instead of raising.
|
||||
"""
|
||||
return await self._table.branches.cherry_pick(from_branch, dry_run)
|
||||
return await self._table.branches.merge(from_branch, dry_run)
|
||||
|
||||
@@ -105,7 +105,7 @@ def test_quickstart(tmp_path):
|
||||
tbl.create_index(num_sub_vectors=1)
|
||||
# --8<-- [end:create_index]
|
||||
# --8<-- [start:delete_rows]
|
||||
tbl.delete("item = 'fizz'")
|
||||
tbl.delete('item = "fizz"')
|
||||
# --8<-- [end:delete_rows]
|
||||
# --8<-- [start:drop_table]
|
||||
db.drop_table("my_table")
|
||||
@@ -201,7 +201,7 @@ async def test_quickstart_async(tmp_path):
|
||||
await tbl.create_index("vector")
|
||||
# --8<-- [end:create_index_async]
|
||||
# --8<-- [start:delete_rows_async]
|
||||
await tbl.delete("item = 'fizz'")
|
||||
await tbl.delete('item = "fizz"')
|
||||
# --8<-- [end:delete_rows_async]
|
||||
# --8<-- [start:drop_table_async]
|
||||
await db.drop_table("my_table_async")
|
||||
|
||||
@@ -266,7 +266,7 @@ def test_table():
|
||||
tbl.add(pydantic_model_items)
|
||||
# --8<-- [end:add_table_from_pydantic]
|
||||
# --8<-- [start:delete_row]
|
||||
tbl.delete("item = 'fizz'")
|
||||
tbl.delete('item = "fizz"')
|
||||
# --8<-- [end:delete_row]
|
||||
# --8<-- [start:delete_specific_row]
|
||||
data = [
|
||||
@@ -538,7 +538,7 @@ async def test_table_async():
|
||||
await async_tbl.add(pydantic_model_items)
|
||||
# --8<-- [end:add_table_async_from_pydantic]
|
||||
# --8<-- [start:delete_row_async]
|
||||
await async_tbl.delete("item = 'fizz'")
|
||||
await async_tbl.delete('item = "fizz"')
|
||||
# --8<-- [end:delete_row_async]
|
||||
# --8<-- [start:delete_specific_row_async]
|
||||
data = [
|
||||
|
||||
@@ -2,41 +2,17 @@
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import io
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
|
||||
import lance
|
||||
import pyarrow as pa
|
||||
import pyarrow.compute as pc
|
||||
import pytest
|
||||
from lance.blob import BlobType as LanceBlobType
|
||||
|
||||
import lancedb
|
||||
from lancedb._blob import (
|
||||
blob_v2_projection_sources,
|
||||
read_row_ids_from_hits,
|
||||
stash_auto_row_ids,
|
||||
)
|
||||
from lancedb.expr import col
|
||||
from lancedb._blob import read_row_ids_from_hits, stash_auto_row_ids
|
||||
from lancedb.index import FTS
|
||||
from lancedb.schema import blob_column_paths, blob_v2_column_paths
|
||||
|
||||
|
||||
_HIDE_LANCE_BLOB = """\
|
||||
import importlib.abc
|
||||
import sys
|
||||
|
||||
class _MissingLanceBlob(importlib.abc.MetaPathFinder):
|
||||
def find_spec(self, fullname, path, target=None):
|
||||
if fullname == "lance.blob" or fullname.startswith("lance.blob."):
|
||||
raise ModuleNotFoundError(fullname, name="lance.blob")
|
||||
|
||||
sys.modules.pop("lance.blob", None)
|
||||
sys.meta_path.insert(0, _MissingLanceBlob())
|
||||
"""
|
||||
|
||||
|
||||
def _blob_table(name, rows):
|
||||
db = lancedb.connect("memory:///")
|
||||
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
|
||||
@@ -70,181 +46,6 @@ def test_blob_factory_declares_v2_field():
|
||||
field = lancedb.blob("image")
|
||||
assert isinstance(field.type, pa.ExtensionType)
|
||||
assert field.type.extension_name == "lance.blob.v2"
|
||||
assert lancedb.BlobType is LanceBlobType
|
||||
assert type(field.type) is LanceBlobType
|
||||
|
||||
|
||||
def test_blob_type_works_without_pylance():
|
||||
script = _HIDE_LANCE_BLOB + textwrap.dedent(
|
||||
"""\
|
||||
import lancedb
|
||||
import pyarrow as pa
|
||||
|
||||
field = lancedb.blob("image")
|
||||
if not isinstance(field.type, pa.ExtensionType):
|
||||
raise SystemExit("expected an extension type")
|
||||
if field.type.extension_name != "lance.blob.v2":
|
||||
raise SystemExit(field.type.extension_name)
|
||||
if lancedb.BlobType is not type(field.type):
|
||||
raise SystemExit("BlobType is not the field type class")
|
||||
if lancedb.BlobType.__module__ != "lancedb.schema":
|
||||
raise SystemExit(lancedb.BlobType.__module__)
|
||||
|
||||
db = lancedb.connect("memory:///")
|
||||
table = db.create_table(
|
||||
"images",
|
||||
schema=pa.schema([pa.field("id", pa.int64()), field]),
|
||||
)
|
||||
table.add([{"id": 1, "image": b"hello"}])
|
||||
result = (
|
||||
table.merge_insert("id")
|
||||
.when_matched_update_all()
|
||||
.when_not_matched_insert_all()
|
||||
.execute([{"id": 1, "image": b"updated"}, {"id": 2, "image": b"inserted"}])
|
||||
)
|
||||
if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
|
||||
raise SystemExit(
|
||||
f"merge_insert rows updated={result.num_updated_rows} "
|
||||
f"inserted={result.num_inserted_rows}"
|
||||
)
|
||||
"""
|
||||
)
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def test_blob_resolves_pylance_type_without_eager_import():
|
||||
script = textwrap.dedent(
|
||||
"""\
|
||||
import sys
|
||||
import lancedb
|
||||
|
||||
if "lance.blob" in sys.modules:
|
||||
raise SystemExit("import lancedb imported lance.blob")
|
||||
field = lancedb.blob("image")
|
||||
from lance.blob import BlobType
|
||||
|
||||
if type(field.type) is not BlobType:
|
||||
raise SystemExit(f"{type(field.type)} is not {BlobType}")
|
||||
import lance
|
||||
|
||||
image = lance.blob_array([b"x"])
|
||||
if type(image.type) is not BlobType:
|
||||
raise SystemExit("blob_array used a different class")
|
||||
if type(image.type) is not type(field.type):
|
||||
raise SystemExit("field and array classes differ")
|
||||
"""
|
||||
)
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def test_blob_fallback_fails_if_name_already_registered():
|
||||
script = _HIDE_LANCE_BLOB + textwrap.dedent(
|
||||
"""\
|
||||
import pyarrow as pa
|
||||
|
||||
class OtherBlobType(pa.ExtensionType):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
pa.struct([pa.field("data", pa.large_binary())]),
|
||||
"lance.blob.v2",
|
||||
)
|
||||
|
||||
def __arrow_ext_serialize__(self):
|
||||
return b""
|
||||
|
||||
@classmethod
|
||||
def __arrow_ext_deserialize__(cls, storage_type, serialized):
|
||||
return cls()
|
||||
|
||||
pa.register_extension_type(OtherBlobType())
|
||||
import lancedb
|
||||
|
||||
try:
|
||||
lancedb.blob("image")
|
||||
except ValueError as err:
|
||||
if "already registered" not in str(err):
|
||||
raise SystemExit(err)
|
||||
else:
|
||||
raise SystemExit("expected ValueError")
|
||||
"""
|
||||
)
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def test_blob_type_rejects_competing_registration_with_pylance():
|
||||
script = textwrap.dedent(
|
||||
"""\
|
||||
import pyarrow as pa
|
||||
import pyarrow.ipc
|
||||
|
||||
class OtherBlobType(pa.ExtensionType):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
pa.struct(
|
||||
[
|
||||
pa.field("data", pa.large_binary()),
|
||||
pa.field("uri", pa.utf8()),
|
||||
pa.field("position", pa.uint64()),
|
||||
pa.field("size", pa.uint64()),
|
||||
]
|
||||
),
|
||||
"lance.blob.v2",
|
||||
)
|
||||
|
||||
def __arrow_ext_serialize__(self):
|
||||
return b""
|
||||
|
||||
@classmethod
|
||||
def __arrow_ext_deserialize__(cls, storage_type, serialized):
|
||||
return cls()
|
||||
|
||||
pa.register_extension_type(OtherBlobType())
|
||||
|
||||
from lance.blob import BlobType
|
||||
|
||||
if BlobType is OtherBlobType:
|
||||
raise SystemExit("pylance BlobType was replaced")
|
||||
schema = pa.schema([pa.field("value", BlobType())])
|
||||
restored = pa.ipc.read_schema(schema.serialize())
|
||||
if type(restored.field("value").type) is not OtherBlobType:
|
||||
raise SystemExit(type(restored.field("value").type))
|
||||
|
||||
import lancedb
|
||||
|
||||
try:
|
||||
lancedb.blob("image")
|
||||
except ValueError as err:
|
||||
if "__main__.OtherBlobType" not in str(err):
|
||||
raise SystemExit(err)
|
||||
else:
|
||||
raise SystemExit("expected ValueError")
|
||||
"""
|
||||
)
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def test_blob_v2_column_paths_include_list_children():
|
||||
@@ -269,14 +70,6 @@ def test_blob_v2_column_paths_include_list_children():
|
||||
]
|
||||
|
||||
|
||||
def test_blob_v2_projection_sources_use_typed_column_name():
|
||||
schema = pa.schema([lancedb.blob("blob")])
|
||||
|
||||
assert blob_v2_projection_sources(schema, {"blob_alias": col("blob")}) == {
|
||||
"blob_alias": "blob"
|
||||
}
|
||||
|
||||
|
||||
def _legacy_v1_table(name):
|
||||
db = lancedb.connect("memory:///")
|
||||
schema = pa.schema(
|
||||
@@ -373,20 +166,6 @@ async def test_async_table_to_pandas_descriptions_mode_omits_row_id():
|
||||
assert set(descriptor.keys()) == {"kind", "position", "size", "blob_id", "blob_uri"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_typed_blob_projection_preserves_source_column():
|
||||
db = await lancedb.connect_async("memory:///typed_blob_projection")
|
||||
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("blob")])
|
||||
table = await db.create_table("typed_blob_projection", schema=schema)
|
||||
await table.add([{"id": 1, "blob": b"alpha"}])
|
||||
|
||||
hits = await table.query().select({"blob_alias": col("blob")}).to_arrow()
|
||||
|
||||
assert "_lance_row_id" in hits.schema.field("blob_alias").type.names
|
||||
blobs = await table.fetch_blobs("blob", hits)
|
||||
assert blobs.to_pylist() == [b"alpha"]
|
||||
|
||||
|
||||
def test_fetch_blobs_round_trip():
|
||||
table = _blob_table(
|
||||
"round_trip",
|
||||
@@ -397,292 +176,6 @@ def test_fetch_blobs_round_trip():
|
||||
assert [blobs[0].as_py(), blobs[1].as_py()] == [b"alpha", b"beta"]
|
||||
|
||||
|
||||
def test_merge_insert_writes_python_bytes():
|
||||
table = _blob_table("merge_bytes", [{"id": 1, "image": b"before"}])
|
||||
result = (
|
||||
table.merge_insert("id")
|
||||
.when_matched_update_all()
|
||||
.when_not_matched_insert_all()
|
||||
.execute([{"id": 1, "image": b"updated"}, {"id": 2, "image": b"inserted"}])
|
||||
)
|
||||
assert result.num_updated_rows == 1
|
||||
assert result.num_inserted_rows == 1
|
||||
by_id = _row_ids_by_id(table)
|
||||
blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
|
||||
assert blobs.to_pylist() == [b"updated", b"inserted"]
|
||||
|
||||
|
||||
def test_merge_insert_bytes_after_reopen_without_touching_blob_type(tmp_path):
|
||||
db = lancedb.connect(tmp_path)
|
||||
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
|
||||
table = db.create_table("images", schema=schema)
|
||||
table.add([{"id": 1, "image": b"hello"}])
|
||||
|
||||
script = textwrap.dedent(
|
||||
f"""\
|
||||
import lancedb
|
||||
|
||||
db = lancedb.connect({str(tmp_path)!r})
|
||||
table = db.open_table("images")
|
||||
image_type = table.schema.field("image").type
|
||||
if type(image_type).__name__ != "StructType":
|
||||
raise SystemExit(f"expected StructType, got {{type(image_type)}}")
|
||||
result = (
|
||||
table.merge_insert("id")
|
||||
.when_matched_update_all()
|
||||
.when_not_matched_insert_all()
|
||||
.execute(
|
||||
[{{"id": 1, "image": b"updated"}}, {{"id": 2, "image": b"inserted"}}]
|
||||
)
|
||||
)
|
||||
if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
|
||||
raise SystemExit(
|
||||
f"rows updated={{result.num_updated_rows}} "
|
||||
f"inserted={{result.num_inserted_rows}}"
|
||||
)
|
||||
hits = table.search().with_row_id(True).limit(10).to_arrow()
|
||||
by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
|
||||
blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
|
||||
if blobs.to_pylist() != [b"updated", b"inserted"]:
|
||||
raise SystemExit(blobs.to_pylist())
|
||||
"""
|
||||
)
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def test_merge_insert_bytes_after_reopen_without_pylance(tmp_path):
|
||||
db = lancedb.connect(tmp_path)
|
||||
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
|
||||
table = db.create_table("images", schema=schema)
|
||||
table.add([{"id": 1, "image": b"hello"}])
|
||||
|
||||
script = _HIDE_LANCE_BLOB + textwrap.dedent(
|
||||
f"""\
|
||||
import lancedb
|
||||
|
||||
db = lancedb.connect({str(tmp_path)!r})
|
||||
table = db.open_table("images")
|
||||
image_type = table.schema.field("image").type
|
||||
if type(image_type).__name__ != "StructType":
|
||||
raise SystemExit(f"expected StructType, got {{type(image_type)}}")
|
||||
result = (
|
||||
table.merge_insert("id")
|
||||
.when_matched_update_all()
|
||||
.when_not_matched_insert_all()
|
||||
.execute(
|
||||
[{{"id": 1, "image": b"updated"}}, {{"id": 2, "image": b"inserted"}}]
|
||||
)
|
||||
)
|
||||
if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
|
||||
raise SystemExit(
|
||||
f"rows updated={{result.num_updated_rows}} "
|
||||
f"inserted={{result.num_inserted_rows}}"
|
||||
)
|
||||
hits = table.search().with_row_id(True).limit(10).to_arrow()
|
||||
by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
|
||||
blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
|
||||
if blobs.to_pylist() != [b"updated", b"inserted"]:
|
||||
raise SystemExit(blobs.to_pylist())
|
||||
"""
|
||||
)
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def test_merge_insert_blob_array_into_reopened_unregistered_table(tmp_path):
|
||||
db = lancedb.connect(tmp_path)
|
||||
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
|
||||
table = db.create_table("images", schema=schema)
|
||||
table.add([{"id": 1, "image": b"before"}])
|
||||
|
||||
script = textwrap.dedent(
|
||||
f"""\
|
||||
import pyarrow as pa
|
||||
import lancedb
|
||||
|
||||
db = lancedb.connect({str(tmp_path)!r})
|
||||
table = db.open_table("images")
|
||||
image_type = table.schema.field("image").type
|
||||
if type(image_type).__name__ != "StructType":
|
||||
raise SystemExit(
|
||||
f"expected StructType before lance import, got {{type(image_type)}}"
|
||||
)
|
||||
|
||||
import lance
|
||||
|
||||
updates = pa.Table.from_arrays(
|
||||
[
|
||||
pa.array([1, 2], type=pa.int64()),
|
||||
lance.blob_array([b"updated", b"inserted"]),
|
||||
],
|
||||
names=["id", "image"],
|
||||
)
|
||||
result = (
|
||||
table.merge_insert("id")
|
||||
.when_matched_update_all()
|
||||
.when_not_matched_insert_all()
|
||||
.execute(updates)
|
||||
)
|
||||
if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
|
||||
raise SystemExit(
|
||||
f"rows updated={{result.num_updated_rows}} "
|
||||
f"inserted={{result.num_inserted_rows}}"
|
||||
)
|
||||
hits = table.search().with_row_id(True).limit(10).to_arrow()
|
||||
by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
|
||||
blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
|
||||
if blobs.to_pylist() != [b"updated", b"inserted"]:
|
||||
raise SystemExit(blobs.to_pylist())
|
||||
"""
|
||||
)
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def test_add_all_null_blob_column():
|
||||
db = lancedb.connect("memory:///")
|
||||
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
|
||||
table = db.create_table("all_null", schema=schema)
|
||||
table.add([{"id": 1, "image": None}, {"id": 2, "image": None}])
|
||||
by_id = _row_ids_by_id(table)
|
||||
blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
|
||||
assert blobs.to_pylist() == [None, None]
|
||||
|
||||
|
||||
def test_create_table_nested_blob_schema_without_rows():
|
||||
db = lancedb.connect("memory:///")
|
||||
schema = pa.schema(
|
||||
[
|
||||
pa.field("id", pa.int64()),
|
||||
pa.field("info", pa.struct([lancedb.blob("blob")])),
|
||||
pa.field("images", pa.list_(lancedb.blob("image"))),
|
||||
]
|
||||
)
|
||||
table = db.create_table("nested_empty", schema=schema)
|
||||
assert table.count_rows() == 0
|
||||
|
||||
|
||||
def test_merge_insert_nested_blob_dicts():
|
||||
db = lancedb.connect("memory:///")
|
||||
info = pa.StructArray.from_arrays(
|
||||
[
|
||||
pa.array(["first"], type=pa.string()),
|
||||
_blob_array("blob", [b"before"]),
|
||||
],
|
||||
names=["name", "blob"],
|
||||
)
|
||||
data = pa.Table.from_arrays(
|
||||
[pa.array([1], type=pa.int64()), info],
|
||||
names=["id", "info"],
|
||||
)
|
||||
table = db.create_table("nested_merge", data=data)
|
||||
result = (
|
||||
table.merge_insert("id")
|
||||
.when_matched_update_all()
|
||||
.execute([{"id": 1, "info": {"name": "first", "blob": b"after"}}])
|
||||
)
|
||||
assert result.num_updated_rows == 1
|
||||
by_id = _row_ids_by_id(table)
|
||||
blobs = table.fetch_blobs("info.blob", [by_id[1]])
|
||||
assert blobs.to_pylist() == [b"after"]
|
||||
|
||||
|
||||
def _list_blob_table(name):
|
||||
db = lancedb.connect("memory:///")
|
||||
blob_field = lancedb.blob("image")
|
||||
images = pa.ListArray.from_arrays(
|
||||
pa.array([0, 1], type=pa.int32()), _blob_array("image", [b"before"])
|
||||
)
|
||||
data = pa.Table.from_arrays(
|
||||
[pa.array([1], type=pa.int64()), images],
|
||||
schema=pa.schema(
|
||||
[pa.field("id", pa.int64()), pa.field("images", pa.list_(blob_field))]
|
||||
),
|
||||
)
|
||||
return db.create_table(name, data=data)
|
||||
|
||||
|
||||
def test_merge_insert_list_blob_dicts():
|
||||
table = _list_blob_table("list_merge")
|
||||
result = (
|
||||
table.merge_insert("id")
|
||||
.when_matched_update_all()
|
||||
.when_not_matched_insert_all()
|
||||
.execute([{"id": 1, "images": [b"one", b"two"]}, {"id": 2, "images": None}])
|
||||
)
|
||||
assert result.num_updated_rows == 1
|
||||
assert result.num_inserted_rows == 1
|
||||
hits = table.search().limit(10).to_arrow()
|
||||
sizes = {
|
||||
row["id"]: None if row["images"] is None else [d["size"] for d in row["images"]]
|
||||
for row in hits.to_pylist()
|
||||
}
|
||||
assert sizes == {1: [3, 3], 2: None}
|
||||
|
||||
|
||||
def test_list_blob_column_queries_as_raw_descriptors():
|
||||
table = _list_blob_table("list_query")
|
||||
hits = table.search().limit(10).to_arrow()
|
||||
element = hits.schema.field("images").type.value_type
|
||||
assert pa.types.is_struct(element)
|
||||
assert "_lance_row_id" not in element.names
|
||||
with pytest.raises(ValueError, match="expected struct before segment"):
|
||||
table.fetch_blobs("images.image", [0])
|
||||
|
||||
|
||||
def test_row_addressable_paths_exclude_list_children():
|
||||
from lancedb.schema import row_addressable_blob_v2_paths
|
||||
|
||||
schema = pa.schema(
|
||||
[
|
||||
pa.field("id", pa.int64()),
|
||||
pa.field("info", pa.struct([lancedb.blob("blob")])),
|
||||
pa.field("images", pa.list_(lancedb.blob("image"))),
|
||||
]
|
||||
)
|
||||
assert blob_v2_column_paths(schema) == ["info.blob", "images.image"]
|
||||
assert row_addressable_blob_v2_paths(schema) == ["info.blob"]
|
||||
|
||||
|
||||
def test_merge_insert_writes_pylance_blob_array():
|
||||
table = _blob_table("merge_pylance", [{"id": 1, "image": b"before"}])
|
||||
image = lance.blob_array([b"updated", b"inserted"])
|
||||
assert type(image.type) is LanceBlobType
|
||||
assert type(image.type) is type(lancedb.BlobType())
|
||||
updates = pa.Table.from_arrays(
|
||||
[pa.array([1, 2], type=pa.int64()), image], names=["id", "image"]
|
||||
)
|
||||
|
||||
result = (
|
||||
table.merge_insert("id")
|
||||
.when_matched_update_all()
|
||||
.when_not_matched_insert_all()
|
||||
.execute(updates)
|
||||
)
|
||||
|
||||
assert result.num_updated_rows == 1
|
||||
assert result.num_inserted_rows == 1
|
||||
by_id = _row_ids_by_id(table)
|
||||
blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
|
||||
assert blobs.to_pylist() == [b"updated", b"inserted"]
|
||||
|
||||
|
||||
def test_fetch_blobs_accepts_query_result():
|
||||
table = _blob_table("from_result", [{"id": 1, "image": b"gamma"}])
|
||||
hits = table.search().limit(10).to_arrow()
|
||||
@@ -910,50 +403,6 @@ async def test_blob_v2_hybrid_fetch_blobs_async():
|
||||
assert {blobs[i].as_py() for i in range(len(blobs))} == {b"alpha", b"beta"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_hybrid_typed_blob_projection_preserves_source_column():
|
||||
db = await lancedb.connect_async("memory:///hybrid_typed_blob")
|
||||
schema = pa.schema(
|
||||
[
|
||||
pa.field("id", pa.int64()),
|
||||
pa.field("text", pa.utf8()),
|
||||
pa.field("vector", pa.list_(pa.float32(), list_size=2)),
|
||||
lancedb.blob("blob"),
|
||||
]
|
||||
)
|
||||
table = await db.create_table("hybrid_typed_blob", schema=schema)
|
||||
await table.add(
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"text": "hello alpha",
|
||||
"vector": [1.0, 0.0],
|
||||
"blob": b"alpha",
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"text": "hello beta",
|
||||
"vector": [0.9, 0.1],
|
||||
"blob": b"beta",
|
||||
},
|
||||
]
|
||||
)
|
||||
await table.create_index("text", config=FTS(with_position=False))
|
||||
|
||||
hits = await (
|
||||
table.query()
|
||||
.nearest_to([1.0, 0.0])
|
||||
.nearest_to_text("hello")
|
||||
.select({"blob_alias": col("blob")})
|
||||
.limit(2)
|
||||
.to_arrow()
|
||||
)
|
||||
|
||||
assert "_lance_row_id" in hits.schema.field("blob_alias").type.names
|
||||
blobs = await table.fetch_blobs("blob", hits)
|
||||
assert {blobs[i].as_py() for i in range(len(blobs))} == {b"alpha", b"beta"}
|
||||
|
||||
|
||||
def test_blob_file_seek_read_and_read_range():
|
||||
payload = _identifiable_payload(1024)
|
||||
table = _blob_table("seek_read", [{"id": 1, "image": payload}])
|
||||
@@ -1168,71 +617,3 @@ def test_fetch_blobs_nested_path_survives_sort_after_query():
|
||||
def _identifiable_payload(size: int) -> bytes:
|
||||
block = 256
|
||||
return b"".join(bytes([i % 256]) * block for i in range(size // block))
|
||||
|
||||
|
||||
def _external_uri_blob_array(uris):
|
||||
blob_type = lancedb.blob("image").type
|
||||
storage_type = blob_type.storage_type
|
||||
child_names = [field.name for field in storage_type]
|
||||
assert "uri" in child_names, "blob layout no longer has a uri child"
|
||||
children = [
|
||||
pa.array(uris if field.name == "uri" else [None] * len(uris), type=field.type)
|
||||
for field in storage_type
|
||||
]
|
||||
storage = pa.StructArray.from_arrays(children, fields=list(storage_type))
|
||||
return pa.ExtensionArray.from_storage(blob_type, storage)
|
||||
|
||||
|
||||
def _external_uri_table_and_rows(name, uris):
|
||||
db = lancedb.connect("memory:///")
|
||||
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
|
||||
table = db.create_table(name, schema=schema)
|
||||
rows = pa.Table.from_arrays(
|
||||
[
|
||||
pa.array(range(len(uris)), type=pa.int64()),
|
||||
_external_uri_blob_array(uris),
|
||||
],
|
||||
schema=schema,
|
||||
)
|
||||
return table, rows
|
||||
|
||||
|
||||
def test_add_external_uri_struct_round_trips_with_flag(tmp_path):
|
||||
payload = b"external-uri-bytes"
|
||||
blob_path = tmp_path / "payload.bin"
|
||||
blob_path.write_bytes(payload)
|
||||
|
||||
table, rows = _external_uri_table_and_rows("external_struct", [blob_path.as_uri()])
|
||||
table.add(rows, allow_external_blob_outside_bases=True)
|
||||
|
||||
hits = table.search().to_arrow()
|
||||
blobs = table.fetch_blobs("image", hits)
|
||||
assert blobs[0].as_py() == payload
|
||||
|
||||
|
||||
def test_add_external_uri_without_flag_raises(tmp_path):
|
||||
blob_path = tmp_path / "payload.bin"
|
||||
blob_path.write_bytes(b"unreachable")
|
||||
|
||||
table, rows = _external_uri_table_and_rows("external_no_flag", [blob_path.as_uri()])
|
||||
with pytest.raises(ValueError, match="allow_external_blob_outside_bases"):
|
||||
table.add(rows)
|
||||
assert table.count_rows() == 0
|
||||
|
||||
|
||||
def test_add_external_uri_string_round_trips_with_flag(tmp_path):
|
||||
payload = b"external-uri-bytes"
|
||||
blob_path = tmp_path / "payload.bin"
|
||||
blob_path.write_bytes(payload)
|
||||
|
||||
db = lancedb.connect("memory:///")
|
||||
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
|
||||
table = db.create_table("external_string", schema=schema)
|
||||
table.add(
|
||||
[{"id": 1, "image": blob_path.as_uri()}],
|
||||
allow_external_blob_outside_bases=True,
|
||||
)
|
||||
|
||||
hits = table.search().to_arrow()
|
||||
blobs = table.fetch_blobs("image", hits)
|
||||
assert blobs[0].as_py() == payload
|
||||
|
||||
@@ -774,7 +774,7 @@ def test_drop_table_async(tmp_db: lancedb.DBConnection):
|
||||
job = tmp_db.drop_table_async("test")
|
||||
assert job.id is None
|
||||
assert job.status() == "finished"
|
||||
assert job.wait() is None
|
||||
job.wait()
|
||||
assert tmp_db.table_names() == []
|
||||
|
||||
tmp_db.create_table("test", data=data)
|
||||
@@ -790,7 +790,7 @@ async def test_drop_table_async_connection(tmp_db_async: lancedb.AsyncConnection
|
||||
job = await tmp_db_async.drop_table_async("test")
|
||||
assert job.id is None
|
||||
assert await job.status() == "finished"
|
||||
assert await job.wait() is None
|
||||
await job.wait()
|
||||
assert await tmp_db_async.table_names() == []
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -52,7 +52,7 @@ class TestExprConstruction:
|
||||
def test_func(self):
|
||||
e = func("lower", col("name"))
|
||||
assert isinstance(e, Expr)
|
||||
assert e.to_sql() == "lower(`name`)"
|
||||
assert e.to_sql() == "lower(name)"
|
||||
|
||||
def test_func_unknown_raises(self):
|
||||
with pytest.raises(Exception):
|
||||
@@ -115,7 +115,7 @@ class TestExprOperators:
|
||||
def test_and_operator(self):
|
||||
e = (col("age") > lit(18)) & (col("status") == lit("active"))
|
||||
assert isinstance(e, Expr)
|
||||
assert e.to_sql() == "((age > 18) AND (`status` = 'active'))"
|
||||
assert e.to_sql() == "((age > 18) AND (status = 'active'))"
|
||||
|
||||
def test_or_operator(self):
|
||||
e = (col("a") == lit(1)) | (col("b") == lit(2))
|
||||
@@ -166,7 +166,7 @@ class TestExprOperators:
|
||||
def test_coerce_plain_str(self):
|
||||
e = col("name") == "alice"
|
||||
assert isinstance(e, Expr)
|
||||
assert e.to_sql() == "(`name` = 'alice')"
|
||||
assert e.to_sql() == "(name = 'alice')"
|
||||
|
||||
def test_reflexive_comparisons(self):
|
||||
# 10 < col("age") swaps to col("age") > 10
|
||||
@@ -198,85 +198,85 @@ class TestExprBytesLiteral:
|
||||
|
||||
def test_bytes_equality_expr_sql(self):
|
||||
e = col("data") == lit(b"\xca\xfe")
|
||||
assert e.to_sql() == "(`data` = X'CAFE')"
|
||||
assert e.to_sql() == "(data = X'CAFE')"
|
||||
|
||||
def test_bytes_ne_expr_sql(self):
|
||||
e = col("data") != lit(b"\xff")
|
||||
assert e.to_sql() == "(`data` <> X'FF')"
|
||||
assert e.to_sql() == "(data <> X'FF')"
|
||||
|
||||
def test_bytes_compound_expr_sql(self):
|
||||
e = (col("data") == lit(b"\x01")) & (col("id") > lit(5))
|
||||
assert e.to_sql() == "((`data` = X'01') AND (id > 5))"
|
||||
assert e.to_sql() == "((data = X'01') AND (id > 5))"
|
||||
|
||||
def test_bytes_in_function_call(self):
|
||||
# Regression test: binary literals inside scalar function calls
|
||||
# used to fail because DataFusion's unparser does not support Binary
|
||||
# scalars. Now handled via a placeholder-substitution rewrite.
|
||||
e = func("contains", col("data"), lit(b"\xff"))
|
||||
assert e.to_sql() == "contains(`data`, X'FF')"
|
||||
assert e.to_sql() == "contains(data, X'FF')"
|
||||
|
||||
def test_bytes_in_not(self):
|
||||
e = ~(col("data") == lit(b"\xff"))
|
||||
assert e.to_sql() == "NOT (`data` = X'FF')"
|
||||
assert e.to_sql() == "NOT (data = X'FF')"
|
||||
|
||||
|
||||
class TestExprStringMethods:
|
||||
def test_lower(self):
|
||||
e = col("name").lower()
|
||||
assert isinstance(e, Expr)
|
||||
assert e.to_sql() == "lower(`name`)"
|
||||
assert e.to_sql() == "lower(name)"
|
||||
|
||||
def test_upper(self):
|
||||
e = col("name").upper()
|
||||
assert isinstance(e, Expr)
|
||||
assert e.to_sql() == "upper(`name`)"
|
||||
assert e.to_sql() == "upper(name)"
|
||||
|
||||
def test_contains(self):
|
||||
e = col("text").contains(lit("hello"))
|
||||
assert isinstance(e, Expr)
|
||||
assert e.to_sql() == "contains(`text`, 'hello')"
|
||||
assert e.to_sql() == "contains(text, 'hello')"
|
||||
|
||||
def test_contains_with_str_coerce(self):
|
||||
e = col("text").contains("hello")
|
||||
assert isinstance(e, Expr)
|
||||
assert e.to_sql() == "contains(`text`, 'hello')"
|
||||
assert e.to_sql() == "contains(text, 'hello')"
|
||||
|
||||
def test_chained_lower_eq(self):
|
||||
e = col("name").lower() == lit("alice")
|
||||
assert isinstance(e, Expr)
|
||||
assert e.to_sql() == "(lower(`name`) = 'alice')"
|
||||
assert e.to_sql() == "(lower(name) = 'alice')"
|
||||
|
||||
|
||||
class TestExprCast:
|
||||
def test_cast_string(self):
|
||||
e = col("id").cast("string")
|
||||
assert isinstance(e, Expr)
|
||||
assert e.to_sql() == "arrow_cast(id, 'Utf8')"
|
||||
assert e.to_sql() == "CAST(id AS VARCHAR)"
|
||||
|
||||
def test_cast_int32(self):
|
||||
e = col("score").cast("int32")
|
||||
assert isinstance(e, Expr)
|
||||
assert e.to_sql() == "arrow_cast(score, 'Int32')"
|
||||
assert e.to_sql() == "CAST(score AS INTEGER)"
|
||||
|
||||
def test_cast_float64(self):
|
||||
e = col("val").cast("float64")
|
||||
assert isinstance(e, Expr)
|
||||
assert e.to_sql() == "arrow_cast(val, 'Float64')"
|
||||
assert e.to_sql() == "CAST(val AS DOUBLE)"
|
||||
|
||||
def test_cast_pyarrow_type(self):
|
||||
e = col("score").cast(pa.int32())
|
||||
assert isinstance(e, Expr)
|
||||
assert e.to_sql() == "arrow_cast(score, 'Int32')"
|
||||
assert e.to_sql() == "CAST(score AS INTEGER)"
|
||||
|
||||
def test_cast_pyarrow_float64(self):
|
||||
e = col("val").cast(pa.float64())
|
||||
assert isinstance(e, Expr)
|
||||
assert e.to_sql() == "arrow_cast(val, 'Float64')"
|
||||
assert e.to_sql() == "CAST(val AS DOUBLE)"
|
||||
|
||||
def test_cast_pyarrow_string(self):
|
||||
e = col("id").cast(pa.string())
|
||||
assert isinstance(e, Expr)
|
||||
assert e.to_sql() == "arrow_cast(id, 'Utf8')"
|
||||
assert e.to_sql() == "CAST(id AS VARCHAR)"
|
||||
|
||||
def test_cast_pyarrow_and_string_equivalent(self):
|
||||
# pa.int32() and "int32" should produce equivalent SQL
|
||||
@@ -597,14 +597,14 @@ class TestExprIsin:
|
||||
def test_isin_strs(self):
|
||||
assert (
|
||||
col("status").isin(["active", "pending"]).to_sql()
|
||||
== "`status` IN ('active', 'pending')"
|
||||
== "status IN ('active', 'pending')"
|
||||
)
|
||||
|
||||
def test_isin_coerces_and_mixes(self):
|
||||
assert col("id").isin([lit(1), 2]).to_sql() == "id IN (1, 2)"
|
||||
|
||||
def test_isin_empty(self):
|
||||
assert col("id").isin([]).to_sql() == "false"
|
||||
assert col("id").isin([]).to_sql() == "id IN ()"
|
||||
|
||||
def test_isin_filter(self, simple_table):
|
||||
result = simple_table.search().where(col("id").isin([1, 3, 5])).to_arrow()
|
||||
|
||||
@@ -121,7 +121,7 @@ def test_function_version_identity_is_immutable_and_exact():
|
||||
assert FunctionVersion(**changed) != version
|
||||
|
||||
|
||||
def test_function_version_binds_named_columns_as_one_immutable_application():
|
||||
def test_function_version_binds_named_columns_as_one_immutable_group():
|
||||
version = FunctionVersion.from_json(
|
||||
json.dumps(job_result("remote_function_job.json"))
|
||||
)
|
||||
@@ -131,10 +131,13 @@ def test_function_version_binds_named_columns_as_one_immutable_application():
|
||||
assert application.function.name == version.name
|
||||
assert application.function.version == version.version
|
||||
assert application.output is version.signature.output
|
||||
assert application.group_id.startswith("fg_")
|
||||
assert [
|
||||
(value.parameter, value.kind, value.value["path"])
|
||||
for value in application.inputs
|
||||
] == [("text", "column", "documents.body")]
|
||||
with pytest.raises((TypeError, ValueError)):
|
||||
application.group_id = "fg_changed"
|
||||
|
||||
|
||||
def test_function_version_binding_validates_names_and_direct_columns():
|
||||
@@ -153,7 +156,7 @@ def test_function_version_binding_validates_names_and_direct_columns():
|
||||
def test_function_version_keeps_named_struct_outputs_in_one_application():
|
||||
value = job_result("remote_function_job.json")
|
||||
value["name"] = "text_features"
|
||||
value["version"] = "fv_multi_output"
|
||||
value["version"] = "fv_grouped"
|
||||
value["signature"] = {
|
||||
"inputs": [
|
||||
{"name": "title", "arrow_type": "utf8", "nullable": True},
|
||||
@@ -218,6 +221,7 @@ def test_function_application_uses_rename_columns_only():
|
||||
assert application.columns["normalized_text"] == "search_text"
|
||||
assert renamed.columns["normalized_text"] == "body_normalized"
|
||||
assert renamed.function == application.function
|
||||
assert renamed.group_id == application.group_id
|
||||
assert not hasattr(application, "rename_outputs")
|
||||
with pytest.raises(TypeError, match="immutable"):
|
||||
renamed.columns["normalized_text"] = "changed"
|
||||
@@ -238,6 +242,7 @@ def test_function_application_uses_rename_columns_only():
|
||||
|
||||
def test_binding_and_refresh_result_keep_stable_remote_fields():
|
||||
binding = FunctionBinding.from_json(fixture("remote_function_binding.json"))
|
||||
assert binding.revision == 3
|
||||
assert binding.function.version == "fv_01K3TEXT"
|
||||
assert [output.output_ordinal for output in binding.outputs] == [0, 1]
|
||||
assert binding.input_schema is not None
|
||||
@@ -317,7 +322,7 @@ def known_application() -> FunctionApplication:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_columns_routes_struct_as_one_and_multi_output_binding_atomically():
|
||||
async def test_add_columns_routes_struct_as_one_and_grouped_expansion_atomically():
|
||||
inner = _FunctionDeclarationInner()
|
||||
table = AsyncTable(inner)
|
||||
application = known_application()
|
||||
@@ -338,12 +343,12 @@ async def test_add_columns_routes_struct_as_one_and_multi_output_binding_atomica
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_columns_rejects_multiple_bindings_and_unknown_newer_application():
|
||||
async def test_add_columns_rejects_mixed_groups_and_unknown_newer_application():
|
||||
inner = _FunctionDeclarationInner()
|
||||
table = AsyncTable(inner)
|
||||
application = known_application()
|
||||
|
||||
with pytest.raises(ValueError, match="exactly one Function binding"):
|
||||
with pytest.raises(ValueError, match="exactly one Function sibling group"):
|
||||
await table.add_columns({"a": application, "b": application})
|
||||
|
||||
future = json.loads(fixture("remote_function_application.json"))
|
||||
@@ -371,6 +376,7 @@ def test_rename_requires_named_struct_and_keeps_partial_mapping_immutable():
|
||||
"arrow_type": "list<float32>",
|
||||
"nullable": False,
|
||||
},
|
||||
"group_id": "fg_scalar",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
@@ -3,12 +3,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import contextlib
|
||||
import functools
|
||||
import importlib.util
|
||||
import types
|
||||
from datetime import date
|
||||
import http.server
|
||||
import json
|
||||
from pathlib import Path
|
||||
@@ -19,16 +14,7 @@ import pyarrow as pa
|
||||
import pytest
|
||||
|
||||
import lancedb
|
||||
from lancedb.functions import (
|
||||
_MAX_FUNCTION_SECRET_VALUE_BYTES,
|
||||
_MAX_FUNCTION_SECRET_VALUES_BYTES,
|
||||
FunctionRegistrationRequest,
|
||||
UdfDefinition,
|
||||
udf,
|
||||
)
|
||||
|
||||
THRESHOLD = 20
|
||||
_CACHE = None
|
||||
from lancedb.functions import UdfDefinition, udf
|
||||
|
||||
|
||||
FIXTURES = (
|
||||
@@ -85,416 +71,9 @@ def test_scalar_udf_matches_shared_registration_golden_and_remains_callable():
|
||||
_assert_no_secret_values(request)
|
||||
|
||||
|
||||
def _run_packaged(definition, *args):
|
||||
"""Execute the shipped artifact in a fresh namespace, as a worker would."""
|
||||
source = base64.b64decode(definition.registration_request.artifact.content.data)
|
||||
namespace: dict = {}
|
||||
exec(compile(source, "<udf>", "exec"), namespace)
|
||||
return namespace[definition.registration_request.artifact.entrypoint](*args)
|
||||
|
||||
|
||||
def test_udf_conda_environment():
|
||||
@udf(conda=["scipy", "numpy"], conda_channels=["conda-forge", "defaults"])
|
||||
def halve(value: float) -> float:
|
||||
return value / 2
|
||||
|
||||
request = json.loads(halve.registration_request.to_canonical_json())
|
||||
assert request["runtime"]["environment"] == {
|
||||
"kind": "conda",
|
||||
"packages": ["numpy", "scipy"],
|
||||
"channels": ["conda-forge", "defaults"],
|
||||
}
|
||||
pip_request = json.loads(normalize_score.registration_request.to_canonical_json())
|
||||
assert "channels" not in pip_request["runtime"]["environment"]
|
||||
|
||||
with pytest.raises(ValueError, match="not both"):
|
||||
udf(name="both", pip=["numpy"], conda=["numpy"])(lambda value: value)
|
||||
with pytest.raises(ValueError, match="requires conda"):
|
||||
udf(name="channels", conda_channels=["conda-forge"])(lambda value: value)
|
||||
|
||||
|
||||
def test_udf_packages_attribute_access_and_body_imports():
|
||||
@udf
|
||||
def word_norm(body: str) -> float:
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
words = body.split()
|
||||
except AttributeError as error:
|
||||
raise ValueError(str(error)) from error
|
||||
return float(np.linalg.norm([len(w) for w in words]))
|
||||
|
||||
assert _run_packaged(word_norm, "aa bb") == pytest.approx(8**0.5)
|
||||
|
||||
|
||||
def test_udf_packages_module_globals_and_global_caches():
|
||||
@udf
|
||||
def label(value: int) -> str:
|
||||
return "big" if value >= THRESHOLD else "small"
|
||||
|
||||
assert _run_packaged(label, 21) == "big"
|
||||
|
||||
@udf
|
||||
def cached(value: int) -> int:
|
||||
global _CACHE
|
||||
if _CACHE is None:
|
||||
_CACHE = 40
|
||||
return _CACHE + value
|
||||
|
||||
assert _run_packaged(cached, 2) == 42
|
||||
|
||||
|
||||
def test_udf_annotations_are_not_runtime_names():
|
||||
@udf
|
||||
def identity(value: date) -> date:
|
||||
return value
|
||||
|
||||
assert _run_packaged(identity, date(2026, 8, 25)) == date(2026, 8, 25)
|
||||
|
||||
|
||||
def test_udf_nested_scopes_resolve_lexically():
|
||||
@udf
|
||||
def score(value: int) -> int:
|
||||
offset = 2
|
||||
|
||||
def add_offset() -> int:
|
||||
return value + offset
|
||||
|
||||
return add_offset() + sum(v for v in [0])
|
||||
|
||||
assert _run_packaged(score, 3) == 5
|
||||
|
||||
|
||||
def test_udf_resolves_module_globals_before_builtins(tmp_path):
|
||||
module_path = tmp_path / "shadowing_udfs.py"
|
||||
module_path.write_text(
|
||||
"max = 7\n"
|
||||
"len = lambda _: 99\n"
|
||||
"\n"
|
||||
"def uses_literal_shadow(value: int) -> int:\n"
|
||||
" def nested() -> int:\n"
|
||||
" return max\n"
|
||||
" return nested() + value\n"
|
||||
"\n"
|
||||
"def uses_callable_shadow(value: int) -> int:\n"
|
||||
" def nested() -> int:\n"
|
||||
" return len([1])\n"
|
||||
" return nested() + value\n"
|
||||
)
|
||||
spec = importlib.util.spec_from_file_location("shadowing_udfs", module_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
# The module's `max = 7` is what the interpreter would use, so it ships.
|
||||
assert _run_packaged(udf(module.uses_literal_shadow), 1) == 8
|
||||
# A callable global cannot ship; it must not be silently swapped for the builtin.
|
||||
with pytest.raises(TypeError, match="unsupported global value of type function"):
|
||||
udf(module.uses_callable_shadow)
|
||||
|
||||
|
||||
def test_canonical_arrow_type_is_exactly_the_grammar():
|
||||
from lancedb.functions import _GRAMMAR_PRIMITIVES, _canonical_arrow_type
|
||||
|
||||
golden = json.loads(
|
||||
(
|
||||
Path(__file__).parents[3]
|
||||
/ "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json"
|
||||
).read_text()
|
||||
)
|
||||
primitives = [
|
||||
case["arrow_type"] for case in golden["valid"] if "<" not in case["arrow_type"]
|
||||
]
|
||||
assert [name for _, name in _GRAMMAR_PRIMITIVES] == primitives
|
||||
for outside in [
|
||||
pa.timestamp("us"),
|
||||
pa.decimal128(10, 2),
|
||||
pa.large_string(),
|
||||
pa.large_binary(),
|
||||
pa.binary(4),
|
||||
pa.duration("s"),
|
||||
pa.struct([pa.field("a", pa.int32())]),
|
||||
pa.list_(pa.float32(), 0),
|
||||
pa.list_(pa.timestamp("us")),
|
||||
]:
|
||||
with pytest.raises(TypeError, match="unsupported Arrow type"):
|
||||
_canonical_arrow_type(outside)
|
||||
|
||||
|
||||
def test_udf_nested_annotations_are_postponed_in_the_artifact():
|
||||
@udf
|
||||
def score(value: int) -> int:
|
||||
def identity(item: date) -> date:
|
||||
return item
|
||||
|
||||
identity(date(2026, 8, 25))
|
||||
return value
|
||||
|
||||
assert _run_packaged(score, 3) == 3
|
||||
|
||||
|
||||
def test_udf_ships_globals_the_body_deletes():
|
||||
@udf
|
||||
def clear(value: int) -> int:
|
||||
global _CACHE
|
||||
del _CACHE
|
||||
return value
|
||||
|
||||
assert _run_packaged(clear, 3) == 3
|
||||
|
||||
|
||||
def test_udf_rejects_a_module_global_that_does_not_import_as_itself(tmp_path):
|
||||
module_path = tmp_path / "fake_module_udfs.py"
|
||||
module_path.write_text(
|
||||
"import types\n"
|
||||
"np = types.ModuleType('numpy')\n"
|
||||
"np.sqrt = lambda x: 0\n"
|
||||
"\n"
|
||||
"def score(value: int) -> int:\n"
|
||||
" return int(np.sqrt(value))\n"
|
||||
)
|
||||
spec = importlib.util.spec_from_file_location("fake_module_udfs", module_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
with pytest.raises(TypeError, match="does not import as 'numpy'"):
|
||||
udf(module.score)
|
||||
|
||||
|
||||
def test_udf_rejects_a_module_level_namespace_alias(tmp_path):
|
||||
module_path = tmp_path / "aliasing_udfs.py"
|
||||
module_path.write_text(
|
||||
"import builtins as b\n"
|
||||
"THRESHOLD = 5\n"
|
||||
"\n"
|
||||
"def score(value: int) -> int:\n"
|
||||
" return value + b.vars(b.__import__('aliasing_udfs'))['THRESHOLD']\n"
|
||||
)
|
||||
spec = importlib.util.spec_from_file_location("aliasing_udfs", module_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
with pytest.raises(ValueError, match="dynamic namespace access"):
|
||||
udf(module.score)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"access",
|
||||
[
|
||||
"globals()['THRESHOLD']",
|
||||
"eval('THRESHOLD')",
|
||||
"(lambda g: g()['THRESHOLD'])(globals)",
|
||||
"__import__('sys').modules[__name__].THRESHOLD",
|
||||
"sys.modules[__name__].THRESHOLD",
|
||||
],
|
||||
)
|
||||
def test_udf_rejects_dynamic_namespace_access(access):
|
||||
namespace: dict = {}
|
||||
exec(
|
||||
f"def score(value: int) -> int:\n return value + {access}\n",
|
||||
{"THRESHOLD": 5},
|
||||
namespace,
|
||||
)
|
||||
with pytest.raises(ValueError, match="dynamic namespace access"):
|
||||
_package_from_text(
|
||||
"def score(value: int) -> int:\n"
|
||||
" import sys\n"
|
||||
f" return value + {access}\n"
|
||||
)
|
||||
|
||||
|
||||
def _package_from_text(source: str, module_globals: dict | None = None):
|
||||
"""Load `source` as a real module file so the packager can inspect it."""
|
||||
import tempfile
|
||||
|
||||
directory = tempfile.mkdtemp()
|
||||
path = Path(directory) / "generated_udf_module.py"
|
||||
path.write_text(source)
|
||||
spec = importlib.util.spec_from_file_location(f"generated_udf_{id(source)}", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
if module_globals:
|
||||
module.__dict__.update(module_globals)
|
||||
spec.loader.exec_module(module)
|
||||
functions = [
|
||||
value
|
||||
for value in vars(module).values()
|
||||
if callable(value) and getattr(value, "__module__", None) == module.__name__
|
||||
]
|
||||
return udf(functions[0])
|
||||
|
||||
|
||||
def test_udf_rejects_a_non_standard_builtins_environment():
|
||||
def score(value: int) -> int:
|
||||
return len([1]) + value
|
||||
|
||||
score.__globals__ # noqa: B018 -- real function, real globals
|
||||
import builtins
|
||||
|
||||
patched = types.FunctionType(
|
||||
score.__code__,
|
||||
{"__builtins__": {**vars(builtins), "len": lambda _: 99}},
|
||||
"score",
|
||||
)
|
||||
patched.__annotations__ = score.__annotations__
|
||||
assert patched(3) == 102
|
||||
with pytest.raises(ValueError, match="non-standard builtins environment"):
|
||||
udf(patched)
|
||||
|
||||
class ReportingDict(dict): # reports standard entries, resolves differently
|
||||
def __missing__(self, key):
|
||||
return vars(builtins)[key]
|
||||
|
||||
disguised = types.FunctionType(
|
||||
score.__code__, {"__builtins__": ReportingDict(len=lambda _: 99)}, "score"
|
||||
)
|
||||
disguised.__annotations__ = score.__annotations__
|
||||
assert disguised(3) == 102
|
||||
with pytest.raises(ValueError, match="non-standard builtins environment"):
|
||||
udf(disguised)
|
||||
|
||||
hooked = types.FunctionType(
|
||||
score.__code__,
|
||||
{"__builtins__": {**vars(builtins), "__import__": lambda *a, **k: None}},
|
||||
"score",
|
||||
)
|
||||
hooked.__annotations__ = score.__annotations__
|
||||
with pytest.raises(ValueError, match="non-standard builtins environment"):
|
||||
udf(hooked)
|
||||
|
||||
|
||||
def test_udf_recursion_versus_a_rebound_module_name(tmp_path):
|
||||
module_path = tmp_path / "rebound_udfs.py"
|
||||
module_path.write_text(
|
||||
"def fact(value: int) -> int:\n"
|
||||
" return 1 if value <= 1 else value * fact(value - 1)\n"
|
||||
"\n"
|
||||
"def score(value: int) -> int:\n"
|
||||
" return score + value\n"
|
||||
)
|
||||
spec = importlib.util.spec_from_file_location("rebound_udfs", module_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
assert _run_packaged(udf(module.fact), 5) == 120
|
||||
raw = module.score
|
||||
module.score = 10
|
||||
with pytest.raises(ValueError, match="binds that name to another value"):
|
||||
udf(raw)
|
||||
# A wrapper that merely exposes __wrapped__ is not the function.
|
||||
module.score = functools.wraps(raw)(lambda value: 41)
|
||||
with pytest.raises(ValueError, match="binds that name to another value"):
|
||||
udf(raw)
|
||||
# The decorator's own result is; a subclass of it is not.
|
||||
module.fact = udf(module.fact)
|
||||
assert _run_packaged(module.fact, 4) == 24
|
||||
|
||||
class Twisted(UdfDefinition):
|
||||
def __call__(self, *args, **kwargs):
|
||||
return 41
|
||||
|
||||
raw_fact = module.fact._function
|
||||
module.fact = Twisted(
|
||||
raw_fact,
|
||||
name=None,
|
||||
input_schema=None,
|
||||
output_schema=None,
|
||||
pip=(),
|
||||
env={},
|
||||
secrets=(),
|
||||
python_version=None,
|
||||
)
|
||||
with pytest.raises(ValueError, match="binds that name to another value"):
|
||||
udf(raw_fact)
|
||||
|
||||
|
||||
def test_canonical_arrow_type_rejects_unrepresentable_list_children():
|
||||
from lancedb.functions import _canonical_arrow_type
|
||||
|
||||
for outside in [
|
||||
pa.list_(pa.float32()), # pyarrow default: nullable child
|
||||
pa.list_(pa.field("custom", pa.float32(), nullable=False)),
|
||||
pa.list_(pa.field("item", pa.float32(), nullable=False, metadata={"k": "v"})),
|
||||
pa.list_(pa.field("item", pa.float32(), nullable=False), 0),
|
||||
]:
|
||||
with pytest.raises(TypeError, match="unsupported Arrow type"):
|
||||
_canonical_arrow_type(outside)
|
||||
assert (
|
||||
_canonical_arrow_type(
|
||||
pa.list_(pa.field("item", pa.float32(), nullable=False), 3)
|
||||
)
|
||||
== "fixed_size_list<float32, 3>"
|
||||
)
|
||||
|
||||
|
||||
def _calls_missing(value: int) -> int:
|
||||
return missing(value) # noqa: F821
|
||||
|
||||
|
||||
def _shadows_missing_in_a_comprehension(value: int) -> int:
|
||||
return missing(value) + sum(missing for missing in ()) # noqa: F821
|
||||
|
||||
|
||||
def _shadows_missing_in_a_lambda(value: int) -> int:
|
||||
return (lambda missing: missing)(value) + missing # noqa: F821
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"function",
|
||||
[_calls_missing, _shadows_missing_in_a_comprehension, _shadows_missing_in_a_lambda],
|
||||
)
|
||||
def test_udf_rejects_a_truly_unresolved_global(function):
|
||||
with pytest.raises(ValueError, match=r"unresolved global names: \['missing'\]"):
|
||||
udf(function)
|
||||
|
||||
|
||||
def _arrow_type_from_golden(spec: dict) -> pa.DataType:
|
||||
kind = spec["type"]
|
||||
if kind in ("list", "large_list", "fixed_size_list"):
|
||||
item = _arrow_type_from_golden(spec["fields"][0]["type"])
|
||||
field = pa.field("item", item, nullable=False)
|
||||
if kind == "list":
|
||||
return pa.list_(field)
|
||||
if kind == "large_list":
|
||||
return pa.large_list(field)
|
||||
return pa.list_(field, spec["length"])
|
||||
return {
|
||||
"null": pa.null(),
|
||||
"bool": pa.bool_(),
|
||||
"utf8": pa.string(),
|
||||
"binary": pa.binary(),
|
||||
"float16": pa.float16(),
|
||||
"float32": pa.float32(),
|
||||
"float64": pa.float64(),
|
||||
"date32": pa.date32(),
|
||||
"date64": pa.date64(),
|
||||
}.get(kind) or getattr(pa, kind)()
|
||||
|
||||
|
||||
def test_arrow_type_grammar_matches_the_shared_golden():
|
||||
golden = json.loads(
|
||||
(
|
||||
Path(__file__).parents[3]
|
||||
/ "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json"
|
||||
).read_text()
|
||||
)
|
||||
from lancedb.functions import _canonical_arrow_type
|
||||
|
||||
emitted = {
|
||||
case["arrow_type"]: _canonical_arrow_type(_arrow_type_from_golden(case["json"]))
|
||||
for case in golden["valid"]
|
||||
}
|
||||
assert emitted == {
|
||||
case["arrow_type"]: case["arrow_type"] for case in golden["valid"]
|
||||
}
|
||||
assert not set(emitted) & set(golden["invalid"])
|
||||
for case in golden["server_only"]:
|
||||
with pytest.raises(TypeError, match="unsupported Arrow type"):
|
||||
_canonical_arrow_type(_arrow_type_from_golden(case["json"]))
|
||||
|
||||
|
||||
def test_explicit_arrow_schema_is_deterministic():
|
||||
input_schema = pa.schema([pa.field("value", pa.float32(), nullable=True)])
|
||||
output_schema = pa.field(
|
||||
"embedding",
|
||||
pa.list_(pa.field("item", pa.float32(), nullable=False), 3),
|
||||
nullable=False,
|
||||
)
|
||||
output_schema = pa.field("embedding", pa.list_(pa.float32(), 3), nullable=False)
|
||||
|
||||
@udf(input_schema=input_schema, output_schema=output_schema)
|
||||
def explicit(value):
|
||||
@@ -503,7 +82,7 @@ def test_explicit_arrow_schema_is_deterministic():
|
||||
signature = explicit.registration_request.signature
|
||||
assert signature.inputs[0].arrow_type == "float32"
|
||||
assert signature.inputs[0].nullable is True
|
||||
assert signature.output.arrow_type == "fixed_size_list<float32, 3>"
|
||||
assert signature.output.arrow_type == "fixed_size_list<float32>[3]"
|
||||
assert signature.output.nullable is False
|
||||
|
||||
|
||||
@@ -551,16 +130,7 @@ def test_annotation_and_explicit_schema_validation_fail_closed():
|
||||
return value
|
||||
|
||||
|
||||
def test_secret_names_are_canonical_and_disjoint_from_environment():
|
||||
@udf(secrets=["Z_TOKEN", "A_TOKEN", "Z_TOKEN"])
|
||||
def canonical_secrets(value: int) -> int:
|
||||
return value
|
||||
|
||||
assert canonical_secrets.registration_request.required_secrets == (
|
||||
"A_TOKEN",
|
||||
"Z_TOKEN",
|
||||
)
|
||||
|
||||
def test_environment_rejects_secret_value_overlap():
|
||||
with pytest.raises(ValueError, match="must be disjoint"):
|
||||
|
||||
@udf(env={"TOKEN": "plaintext"}, secrets=["TOKEN"])
|
||||
@@ -568,37 +138,13 @@ def test_secret_names_are_canonical_and_disjoint_from_environment():
|
||||
return value
|
||||
|
||||
|
||||
def test_declared_secret_api_still_requires_explicit_create_values():
|
||||
@udf(secrets=["API_TOKEN"])
|
||||
def declared_secret(value: int) -> int:
|
||||
return value
|
||||
|
||||
with pytest.raises(ValueError, match="missing"):
|
||||
declared_secret._submission_json(None)
|
||||
submission = json.loads(
|
||||
declared_secret._submission_json({"API_TOKEN": "explicit-secret"})
|
||||
)
|
||||
assert submission["required_secrets"] == ["API_TOKEN"]
|
||||
assert submission["secret_values"] == {"API_TOKEN": "explicit-secret"}
|
||||
|
||||
|
||||
def test_no_secrets_preserve_canonical_registration_shape():
|
||||
@udf
|
||||
def no_secrets(value: int) -> int:
|
||||
return value
|
||||
|
||||
canonical = json.loads(no_secrets.registration_request.to_canonical_json())
|
||||
assert "required_secrets" not in canonical
|
||||
assert json.loads(no_secrets._submission_json(None)) == canonical
|
||||
|
||||
|
||||
def test_local_function_catalog_operations_are_not_supported(tmp_path):
|
||||
db = lancedb.connect(tmp_path)
|
||||
message = "Function catalog operations are not supported by this database"
|
||||
with pytest.raises(NotImplementedError, match=message):
|
||||
db.create_function(normalize_score, secrets={"API_TOKEN": "value"})
|
||||
db.create_function(normalize_score)
|
||||
with pytest.raises(NotImplementedError, match=message):
|
||||
db.create_function_async(normalize_score, secrets={"API_TOKEN": "value"})
|
||||
db.create_function_async(normalize_score)
|
||||
with pytest.raises(NotImplementedError, match=message):
|
||||
db.get_function("normalize_score", version="fv_exact")
|
||||
|
||||
@@ -641,7 +187,7 @@ def _mock_remote_function_catalog():
|
||||
"job_state": "DONE",
|
||||
"result": state["version"],
|
||||
}
|
||||
elif self.path == "/v1/functions/describe":
|
||||
elif self.path == "/v1/functions/get":
|
||||
assert body == {
|
||||
"name": "normalize_score",
|
||||
"version": "fv_exact",
|
||||
@@ -675,9 +221,7 @@ def test_remote_registration_job_and_exact_version_reopen_round_trip():
|
||||
host_override=host,
|
||||
client_config={"retry_config": {"retries": 0}},
|
||||
)
|
||||
registration = db.create_function_async(
|
||||
normalize_score, secrets={"API_TOKEN": "secret-value"}
|
||||
)
|
||||
registration = db.create_function_async(normalize_score)
|
||||
assert registration.id == "job-register"
|
||||
created = registration.wait()
|
||||
reopened = db.get_function("normalize_score", version=created.version)
|
||||
@@ -686,18 +230,10 @@ def test_remote_registration_job_and_exact_version_reopen_round_trip():
|
||||
assert reopened.name == "normalize_score"
|
||||
assert reopened.version == "fv_exact"
|
||||
create_request = state["requests"][0][1]
|
||||
expected = json.loads(normalize_score.registration_request.to_canonical_json())
|
||||
expected["secret_values"] = {"API_TOKEN": "secret-value"}
|
||||
assert create_request == expected
|
||||
durable_request = FunctionRegistrationRequest.from_json(json.dumps(create_request))
|
||||
assert not hasattr(durable_request, "secret_values")
|
||||
assert "secret_values" not in json.loads(durable_request.to_canonical_json())
|
||||
assert "secret_values" not in json.loads(
|
||||
assert create_request == json.loads(
|
||||
normalize_score.registration_request.to_canonical_json()
|
||||
)
|
||||
assert "secret-value" not in repr(normalize_score)
|
||||
assert "secret-value" not in repr(normalize_score.registration_request)
|
||||
assert not hasattr(created, "secret_values")
|
||||
_assert_no_secret_values(create_request)
|
||||
|
||||
|
||||
def test_blocking_remote_registration_returns_function_version():
|
||||
@@ -708,9 +244,7 @@ def test_blocking_remote_registration_returns_function_version():
|
||||
host_override=host,
|
||||
client_config={"retry_config": {"retries": 0}},
|
||||
)
|
||||
created = db.create_function(
|
||||
normalize_score, secrets={"API_TOKEN": "blocking-secret"}
|
||||
)
|
||||
created = db.create_function(normalize_score)
|
||||
|
||||
assert created.name == "normalize_score"
|
||||
assert created.version == "fv_exact"
|
||||
@@ -718,121 +252,3 @@ def test_blocking_remote_registration_returns_function_version():
|
||||
"/v1/functions/create",
|
||||
"/v1/jobs/describe",
|
||||
]
|
||||
assert state["requests"][0][1]["secret_values"] == {"API_TOKEN": "blocking-secret"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("secret_values", "error_type", "message"),
|
||||
[
|
||||
(None, ValueError, "missing"),
|
||||
({}, ValueError, "missing"),
|
||||
({"OTHER": "value"}, ValueError, "missing.*unexpected"),
|
||||
({"API_TOKEN": ""}, ValueError, "non-empty"),
|
||||
({"API_TOKEN": "bad\0value"}, ValueError, "NUL"),
|
||||
({"API_TOKEN": 123}, TypeError, "must be a string"),
|
||||
([("API_TOKEN", "value")], TypeError, "must be a mapping"),
|
||||
],
|
||||
)
|
||||
def test_secret_values_are_validated_before_remote_request(
|
||||
secret_values, error_type, message
|
||||
):
|
||||
with _mock_remote_function_catalog() as (host, state):
|
||||
db = lancedb.connect(
|
||||
"db://dev",
|
||||
api_key="fake",
|
||||
host_override=host,
|
||||
client_config={"retry_config": {"retries": 0}},
|
||||
)
|
||||
with pytest.raises(error_type, match=message):
|
||||
db.create_function_async(normalize_score, secrets=secret_values)
|
||||
assert state["requests"] == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value",
|
||||
[
|
||||
"x" * _MAX_FUNCTION_SECRET_VALUE_BYTES,
|
||||
"é" * (_MAX_FUNCTION_SECRET_VALUE_BYTES // len("é".encode("utf-8"))),
|
||||
],
|
||||
ids=["ascii", "multibyte"],
|
||||
)
|
||||
def test_secret_value_accepts_exact_utf8_byte_limit(value):
|
||||
submission = json.loads(normalize_score._submission_json({"API_TOKEN": value}))
|
||||
assert submission["secret_values"]["API_TOKEN"] == value
|
||||
assert len(value.encode("utf-8")) == _MAX_FUNCTION_SECRET_VALUE_BYTES
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value",
|
||||
[
|
||||
"x" * (_MAX_FUNCTION_SECRET_VALUE_BYTES + 1),
|
||||
"é" * (_MAX_FUNCTION_SECRET_VALUE_BYTES // len("é".encode("utf-8")) + 1),
|
||||
],
|
||||
ids=["ascii", "multibyte"],
|
||||
)
|
||||
def test_secret_value_rejects_over_utf8_byte_limit_before_json_construction(
|
||||
monkeypatch, value
|
||||
):
|
||||
def fail_if_json_construction_starts(self):
|
||||
pytest.fail("oversized secret reached JSON construction")
|
||||
|
||||
monkeypatch.setattr(
|
||||
FunctionRegistrationRequest, "_known_dict", fail_if_json_construction_starts
|
||||
)
|
||||
with pytest.raises(ValueError, match=r"exceeds the 65536-byte limit"):
|
||||
normalize_score._submission_json({"API_TOKEN": value})
|
||||
|
||||
|
||||
def test_secret_values_accept_exact_aggregate_utf8_byte_limit(monkeypatch):
|
||||
names = tuple(f"SECRET_{index}" for index in range(8))
|
||||
value = "é" * (_MAX_FUNCTION_SECRET_VALUE_BYTES // len("é".encode("utf-8")))
|
||||
values = {name: value for name in names}
|
||||
monkeypatch.setattr(
|
||||
normalize_score,
|
||||
"_request",
|
||||
normalize_score._request._copy(update={"required_secrets": names}),
|
||||
)
|
||||
|
||||
submission = json.loads(normalize_score._submission_json(values))
|
||||
|
||||
assert submission["secret_values"] == values
|
||||
assert sum(len(item.encode("utf-8")) for item in values.values()) == (
|
||||
_MAX_FUNCTION_SECRET_VALUES_BYTES
|
||||
)
|
||||
|
||||
|
||||
def test_secret_values_reject_aggregate_over_limit_before_construction(monkeypatch):
|
||||
names = tuple(f"SECRET_{index}" for index in range(9))
|
||||
values = {name: "x" * _MAX_FUNCTION_SECRET_VALUE_BYTES for name in names}
|
||||
monkeypatch.setattr(
|
||||
normalize_score,
|
||||
"_request",
|
||||
normalize_score._request._copy(update={"required_secrets": names}),
|
||||
)
|
||||
|
||||
def fail_if_json_construction_starts(self):
|
||||
pytest.fail("oversized aggregate reached JSON construction")
|
||||
|
||||
monkeypatch.setattr(
|
||||
FunctionRegistrationRequest, "_known_dict", fail_if_json_construction_starts
|
||||
)
|
||||
with pytest.raises(ValueError, match=r"exceed.*524288-byte request limit"):
|
||||
normalize_score._submission_json(values)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_remote_registration_submits_secret_values_only_once():
|
||||
with _mock_remote_function_catalog() as (host, state):
|
||||
db = await lancedb.connect_async(
|
||||
"db://dev",
|
||||
api_key="fake",
|
||||
host_override=host,
|
||||
client_config={"retry_config": {"retries": 0}},
|
||||
)
|
||||
registration = await db.create_function_async(
|
||||
normalize_score, secrets={"API_TOKEN": "async-secret"}
|
||||
)
|
||||
created = await registration.wait()
|
||||
|
||||
assert state["requests"][0][1]["secret_values"] == {"API_TOKEN": "async-secret"}
|
||||
assert not hasattr(created, "secret_values")
|
||||
|
||||
@@ -25,7 +25,6 @@ from lancedb.db import DBConnection
|
||||
from lancedb.index import FTS
|
||||
from lancedb.query import (
|
||||
BoostQuery,
|
||||
DocumentGranularity,
|
||||
MatchQuery,
|
||||
MultiMatchQuery,
|
||||
PhraseQuery,
|
||||
@@ -246,63 +245,6 @@ def test_create_inverted_index_rejects_invalid_block_size(table):
|
||||
table.create_index("text", config=FTS(block_size=129))
|
||||
|
||||
|
||||
def test_list_element_document_granularity(tmp_path):
|
||||
docs_type = pa.list_(pa.struct([pa.field("content", pa.string())]))
|
||||
docs = pa.array(
|
||||
[
|
||||
[
|
||||
{"content": "alpha beta"},
|
||||
None,
|
||||
{"content": ""},
|
||||
{"content": "the and"},
|
||||
{"content": "alpha beta"},
|
||||
]
|
||||
],
|
||||
type=docs_type,
|
||||
)
|
||||
table = ldb.connect(tmp_path).create_table(
|
||||
"list_element_docs", pa.table({"id": [0], "docs": docs})
|
||||
)
|
||||
row_table = ldb.connect(tmp_path).create_table(
|
||||
"row_docs", pa.table({"id": [0], "docs": docs})
|
||||
)
|
||||
row_table.create_index("docs.content", config=FTS())
|
||||
row_result = row_table.search(MatchQuery("alpha", "docs.content")).to_arrow()
|
||||
assert row_result.num_rows == 1
|
||||
assert "_doc_index" not in row_result.column_names
|
||||
|
||||
granularity = DocumentGranularity.LIST_ELEMENT
|
||||
table.create_index(
|
||||
"docs.content",
|
||||
config=FTS(with_position=True, document_granularity=granularity),
|
||||
)
|
||||
assert table.list_indices()[0].columns == ["docs.content"]
|
||||
|
||||
def coordinates(query):
|
||||
result = table.search(query).limit(10).to_arrow()
|
||||
doc_index_type = result.schema.field("_doc_index").type
|
||||
assert pa.types.is_list(doc_index_type)
|
||||
assert doc_index_type.value_type == pa.uint32()
|
||||
return sorted(result["_doc_index"].to_pylist())
|
||||
|
||||
assert coordinates(
|
||||
MatchQuery("alpha", "docs.content", document_granularity=granularity)
|
||||
) == [[0], [4]]
|
||||
assert coordinates(
|
||||
PhraseQuery("alpha beta", "docs.content", document_granularity=granularity)
|
||||
) == [[0], [4]]
|
||||
assert coordinates(MatchQuery("alpha", "docs.content")) == [[0], [4]]
|
||||
assert FTS().document_granularity is DocumentGranularity.ROW
|
||||
|
||||
|
||||
def test_create_inverted_index_respects_build_memory_limit(table):
|
||||
with pytest.raises(ValueError, match="exceeds worker memory limit"):
|
||||
table.create_index(
|
||||
"text",
|
||||
config=FTS(memory_limit=0, num_workers=1),
|
||||
)
|
||||
|
||||
|
||||
def test_custom_stop_words_list(table):
|
||||
table.create_index(
|
||||
"text",
|
||||
@@ -1139,20 +1081,6 @@ def test_fts_query_to_json():
|
||||
)
|
||||
assert json_str == expected
|
||||
|
||||
# Test MatchQuery with list-element document granularity
|
||||
match_query = MatchQuery(
|
||||
"hello world",
|
||||
"text",
|
||||
document_granularity=DocumentGranularity.LIST_ELEMENT,
|
||||
)
|
||||
json_str = match_query.to_json()
|
||||
expected = (
|
||||
'{"match":{"column":"text","terms":"hello world","boost":1.0,'
|
||||
'"fuzziness":0,"max_expansions":50,"operator":"Or","prefix_length":0,'
|
||||
'"document_granularity":"list_element"}}'
|
||||
)
|
||||
assert json_str == expected
|
||||
|
||||
# Test MatchQuery with options
|
||||
match_query = MatchQuery("puppy", "text", fuzziness=2, boost=1.5, prefix_length=3)
|
||||
json_str = match_query.to_json()
|
||||
@@ -1162,19 +1090,6 @@ def test_fts_query_to_json():
|
||||
)
|
||||
assert json_str == expected
|
||||
|
||||
# Test PhraseQuery with list-element document granularity
|
||||
phrase_query = PhraseQuery(
|
||||
"quick brown fox",
|
||||
"title",
|
||||
document_granularity=DocumentGranularity.LIST_ELEMENT,
|
||||
)
|
||||
json_str = phrase_query.to_json()
|
||||
expected = (
|
||||
'{"phrase":{"column":"title","terms":"quick brown fox","slop":0,'
|
||||
'"document_granularity":"list_element"}}'
|
||||
)
|
||||
assert json_str == expected
|
||||
|
||||
# Test PhraseQuery
|
||||
phrase_query = PhraseQuery("quick brown fox", "title")
|
||||
json_str = phrase_query.to_json()
|
||||
|
||||
@@ -88,7 +88,7 @@ async def binary_table(db_async):
|
||||
async def test_create_index_async_returns_done_job(some_table: AsyncTable):
|
||||
job = await some_table.create_index_async("id", config=BTree())
|
||||
assert job.id is None
|
||||
assert await job.wait() is None
|
||||
await job.wait()
|
||||
assert len(await some_table.list_indices()) == 1
|
||||
await job.cancel()
|
||||
|
||||
|
||||
@@ -1,268 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import lancedb
|
||||
import pytest
|
||||
from lancedb.materialized_view import MaterializedViewDefinition
|
||||
|
||||
|
||||
STABLE_ROW_IDS = {"new_table_enable_stable_row_ids": "true"}
|
||||
|
||||
|
||||
def make_db(tmp_path):
|
||||
db = lancedb.connect(tmp_path, storage_options=STABLE_ROW_IDS)
|
||||
db.create_table(
|
||||
"people",
|
||||
[
|
||||
{"name": "ada", "age": 36},
|
||||
{"name": "kid", "age": 7},
|
||||
{"name": "grace", "age": 85},
|
||||
],
|
||||
)
|
||||
return db
|
||||
|
||||
|
||||
def test_create_refresh_and_query(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
view = db.create_materialized_view(
|
||||
"adults",
|
||||
"people",
|
||||
select=["name", ("shout", "upper(name)")],
|
||||
where="age >= 18",
|
||||
)
|
||||
assert view.name == "adults"
|
||||
assert view.table.count_rows() == 0
|
||||
|
||||
result = view.refresh()
|
||||
assert result.mode == "rebuild"
|
||||
assert result.rows_written == 2
|
||||
|
||||
rows = view.table.search().to_list()
|
||||
assert sorted(row["shout"] for row in rows) == ["ADA", "GRACE"]
|
||||
|
||||
|
||||
def test_definition_round_trips(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
db.create_materialized_view("adults", "people", where="age >= 18")
|
||||
|
||||
view = db.open_materialized_view("adults")
|
||||
assert view.definition == MaterializedViewDefinition(
|
||||
source_table="people",
|
||||
projections=[("name", "`name`"), ("age", "`age`")],
|
||||
filter="age >= 18",
|
||||
inputs=["age", "name"],
|
||||
)
|
||||
|
||||
|
||||
def test_incremental_refresh_after_append(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
view = db.create_materialized_view("copy", "people")
|
||||
view.refresh()
|
||||
|
||||
db.open_table("people").add([{"name": "alan", "age": 41}])
|
||||
result = view.refresh()
|
||||
assert result.mode == "incremental"
|
||||
assert result.rows_written == 1
|
||||
assert view.table.count_rows() == 4
|
||||
|
||||
assert view.refresh().mode == "no_op"
|
||||
|
||||
|
||||
def test_incremental_refresh_after_update(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
view = db.create_materialized_view("copy", "people")
|
||||
view.refresh()
|
||||
|
||||
db.open_table("people").update(where="name = 'kid'", values={"age": 8})
|
||||
result = view.refresh()
|
||||
assert result.mode == "incremental"
|
||||
assert result.rows_written == 1
|
||||
rows = view.table.search().to_list()
|
||||
assert sorted(row["age"] for row in rows) == [8, 36, 85]
|
||||
|
||||
|
||||
def test_legacy_storage_source_update_rebuilds(tmp_path):
|
||||
db = lancedb.connect(
|
||||
tmp_path,
|
||||
storage_options={**STABLE_ROW_IDS, "new_table_data_storage_version": "legacy"},
|
||||
)
|
||||
db.create_table("people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}])
|
||||
view = db.create_materialized_view("copy", "people")
|
||||
view.refresh()
|
||||
|
||||
db.open_table("people").update(where="name = 'kid'", values={"age": 8})
|
||||
result = view.refresh()
|
||||
assert result.mode == "rebuild"
|
||||
rows = view.table.search().to_list()
|
||||
assert sorted(row["age"] for row in rows) == [8, 36]
|
||||
|
||||
|
||||
def test_list_and_not_a_view(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
db.create_materialized_view("adults", "people", where="age >= 18")
|
||||
|
||||
assert db.list_materialized_views() == ["adults"]
|
||||
with pytest.raises(ValueError, match="not a materialized view"):
|
||||
db.open_materialized_view("people")
|
||||
|
||||
|
||||
def test_invalid_expression_fails_at_create(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
with pytest.raises(Exception, match="missing"):
|
||||
db.create_materialized_view("bad", "people", select=[("x", "missing + 1")])
|
||||
assert "bad" not in db.list_tables().tables
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_create_refresh_and_open(tmp_path):
|
||||
db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS)
|
||||
await db.create_table("people", [{"name": "ada", "age": 36}])
|
||||
|
||||
view = await db.create_materialized_view(
|
||||
"shouts", "people", select=[("shout", "upper(name)")]
|
||||
)
|
||||
result = await view.refresh()
|
||||
assert result.mode == "rebuild"
|
||||
assert result.rows_written == 1
|
||||
|
||||
reopened = await db.open_materialized_view("shouts")
|
||||
definition = await reopened.definition()
|
||||
assert definition.projections == [("shout", "upper(name)")]
|
||||
assert await db.list_materialized_views() == ["shouts"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_incremental(tmp_path):
|
||||
db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS)
|
||||
await db.create_table("people", [{"name": "ada", "age": 36}])
|
||||
view = await db.create_materialized_view("copy", "people")
|
||||
await view.refresh()
|
||||
|
||||
table = await db.open_table("people")
|
||||
await table.add([{"name": "alan", "age": 41}])
|
||||
result = await view.refresh()
|
||||
assert result.mode == "incremental"
|
||||
assert result.rows_written == 1
|
||||
|
||||
|
||||
def test_source_requires_stable_row_ids(tmp_path):
|
||||
db = lancedb.connect(tmp_path)
|
||||
db.create_table("plain", [{"x": 1}])
|
||||
with pytest.raises(Exception, match="stable row ids"):
|
||||
db.create_materialized_view("v", "plain")
|
||||
|
||||
|
||||
def test_bare_select_names_are_quoted(tmp_path):
|
||||
db = lancedb.connect(tmp_path, storage_options=STABLE_ROW_IDS)
|
||||
db.create_table("odd_names", [{"order item": "widget", "select": 2}])
|
||||
|
||||
view = db.create_materialized_view(
|
||||
"quoted", "odd_names", select=["order item", "select"]
|
||||
)
|
||||
result = view.refresh()
|
||||
assert result.rows_written == 1
|
||||
rows = view.table.search().to_list()
|
||||
assert rows[0]["order item"] == "widget"
|
||||
assert rows[0]["select"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_remote_is_refused_without_network():
|
||||
db = await lancedb.connect_async(
|
||||
"db://nowhere", api_key="sk_test", region="us-east-1"
|
||||
)
|
||||
with pytest.raises(NotImplementedError, match="local"):
|
||||
await db.create_materialized_view("v", "src")
|
||||
with pytest.raises(NotImplementedError, match="local"):
|
||||
await db.open_materialized_view("v")
|
||||
with pytest.raises(NotImplementedError, match="local"):
|
||||
await db.list_materialized_views()
|
||||
|
||||
|
||||
def test_scalar_select_is_one_column(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
view = db.create_materialized_view("just_name", "people", select="name")
|
||||
view.refresh()
|
||||
rows = view.table.search().to_list()
|
||||
assert set(rows[0]) - {"__source_row_id"} == {"name"}
|
||||
assert sorted(row["name"] for row in rows) == ["ada", "grace", "kid"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_scalar_select_is_one_column(tmp_path):
|
||||
db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS)
|
||||
await db.create_table("people", [{"name": "ada", "age": 36}])
|
||||
view = await db.create_materialized_view("just_name", "people", select="name")
|
||||
await view.refresh()
|
||||
rows = await view.table.query().to_list()
|
||||
assert set(rows[0]) - {"__source_row_id"} == {"name"}
|
||||
|
||||
|
||||
def test_limit_above_i64_max_is_refused(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
with pytest.raises(ValueError, match="exceeds the maximum"):
|
||||
db.create_materialized_view("too_big", "people", limit=2**63)
|
||||
# The boundary is fine, and zero still means an empty view.
|
||||
db.create_materialized_view("at_max", "people", limit=2**63 - 1)
|
||||
empty = db.create_materialized_view("none", "people", limit=0)
|
||||
empty.refresh()
|
||||
assert empty.table.count_rows() == 0
|
||||
|
||||
|
||||
def _namespace_db(tmp_path):
|
||||
return lancedb.connect_namespace(
|
||||
"dir",
|
||||
{"root": str(tmp_path)},
|
||||
storage_options=STABLE_ROW_IDS,
|
||||
)
|
||||
|
||||
|
||||
def test_namespace_connection_materialized_views(tmp_path):
|
||||
db = _namespace_db(tmp_path)
|
||||
db.create_table(
|
||||
"people",
|
||||
[{"name": "ada", "age": 36}, {"name": "kid", "age": 7}],
|
||||
storage_options=STABLE_ROW_IDS,
|
||||
)
|
||||
|
||||
view = db.create_materialized_view("adults", "people", where="age >= 18")
|
||||
view.refresh()
|
||||
assert view.table.count_rows() == 1
|
||||
assert db.list_materialized_views() == ["adults"]
|
||||
|
||||
reopened = db.open_materialized_view("adults")
|
||||
assert reopened.definition.source_table == "people"
|
||||
with pytest.raises(ValueError, match="not a materialized view"):
|
||||
db.open_materialized_view("people")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_namespace_connection_materialized_views(tmp_path):
|
||||
db = lancedb.connect_namespace_async(
|
||||
"dir",
|
||||
{"root": str(tmp_path)},
|
||||
storage_options=STABLE_ROW_IDS,
|
||||
)
|
||||
await db.create_table(
|
||||
"people",
|
||||
[{"name": "ada", "age": 36}, {"name": "kid", "age": 7}],
|
||||
storage_options=STABLE_ROW_IDS,
|
||||
)
|
||||
|
||||
view = await db.create_materialized_view("adults", "people", where="age >= 18")
|
||||
await view.refresh()
|
||||
assert await view.table.count_rows() == 1
|
||||
assert await db.list_materialized_views() == ["adults"]
|
||||
|
||||
reopened = await db.open_materialized_view("adults")
|
||||
assert (await reopened.definition()).source_table == "people"
|
||||
|
||||
# The view's table came through the namespace, not straight from the
|
||||
# inner connection: a bare inner table carries no namespace context, so
|
||||
# its pushdown routing differs from a table the namespace opened.
|
||||
through_namespace = await db.open_table("adults")
|
||||
for handle in (view.table, reopened.table):
|
||||
assert (
|
||||
handle._route_pushdown_to_rust == through_namespace._route_pushdown_to_rust
|
||||
)
|
||||
assert handle._namespace_path == through_namespace._namespace_path
|
||||
@@ -56,31 +56,6 @@ def test_execute_does_not_reenter_background_loop(tmp_path, monkeypatch):
|
||||
assert permutation_tbl._conn.read_consistency_interval is None
|
||||
|
||||
|
||||
def test_pickled_permutation_reads_pinned_version(tmp_path):
|
||||
"""An unpickled copy must still read the pinned version, which also covers the
|
||||
version surviving the ``to_arrow()`` round trip in ``__getstate__``."""
|
||||
import pickle
|
||||
|
||||
db = connect(tmp_path)
|
||||
tbl = db.create_table("base", pa.table({"idx": range(20)}))
|
||||
permutation_tbl = permutation_builder(tbl).execute()
|
||||
perm = Permutation.from_tables(tbl, permutation_tbl)
|
||||
|
||||
payload = pickle.dumps(perm)
|
||||
|
||||
# Compact so the stored row addresses no longer describe these rows at latest.
|
||||
tbl.delete("true")
|
||||
tbl.optimize()
|
||||
assert tbl.count_rows() == 0
|
||||
|
||||
# Unpickle after the mutation: __setstate__ reopens at latest, so this only
|
||||
# passes if the recorded version is applied on reopen.
|
||||
restored = pickle.loads(payload)
|
||||
assert len(restored) == 20
|
||||
rows = restored.__getitems__(list(range(20)))
|
||||
assert sorted(row["idx"] for row in rows) == list(range(20))
|
||||
|
||||
|
||||
def test_split_random_counts(mem_db):
|
||||
"""Test random splitting with absolute counts."""
|
||||
tbl = mem_db.create_table(
|
||||
|
||||
@@ -675,21 +675,6 @@ def test_distance_range(table: lancedb.table.Table):
|
||||
assert res["_distance"].to_pylist() == [min_dist, max_dist]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("expression", ["1 - _distance", "1.0 - _distance"])
|
||||
def test_select_arithmetic_with_distance(table, expression):
|
||||
result = (
|
||||
table.search([10, 10])
|
||||
.select({"similarity": expression, "_distance": "_distance"})
|
||||
.distance_type("cosine")
|
||||
.to_arrow()
|
||||
)
|
||||
|
||||
assert result.schema.field("similarity").type == pa.float32()
|
||||
assert result["similarity"].to_pylist() == pytest.approx(
|
||||
[1 - distance for distance in result["_distance"].to_pylist()]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_distance_range_async(table_async: AsyncTable):
|
||||
q = [0, 0]
|
||||
@@ -912,23 +897,6 @@ def test_query_builder_batches(table):
|
||||
assert rs_list["id"][1] == 2
|
||||
|
||||
|
||||
def test_batch_vector_query_shares_filtered_flat_scan(table):
|
||||
query = (
|
||||
table.search([[1.0, 2.0], [3.0, 4.0]])
|
||||
.where("id > 0", prefilter=True)
|
||||
.limit(1)
|
||||
.select(["id"])
|
||||
)
|
||||
|
||||
plan = query.explain_plan(verbose=True)
|
||||
assert "KNNVectorDistance: queries=2" in plan
|
||||
assert "UnionExec" not in plan
|
||||
|
||||
results = query.to_arrow()
|
||||
assert len(results) == 2
|
||||
assert results["query_index"].to_pylist() == [0, 1]
|
||||
|
||||
|
||||
def test_dynamic_projection(table):
|
||||
rs = (
|
||||
LanceVectorQueryBuilder(table, [0, 0], "vector")
|
||||
|
||||
@@ -242,8 +242,8 @@ def test_remote_table_branches_sync():
|
||||
table.branches.delete("exp")
|
||||
|
||||
|
||||
def test_remote_table_cherry_pick_defaults_to_execute():
|
||||
cherry_pick_bodies = []
|
||||
def test_remote_table_branch_merge_defaults_to_execute():
|
||||
merge_bodies = []
|
||||
diff = {
|
||||
"fromBranch": "exp",
|
||||
"parentVersion": 1,
|
||||
@@ -265,7 +265,8 @@ def test_remote_table_cherry_pick_defaults_to_execute():
|
||||
"changedColumns": [],
|
||||
"addedIndexes": [],
|
||||
"removedIndexes": [],
|
||||
"errors": [],
|
||||
"mergeable": True,
|
||||
"mergeBlockers": [],
|
||||
}
|
||||
|
||||
def handler(request):
|
||||
@@ -275,11 +276,11 @@ def test_remote_table_cherry_pick_defaults_to_execute():
|
||||
else:
|
||||
content_len = int(request.headers.get("Content-Length"))
|
||||
request_body = json.loads(request.rfile.read(content_len))
|
||||
cherry_pick_bodies.append(request_body)
|
||||
merge_bodies.append(request_body)
|
||||
dry_run = request_body["dry_run"]
|
||||
status = 200 if dry_run else 409
|
||||
body = {
|
||||
"status": "ready" if dry_run else "failed",
|
||||
"status": "ready" if dry_run else "rejected",
|
||||
"diff": diff,
|
||||
"preview": {"promotedColumns": []},
|
||||
}
|
||||
@@ -291,10 +292,10 @@ def test_remote_table_cherry_pick_defaults_to_execute():
|
||||
|
||||
with mock_lancedb_connection(handler) as db:
|
||||
branches = db.open_table("test").branches
|
||||
assert branches.cherry_pick("exp")["status"] == "failed"
|
||||
assert branches.cherry_pick("exp", dry_run=True)["status"] == "ready"
|
||||
assert branches.merge("exp")["status"] == "rejected"
|
||||
assert branches.merge("exp", dry_run=True)["status"] == "ready"
|
||||
|
||||
assert cherry_pick_bodies == [
|
||||
assert merge_bodies == [
|
||||
{"from_branch": "exp", "dry_run": False},
|
||||
{"from_branch": "exp", "dry_run": True},
|
||||
]
|
||||
@@ -875,85 +876,11 @@ def test_remote_create_index_async_returns_job():
|
||||
table = db.create_table("test", [{"id": 1}])
|
||||
job = table.create_index_async("id", config=BTree())
|
||||
assert job.id == "job-1"
|
||||
assert job.wait(timeout=timedelta(seconds=30)) is None
|
||||
job.wait(timeout=timedelta(seconds=30))
|
||||
assert len(describe_calls) == 2
|
||||
job.cancel()
|
||||
|
||||
|
||||
def test_remote_refresh_async_returns_typed_terminal_result():
|
||||
terminal_result = {
|
||||
"rows_assigned": 12,
|
||||
"rows_failed": 0,
|
||||
"rows_remaining": 0,
|
||||
"source_version": 7,
|
||||
"published_version": 8,
|
||||
}
|
||||
|
||||
def handler(request):
|
||||
content_len = int(request.headers.get("Content-Length", 0))
|
||||
body = request.rfile.read(content_len) if content_len > 0 else b""
|
||||
if request.path == "/v1/table/test/backfill_column":
|
||||
assert json.loads(body)["column"] == "derived"
|
||||
request.send_response(202)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.end_headers()
|
||||
request.wfile.write(b'{"job_id": "refresh-1"}')
|
||||
elif request.path == "/v1/jobs/describe":
|
||||
assert json.loads(body)["job_id"] == "refresh-1"
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.end_headers()
|
||||
request.wfile.write(
|
||||
json.dumps(
|
||||
{
|
||||
"job_id": "refresh-1",
|
||||
"job_type": "function_refresh",
|
||||
"job_state": "DONE",
|
||||
"result": terminal_result,
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
elif request.path == "/v1/table/test/create/?mode=create":
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.end_headers()
|
||||
request.wfile.write(b"{}")
|
||||
elif request.path == "/v1/table/test/describe/":
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.end_headers()
|
||||
request.wfile.write(
|
||||
json.dumps(
|
||||
{
|
||||
"version": 1,
|
||||
"schema": {
|
||||
"fields": [
|
||||
{
|
||||
"name": "id",
|
||||
"type": {"type": "int64"},
|
||||
"nullable": False,
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
else:
|
||||
request.send_response(404)
|
||||
request.end_headers()
|
||||
|
||||
with mock_lancedb_connection(handler) as db:
|
||||
table = db.create_table("test", [{"id": 1}])
|
||||
job = table.refresh_column_async("derived")
|
||||
assert job.id == "refresh-1"
|
||||
result = job.wait(timeout=timedelta(seconds=30))
|
||||
|
||||
assert isinstance(result, lancedb.RefreshColumnResult)
|
||||
assert result.model_dump() == terminal_result
|
||||
assert result.rows_filled == 12
|
||||
assert result.version == 8
|
||||
|
||||
|
||||
def test_remote_job_wait_raises_on_failure():
|
||||
from lancedb.exceptions import JobFailedError
|
||||
from lancedb.index import BTree
|
||||
@@ -1618,49 +1545,6 @@ def test_query_sync_fts():
|
||||
)
|
||||
|
||||
|
||||
def test_query_sync_fts_document_granularity():
|
||||
from lancedb.query import DocumentGranularity, MatchQuery
|
||||
|
||||
def handler(body):
|
||||
assert body == {
|
||||
"full_text_query": {
|
||||
"query": {
|
||||
"match": {
|
||||
"column": "docs.content",
|
||||
"terms": "alpha",
|
||||
"boost": 1.0,
|
||||
"fuzziness": 0,
|
||||
"max_expansions": 50,
|
||||
"operator": "Or",
|
||||
"prefix_length": 0,
|
||||
"document_granularity": "list_element",
|
||||
}
|
||||
}
|
||||
},
|
||||
"k": 10,
|
||||
"prefilter": True,
|
||||
"vector": [],
|
||||
"version": None,
|
||||
}
|
||||
return pa.table(
|
||||
{
|
||||
"id": [1, 1],
|
||||
"_doc_index": pa.array([[0], [4]], type=pa.list_(pa.uint32())),
|
||||
}
|
||||
)
|
||||
|
||||
with query_test_table(handler, server_version=Version("0.6.0")) as table:
|
||||
result = table.search(
|
||||
MatchQuery(
|
||||
"alpha",
|
||||
"docs.content",
|
||||
document_granularity=DocumentGranularity.LIST_ELEMENT,
|
||||
)
|
||||
).to_arrow()
|
||||
|
||||
assert result["_doc_index"].to_pylist() == [[0], [4]]
|
||||
|
||||
|
||||
def test_query_sync_hybrid():
|
||||
def handler(body):
|
||||
if "full_text_query" in body:
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import timedelta
|
||||
import threading
|
||||
|
||||
@@ -87,25 +86,6 @@ def test_s3_lifecycle(s3_bucket: str):
|
||||
asyncio.run(test())
|
||||
|
||||
|
||||
@pytest.mark.s3_test
|
||||
def test_concurrent_open_table(s3_bucket: str):
|
||||
uri = f"s3://{s3_bucket}/test_concurrent_open_table"
|
||||
db = lancedb.connect(uri, storage_options=copy.copy(CONFIG))
|
||||
db.create_table("test", pa.table({"x": [1, 2, 3]}))
|
||||
|
||||
num_workers = 32
|
||||
barrier = threading.Barrier(num_workers)
|
||||
|
||||
def open_and_count(_):
|
||||
barrier.wait()
|
||||
return db.open_table("test").count_rows()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=num_workers) as pool:
|
||||
row_counts = list(pool.map(open_and_count, range(num_workers)))
|
||||
|
||||
assert row_counts == [3] * num_workers
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def kms_key():
|
||||
kms = get_boto3_client("kms", endpoint_url=CONFIG["aws_endpoint"])
|
||||
|
||||
@@ -11,7 +11,6 @@ import warnings
|
||||
import weakref
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import date, datetime, timedelta
|
||||
from decimal import Decimal
|
||||
from time import sleep
|
||||
from typing import List
|
||||
from unittest.mock import patch
|
||||
@@ -337,21 +336,6 @@ async def test_update_async(mem_db_async: AsyncConnection):
|
||||
assert await table.count_rows("id == 10") == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_expr_filter_literals_async(mem_db_async: AsyncConnection):
|
||||
values = ["5", "4.66e-84", "it's"]
|
||||
table = await mem_db_async.create_table(
|
||||
"update_expr_literals",
|
||||
data=[{"field": value, "result": "original"} for value in values],
|
||||
)
|
||||
|
||||
for value in values:
|
||||
update_res = await table.update({"result": value}, where=col("field") == value)
|
||||
assert update_res.rows_updated == 1
|
||||
|
||||
assert (await table.to_arrow())["result"].to_pylist() == values
|
||||
|
||||
|
||||
def test_create_table(mem_db: DBConnection):
|
||||
schema = pa.schema(
|
||||
{
|
||||
@@ -1483,7 +1467,7 @@ def test_create_index_async_returns_done_job(mem_db: DBConnection):
|
||||
table = mem_db.create_table("job_test", [{"id": i} for i in range(10)])
|
||||
job = table.create_index_async("id", config=BTree())
|
||||
assert job.id is None
|
||||
assert job.wait() is None
|
||||
job.wait()
|
||||
assert len(table.list_indices()) == 1
|
||||
job.cancel()
|
||||
|
||||
@@ -2359,148 +2343,6 @@ def test_update(mem_db: DBConnection):
|
||||
assert np.allclose(v, np.array([[1.2, 1.9], [1.1, 1.1]]))
|
||||
|
||||
|
||||
def test_update_expr_filter_literals(mem_db: DBConnection):
|
||||
values = ["5", "4.66e-84", "it's"]
|
||||
table = mem_db.create_table(
|
||||
"update_expr_literals",
|
||||
data=[{"field": value, "result": "original"} for value in values],
|
||||
)
|
||||
|
||||
for value in values:
|
||||
update_res = table.update(where=col("field") == value, values={"result": value})
|
||||
assert update_res.rows_updated == 1
|
||||
|
||||
assert table.to_arrow()["result"].to_pylist() == values
|
||||
|
||||
|
||||
def test_update_expr_filter_preserves_typed_semantics(mem_db: DBConnection):
|
||||
low = Decimal("1.234567890123456789")
|
||||
high = Decimal("1.234567890123456790")
|
||||
decimal_schema = pa.schema(
|
||||
[("val", pa.decimal128(19, 18)), ("result", pa.string())]
|
||||
)
|
||||
decimal_table = mem_db.create_table(
|
||||
"update_expr_decimal",
|
||||
pa.table(
|
||||
{"val": [low, high], "result": ["old", "old"]},
|
||||
schema=decimal_schema,
|
||||
),
|
||||
)
|
||||
predicate = col("val") < lit(high)
|
||||
assert decimal_table.search().where(predicate).to_arrow().num_rows == 1
|
||||
result = decimal_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 1
|
||||
|
||||
keyword_table = mem_db.create_table(
|
||||
"update_expr_keyword", [{"null": 1, "result": "old"}]
|
||||
)
|
||||
predicate = col("null") == 1
|
||||
assert keyword_table.search().where(predicate).to_arrow().num_rows == 1
|
||||
result = keyword_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 1
|
||||
|
||||
empty_in_table = mem_db.create_table(
|
||||
"update_expr_empty_in", [{"id": 1, "result": "old"}]
|
||||
)
|
||||
predicate = col("id").isin([])
|
||||
assert empty_in_table.search().where(predicate).to_arrow().num_rows == 0
|
||||
result = empty_in_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 0
|
||||
|
||||
marker = "__lancedb_binary_placeholder_0__"
|
||||
binary_schema = pa.schema(
|
||||
[("payload", pa.binary()), ("text", pa.string()), ("result", pa.string())]
|
||||
)
|
||||
binary_table = mem_db.create_table(
|
||||
"update_expr_binary",
|
||||
pa.table(
|
||||
{
|
||||
"payload": [b"\x01", b"\x02"],
|
||||
"text": ["other", marker],
|
||||
"result": ["old", "old"],
|
||||
},
|
||||
schema=binary_schema,
|
||||
),
|
||||
)
|
||||
predicate = (col("payload") == lit(b"\x01")) | (col("text") == marker)
|
||||
assert binary_table.search().where(predicate).to_arrow().num_rows == 2
|
||||
result = binary_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 2
|
||||
|
||||
nonfinite_table = mem_db.create_table(
|
||||
"update_expr_nonfinite",
|
||||
[{"x": 1.0, "result": "old"}, {"x": 2.0, "result": "old"}],
|
||||
)
|
||||
predicate = col("x") < float("inf")
|
||||
assert nonfinite_table.search().where(predicate).to_arrow().num_rows == 2
|
||||
result = nonfinite_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 2
|
||||
|
||||
float16_table = mem_db.create_table(
|
||||
"update_expr_float16",
|
||||
[{"x": 1.0, "result": "old"}, {"x": 3.0, "result": "old"}],
|
||||
)
|
||||
predicate = col("x").cast(pa.float16()) < 2.0
|
||||
assert float16_table.search().where(predicate).to_arrow().num_rows == 1
|
||||
result = float16_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 1
|
||||
|
||||
string_cast_table = mem_db.create_table(
|
||||
"update_expr_string_cast",
|
||||
[{"x": 1, "result": "old"}, {"x": 2, "result": "old"}],
|
||||
)
|
||||
predicate = col("x").cast("string") == "1"
|
||||
assert string_cast_table.search().where(predicate).to_arrow().num_rows == 1
|
||||
result = string_cast_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 1
|
||||
|
||||
quoted_identifier_schema = pa.schema(
|
||||
[("payload", pa.binary()), ("odd'name", pa.int64()), ("result", pa.string())]
|
||||
)
|
||||
quoted_identifier_table = mem_db.create_table(
|
||||
"update_expr_quoted_identifier",
|
||||
pa.table(
|
||||
{"payload": [b"\x01"], "odd'name": [1], "result": ["old"]},
|
||||
schema=quoted_identifier_schema,
|
||||
),
|
||||
)
|
||||
predicate = (col("payload") == lit(b"\x01")) & (col("odd'name") == 1)
|
||||
assert quoted_identifier_table.search().where(predicate).to_arrow().num_rows == 1
|
||||
result = quoted_identifier_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 1
|
||||
|
||||
decimal256_schema = pa.schema(
|
||||
[("val", pa.decimal256(40, 2)), ("result", pa.string())]
|
||||
)
|
||||
decimal256_table = mem_db.create_table(
|
||||
"update_expr_decimal256",
|
||||
pa.table(
|
||||
{
|
||||
"val": [Decimal("1.00"), Decimal("3.00")],
|
||||
"result": ["old", "old"],
|
||||
},
|
||||
schema=decimal256_schema,
|
||||
),
|
||||
)
|
||||
predicate = col("val") < lit(Decimal("2.00")).cast(pa.decimal256(40, 2))
|
||||
assert decimal256_table.search().where(predicate).to_arrow().num_rows == 1
|
||||
result = decimal256_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 1
|
||||
|
||||
binary_empty_table = mem_db.create_table(
|
||||
"update_expr_binary_empty",
|
||||
pa.table(
|
||||
{"payload": [b"\x01", b"\x02"], "result": ["old", "old"]},
|
||||
schema=pa.schema([("payload", pa.binary()), ("result", pa.string())]),
|
||||
),
|
||||
)
|
||||
predicate = (col("payload") == lit(b"\x01")).isin([])
|
||||
assert binary_empty_table.search().where(predicate).to_arrow().num_rows == 0
|
||||
assert predicate.to_sql() == "false"
|
||||
result = binary_empty_table.update(where=predicate, values={"result": "new"})
|
||||
assert result.rows_updated == 0
|
||||
|
||||
|
||||
def test_update_with_arrow_scalar(mem_db: DBConnection):
|
||||
schema = pa.schema({"id": pa.int64(), "vector": pa.list_(pa.float32(), 4)})
|
||||
table = mem_db.create_table("my_table", schema=schema)
|
||||
@@ -4105,21 +3947,10 @@ def test_refresh_column_async_returns_job(tmp_path):
|
||||
|
||||
job = table.refresh_column_async("doubled")
|
||||
assert job.id is None # in-process jobs have no server id
|
||||
result = job.wait()
|
||||
assert isinstance(result, lancedb.RefreshColumnResult)
|
||||
assert result.rows_assigned == 2
|
||||
assert result.rows_failed == 0
|
||||
assert result.rows_remaining == 0
|
||||
assert result.source_version == 2
|
||||
assert result.published_version == 3
|
||||
assert job.wait() is None
|
||||
assert job.status() == "finished"
|
||||
assert sorted(table.to_arrow()["doubled"].to_pylist()) == [2, 4]
|
||||
|
||||
no_op = table.refresh_column_async("doubled").wait()
|
||||
assert no_op.rows_assigned == 0
|
||||
assert no_op.source_version == 3
|
||||
assert no_op.published_version is None
|
||||
|
||||
# Bad input raises at the call, not through the job.
|
||||
with pytest.raises(Exception, match="not a computed column"):
|
||||
table.refresh_column_async("x")
|
||||
@@ -4132,10 +3963,6 @@ async def test_refresh_column_async_job_async_table(tmp_path):
|
||||
await table.add_columns(computed={"tripled": "x * 3"})
|
||||
|
||||
job = await table.refresh_column_async("tripled")
|
||||
result = await job.wait()
|
||||
assert isinstance(result, lancedb.RefreshColumnResult)
|
||||
assert result.rows_assigned == 1
|
||||
assert result.source_version == 2
|
||||
assert result.published_version == 3
|
||||
assert await job.wait() is None
|
||||
assert await job.status() == "finished"
|
||||
assert (await table.to_arrow())["tripled"].to_pylist() == [9]
|
||||
|
||||
@@ -7,7 +7,6 @@ import pathlib
|
||||
from typing import Optional
|
||||
|
||||
import lance
|
||||
from lance.blob import BlobType as LanceBlobType
|
||||
from lancedb.conftest import MockTextEmbeddingFunction
|
||||
from lancedb.embeddings.base import EmbeddingFunctionConfig
|
||||
from lancedb.embeddings.registry import EmbeddingFunctionRegistry
|
||||
@@ -908,165 +907,6 @@ def test_cast_to_target_schema():
|
||||
assert output == expected
|
||||
|
||||
|
||||
def test_cast_to_target_schema_coerces_binary_to_blob_v2():
|
||||
data = pa.table({"image": pa.array([b"hello", None], type=pa.binary())})
|
||||
target = pa.schema([lancedb.blob("image")])
|
||||
|
||||
output = _cast_to_target_schema(data.to_reader(), target).read_all()
|
||||
|
||||
image = output["image"].chunk(0)
|
||||
assert type(image.type) is lancedb.BlobType
|
||||
assert image.storage.to_pylist() == [
|
||||
{"data": b"hello", "uri": None, "position": None, "size": None},
|
||||
None,
|
||||
]
|
||||
|
||||
|
||||
def test_cast_to_target_schema_coerces_binary_to_metadata_blob_struct():
|
||||
storage = lancedb.blob("image").type.storage_type
|
||||
target = pa.schema(
|
||||
[
|
||||
pa.field(
|
||||
"image",
|
||||
storage,
|
||||
metadata={
|
||||
b"ARROW:extension:name": b"lance.blob.v2",
|
||||
b"ARROW:extension:metadata": b"",
|
||||
},
|
||||
)
|
||||
]
|
||||
)
|
||||
data = pa.table({"image": pa.array([b"hello", None], type=pa.binary())})
|
||||
|
||||
output = _cast_to_target_schema(data.to_reader(), target).read_all()
|
||||
|
||||
image = output["image"].chunk(0)
|
||||
assert not isinstance(image.type, pa.ExtensionType)
|
||||
assert image.to_pylist() == [
|
||||
{"data": b"hello", "uri": None, "position": None, "size": None},
|
||||
None,
|
||||
]
|
||||
|
||||
|
||||
def test_cast_to_target_schema_coerces_nested_binary_blob():
|
||||
data = pa.table(
|
||||
{
|
||||
"info": pa.array(
|
||||
[{"blob": b"hello"}, {"blob": None}],
|
||||
type=pa.struct([pa.field("blob", pa.binary())]),
|
||||
)
|
||||
}
|
||||
)
|
||||
target = pa.schema([pa.field("info", pa.struct([lancedb.blob("blob")]))])
|
||||
|
||||
output = _cast_to_target_schema(data.to_reader(), target).read_all()
|
||||
|
||||
blob = output["info"].chunk(0).field("blob")
|
||||
assert type(blob.type) is lancedb.BlobType
|
||||
assert blob.storage.to_pylist() == [
|
||||
{"data": b"hello", "uri": None, "position": None, "size": None},
|
||||
None,
|
||||
]
|
||||
|
||||
|
||||
def test_cast_to_target_schema_coerces_list_binary_blob_with_inferred_child_name():
|
||||
data = pa.table(
|
||||
{"images": pa.array([[b"a", b"b"], None], type=pa.list_(pa.binary()))}
|
||||
)
|
||||
target = pa.schema([pa.field("images", pa.list_(lancedb.blob("image")))])
|
||||
|
||||
output = _cast_to_target_schema(data.to_reader(), target).read_all()
|
||||
|
||||
images = output["images"].chunk(0)
|
||||
assert images.type.value_field.name == "image"
|
||||
assert type(images.type.value_type) is lancedb.BlobType
|
||||
assert images.to_pylist()[1] is None
|
||||
assert images.values.storage.to_pylist() == [
|
||||
{"data": b"a", "uri": None, "position": None, "size": None},
|
||||
{"data": b"b", "uri": None, "position": None, "size": None},
|
||||
]
|
||||
|
||||
|
||||
def test_list_blob_coercion_preserves_null_slots_with_nonzero_extent():
|
||||
child = pa.field("image", pa.binary())
|
||||
source = pa.ListArray.from_arrays(
|
||||
pa.array([0, 2, 4], type=pa.int32()),
|
||||
pa.array([b"a", b"b", b"dead", b"beef"], type=pa.binary()),
|
||||
mask=pa.array([False, True]),
|
||||
).cast(pa.list_(child))
|
||||
target = pa.schema([pa.field("images", pa.list_(lancedb.blob("image")))])
|
||||
|
||||
output = _cast_to_target_schema(
|
||||
pa.table({"images": source}).to_reader(), target
|
||||
).read_all()
|
||||
|
||||
images = output["images"].chunk(0)
|
||||
assert images.to_pylist()[1] is None
|
||||
assert [b["data"] for b in images.to_pylist()[0]] == [b"a", b"b"]
|
||||
|
||||
|
||||
def test_fixed_size_list_blob_coercion_keeps_null_rows():
|
||||
child = pa.field("frame", pa.binary())
|
||||
source = (
|
||||
pa.FixedSizeListArray.from_arrays(
|
||||
pa.array([b"a", b"b", b"c", b"d"], type=pa.binary()), 2
|
||||
)
|
||||
.take(pa.array([0, None], type=pa.int32()))
|
||||
.cast(pa.list_(child, 2))
|
||||
)
|
||||
target = pa.schema([pa.field("frames", pa.list_(lancedb.blob("frame"), 2))])
|
||||
|
||||
output = _cast_to_target_schema(
|
||||
pa.table({"frames": source}).to_reader(), target
|
||||
).read_all()
|
||||
|
||||
frames = output["frames"].chunk(0)
|
||||
assert frames.to_pylist()[1] is None
|
||||
assert [b["data"] for b in frames.to_pylist()[0]] == [b"a", b"b"]
|
||||
|
||||
|
||||
def test_cast_to_target_schema_accepts_pylance_blob_v2():
|
||||
target_type = lancedb.BlobType()
|
||||
source = lance.blob_array([b"hello", None])
|
||||
assert type(source.type) is LanceBlobType
|
||||
assert type(source.type) is type(target_type)
|
||||
data = pa.table({"image": source})
|
||||
target = pa.schema([pa.field("image", target_type)])
|
||||
|
||||
output = _cast_to_target_schema(data.to_reader(), target).read_all()
|
||||
|
||||
image = output["image"].chunk(0)
|
||||
assert type(image.type) is LanceBlobType
|
||||
assert image.type == target_type
|
||||
assert image.storage.to_pylist() == [
|
||||
{"data": b"hello", "uri": None, "position": None, "size": None},
|
||||
None,
|
||||
]
|
||||
|
||||
|
||||
def test_cast_to_target_schema_rejects_different_blob_v2_class():
|
||||
class OtherBlobType(pa.ExtensionType):
|
||||
def __init__(self):
|
||||
super().__init__(lancedb.BlobType().storage_type, "lance.blob.v2")
|
||||
|
||||
def __arrow_ext_serialize__(self) -> bytes:
|
||||
return b""
|
||||
|
||||
@classmethod
|
||||
def __arrow_ext_deserialize__(
|
||||
cls, storage_type: pa.DataType, serialized: bytes
|
||||
) -> "OtherBlobType":
|
||||
return cls()
|
||||
|
||||
storage = lance.blob_array([b"hello"]).storage
|
||||
source = pa.ExtensionArray.from_storage(OtherBlobType(), storage)
|
||||
data = pa.table({"image": source})
|
||||
target = pa.schema([lancedb.blob("image")])
|
||||
|
||||
with pytest.raises(pa.ArrowTypeError, match="different extension type"):
|
||||
_cast_to_target_schema(data.to_reader(), target).read_all()
|
||||
|
||||
|
||||
def test_sanitize_data_stream():
|
||||
# Make sure we don't collect the whole stream when running sanitize_data
|
||||
schema = pa.schema({"a": pa.int32()})
|
||||
|
||||
@@ -333,40 +333,6 @@ impl Connection {
|
||||
})
|
||||
}
|
||||
|
||||
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None))]
|
||||
pub fn create_materialized_view(
|
||||
self_: PyRef<'_, Self>,
|
||||
name: String,
|
||||
source: String,
|
||||
projections: Option<Vec<(String, String)>>,
|
||||
filter: Option<String>,
|
||||
limit: Option<u64>,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let mut builder = inner.create_materialized_view(name, source);
|
||||
if let Some(projections) = projections {
|
||||
builder = builder.select(projections);
|
||||
}
|
||||
if let Some(filter) = filter {
|
||||
builder = builder.only_if(filter);
|
||||
}
|
||||
if let Some(limit) = limit {
|
||||
builder = builder.limit(limit);
|
||||
}
|
||||
let view = builder.execute().await.infer_error()?;
|
||||
Ok(Table::new(view.table().clone()))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn list_materialized_views(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let views = inner.list_materialized_views().await.infer_error()?;
|
||||
Ok(views.into_iter().map(|view| view.name).collect::<Vec<_>>())
|
||||
})
|
||||
}
|
||||
|
||||
#[pyo3(signature = (name, namespace_path=None))]
|
||||
pub fn drop_table(
|
||||
self_: PyRef<'_, Self>,
|
||||
@@ -609,7 +575,7 @@ impl Connection {
|
||||
.create_function_async(request)
|
||||
.await
|
||||
.infer_error()
|
||||
.map(crate::job::Job::new_typed)
|
||||
.map(crate::job::FunctionJob::new)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user