mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-27 08:28:28 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2bab493265 |
+1
-1
@@ -1,5 +1,5 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "0.38.0-beta.11"
|
||||
current_version = "0.38.0-beta.10"
|
||||
parse = """(?x)
|
||||
(?P<major>0|[1-9]\\d*)\\.
|
||||
(?P<minor>0|[1-9]\\d*)\\.
|
||||
|
||||
Generated
+45
-45
@@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.5",
|
||||
@@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -4888,8 +4888,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4911,7 +4911,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4925,7 +4925,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -4934,8 +4934,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -4945,8 +4945,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4983,8 +4983,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5013,8 +5013,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5031,8 +5031,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5041,8 +5041,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5075,8 +5075,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5107,8 +5107,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5172,8 +5172,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index-core"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5195,8 +5195,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5236,8 +5236,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5251,8 +5251,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5264,8 +5264,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -5318,8 +5318,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5333,8 +5333,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5374,8 +5374,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5388,8 +5388,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "12.0.0-beta.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
|
||||
version = "11.0.0-beta.22"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
|
||||
dependencies = [
|
||||
"frostem",
|
||||
"icu_segmenter",
|
||||
@@ -5402,7 +5402,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb"
|
||||
version = "0.38.0-beta.11"
|
||||
version = "0.38.0-beta.9"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"anyhow",
|
||||
@@ -5490,7 +5490,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-nodejs"
|
||||
version = "0.38.0-beta.11"
|
||||
version = "0.38.0-beta.9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5515,7 +5515,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-python"
|
||||
version = "0.38.0-beta.11"
|
||||
version = "0.38.0-beta.9"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
|
||||
+14
-14
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lancedb = { path = "rust/lancedb", default-features = false }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
|
||||
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
|
||||
<dependency>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-core</artifactId>
|
||||
<version>0.38.0-beta.11</version>
|
||||
<version>0.38.0-beta.10</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
|
||||
@@ -1,518 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / AutoQuery
|
||||
|
||||
# Class: AutoQuery
|
||||
|
||||
A builder for automatic string searches.
|
||||
|
||||
Automatic search determines whether to use full-text or vector search from
|
||||
the table revision selected for each execution. This builder exposes the
|
||||
common operations supported by both query families.
|
||||
|
||||
## Extends
|
||||
|
||||
- `StandardQueryBase`<`NativeQuery` \| `NativeVectorQuery`>
|
||||
|
||||
## Properties
|
||||
|
||||
### inner
|
||||
|
||||
```ts
|
||||
protected inner: Query | VectorQuery | Promise<Query | VectorQuery>;
|
||||
```
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.inner`
|
||||
|
||||
## Methods
|
||||
|
||||
### analyzePlan()
|
||||
|
||||
```ts
|
||||
analyzePlan(distributedMetrics?): Promise<string>
|
||||
```
|
||||
|
||||
Executes the query and returns the physical query plan annotated with runtime metrics.
|
||||
|
||||
This is useful for debugging and performance analysis, as it shows how the query was executed
|
||||
and includes metrics such as elapsed time, rows processed, and I/O statistics.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **distributedMetrics?**: [`AnalyzePlanDistributedMetrics`](../type-aliases/AnalyzePlanDistributedMetrics.md)
|
||||
How distributed worker metrics are displayed for remote query plans.
|
||||
Defaults to `"aggregate"`.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`string`>
|
||||
|
||||
A query execution plan with runtime metrics for each step.
|
||||
|
||||
#### Example
|
||||
|
||||
```ts
|
||||
import * as lancedb from "@lancedb/lancedb"
|
||||
|
||||
const db = await lancedb.connect("./.lancedb");
|
||||
const table = await db.createTable("my_table", [
|
||||
{ vector: [1.1, 0.9], id: "1" },
|
||||
]);
|
||||
|
||||
const plan = await table.query().nearestTo([0.5, 0.2]).analyzePlan();
|
||||
|
||||
Example output (with runtime metrics inlined):
|
||||
AnalyzeExec verbose=true, metrics=[]
|
||||
ProjectionExec: expr=[id@3 as id, vector@0 as vector, _distance@2 as _distance], metrics=[output_rows=1, elapsed_compute=3.292µs]
|
||||
Take: columns="vector, _rowid, _distance, (id)", metrics=[output_rows=1, elapsed_compute=66.001µs, batches_processed=1, bytes_read=8, iops=1, requests=1]
|
||||
CoalesceBatchesExec: target_batch_size=1024, metrics=[output_rows=1, elapsed_compute=3.333µs]
|
||||
GlobalLimitExec: skip=0, fetch=10, metrics=[output_rows=1, elapsed_compute=167ns]
|
||||
FilterExec: _distance@2 IS NOT NULL, metrics=[output_rows=1, elapsed_compute=8.542µs]
|
||||
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=63.25µs, row_replacements=1]
|
||||
KNNVectorDistance: metric=l2, metrics=[output_rows=1, elapsed_compute=114.333µs, output_batches=1]
|
||||
LanceScan: uri=/path/to/data, projection=[vector], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1, elapsed_compute=103.626µs, bytes_read=549, iops=2, requests=2]
|
||||
```
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.analyzePlan`
|
||||
|
||||
***
|
||||
|
||||
### execute()
|
||||
|
||||
```ts
|
||||
protected execute(options?): AsyncGenerator<RecordBatch<any>, void, unknown>
|
||||
```
|
||||
|
||||
Execute the query and return the results as an
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **options?**: `Partial`<[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)>
|
||||
|
||||
#### Returns
|
||||
|
||||
`AsyncGenerator`<`RecordBatch`<`any`>, `void`, `unknown`>
|
||||
|
||||
#### See
|
||||
|
||||
- AsyncIterator
|
||||
of
|
||||
- RecordBatch.
|
||||
|
||||
By default, LanceDb will use many threads to calculate results and, when
|
||||
the result set is large, multiple batches will be processed at one time.
|
||||
This readahead is limited however and backpressure will be applied if this
|
||||
stream is consumed slowly (this constrains the maximum memory used by a
|
||||
single query)
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.execute`
|
||||
|
||||
***
|
||||
|
||||
### explainPlan()
|
||||
|
||||
```ts
|
||||
explainPlan(verbose): Promise<string>
|
||||
```
|
||||
|
||||
Generates an explanation of the query execution plan.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **verbose**: `boolean` = `false`
|
||||
If true, provides a more detailed explanation. Defaults to false.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`string`>
|
||||
|
||||
A Promise that resolves to a string containing the query execution plan explanation.
|
||||
|
||||
#### Example
|
||||
|
||||
```ts
|
||||
import * as lancedb from "@lancedb/lancedb"
|
||||
const db = await lancedb.connect("./.lancedb");
|
||||
const table = await db.createTable("my_table", [
|
||||
{ vector: [1.1, 0.9], id: "1" },
|
||||
]);
|
||||
const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
|
||||
```
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.explainPlan`
|
||||
|
||||
***
|
||||
|
||||
### fastSearch()
|
||||
|
||||
```ts
|
||||
fastSearch(): this
|
||||
```
|
||||
|
||||
Skip searching un-indexed data. This can make search faster, but will miss
|
||||
any data that is not yet indexed.
|
||||
|
||||
Use [Table#optimize](Table.md#optimize) to index all un-indexed data.
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.fastSearch`
|
||||
|
||||
***
|
||||
|
||||
### ~~filter()~~
|
||||
|
||||
```ts
|
||||
filter(predicate): this
|
||||
```
|
||||
|
||||
A filter statement to be applied to this query.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **predicate**: `string`
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
#### See
|
||||
|
||||
where
|
||||
|
||||
#### Deprecated
|
||||
|
||||
Use `where` instead
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.filter`
|
||||
|
||||
***
|
||||
|
||||
### fullTextSearch()
|
||||
|
||||
```ts
|
||||
fullTextSearch(query, options?): this
|
||||
```
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **query**: `string` \| [`FullTextQuery`](../interfaces/FullTextQuery.md)
|
||||
|
||||
* **options?**: `Partial`<[`FullTextSearchOptions`](../interfaces/FullTextSearchOptions.md)>
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.fullTextSearch`
|
||||
|
||||
***
|
||||
|
||||
### limit()
|
||||
|
||||
```ts
|
||||
limit(limit): this
|
||||
```
|
||||
|
||||
Set the maximum number of results to return.
|
||||
|
||||
By default, a plain search has no limit. If this method is not
|
||||
called then every valid row from the table will be returned.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **limit**: `number`
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.limit`
|
||||
|
||||
***
|
||||
|
||||
### offset()
|
||||
|
||||
```ts
|
||||
offset(offset): this
|
||||
```
|
||||
|
||||
Set the number of rows to skip before returning results.
|
||||
|
||||
This is useful for pagination.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **offset**: `number`
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.offset`
|
||||
|
||||
***
|
||||
|
||||
### orderBy()
|
||||
|
||||
```ts
|
||||
orderBy(ordering): this
|
||||
```
|
||||
|
||||
Sort the results by the specified column(s).
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **ordering**: [`ColumnOrdering`](../interfaces/ColumnOrdering.md) \| [`ColumnOrdering`](../interfaces/ColumnOrdering.md)[]
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
This query builder.
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.orderBy`
|
||||
|
||||
***
|
||||
|
||||
### outputSchema()
|
||||
|
||||
```ts
|
||||
outputSchema(): Promise<Schema<any>>
|
||||
```
|
||||
|
||||
Returns the schema of the output that will be returned by this query.
|
||||
|
||||
This can be used to inspect the types and names of the columns that will be
|
||||
returned by the query before executing it.
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`Schema`<`any`>>
|
||||
|
||||
An Arrow Schema describing the output columns.
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.outputSchema`
|
||||
|
||||
***
|
||||
|
||||
### select()
|
||||
|
||||
```ts
|
||||
select(columns): this
|
||||
```
|
||||
|
||||
Return only the specified columns.
|
||||
|
||||
By default a query will return all columns from the table. However, this can have
|
||||
a very significant impact on latency. LanceDb stores data in a columnar fashion. This
|
||||
means we can finely tune our I/O to select exactly the columns we need.
|
||||
|
||||
As a best practice you should always limit queries to the columns that you need. If you
|
||||
pass in an array of column names then only those columns will be returned.
|
||||
|
||||
You can also use this method to create new "dynamic" columns based on your existing columns.
|
||||
For example, you may not care about "a" or "b" but instead simply want "a + b". This is often
|
||||
seen in the SELECT clause of an SQL query (e.g. `SELECT a+b FROM my_table`).
|
||||
|
||||
To create dynamic columns you can pass in a Map<string, string>. A column will be returned
|
||||
for each entry in the map. The key provides the name of the column. The value is
|
||||
an SQL string used to specify how the column is calculated.
|
||||
|
||||
For example, an SQL query might state `SELECT a + b AS combined, c`. The equivalent
|
||||
input to this method would be:
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **columns**: `string` \| `string`[] \| `Record`<`string`, `string`> \| `Map`<`string`, `string`>
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
#### Example
|
||||
|
||||
```ts
|
||||
new Map([["combined", "a + b"], ["c", "c"]])
|
||||
|
||||
Columns will always be returned in the order given, even if that order is different than
|
||||
the order used when adding the data.
|
||||
|
||||
Note that you can pass in a `Record<string, string>` (e.g. an object literal). This method
|
||||
uses `Object.entries` which should preserve the insertion order of the object. However,
|
||||
object insertion order is easy to get wrong and `Map` is more foolproof.
|
||||
```
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.select`
|
||||
|
||||
***
|
||||
|
||||
### toArray()
|
||||
|
||||
```ts
|
||||
toArray(options?): Promise<any[]>
|
||||
```
|
||||
|
||||
Collect the results as an array of objects.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **options?**: `Partial`<[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)>
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`any`[]>
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.toArray`
|
||||
|
||||
***
|
||||
|
||||
### toArrow()
|
||||
|
||||
```ts
|
||||
toArrow(options?): Promise<Table<any>>
|
||||
```
|
||||
|
||||
Collect the results as an Arrow
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **options?**: `Partial`<[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)>
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`Table`<`any`>>
|
||||
|
||||
#### See
|
||||
|
||||
ArrowTable.
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.toArrow`
|
||||
|
||||
***
|
||||
|
||||
### useLsm()
|
||||
|
||||
```ts
|
||||
useLsm(enable): this
|
||||
```
|
||||
|
||||
Control MemWAL read routing for this query.
|
||||
|
||||
By default (unset), when the table carries a MemWAL write spec (see
|
||||
[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so
|
||||
they also return data written via the `mergeInsert` LSM path that has not yet
|
||||
been compacted into the base table (the active/frozen in-memory memtables and
|
||||
the flushed generations), deduplicated by primary key; a table without a spec
|
||||
reads the base table.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **enable**: `boolean`
|
||||
`true` forces the LSM scanner and errors if the table has no
|
||||
MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
|
||||
even when a spec is present.
|
||||
Note: the LSM scanner does not support every query shape (e.g. reranking,
|
||||
hybrid search, `orderBy`). On a MemWAL table those shapes error unless
|
||||
`useLsm(false)` is set, because a base-only read would silently exclude
|
||||
un-compacted MemWAL data.
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.useLsm`
|
||||
|
||||
***
|
||||
|
||||
### where()
|
||||
|
||||
```ts
|
||||
where(predicate): this
|
||||
```
|
||||
|
||||
A filter statement to be applied to this query.
|
||||
|
||||
The filter should be supplied as an SQL query string. For example:
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **predicate**: `string`
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
#### Example
|
||||
|
||||
```ts
|
||||
x > 10
|
||||
y > 0 AND y < 100
|
||||
x > 5 OR y = 'test'
|
||||
|
||||
Filtering performance can often be improved by creating a scalar index
|
||||
on the filter column(s).
|
||||
|
||||
Calling this multiple times combines the filters with a logical AND rather
|
||||
than replacing the previous filter.
|
||||
```
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.where`
|
||||
|
||||
***
|
||||
|
||||
### withRowId()
|
||||
|
||||
```ts
|
||||
withRowId(): this
|
||||
```
|
||||
|
||||
Whether to return the row id in the results.
|
||||
|
||||
This column can be used to match results between different queries. For
|
||||
example, to match results from a full text search and a vector search in
|
||||
order to perform hybrid search.
|
||||
|
||||
#### Returns
|
||||
|
||||
`this`
|
||||
|
||||
#### Inherited from
|
||||
|
||||
`StandardQueryBase.withRowId`
|
||||
@@ -942,7 +942,7 @@ Get the schema of the table.
|
||||
abstract search(
|
||||
query,
|
||||
queryType?,
|
||||
ftsColumns?): Query | VectorQuery | AutoQuery
|
||||
ftsColumns?): Query | VectorQuery
|
||||
```
|
||||
|
||||
Create a search query to find the nearest neighbors
|
||||
@@ -964,7 +964,7 @@ of the given query
|
||||
|
||||
#### Returns
|
||||
|
||||
[`Query`](Query.md) \| [`VectorQuery`](VectorQuery.md) \| [`AutoQuery`](AutoQuery.md)
|
||||
[`Query`](Query.md) \| [`VectorQuery`](VectorQuery.md)
|
||||
|
||||
***
|
||||
|
||||
@@ -1292,18 +1292,6 @@ abstract updateFieldMetadata(updates): Promise<UpdateFieldMetadataResult>
|
||||
|
||||
Update per-field (column) metadata.
|
||||
|
||||
The following keys are treated specially, by convention, and should be
|
||||
used when appropriate:
|
||||
|
||||
- `lancedb:description`: for a human-readable description of a field.
|
||||
- `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
|
||||
names the tag category; e.g. `lancedb:tag:model: "clip"`.
|
||||
- `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
|
||||
`feature_v2` might be in the same logical column.
|
||||
- `lancedb:status`: for status options (`production`, `candidate`,
|
||||
`deprecated`, `archived`) to designate the current life cycle state of
|
||||
this column.
|
||||
|
||||
#### Parameters
|
||||
|
||||
* **updates**: [`FieldMetadataUpdate`](../interfaces/FieldMetadataUpdate.md)[]
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
|
||||
## Classes
|
||||
|
||||
- [AutoQuery](classes/AutoQuery.md)
|
||||
- [BooleanQuery](classes/BooleanQuery.md)
|
||||
- [BoostQuery](classes/BoostQuery.md)
|
||||
- [BranchContents](classes/BranchContents.md)
|
||||
|
||||
@@ -17,8 +17,7 @@ metadata: Record<string, null | string>;
|
||||
```
|
||||
|
||||
Metadata key/value pairs. Merged into the field's existing metadata by
|
||||
default; a value of `null` deletes that key. See
|
||||
[Table.updateFieldMetadata](../classes/Table.md#updatefieldmetadata) for the conventional `lancedb:*` keys.
|
||||
default; a value of `null` deletes that key.
|
||||
|
||||
***
|
||||
|
||||
|
||||
@@ -10,12 +10,16 @@
|
||||
function getRegistry(): EmbeddingFunctionRegistry
|
||||
```
|
||||
|
||||
Get the global embedding function registry.
|
||||
|
||||
LanceDB built-in providers are initialized when this public API is first
|
||||
used, so importing the root package does not change automatic search
|
||||
selection for tables without embedding metadata.
|
||||
Utility function to get the global instance of the registry
|
||||
|
||||
## Returns
|
||||
|
||||
[`EmbeddingFunctionRegistry`](../classes/EmbeddingFunctionRegistry.md)
|
||||
|
||||
`EmbeddingFunctionRegistry` The global instance of the registry
|
||||
|
||||
## Example
|
||||
|
||||
```ts
|
||||
const registry = getRegistry();
|
||||
const openai = registry.get("openai").create();
|
||||
|
||||
@@ -159,8 +159,6 @@ and combined with [BooleanQuery][lancedb.query.BooleanQuery].
|
||||
|
||||
::: lancedb.query.FullTextOperator
|
||||
|
||||
::: lancedb.query.DocumentGranularity
|
||||
|
||||
::: lancedb.query.Occur
|
||||
|
||||
## Embeddings
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
<parent>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.38.0-beta.11</version>
|
||||
<version>0.38.0-beta.10</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
|
||||
+2
-2
@@ -6,7 +6,7 @@
|
||||
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.38.0-beta.11</version>
|
||||
<version>0.38.0-beta.10</version>
|
||||
<packaging>pom</packaging>
|
||||
<name>${project.artifactId}</name>
|
||||
<description>LanceDB Java SDK Parent POM</description>
|
||||
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>12.0.0-beta.2</lance-core.version>
|
||||
<lance-core.version>11.0.0-beta.22</lance-core.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[package]
|
||||
name = "lancedb-nodejs"
|
||||
edition.workspace = true
|
||||
version = "0.38.0-beta.11"
|
||||
version = "0.38.0-beta.10"
|
||||
publish = false
|
||||
license.workspace = true
|
||||
description.workspace = true
|
||||
|
||||
@@ -6,9 +6,6 @@ import * as arrow17 from "apache-arrow-17";
|
||||
import * as arrow18 from "apache-arrow-18";
|
||||
|
||||
import {
|
||||
Field as CurrentField,
|
||||
LargeBinary as CurrentLargeBinary,
|
||||
Schema as CurrentSchema,
|
||||
Vector as CurrentVector,
|
||||
convertToTable,
|
||||
tableFromIPC as currentTableFromIPC,
|
||||
@@ -39,24 +36,6 @@ function sampleRecords(): Array<Record<string, any>> {
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
it("preserves field metadata from a provided schema", async function () {
|
||||
const jsonMetadata = new Map([["ARROW:extension:name", "lance.json"]]);
|
||||
const schema = new CurrentSchema([
|
||||
new CurrentField("meta", new CurrentLargeBinary(), true, jsonMetadata),
|
||||
]);
|
||||
|
||||
const table = makeArrowTable(
|
||||
[{ meta: Buffer.from(JSON.stringify({ source: "test" })) }],
|
||||
{ schema },
|
||||
);
|
||||
|
||||
expect(table.schema.fields[0].metadata).toEqual(jsonMetadata);
|
||||
|
||||
const roundTripped = currentTableFromIPC(await fromTableToBuffer(table));
|
||||
expect(roundTripped.schema.fields[0].metadata).toEqual(jsonMetadata);
|
||||
});
|
||||
|
||||
describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
"Arrow",
|
||||
(
|
||||
|
||||
@@ -187,58 +187,6 @@ describe("embedding functions", () => {
|
||||
const vector0 = JSON.parse(JSON.stringify(arr[0].vector));
|
||||
expect(vector0).toEqual([1, 2, 3]);
|
||||
});
|
||||
it("should append multiple Python embeddings with the same alias", async () => {
|
||||
@register("python-mock")
|
||||
// biome-ignore lint/correctness/noUnusedVariables: the decorator registers this class
|
||||
class MockEmbeddingFunction extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 3;
|
||||
}
|
||||
embeddingDataType(): Float {
|
||||
return new Float32();
|
||||
}
|
||||
async computeQueryEmbeddings(_data: string) {
|
||||
return [1, 2, 3];
|
||||
}
|
||||
async computeSourceEmbeddings(data: string[]) {
|
||||
return data.map((value) =>
|
||||
value === "hello world" ? [1, 2, 3] : [4, 5, 6],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const metadata = new Map([
|
||||
[
|
||||
"embedding_functions",
|
||||
'[{"source_column":"text1","vector_column":"vector1","name":"python-mock","model":{}},{"source_column":"text2","vector_column":"vector2","name":"python-mock","model":{}}]',
|
||||
],
|
||||
]);
|
||||
const schema = new Schema(
|
||||
[
|
||||
new Field("text1", new Utf8(), true),
|
||||
new Field("text2", new Utf8(), true),
|
||||
new Field(
|
||||
"vector1",
|
||||
new FixedSizeList(3, new Field("item", new Float32(), true)),
|
||||
true,
|
||||
),
|
||||
new Field(
|
||||
"vector2",
|
||||
new FixedSizeList(3, new Field("item", new Float32(), true)),
|
||||
true,
|
||||
),
|
||||
],
|
||||
metadata,
|
||||
);
|
||||
|
||||
const db = await connect(tmpDir.name);
|
||||
const table = await db.createEmptyTable("test", schema);
|
||||
await table.add([{ text1: "hello world", text2: "goodbye world" }]);
|
||||
|
||||
const rows = await table.query().toArray();
|
||||
expect(JSON.parse(JSON.stringify(rows[0].vector1))).toEqual([1, 2, 3]);
|
||||
expect(JSON.parse(JSON.stringify(rows[0].vector2))).toEqual([4, 5, 6]);
|
||||
});
|
||||
|
||||
it("should append generated vectors to a non-nullable schema", async () => {
|
||||
@register("non_nullable_schema_test")
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import type { OpenAIEmbeddingFunction } from "../lancedb/embedding/openai";
|
||||
import type { EmbeddingFunctionRegistry } from "../lancedb/embedding/registry";
|
||||
|
||||
type EmbeddingModule = typeof import("../lancedb/embedding");
|
||||
type OpenAIModule = typeof import("../lancedb/embedding/openai");
|
||||
type RegistryModule = typeof import("../lancedb/embedding/registry");
|
||||
|
||||
describe("embedding function registry", () => {
|
||||
const registries: EmbeddingFunctionRegistry[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const registry of registries) {
|
||||
registry.reset();
|
||||
}
|
||||
registries.length = 0;
|
||||
});
|
||||
|
||||
it("defers built-in providers until the public registry API is used", () => {
|
||||
jest.isolateModules(() => {
|
||||
const embedding = require("../lancedb/embedding") as EmbeddingModule;
|
||||
const { getRegistry: getInternalRegistry } =
|
||||
require("../lancedb/embedding/registry") as RegistryModule;
|
||||
const registry = getInternalRegistry();
|
||||
registries.push(registry);
|
||||
|
||||
expect(registry.length()).toBe(0);
|
||||
expect(embedding.getRegistry()).toBe(registry);
|
||||
expect(registry.get("openai")).toBeDefined();
|
||||
expect(registry.get("huggingface")).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves automatic FTS search in a fresh process", () => {
|
||||
execFileSync(
|
||||
process.execPath,
|
||||
[resolve(__dirname, "fixtures", "auto_fts_search.cjs")],
|
||||
{ stdio: "pipe" },
|
||||
);
|
||||
});
|
||||
|
||||
it("shares registrations across duplicated provider module graphs", () => {
|
||||
let registeringRegistry: EmbeddingFunctionRegistry | undefined;
|
||||
let latestOpenAIConstructor: typeof OpenAIEmbeddingFunction | undefined;
|
||||
|
||||
jest.isolateModules(() => {
|
||||
require("../lancedb/embedding/openai");
|
||||
const { getRegistry } =
|
||||
require("../lancedb/embedding/registry") as RegistryModule;
|
||||
registeringRegistry = getRegistry();
|
||||
registries.push(registeringRegistry);
|
||||
expect(registeringRegistry.get("openai")).toBeDefined();
|
||||
});
|
||||
|
||||
expect(() => {
|
||||
jest.isolateModules(() => {
|
||||
const { OpenAIEmbeddingFunction } =
|
||||
require("../lancedb/embedding/openai") as OpenAIModule;
|
||||
latestOpenAIConstructor = OpenAIEmbeddingFunction;
|
||||
const { getRegistry } =
|
||||
require("../lancedb/embedding/registry") as RegistryModule;
|
||||
registries.push(getRegistry());
|
||||
});
|
||||
}).not.toThrow();
|
||||
|
||||
const previousApiKey = process.env.OPENAI_API_KEY;
|
||||
process.env.OPENAI_API_KEY = "test";
|
||||
try {
|
||||
const latestOpenAI = registeringRegistry!
|
||||
.get<OpenAIEmbeddingFunction>("openai")!
|
||||
.create();
|
||||
expect(latestOpenAI).toBeInstanceOf(latestOpenAIConstructor!);
|
||||
} finally {
|
||||
if (previousApiKey === undefined) {
|
||||
delete process.env.OPENAI_API_KEY;
|
||||
} else {
|
||||
process.env.OPENAI_API_KEY = previousApiKey;
|
||||
}
|
||||
}
|
||||
|
||||
jest.isolateModules(() => {
|
||||
const { getRegistry } =
|
||||
require("../lancedb/embedding") as EmbeddingModule;
|
||||
const publicRegistry = getRegistry();
|
||||
registries.push(publicRegistry);
|
||||
expect(publicRegistry).toBe(registeringRegistry);
|
||||
expect(publicRegistry.get("openai")).toBeDefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,33 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
const assert = require("node:assert/strict");
|
||||
const tmp = require("tmp");
|
||||
const { connect, embedding, Index } = require("../../dist");
|
||||
const { getRegistry } = require("../../dist/embedding/registry");
|
||||
|
||||
async function main() {
|
||||
assert.equal(typeof embedding.getRegistry, "function");
|
||||
assert.equal(getRegistry().length(), 0);
|
||||
assert.equal(embedding.getRegistry(), getRegistry());
|
||||
assert.equal(getRegistry().length(), 2);
|
||||
|
||||
const dir = tmp.dirSync({ unsafeCleanup: true });
|
||||
let db;
|
||||
try {
|
||||
db = await connect(dir.name);
|
||||
const table = await db.createTable("docs", [{ text: "hello world" }]);
|
||||
await table.createIndex("text", { config: Index.fts() });
|
||||
|
||||
const rows = await table.search("hello").toArray();
|
||||
assert.equal(rows[0].text, "hello world");
|
||||
} finally {
|
||||
db?.close();
|
||||
dir.removeCallback();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -11,13 +11,10 @@ import * as arrow17 from "apache-arrow-17";
|
||||
import * as arrow18 from "apache-arrow-18";
|
||||
|
||||
import {
|
||||
AutoQuery,
|
||||
Connection,
|
||||
MatchQuery,
|
||||
PhraseQuery,
|
||||
Query,
|
||||
Table,
|
||||
VectorQuery,
|
||||
connect,
|
||||
tokenize,
|
||||
} from "../lancedb";
|
||||
@@ -685,56 +682,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
},
|
||||
);
|
||||
|
||||
// https://github.com/lancedb/lancedb/issues/1963
|
||||
it("should query documents with LangChain PDF metadata", async () => {
|
||||
const tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
try {
|
||||
const db = await connect(tmpDir.name);
|
||||
const documents = [
|
||||
{
|
||||
text: "first page",
|
||||
vector: [1, 0],
|
||||
source: "first.pdf",
|
||||
loc: { pageNumber: 1, lines: { from: 1, to: 12 } },
|
||||
pdf: {
|
||||
version: "1.10.100",
|
||||
info: {
|
||||
format: "PDF 1.7",
|
||||
producer: "pdf.js",
|
||||
creator: "Writer",
|
||||
},
|
||||
totalPages: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
text: "second page",
|
||||
vector: [0, 1],
|
||||
source: "second.pdf",
|
||||
loc: { pageNumber: 2, lines: { from: 13, to: 24 } },
|
||||
pdf: {
|
||||
version: "1.10.100",
|
||||
info: {
|
||||
format: "PDF 1.7",
|
||||
producer: "pdf.js",
|
||||
creator: "Writer",
|
||||
},
|
||||
totalPages: 2,
|
||||
},
|
||||
},
|
||||
];
|
||||
const documentsTable = await db.createTable("documents", documents);
|
||||
|
||||
const results = await documentsTable.query().toArray();
|
||||
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results[0].source).toBe("first.pdf");
|
||||
expect(results[0].pdf.info.producer).toBe("pdf.js");
|
||||
expect(results[1].loc.pageNumber).toBe(2);
|
||||
} finally {
|
||||
tmpDir.removeCallback();
|
||||
}
|
||||
});
|
||||
|
||||
describe("merge insert", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
let table: Table;
|
||||
@@ -1830,194 +1777,6 @@ describe("Read consistency interval", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("automatic search schema consistency", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
|
||||
class SchemaRefreshEmbedding extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 2;
|
||||
}
|
||||
|
||||
embeddingDataType() {
|
||||
return new Float32();
|
||||
}
|
||||
|
||||
async computeSourceEmbeddings(data: string[]) {
|
||||
return data.map((value) => [value.length, 1]);
|
||||
}
|
||||
|
||||
async computeQueryEmbeddings(value: string) {
|
||||
return [value.length, 1];
|
||||
}
|
||||
}
|
||||
|
||||
function embeddingSchema() {
|
||||
const func = new SchemaRefreshEmbedding();
|
||||
return LanceSchema({
|
||||
text: func.sourceField(new Utf8()),
|
||||
vector: func.vectorField(),
|
||||
});
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
getRegistry().reset();
|
||||
register("schema-refresh")(SchemaRefreshEmbedding);
|
||||
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
getRegistry().reset();
|
||||
tmpDir.removeCallback();
|
||||
});
|
||||
|
||||
it("uses the schema refreshed from another connection", async () => {
|
||||
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
|
||||
try {
|
||||
const stale = await first.createTable("docs", [{ text: "before" }], {
|
||||
schema: embeddingSchema(),
|
||||
});
|
||||
const replacement = await second.createTable(
|
||||
"docs",
|
||||
[{ text: "after hello" }],
|
||||
{ mode: "overwrite" },
|
||||
);
|
||||
await replacement.createIndex("text", { config: Index.fts() });
|
||||
|
||||
const search = stale.search("hello");
|
||||
expect(search).toBeInstanceOf(AutoQuery);
|
||||
expect(search).not.toBeInstanceOf(Query);
|
||||
expect(search).not.toBeInstanceOf(VectorQuery);
|
||||
expect("nprobes" in search).toBe(false);
|
||||
|
||||
const rows = await search.toArray();
|
||||
expect(rows[0].text).toBe("after hello");
|
||||
expect((await stale.schema()).metadata.has("embedding_functions")).toBe(
|
||||
false,
|
||||
);
|
||||
} finally {
|
||||
first.close();
|
||||
second.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("tracks embedding metadata across checkout and restore", async () => {
|
||||
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
|
||||
try {
|
||||
await first.createTable("docs", [{ text: "before" }], {
|
||||
schema: embeddingSchema(),
|
||||
});
|
||||
const table = await second.createTable(
|
||||
"docs",
|
||||
[{ text: "after hello" }],
|
||||
{ mode: "overwrite" },
|
||||
);
|
||||
await table.createIndex("text", { config: Index.fts() });
|
||||
|
||||
await table.checkout(1);
|
||||
expect((await table.search("before").toArray())[0].text).toBe("before");
|
||||
|
||||
await table.checkoutLatest();
|
||||
expect((await table.search("hello").toArray())[0].text).toBe(
|
||||
"after hello",
|
||||
);
|
||||
|
||||
await table.checkout(1);
|
||||
await table.restore();
|
||||
expect((await table.search("before").toArray())[0].text).toBe("before");
|
||||
} finally {
|
||||
first.close();
|
||||
second.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("pins automatic search while computing an embedding", async () => {
|
||||
let markStarted!: () => void;
|
||||
let releaseEmbedding!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
markStarted = resolve;
|
||||
});
|
||||
const released = new Promise<void>((resolve) => {
|
||||
releaseEmbedding = resolve;
|
||||
});
|
||||
|
||||
class BlockingEmbedding extends SchemaRefreshEmbedding {
|
||||
async computeQueryEmbeddings(value: string) {
|
||||
markStarted();
|
||||
await released;
|
||||
return [value.length, 1];
|
||||
}
|
||||
}
|
||||
|
||||
register("schema-refresh-blocking")(BlockingEmbedding);
|
||||
const func = new BlockingEmbedding();
|
||||
const schema = LanceSchema({
|
||||
text: func.sourceField(new Utf8()),
|
||||
vector: func.vectorField(),
|
||||
});
|
||||
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
|
||||
try {
|
||||
const table = await first.createTable(
|
||||
"docs",
|
||||
[{ text: "hello before" }],
|
||||
{ schema },
|
||||
);
|
||||
const pending = table.search("hello").toArray();
|
||||
await started;
|
||||
|
||||
const replacement = await second.createTable(
|
||||
"docs",
|
||||
[{ text: "hello after" }],
|
||||
{ mode: "overwrite" },
|
||||
);
|
||||
await replacement.createIndex("text", { config: Index.fts() });
|
||||
releaseEmbedding();
|
||||
|
||||
expect((await pending)[0].text).toBe("hello before");
|
||||
} finally {
|
||||
releaseEmbedding();
|
||||
first.close();
|
||||
second.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("refreshes a reused automatic search for every execution", async () => {
|
||||
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
|
||||
try {
|
||||
const table = await first.createTable("docs", [
|
||||
{ text: "hello before", marker: "before" },
|
||||
]);
|
||||
await table.createIndex("text", { config: Index.fts() });
|
||||
const search = table.search("hello").select(["text"]);
|
||||
|
||||
const before = (await search.toArray())[0];
|
||||
expect(before.text).toBe("hello before");
|
||||
expect(before.marker).toBeUndefined();
|
||||
|
||||
const replacement = await second.createTable(
|
||||
"docs",
|
||||
[{ text: "hello after", marker: "after" }],
|
||||
{ mode: "overwrite" },
|
||||
);
|
||||
await replacement.createIndex("text", { config: Index.fts() });
|
||||
|
||||
const after = (await search.toArray())[0];
|
||||
expect(after.text).toBe("hello after");
|
||||
expect(after.marker).toBeUndefined();
|
||||
} finally {
|
||||
first.close();
|
||||
second.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("schema evolution", function () {
|
||||
let tmpDir: tmp.DirResult;
|
||||
beforeEach(() => {
|
||||
@@ -2585,24 +2344,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
);
|
||||
});
|
||||
|
||||
test("full text search if only an unrelated embedding function is registered", async () => {
|
||||
register("unused")(
|
||||
class extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 3;
|
||||
}
|
||||
embeddingDataType() {
|
||||
return new Float32();
|
||||
}
|
||||
async computeQueryEmbeddings(_data: string) {
|
||||
return [1, 2, 3];
|
||||
}
|
||||
async computeSourceEmbeddings(data: string[]) {
|
||||
return data.map(() => [1, 2, 3]);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
test("full text search if no embedding function provided", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const data = [
|
||||
{ text: "hello world", vector: [0.1, 0.2, 0.3] },
|
||||
@@ -2624,306 +2366,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
expect(results2[0].text).toBe(data[1].text);
|
||||
});
|
||||
|
||||
test("auto search stays consistent with the active revision", async () => {
|
||||
let initCalls = 0;
|
||||
let queryCalls = 0;
|
||||
let markStarted!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
markStarted = resolve;
|
||||
});
|
||||
let releaseEmbedding!: () => void;
|
||||
const embeddingReleased = new Promise<void>((resolve) => {
|
||||
releaseEmbedding = resolve;
|
||||
});
|
||||
|
||||
@register("refresh-test")
|
||||
class TestEmbedding extends EmbeddingFunction<string> {
|
||||
async init() {
|
||||
initCalls += 1;
|
||||
}
|
||||
ndims() {
|
||||
return 1;
|
||||
}
|
||||
embeddingDataType() {
|
||||
return new arrow.Float32();
|
||||
}
|
||||
async computeQueryEmbeddings(value: string) {
|
||||
queryCalls += 1;
|
||||
if (value === "blocked") {
|
||||
markStarted();
|
||||
await embeddingReleased;
|
||||
}
|
||||
return value === "greetings" ? [0.1] : [0.2];
|
||||
}
|
||||
async computeSourceEmbeddings(values: string[]) {
|
||||
return values.map((value) =>
|
||||
value === "hello world" ? [0.1] : [0.2],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const writer = await connect(tmpDir.name);
|
||||
await writer.createTable("test", [{ text: "plain", vector: [0.0] }]);
|
||||
const reader = await connect(tmpDir.name, {
|
||||
readConsistencyInterval: 0,
|
||||
});
|
||||
const tracked = await reader.openTable("test");
|
||||
type SnapshotCountingNative = {
|
||||
querySnapshot: () => Promise<unknown>;
|
||||
};
|
||||
const native = (tracked as unknown as { inner: SnapshotCountingNative })
|
||||
.inner;
|
||||
const querySnapshot = native.querySnapshot.bind(native);
|
||||
let snapshotCalls = 0;
|
||||
native.querySnapshot = async () => {
|
||||
snapshotCalls += 1;
|
||||
return await querySnapshot();
|
||||
};
|
||||
const autoQuery = tracked.search("greetings").select(["text"]).limit(1);
|
||||
|
||||
const func = new TestEmbedding();
|
||||
const schema = LanceSchema({
|
||||
text: func.sourceField(new arrow.Utf8()),
|
||||
vector: func.vectorField(),
|
||||
});
|
||||
const data = [{ text: "hello world" }, { text: "goodbye world" }];
|
||||
await writer.createTable("test", data, { mode: "overwrite", schema });
|
||||
const baselineInitCalls = initCalls;
|
||||
|
||||
expect(
|
||||
(await tracked.schema()).metadata.get("embedding_functions"),
|
||||
).toBeDefined();
|
||||
const results = await autoQuery.toArray();
|
||||
expect(results[0].text).toBe(data[0].text);
|
||||
expect(initCalls).toBe(baselineInitCalls + 1);
|
||||
expect(queryCalls).toBe(1);
|
||||
expect(snapshotCalls).toBe(1);
|
||||
|
||||
const repeatedResults = await autoQuery.toArray();
|
||||
expect(repeatedResults[0].text).toBe(data[0].text);
|
||||
expect(initCalls).toBe(baselineInitCalls + 1);
|
||||
expect(queryCalls).toBe(1);
|
||||
expect(snapshotCalls).toBe(2);
|
||||
|
||||
const pending = tracked
|
||||
.search("blocked")
|
||||
.select(["text"])
|
||||
.limit(1)
|
||||
.toArray();
|
||||
await started;
|
||||
|
||||
const ftsData = [
|
||||
{ text: "greetings from full text", vector: [0.0] },
|
||||
{ text: "blocked from full text", vector: [0.0] },
|
||||
];
|
||||
const ftsTable = await writer.createTable("test", ftsData, {
|
||||
mode: "overwrite",
|
||||
});
|
||||
await ftsTable.createIndex("text", { config: Index.fts() });
|
||||
releaseEmbedding();
|
||||
|
||||
const pendingResults = await pending;
|
||||
expect(pendingResults[0].text).toBe(data[1].text);
|
||||
|
||||
expect(
|
||||
(await tracked.schema()).metadata.get("embedding_functions"),
|
||||
).toBeUndefined();
|
||||
const ftsResults = await autoQuery.toArray();
|
||||
expect(ftsResults[0].text).toBe(ftsData[0].text);
|
||||
});
|
||||
|
||||
test("auto search keeps newer preparation during a revision race", async () => {
|
||||
let aCalls = 0;
|
||||
let bCalls = 0;
|
||||
let markAStarted!: () => void;
|
||||
const aStarted = new Promise<void>((resolve) => {
|
||||
markAStarted = resolve;
|
||||
});
|
||||
let releaseA!: () => void;
|
||||
const aReleased = new Promise<void>((resolve) => {
|
||||
releaseA = resolve;
|
||||
});
|
||||
let markBStarted!: () => void;
|
||||
const bStarted = new Promise<void>((resolve) => {
|
||||
markBStarted = resolve;
|
||||
});
|
||||
let releaseB!: () => void;
|
||||
const bReleased = new Promise<void>((resolve) => {
|
||||
releaseB = resolve;
|
||||
});
|
||||
|
||||
@register("race-a")
|
||||
class EmbeddingA extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 1;
|
||||
}
|
||||
embeddingDataType() {
|
||||
return new arrow.Float32();
|
||||
}
|
||||
async computeQueryEmbeddings() {
|
||||
aCalls += 1;
|
||||
markAStarted();
|
||||
await aReleased;
|
||||
return [0.1];
|
||||
}
|
||||
async computeSourceEmbeddings(values: string[]) {
|
||||
return values.map(() => [0.1]);
|
||||
}
|
||||
}
|
||||
|
||||
@register("race-b")
|
||||
class EmbeddingB extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 1;
|
||||
}
|
||||
embeddingDataType() {
|
||||
return new arrow.Float32();
|
||||
}
|
||||
async computeQueryEmbeddings() {
|
||||
bCalls += 1;
|
||||
markBStarted();
|
||||
await bReleased;
|
||||
return [0.2];
|
||||
}
|
||||
async computeSourceEmbeddings(values: string[]) {
|
||||
return values.map(() => [0.2]);
|
||||
}
|
||||
}
|
||||
|
||||
const writer = await connect(tmpDir.name);
|
||||
const embeddingA = new EmbeddingA();
|
||||
const schemaA = LanceSchema({
|
||||
text: embeddingA.sourceField(new arrow.Utf8()),
|
||||
vector: embeddingA.vectorField(),
|
||||
});
|
||||
await writer.createTable("race", [{ text: "revision a" }], {
|
||||
schema: schemaA,
|
||||
});
|
||||
const reader = await connect(tmpDir.name, {
|
||||
readConsistencyInterval: 0,
|
||||
});
|
||||
const tracked = await reader.openTable("race");
|
||||
const query = tracked.search("query");
|
||||
|
||||
const first = query.toArray();
|
||||
await aStarted;
|
||||
|
||||
const embeddingB = new EmbeddingB();
|
||||
const schemaB = LanceSchema({
|
||||
text: embeddingB.sourceField(new arrow.Utf8()),
|
||||
vector: embeddingB.vectorField(),
|
||||
});
|
||||
await writer.createTable("race", [{ text: "revision b" }], {
|
||||
mode: "overwrite",
|
||||
schema: schemaB,
|
||||
});
|
||||
const second = query.toArray();
|
||||
await bStarted;
|
||||
|
||||
releaseA();
|
||||
releaseB();
|
||||
await Promise.all([first, second]);
|
||||
expect(aCalls).toBe(1);
|
||||
expect(bCalls).toBe(1);
|
||||
});
|
||||
|
||||
test("stale FTS routing keeps newer vector preparation", async () => {
|
||||
let vectorCalls = 0;
|
||||
let markVectorStarted!: () => void;
|
||||
const vectorStarted = new Promise<void>((resolve) => {
|
||||
markVectorStarted = resolve;
|
||||
});
|
||||
let releaseVector!: () => void;
|
||||
const vectorReleased = new Promise<void>((resolve) => {
|
||||
releaseVector = resolve;
|
||||
});
|
||||
|
||||
@register("stale-fts-race")
|
||||
class RaceEmbedding extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 1;
|
||||
}
|
||||
embeddingDataType() {
|
||||
return new arrow.Float32();
|
||||
}
|
||||
async computeQueryEmbeddings() {
|
||||
vectorCalls += 1;
|
||||
markVectorStarted();
|
||||
await vectorReleased;
|
||||
return [0.1];
|
||||
}
|
||||
async computeSourceEmbeddings(values: string[]) {
|
||||
return values.map(() => [0.1]);
|
||||
}
|
||||
}
|
||||
|
||||
const writer = await connect(tmpDir.name);
|
||||
const ftsTable = await writer.createTable("stale_fts", [
|
||||
{ text: "hello", vector: [0.0] },
|
||||
]);
|
||||
await ftsTable.createIndex("text", { config: Index.fts() });
|
||||
|
||||
const reader = await connect(tmpDir.name, {
|
||||
readConsistencyInterval: 0,
|
||||
});
|
||||
const tracked = await reader.openTable("stale_fts");
|
||||
type Snapshot = {
|
||||
schema: () => Promise<Buffer>;
|
||||
};
|
||||
type NativeWithSnapshot = {
|
||||
querySnapshot: () => Promise<Snapshot>;
|
||||
};
|
||||
const native = (tracked as unknown as { inner: NativeWithSnapshot })
|
||||
.inner;
|
||||
const querySnapshot = native.querySnapshot.bind(native);
|
||||
let snapshotCalls = 0;
|
||||
let markStaleSchemaStarted!: () => void;
|
||||
const staleSchemaStarted = new Promise<void>((resolve) => {
|
||||
markStaleSchemaStarted = resolve;
|
||||
});
|
||||
let releaseStaleSchema!: () => void;
|
||||
const staleSchemaReleased = new Promise<void>((resolve) => {
|
||||
releaseStaleSchema = resolve;
|
||||
});
|
||||
native.querySnapshot = async () => {
|
||||
const snapshot = await querySnapshot();
|
||||
snapshotCalls += 1;
|
||||
if (snapshotCalls === 1) {
|
||||
const schema = snapshot.schema.bind(snapshot);
|
||||
snapshot.schema = async () => {
|
||||
markStaleSchemaStarted();
|
||||
await staleSchemaReleased;
|
||||
return await schema();
|
||||
};
|
||||
}
|
||||
return snapshot;
|
||||
};
|
||||
|
||||
const query = tracked.search("hello");
|
||||
const staleFtsExecution = query.toArray();
|
||||
await staleSchemaStarted;
|
||||
|
||||
const embedding = new RaceEmbedding();
|
||||
const vectorSchema = LanceSchema({
|
||||
text: embedding.sourceField(new arrow.Utf8()),
|
||||
vector: embedding.vectorField(),
|
||||
});
|
||||
await writer.createTable("stale_fts", [{ text: "hello" }], {
|
||||
mode: "overwrite",
|
||||
schema: vectorSchema,
|
||||
});
|
||||
|
||||
const vectorExecution = query.toArray();
|
||||
await vectorStarted;
|
||||
releaseStaleSchema();
|
||||
await staleFtsExecution;
|
||||
releaseVector();
|
||||
await vectorExecution;
|
||||
|
||||
await query.toArray();
|
||||
expect(vectorCalls).toBe(1);
|
||||
});
|
||||
|
||||
test("tokenizes FTS queries by column or index name", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const data = [
|
||||
@@ -3474,30 +2916,6 @@ describe("column name options", () => {
|
||||
expect(results[1].query_index).toBe(1);
|
||||
});
|
||||
|
||||
test("observes promised additional vectors while the query is pending", async () => {
|
||||
const initialVector = new Promise<number[]>(() => undefined);
|
||||
const query = table.query().nearestTo(initialVector);
|
||||
const unhandled: unknown[] = [];
|
||||
const onUnhandled = (reason: unknown) => unhandled.push(reason);
|
||||
process.on("unhandledRejection", onUnhandled);
|
||||
|
||||
try {
|
||||
query.addQueryVector(Promise.reject(new Error("extra vector failed")));
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(unhandled).toEqual([]);
|
||||
|
||||
const rejectedQuery = table
|
||||
.query()
|
||||
.nearestTo([0.1, 0.2])
|
||||
.addQueryVector(Promise.reject(new Error("consumed vector failed")));
|
||||
await expect(rejectedQuery.toArray()).rejects.toThrow(
|
||||
"consumed vector failed",
|
||||
);
|
||||
} finally {
|
||||
process.off("unhandledRejection", onUnhandled);
|
||||
}
|
||||
});
|
||||
|
||||
test("index and search multivectors", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const data = [];
|
||||
@@ -3561,27 +2979,6 @@ describe("when creating an empty table", () => {
|
||||
expect((actualSchema.fields[1].type as Float64).precision).toBe(2);
|
||||
});
|
||||
|
||||
it("can add and query JSON data", async () => {
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int32(), true),
|
||||
new Field(
|
||||
"meta",
|
||||
new Utf8(),
|
||||
true,
|
||||
new Map([["ARROW:extension:name", "arrow.json"]]),
|
||||
),
|
||||
]);
|
||||
const table = await con.createEmptyTable("json", schema);
|
||||
const meta = JSON.stringify({ x: 1 });
|
||||
|
||||
await table.add([{ id: 1, meta }]);
|
||||
|
||||
const rows = await table.query().toArray();
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].id).toBe(1);
|
||||
expect(rows[0].meta).toBe(meta);
|
||||
});
|
||||
|
||||
it("can create an empty table from schema that specifies field types by name", async () => {
|
||||
const schemaLike = {
|
||||
fields: [
|
||||
|
||||
@@ -170,7 +170,7 @@ test("basic table examples", async () => {
|
||||
// --8<-- [end:create_index]
|
||||
|
||||
// --8<-- [start:delete_rows]
|
||||
await tbl.delete("item = 'fizz'");
|
||||
await tbl.delete('item = "fizz"');
|
||||
// --8<-- [end:delete_rows]
|
||||
|
||||
// --8<-- [start:drop_table]
|
||||
|
||||
@@ -4,15 +4,7 @@
|
||||
import { Field, Schema } from "../arrow";
|
||||
import { sanitizeType } from "../sanitize";
|
||||
import { EmbeddingFunction } from "./embedding_function";
|
||||
import {
|
||||
EmbeddingFunctionConfig,
|
||||
EmbeddingFunctionRegistry,
|
||||
getRegistry as getGlobalRegistry,
|
||||
registerBuiltIn,
|
||||
} from "./registry";
|
||||
|
||||
type OpenAIModule = typeof import("./openai");
|
||||
type TransformersModule = typeof import("./transformers");
|
||||
import { EmbeddingFunctionConfig, getRegistry } from "./registry";
|
||||
|
||||
export {
|
||||
FieldOptions,
|
||||
@@ -22,39 +14,7 @@ export {
|
||||
EmbeddingFunctionConstructor,
|
||||
} from "./embedding_function";
|
||||
|
||||
export {
|
||||
EmbeddingFunctionRegistry,
|
||||
parseEmbeddingMetadata,
|
||||
register,
|
||||
} from "./registry";
|
||||
export type {
|
||||
CreateReturnType,
|
||||
EmbeddingFunctionConfig,
|
||||
EmbeddingFunctionCreate,
|
||||
EmbeddingMetadataEntry,
|
||||
ResolvedEmbeddingFunctionConfig,
|
||||
} from "./registry";
|
||||
|
||||
function initializeBuiltInProviders() {
|
||||
const { OpenAIEmbeddingFunction } = require("./openai") as OpenAIModule;
|
||||
const { TransformersEmbeddingFunction } =
|
||||
require("./transformers") as TransformersModule;
|
||||
|
||||
registerBuiltIn("openai", OpenAIEmbeddingFunction);
|
||||
registerBuiltIn("huggingface", TransformersEmbeddingFunction);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the global embedding function registry.
|
||||
*
|
||||
* LanceDB built-in providers are initialized when this public API is first
|
||||
* used, so importing the root package does not change automatic search
|
||||
* selection for tables without embedding metadata.
|
||||
*/
|
||||
export function getRegistry(): EmbeddingFunctionRegistry {
|
||||
initializeBuiltInProviders();
|
||||
return getGlobalRegistry();
|
||||
}
|
||||
export * from "./registry";
|
||||
|
||||
/**
|
||||
* Create a schema with embedding functions.
|
||||
|
||||
@@ -5,13 +5,14 @@ import type OpenAI from "openai";
|
||||
import type { EmbeddingCreateParams } from "openai/resources/index";
|
||||
import { Float, Float32 } from "../arrow";
|
||||
import { EmbeddingFunction } from "./embedding_function";
|
||||
import { registerBuiltIn } from "./registry";
|
||||
import { register } from "./registry";
|
||||
|
||||
export type OpenAIOptions = {
|
||||
apiKey: string;
|
||||
model: EmbeddingCreateParams["model"];
|
||||
};
|
||||
|
||||
@register("openai")
|
||||
export class OpenAIEmbeddingFunction extends EmbeddingFunction<
|
||||
string,
|
||||
Partial<OpenAIOptions>
|
||||
@@ -99,5 +100,3 @@ export class OpenAIEmbeddingFunction extends EmbeddingFunction<
|
||||
return response.data[0].embedding;
|
||||
}
|
||||
}
|
||||
|
||||
registerBuiltIn("openai", OpenAIEmbeddingFunction);
|
||||
|
||||
@@ -7,10 +7,6 @@ import {
|
||||
} from "./embedding_function";
|
||||
import "reflect-metadata";
|
||||
|
||||
const builtInFunctionsKey = Symbol.for(
|
||||
"@lancedb/lancedb::embedding-built-in-functions::v1",
|
||||
);
|
||||
|
||||
export type CreateReturnType<T> = T extends { init: () => Promise<void> }
|
||||
? Promise<T>
|
||||
: T;
|
||||
@@ -63,15 +59,6 @@ export class EmbeddingFunctionRegistry {
|
||||
};
|
||||
}
|
||||
|
||||
/** @ignore */
|
||||
setBuiltIn<
|
||||
T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
|
||||
>(name: string, ctor: T): T {
|
||||
this.#functions.set(name, ctor);
|
||||
Reflect.defineMetadata("lancedb::embedding::name", name, ctor);
|
||||
return ctor;
|
||||
}
|
||||
|
||||
get<T extends EmbeddingFunction<unknown>>(
|
||||
name: string,
|
||||
): EmbeddingFunctionCreate<T> | undefined;
|
||||
@@ -109,7 +96,6 @@ export class EmbeddingFunctionRegistry {
|
||||
*/
|
||||
reset(this: EmbeddingFunctionRegistry) {
|
||||
this.#functions.clear();
|
||||
getBuiltInFunctions(this).clear();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -197,56 +183,12 @@ export class EmbeddingFunctionRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
function getBuiltInFunctions(registry: EmbeddingFunctionRegistry): Set<string> {
|
||||
const registryWithBuiltIns = registry as EmbeddingFunctionRegistry & {
|
||||
[key: symbol]: Set<string> | undefined;
|
||||
};
|
||||
let builtInFunctions = registryWithBuiltIns[builtInFunctionsKey];
|
||||
if (builtInFunctions === undefined) {
|
||||
builtInFunctions = new Set<string>();
|
||||
registryWithBuiltIns[builtInFunctionsKey] = builtInFunctions;
|
||||
}
|
||||
return builtInFunctions;
|
||||
}
|
||||
|
||||
// Server bundlers can load the side-effect embedding entry points and the public
|
||||
// embedding API from separate module graphs. Keep their registry shared.
|
||||
const registryKey = Symbol.for(
|
||||
"@lancedb/lancedb::embedding-function-registry::v1",
|
||||
);
|
||||
const registryGlobal = globalThis as typeof globalThis & {
|
||||
[key: symbol]: EmbeddingFunctionRegistry | undefined;
|
||||
};
|
||||
|
||||
function getGlobalRegistry(): EmbeddingFunctionRegistry {
|
||||
const existingRegistry = registryGlobal[registryKey];
|
||||
if (existingRegistry !== undefined) {
|
||||
return existingRegistry;
|
||||
}
|
||||
const registry = new EmbeddingFunctionRegistry();
|
||||
registryGlobal[registryKey] = registry;
|
||||
return registry;
|
||||
}
|
||||
|
||||
const _REGISTRY = getGlobalRegistry();
|
||||
const _REGISTRY = new EmbeddingFunctionRegistry();
|
||||
|
||||
export function register(name?: string) {
|
||||
return _REGISTRY.register(name);
|
||||
}
|
||||
|
||||
/** @ignore */
|
||||
export function registerBuiltIn<
|
||||
T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
|
||||
>(name: string, ctor: T): T {
|
||||
const builtInFunctions = getBuiltInFunctions(_REGISTRY);
|
||||
if (builtInFunctions.has(name)) {
|
||||
return _REGISTRY.setBuiltIn(name, ctor);
|
||||
}
|
||||
_REGISTRY.register(name)(ctor);
|
||||
builtInFunctions.add(name);
|
||||
return ctor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility function to get the global instance of the registry
|
||||
* @returns `EmbeddingFunctionRegistry` The global instance of the registry
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import { Float, Float32 } from "../arrow";
|
||||
import { EmbeddingFunction } from "./embedding_function";
|
||||
import { registerBuiltIn } from "./registry";
|
||||
import { register } from "./registry";
|
||||
|
||||
export type XenovaTransformerOptions = {
|
||||
/** The wasm compatible model to use */
|
||||
@@ -31,6 +31,7 @@ export type XenovaTransformerOptions = {
|
||||
};
|
||||
};
|
||||
|
||||
@register("huggingface")
|
||||
export class TransformersEmbeddingFunction extends EmbeddingFunction<
|
||||
string,
|
||||
Partial<XenovaTransformerOptions>
|
||||
@@ -157,8 +158,6 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
|
||||
}
|
||||
}
|
||||
|
||||
registerBuiltIn("huggingface", TransformersEmbeddingFunction);
|
||||
|
||||
const tensorDiv = (
|
||||
src: import("@huggingface/transformers").Tensor,
|
||||
divBy: number,
|
||||
|
||||
@@ -103,7 +103,6 @@ export {
|
||||
} from "./native.js";
|
||||
|
||||
export {
|
||||
AutoQuery,
|
||||
ExecutableQuery,
|
||||
Query,
|
||||
QueryBase,
|
||||
|
||||
+106
-205
@@ -100,29 +100,6 @@ export interface FullTextSearchOptions {
|
||||
columns?: string | string[];
|
||||
}
|
||||
|
||||
function nearestToNative(
|
||||
inner: NativeQuery,
|
||||
vector: Awaited<IntoVector>,
|
||||
): NativeVectorQuery {
|
||||
const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector);
|
||||
if (raw) {
|
||||
return inner.nearestToRaw(raw.data, raw.dtype);
|
||||
}
|
||||
return inner.nearestTo(Float32Array.from(vector as number[]));
|
||||
}
|
||||
|
||||
function addQueryVectorToNative(
|
||||
inner: NativeVectorQuery,
|
||||
vector: Awaited<IntoVector>,
|
||||
) {
|
||||
const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector);
|
||||
if (raw) {
|
||||
inner.addQueryVectorRaw(raw.data, raw.dtype);
|
||||
} else {
|
||||
inner.addQueryVector(Float32Array.from(vector as number[]));
|
||||
}
|
||||
}
|
||||
|
||||
/** Common methods supported by all query types
|
||||
*
|
||||
* @see {@link Query}
|
||||
@@ -134,15 +111,13 @@ export class QueryBase<
|
||||
NativeQueryType extends NativeQuery | NativeVectorQuery | NativeTakeQuery,
|
||||
> implements AsyncIterable<RecordBatch>
|
||||
{
|
||||
protected inner!: NativeQueryType | Promise<NativeQueryType>;
|
||||
|
||||
/**
|
||||
* @hidden
|
||||
*/
|
||||
protected constructor(inner?: NativeQueryType | Promise<NativeQueryType>) {
|
||||
if (inner !== undefined) {
|
||||
this.inner = inner;
|
||||
}
|
||||
protected constructor(
|
||||
protected inner: NativeQueryType | Promise<NativeQueryType>,
|
||||
) {
|
||||
// intentionally empty
|
||||
}
|
||||
|
||||
// call a function on the inner (either a promise or the actual object)
|
||||
@@ -160,15 +135,6 @@ export class QueryBase<
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the native query used by the next terminal operation.
|
||||
*
|
||||
* @hidden
|
||||
*/
|
||||
protected async getInner(): Promise<NativeQueryType> {
|
||||
return this.inner;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return only the specified columns.
|
||||
*
|
||||
@@ -241,11 +207,16 @@ export class QueryBase<
|
||||
/**
|
||||
* @hidden
|
||||
*/
|
||||
protected async nativeExecute(
|
||||
protected nativeExecute(
|
||||
options?: Partial<QueryExecutionOptions>,
|
||||
): Promise<NativeBatchIterator> {
|
||||
const inner = await this.getInner();
|
||||
return inner.execute(options?.maxBatchLength, options?.timeoutMs);
|
||||
if (this.inner instanceof Promise) {
|
||||
return this.inner.then((inner) =>
|
||||
inner.execute(options?.maxBatchLength, options?.timeoutMs),
|
||||
);
|
||||
} else {
|
||||
return this.inner.execute(options?.maxBatchLength, options?.timeoutMs);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -274,7 +245,12 @@ export class QueryBase<
|
||||
/** Collect the results as an Arrow @see {@link ArrowTable}. */
|
||||
async toArrow(options?: Partial<QueryExecutionOptions>): Promise<ArrowTable> {
|
||||
const batches = [];
|
||||
const inner = await this.getInner();
|
||||
let inner;
|
||||
if (this.inner instanceof Promise) {
|
||||
inner = await this.inner;
|
||||
} else {
|
||||
inner = this.inner;
|
||||
}
|
||||
for await (const batch of new RecordBatchIterable(inner, options)) {
|
||||
batches.push(batch);
|
||||
}
|
||||
@@ -303,8 +279,11 @@ export class QueryBase<
|
||||
* @returns A Promise that resolves to a string containing the query execution plan explanation.
|
||||
*/
|
||||
async explainPlan(verbose = false): Promise<string> {
|
||||
const inner = await this.getInner();
|
||||
return inner.explainPlan(verbose);
|
||||
if (this.inner instanceof Promise) {
|
||||
return this.inner.then((inner) => inner.explainPlan(verbose));
|
||||
} else {
|
||||
return this.inner.explainPlan(verbose);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -342,8 +321,13 @@ export class QueryBase<
|
||||
distributedMetrics?: AnalyzePlanDistributedMetrics,
|
||||
): Promise<string> {
|
||||
const distributedMetricsMode = distributedMetrics ?? "aggregate";
|
||||
const inner = await this.getInner();
|
||||
return inner.analyzePlan(distributedMetricsMode);
|
||||
if (this.inner instanceof Promise) {
|
||||
return this.inner.then((inner) =>
|
||||
inner.analyzePlan(distributedMetricsMode),
|
||||
);
|
||||
} else {
|
||||
return this.inner.analyzePlan(distributedMetricsMode);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -355,8 +339,12 @@ export class QueryBase<
|
||||
* @returns An Arrow Schema describing the output columns.
|
||||
*/
|
||||
async outputSchema(): Promise<import("./arrow").Schema> {
|
||||
const inner = await this.getInner();
|
||||
const schemaBuffer = await inner.outputSchema();
|
||||
let schemaBuffer: Buffer;
|
||||
if (this.inner instanceof Promise) {
|
||||
schemaBuffer = await this.inner.then((inner) => inner.outputSchema());
|
||||
} else {
|
||||
schemaBuffer = await this.inner.outputSchema();
|
||||
}
|
||||
const schema = tableFromIPC(schemaBuffer).schema;
|
||||
return schema;
|
||||
}
|
||||
@@ -368,7 +356,7 @@ export class StandardQueryBase<
|
||||
extends QueryBase<NativeQueryType>
|
||||
implements ExecutableQuery
|
||||
{
|
||||
constructor(inner?: NativeQueryType | Promise<NativeQueryType>) {
|
||||
constructor(inner: NativeQueryType | Promise<NativeQueryType>) {
|
||||
super(inner);
|
||||
}
|
||||
|
||||
@@ -522,13 +510,6 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
super(inner);
|
||||
}
|
||||
|
||||
/**
|
||||
* @hidden
|
||||
*/
|
||||
protected doVectorCall(fn: (inner: NativeVectorQuery) => void) {
|
||||
super.doCall(fn);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the number of partitions to search (probe)
|
||||
*
|
||||
@@ -556,7 +537,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* the minimum and maximum to the same value.
|
||||
*/
|
||||
nprobes(nprobes: number): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.nprobes(nprobes));
|
||||
super.doCall((inner) => inner.nprobes(nprobes));
|
||||
|
||||
return this;
|
||||
}
|
||||
@@ -570,7 +551,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* but will also increase latency.
|
||||
*/
|
||||
minimumNprobes(minimumNprobes: number): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.minimumNprobes(minimumNprobes));
|
||||
super.doCall((inner) => inner.minimumNprobes(minimumNprobes));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -584,7 +565,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* potential false negatives.
|
||||
*/
|
||||
maximumNprobes(maximumNprobes: number): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.maximumNprobes(maximumNprobes));
|
||||
super.doCall((inner) => inner.maximumNprobes(maximumNprobes));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -597,7 +578,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* `undefined` means no lower or upper bound.
|
||||
*/
|
||||
distanceRange(lowerBound?: number, upperBound?: number): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.distanceRange(lowerBound, upperBound));
|
||||
super.doCall((inner) => inner.distanceRange(lowerBound, upperBound));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -611,7 +592,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* also increase the latency of your query. The default value is 1.5*limit.
|
||||
*/
|
||||
ef(ef: number): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.ef(ef));
|
||||
super.doCall((inner) => inner.ef(ef));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -625,7 +606,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* whose data type is a fixed-size-list of floats.
|
||||
*/
|
||||
column(column: string): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.column(column));
|
||||
super.doCall((inner) => inner.column(column));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -646,7 +627,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
distanceType(
|
||||
distanceType: Required<IvfPqOptions>["distanceType"],
|
||||
): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.distanceType(distanceType));
|
||||
super.doCall((inner) => inner.distanceType(distanceType));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -680,7 +661,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* distance between the query vector and the actual uncompressed vector.
|
||||
*/
|
||||
refineFactor(refineFactor: number): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.refineFactor(refineFactor));
|
||||
super.doCall((inner) => inner.refineFactor(refineFactor));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -705,7 +686,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* factor can often help restore some of the results lost by post filtering.
|
||||
*/
|
||||
postfilter(): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.postfilter());
|
||||
super.doCall((inner) => inner.postfilter());
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -719,7 +700,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* calculate your recall to select an appropriate value for nprobes.
|
||||
*/
|
||||
bypassVectorIndex(): VectorQuery {
|
||||
this.doVectorCall((inner) => inner.bypassVectorIndex());
|
||||
super.doCall((inner) => inner.bypassVectorIndex());
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -727,39 +708,43 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* Add a query vector to the search
|
||||
*
|
||||
* This method can be called multiple times to add multiple query vectors
|
||||
* to the search. A column called `query_index` will be added to indicate the index
|
||||
* of the query vector that produced the result. Flat searches share one table scan
|
||||
* across the query vectors, avoiding the scan and memory amplification of running
|
||||
* multiple queries concurrently. Indexed searches may still perform per-vector
|
||||
* index work.
|
||||
* to the search. If multiple query vectors are added, then they will be searched
|
||||
* in parallel, and the results will be concatenated. A column called `query_index`
|
||||
* will be added to indicate the index of the query vector that produced the result.
|
||||
*
|
||||
* Performance wise, this is equivalent to running multiple queries concurrently.
|
||||
*/
|
||||
addQueryVector(vector: IntoVector): VectorQuery {
|
||||
if (vector instanceof Promise) {
|
||||
// Observe the promise as soon as it is accepted. The existing native
|
||||
// query may still be pending, and delaying observation until it resolves
|
||||
// can otherwise surface a fast rejection as unhandled.
|
||||
const settledVector = vector.then(
|
||||
(value) => ({ status: "fulfilled" as const, value }),
|
||||
(reason) => ({ status: "rejected" as const, reason }),
|
||||
);
|
||||
const res = (async () => {
|
||||
const inner = await this.getInner();
|
||||
const outcome = await settledVector;
|
||||
if (outcome.status === "rejected") {
|
||||
throw outcome.reason;
|
||||
try {
|
||||
const v = await vector;
|
||||
// biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
|
||||
const value: any = this.addQueryVector(v);
|
||||
const inner = value.inner as
|
||||
| NativeVectorQuery
|
||||
| Promise<NativeVectorQuery>;
|
||||
return inner;
|
||||
} catch (e) {
|
||||
return Promise.reject(e);
|
||||
}
|
||||
addQueryVectorToNative(inner, outcome.value);
|
||||
return inner;
|
||||
})();
|
||||
return new VectorQuery(res);
|
||||
} else {
|
||||
this.doVectorCall((inner) => addQueryVectorToNative(inner, vector));
|
||||
super.doCall((inner) => {
|
||||
const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector);
|
||||
if (raw) {
|
||||
inner.addQueryVectorRaw(raw.data, raw.dtype);
|
||||
} else {
|
||||
inner.addQueryVector(Float32Array.from(vector as number[]));
|
||||
}
|
||||
});
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
rerank(reranker: Reranker): VectorQuery {
|
||||
this.doVectorCall((inner) =>
|
||||
super.doCall((inner) =>
|
||||
inner.rerank(async (args) => {
|
||||
const vecResults = await fromBufferToRecordBatch(args.vecResults);
|
||||
const ftsResults = await fromBufferToRecordBatch(args.ftsResults);
|
||||
@@ -778,71 +763,6 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a string query whose vector/FTS routing is resolved against the active
|
||||
* table schema when the query executes.
|
||||
*
|
||||
* @hidden
|
||||
*/
|
||||
export function createAutoQuery(
|
||||
table: NativeTable,
|
||||
query: string,
|
||||
columns: string[] | null,
|
||||
getVector: (metadata: string) => Promise<Awaited<IntoVector>>,
|
||||
): AutoQuery {
|
||||
type RouteSnapshot = {
|
||||
table: NativeTable;
|
||||
embeddingMetadata: string | undefined;
|
||||
};
|
||||
type CachedPreparation = {
|
||||
metadata: string;
|
||||
vector: Promise<Awaited<IntoVector>>;
|
||||
};
|
||||
|
||||
let cachedPreparation: CachedPreparation | undefined;
|
||||
|
||||
const snapshotRoute = async (): Promise<RouteSnapshot> => {
|
||||
const snapshot = await table.querySnapshot();
|
||||
const schema = tableFromIPC(await snapshot.schema()).schema;
|
||||
return {
|
||||
table: snapshot,
|
||||
embeddingMetadata: schema.metadata.get("embedding_functions"),
|
||||
};
|
||||
};
|
||||
|
||||
const createInner = async (): Promise<NativeQuery | NativeVectorQuery> => {
|
||||
const route = await snapshotRoute();
|
||||
if (route.embeddingMetadata === undefined) {
|
||||
const inner = route.table.query();
|
||||
inner.fullTextSearch({ query, columns });
|
||||
return inner;
|
||||
}
|
||||
|
||||
const metadata = route.embeddingMetadata;
|
||||
if (cachedPreparation?.metadata !== metadata) {
|
||||
cachedPreparation = {
|
||||
metadata,
|
||||
vector: Promise.resolve().then(() => getVector(metadata)),
|
||||
};
|
||||
}
|
||||
|
||||
const preparation = cachedPreparation;
|
||||
let vector: Awaited<IntoVector>;
|
||||
try {
|
||||
vector = await preparation.vector;
|
||||
} catch (error) {
|
||||
if (cachedPreparation === preparation) {
|
||||
cachedPreparation = undefined;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
return nearestToNative(route.table.query(), vector);
|
||||
};
|
||||
|
||||
return new AutoQuery(createInner);
|
||||
}
|
||||
|
||||
/**
|
||||
* A query that returns a subset of the rows in the table.
|
||||
*
|
||||
@@ -868,51 +788,6 @@ export class TakeQuery extends QueryBase<NativeTakeQuery> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A builder for automatic string searches.
|
||||
*
|
||||
* Automatic search determines whether to use full-text or vector search from
|
||||
* the table revision selected for each execution. This builder exposes the
|
||||
* common operations supported by both query families.
|
||||
*
|
||||
* @hideconstructor
|
||||
*/
|
||||
export class AutoQuery extends StandardQueryBase<
|
||||
NativeQuery | NativeVectorQuery
|
||||
> {
|
||||
private readonly calls: Array<
|
||||
(inner: NativeQuery | NativeVectorQuery) => void
|
||||
> = [];
|
||||
|
||||
/** @hidden */
|
||||
constructor(
|
||||
private readonly createInner: () => Promise<
|
||||
NativeQuery | NativeVectorQuery
|
||||
>,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
/** @hidden */
|
||||
protected override doCall(
|
||||
fn: (inner: NativeQuery | NativeVectorQuery) => void,
|
||||
) {
|
||||
this.calls.push(fn);
|
||||
}
|
||||
|
||||
/** @hidden */
|
||||
protected override async getInner(): Promise<
|
||||
NativeQuery | NativeVectorQuery
|
||||
> {
|
||||
const calls = [...this.calls];
|
||||
const inner = await this.createInner();
|
||||
for (const call of calls) {
|
||||
call(inner);
|
||||
}
|
||||
return inner;
|
||||
}
|
||||
}
|
||||
|
||||
/** A builder for LanceDB queries.
|
||||
*
|
||||
* @see {@link Table#query}, {@link Table#search}
|
||||
@@ -965,19 +840,45 @@ export class Query extends StandardQueryBase<NativeQuery> {
|
||||
* a default `limit` of 10 will be used. @see {@link Query#limit}
|
||||
*/
|
||||
nearestTo(vector: IntoVector): VectorQuery {
|
||||
const inner = this.inner;
|
||||
if (inner instanceof Promise) {
|
||||
const nativeQuery = inner.then(async (resolvedInner) =>
|
||||
nearestToNative(resolvedInner, await vector),
|
||||
);
|
||||
const callNearestTo = (
|
||||
inner: NativeQuery,
|
||||
resolved: Float32Array | Float64Array | Uint8Array | number[],
|
||||
): NativeVectorQuery => {
|
||||
const raw = Array.isArray(resolved)
|
||||
? null
|
||||
: extractVectorBuffer(resolved);
|
||||
if (raw) {
|
||||
return inner.nearestToRaw(raw.data, raw.dtype);
|
||||
}
|
||||
return inner.nearestTo(Float32Array.from(resolved as number[]));
|
||||
};
|
||||
|
||||
if (this.inner instanceof Promise) {
|
||||
const nativeQuery = this.inner.then(async (inner) => {
|
||||
const resolved = vector instanceof Promise ? await vector : vector;
|
||||
return callNearestTo(inner, resolved);
|
||||
});
|
||||
return new VectorQuery(nativeQuery);
|
||||
}
|
||||
if (vector instanceof Promise) {
|
||||
return new VectorQuery(
|
||||
vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)),
|
||||
);
|
||||
const res = (async () => {
|
||||
try {
|
||||
const v = await vector;
|
||||
// biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
|
||||
const value: any = this.nearestTo(v);
|
||||
const inner = value.inner as
|
||||
| NativeVectorQuery
|
||||
| Promise<NativeVectorQuery>;
|
||||
return inner;
|
||||
} catch (e) {
|
||||
return Promise.reject(e);
|
||||
}
|
||||
})();
|
||||
return new VectorQuery(res);
|
||||
} else {
|
||||
const vectorQuery = callNearestTo(this.inner, vector);
|
||||
return new VectorQuery(vectorQuery);
|
||||
}
|
||||
return new VectorQuery(nearestToNative(inner, vector));
|
||||
}
|
||||
|
||||
nearestToText(query: string | FullTextQuery, columns?: string[]): Query {
|
||||
|
||||
@@ -406,11 +406,10 @@ function matchingFields(fields: Field[], tree: FieldTree): Field[] {
|
||||
field.name,
|
||||
new Struct(matchingFields(struct.children, value)),
|
||||
field.nullable,
|
||||
field.metadata,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
matches.push(field);
|
||||
matches.push(new Field(field.name, value as DataType, field.nullable));
|
||||
}
|
||||
}
|
||||
return matches;
|
||||
|
||||
+15
-45
@@ -43,12 +43,10 @@ import {
|
||||
Table as _NativeTable,
|
||||
} from "./native";
|
||||
import {
|
||||
AutoQuery,
|
||||
FullTextQuery,
|
||||
Query,
|
||||
TakeQuery,
|
||||
VectorQuery,
|
||||
createAutoQuery,
|
||||
instanceOfFullTextQuery,
|
||||
} from "./query";
|
||||
import { sanitizeType } from "./sanitize";
|
||||
@@ -525,7 +523,7 @@ export abstract class Table {
|
||||
query: string | IntoVector | MultiVector | FullTextQuery,
|
||||
queryType?: string,
|
||||
ftsColumns?: string | string[],
|
||||
): VectorQuery | Query | AutoQuery;
|
||||
): VectorQuery | Query;
|
||||
/**
|
||||
* Search the table with a given query vector.
|
||||
*
|
||||
@@ -630,18 +628,6 @@ export abstract class Table {
|
||||
|
||||
/**
|
||||
* Update per-field (column) metadata.
|
||||
*
|
||||
* The following keys are treated specially, by convention, and should be
|
||||
* used when appropriate:
|
||||
*
|
||||
* - `lancedb:description`: for a human-readable description of a field.
|
||||
* - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
|
||||
* names the tag category; e.g. `lancedb:tag:model: "clip"`.
|
||||
* - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
|
||||
* `feature_v2` might be in the same logical column.
|
||||
* - `lancedb:status`: for status options (`production`, `candidate`,
|
||||
* `deprecated`, `archived`) to designate the current life cycle state of
|
||||
* this column.
|
||||
* @param {FieldMetadataUpdate[]} updates One or more per-field updates. Each
|
||||
* update's metadata is merged into the field's existing metadata by default;
|
||||
* a value of `null` deletes that key, and `replace: true` swaps the whole map.
|
||||
@@ -989,11 +975,10 @@ export class LocalTable extends Table {
|
||||
return this.inner.display();
|
||||
}
|
||||
|
||||
private async getEmbeddingFunctions(
|
||||
inner: _NativeTable = this.inner,
|
||||
): Promise<Map<string, EmbeddingFunctionConfig>> {
|
||||
const schemaBuf = await inner.schema();
|
||||
const schema = tableFromIPC(schemaBuf).schema;
|
||||
private async getEmbeddingFunctions(): Promise<
|
||||
Map<string, EmbeddingFunctionConfig>
|
||||
> {
|
||||
const schema = await this.schema();
|
||||
const registry = getRegistry();
|
||||
return registry.parseFunctions(schema.metadata);
|
||||
}
|
||||
@@ -1175,7 +1160,7 @@ export class LocalTable extends Table {
|
||||
query: string | IntoVector | MultiVector | FullTextQuery,
|
||||
queryType: string = "auto",
|
||||
ftsColumns?: string | string[],
|
||||
): VectorQuery | Query | AutoQuery {
|
||||
): VectorQuery | Query {
|
||||
if (typeof query !== "string" && !instanceOfFullTextQuery(query)) {
|
||||
if (queryType === "fts") {
|
||||
throw new Error("Cannot perform full text search on a vector query");
|
||||
@@ -1190,28 +1175,14 @@ export class LocalTable extends Table {
|
||||
});
|
||||
}
|
||||
|
||||
if (queryType === "auto") {
|
||||
if (instanceOfFullTextQuery(query)) {
|
||||
return this.query().fullTextSearch(query, {
|
||||
columns: ftsColumns,
|
||||
});
|
||||
}
|
||||
|
||||
const columns =
|
||||
typeof ftsColumns === "string" ? [ftsColumns] : (ftsColumns ?? null);
|
||||
return createAutoQuery(this.inner, query, columns, async (metadata) => {
|
||||
const functions = await getRegistry().parseFunctions(
|
||||
new Map([["embedding_functions", metadata]]),
|
||||
);
|
||||
// TODO: Support multiple embedding functions
|
||||
const embeddingFunc: EmbeddingFunctionConfig | undefined = functions
|
||||
.values()
|
||||
.next().value;
|
||||
// The route only calls this callback when embedding metadata exists.
|
||||
// parseFunctions either yields a provider or reports malformed metadata.
|
||||
if (!embeddingFunc)
|
||||
throw new Error("Invalid embedding function metadata");
|
||||
return await embeddingFunc.function.computeQueryEmbeddings(query);
|
||||
// The query type is auto or vector
|
||||
// fall back to full text search if no embedding functions are defined and the query is a string
|
||||
if (
|
||||
queryType === "auto" &&
|
||||
(getRegistry().length() === 0 || instanceOfFullTextQuery(query))
|
||||
) {
|
||||
return this.query().fullTextSearch(query, {
|
||||
columns: ftsColumns,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1567,8 +1538,7 @@ export interface FieldMetadataUpdate {
|
||||
path: string;
|
||||
/**
|
||||
* Metadata key/value pairs. Merged into the field's existing metadata by
|
||||
* default; a value of `null` deletes that key. See
|
||||
* {@link Table.updateFieldMetadata} for the conventional `lancedb:*` keys.
|
||||
* default; a value of `null` deletes that key.
|
||||
*/
|
||||
metadata: Record<string, string | null>;
|
||||
/** If true, replace the field's entire metadata map instead of merging. */
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-darwin-arm64",
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.10",
|
||||
"os": ["darwin"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.darwin-arm64.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.10",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-musl",
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.10",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-gnu",
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.10",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-musl",
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.10",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.10",
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-x64-msvc",
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.10",
|
||||
"os": ["win32"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.win32-x64-msvc.node",
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb",
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.9",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@lancedb/lancedb",
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.9",
|
||||
"cpu": [
|
||||
"x64",
|
||||
"arm64"
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@
|
||||
"ann"
|
||||
],
|
||||
"private": false,
|
||||
"version": "0.38.0-beta.11",
|
||||
"version": "0.38.0-beta.10",
|
||||
"main": "dist/index.js",
|
||||
"exports": {
|
||||
".": "./dist/index.js",
|
||||
|
||||
@@ -278,13 +278,6 @@ impl Table {
|
||||
Ok(Query::new(self.inner_ref()?.query()))
|
||||
}
|
||||
|
||||
/// Return a read-only table handle pinned to the current query revision.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn query_snapshot(&self) -> napi::Result<Self> {
|
||||
let snapshot = self.inner_ref()?.query_snapshot().await.default_error()?;
|
||||
Ok(Self::new(snapshot))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub fn take_offsets(&self, offsets: Vec<i64>) -> napi::Result<TakeQuery> {
|
||||
Ok(TakeQuery::new(
|
||||
@@ -561,12 +554,6 @@ impl Table {
|
||||
.default_error()
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn checkout_current(&self) -> napi::Result<Self> {
|
||||
let table = self.inner_ref()?.checkout_current().await.default_error()?;
|
||||
Ok(Self::new(table))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn checkout(&self, version: i64) -> napi::Result<()> {
|
||||
self.inner_ref()?
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb-python"
|
||||
version = "0.38.0-beta.11"
|
||||
version = "0.38.0-beta.10"
|
||||
publish = false
|
||||
edition.workspace = true
|
||||
description = "Python bindings for LanceDB"
|
||||
|
||||
@@ -101,12 +101,9 @@ azure = ["adlfs>=2024.2.0"]
|
||||
[tool.maturin]
|
||||
python-source = "python"
|
||||
module-name = "lancedb._lancedb"
|
||||
# uv installs the project as an editable package before `uv run`, so keep that
|
||||
# bootstrap build consistent with `maturin develop`.
|
||||
editable-profile = "dev"
|
||||
|
||||
[build-system]
|
||||
requires = ["maturin>=1.10"]
|
||||
requires = ["maturin>=1.9.4"]
|
||||
build-backend = "maturin"
|
||||
|
||||
[tool.ruff.lint]
|
||||
|
||||
@@ -179,18 +179,6 @@ def connect(
|
||||
... },
|
||||
... )
|
||||
|
||||
For Azure Blob Storage, credentials can be passed directly without setting
|
||||
environment variables:
|
||||
|
||||
>>> azure_storage_options = {
|
||||
... "account_name": "some-account",
|
||||
... "account_key": "some-key",
|
||||
... }
|
||||
>>> db = lancedb.connect( # doctest: +SKIP
|
||||
... "az://my-container/my-database",
|
||||
... storage_options=azure_storage_options,
|
||||
... )
|
||||
|
||||
For tests and temporary data, use an in-memory database:
|
||||
|
||||
>>> db = lancedb.connect("memory://")
|
||||
@@ -477,10 +465,6 @@ async def connect_async(
|
||||
--------
|
||||
|
||||
>>> import lancedb
|
||||
>>> azure_storage_options = {
|
||||
... "account_name": "some-account",
|
||||
... "account_key": "some-key",
|
||||
... }
|
||||
>>> async def doctest_example():
|
||||
... # For a local directory, provide a path to the database
|
||||
... db = await lancedb.connect_async("~/.lancedb")
|
||||
@@ -488,11 +472,6 @@ async def connect_async(
|
||||
... db = await lancedb.connect_async("s3://my-bucket/lancedb",
|
||||
... storage_options={
|
||||
... "aws_access_key_id": "***"})
|
||||
... # Azure credentials can also be passed directly
|
||||
... db = await lancedb.connect_async(
|
||||
... "az://my-container/my-database",
|
||||
... storage_options=azure_storage_options,
|
||||
... )
|
||||
... # For tests and temporary data, use an in-memory database
|
||||
... db = await lancedb.connect_async("memory://")
|
||||
... # Connect to LanceDB cloud
|
||||
|
||||
@@ -283,7 +283,6 @@ class Table:
|
||||
mode: Literal["append", "overwrite"],
|
||||
progress: Optional[Any] = None,
|
||||
write_parallelism: Optional[int] = None,
|
||||
allow_external_blob_outside_bases: bool = False,
|
||||
) -> AddResult: ...
|
||||
async def update(
|
||||
self, updates: Dict[str, str], where: Optional[str]
|
||||
|
||||
@@ -222,7 +222,6 @@ class PythonEnvironmentSpec(_RemoteValue):
|
||||
|
||||
kind: str
|
||||
packages: tuple[str, ...] = ()
|
||||
channels: tuple[str, ...] = ()
|
||||
path: Optional[str] = None
|
||||
modules: tuple[str, ...] = ()
|
||||
image: Optional[str] = None
|
||||
@@ -910,25 +909,13 @@ class UdfDefinition:
|
||||
pip: tuple[str, ...],
|
||||
env: Mapping[str, str],
|
||||
python_version: Optional[str],
|
||||
conda: tuple[str, ...] = (),
|
||||
conda_channels: tuple[str, ...] = (),
|
||||
):
|
||||
function_name = name or function.__name__
|
||||
if not _FUNCTION_NAME.fullmatch(function_name):
|
||||
raise ValueError(f"invalid Function name: {function_name!r}")
|
||||
if pip and conda:
|
||||
raise ValueError("a Function environment is pip or conda, not both")
|
||||
if conda_channels and not conda:
|
||||
raise ValueError("conda_channels requires conda packages")
|
||||
packages = tuple(sorted(set(conda if conda else pip)))
|
||||
packages = tuple(sorted(set(pip)))
|
||||
if any(not package or package != package.strip() for package in packages):
|
||||
raise ValueError("package requirements must be non-empty and trimmed")
|
||||
if conda:
|
||||
environment_spec = PythonEnvironmentSpec(
|
||||
kind="conda", packages=packages, channels=tuple(conda_channels)
|
||||
)
|
||||
else:
|
||||
environment_spec = PythonEnvironmentSpec(kind="pip", packages=packages)
|
||||
raise ValueError("pip requirements must be non-empty and trimmed")
|
||||
environment = dict(env)
|
||||
if any(
|
||||
not isinstance(key, str) or not isinstance(value, str)
|
||||
@@ -942,7 +929,7 @@ class UdfDefinition:
|
||||
kind="python",
|
||||
python_version=python_version
|
||||
or f"{sys.version_info.major}.{sys.version_info.minor}",
|
||||
environment=environment_spec,
|
||||
environment=PythonEnvironmentSpec(kind="pip", packages=packages),
|
||||
env=environment,
|
||||
)
|
||||
self._function = function
|
||||
@@ -989,8 +976,6 @@ def udf(
|
||||
pip: tuple[str, ...] | list[str] = (),
|
||||
env: Optional[Mapping[str, str]] = None,
|
||||
python_version: Optional[str] = None,
|
||||
conda: tuple[str, ...] | list[str] = (),
|
||||
conda_channels: tuple[str, ...] | list[str] = (),
|
||||
) -> Callable[[Callable[..., Any]], UdfDefinition]: ...
|
||||
|
||||
|
||||
@@ -1003,8 +988,6 @@ def udf(
|
||||
pip: tuple[str, ...] | list[str] = (),
|
||||
env: Optional[Mapping[str, str]] = None,
|
||||
python_version: Optional[str] = None,
|
||||
conda: tuple[str, ...] | list[str] = (),
|
||||
conda_channels: tuple[str, ...] | list[str] = (),
|
||||
):
|
||||
"""Prepare a scalar Python callable for remote Function registration.
|
||||
|
||||
@@ -1027,10 +1010,6 @@ def udf(
|
||||
provided together with ``input_schema``.
|
||||
pip : sequence of str, optional
|
||||
Pip requirements for the remote environment.
|
||||
conda : sequence of str, optional
|
||||
Conda packages for the remote environment, instead of ``pip``.
|
||||
conda_channels : sequence of str, optional
|
||||
Conda channels in priority order; requires ``conda``.
|
||||
env : mapping of str to str, optional
|
||||
Environment variables included in the Function definition.
|
||||
python_version : str, optional
|
||||
@@ -1070,8 +1049,6 @@ def udf(
|
||||
pip=tuple(pip),
|
||||
env={} if env is None else env,
|
||||
python_version=python_version,
|
||||
conda=tuple(conda),
|
||||
conda_channels=tuple(conda_channels),
|
||||
)
|
||||
|
||||
if function is None:
|
||||
|
||||
@@ -7,7 +7,6 @@ from typing import List, Literal, Optional
|
||||
from ._lancedb import (
|
||||
IndexConfig,
|
||||
)
|
||||
from .query import DocumentGranularity
|
||||
from .types import BaseTokenizerType
|
||||
|
||||
lang_mapping = {
|
||||
@@ -122,11 +121,6 @@ class FTS:
|
||||
|
||||
>>> config = FTS(block_size=256)
|
||||
|
||||
Create an index that treats each deepest-list element as one document:
|
||||
|
||||
>>> from lancedb.query import DocumentGranularity
|
||||
>>> config = FTS(document_granularity=DocumentGranularity.LIST_ELEMENT)
|
||||
|
||||
Attributes
|
||||
----------
|
||||
with_position : bool, default False
|
||||
@@ -178,11 +172,6 @@ class FTS:
|
||||
roughly half of the available CPU cores. The effective value is
|
||||
limited by the available compute capacity. This build-only setting is
|
||||
not persisted with the index and does not apply to remote tables.
|
||||
document_granularity : DocumentGranularity, default ROW
|
||||
``ROW`` treats the selected text in one table row as one document.
|
||||
``LIST_ELEMENT`` treats each element of the deepest list on the indexed
|
||||
field path as one document and returns its physical coordinates in
|
||||
``_doc_index`` for matching queries.
|
||||
|
||||
Notes
|
||||
-----
|
||||
@@ -207,7 +196,6 @@ class FTS:
|
||||
custom_stop_words: Optional[List[str]] = None
|
||||
memory_limit: Optional[int] = None
|
||||
num_workers: Optional[int] = None
|
||||
document_granularity: DocumentGranularity = DocumentGranularity.ROW
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -375,13 +375,6 @@ class FullTextOperator(str, Enum):
|
||||
OR = "OR"
|
||||
|
||||
|
||||
class DocumentGranularity(str, Enum):
|
||||
"""The unit treated as one full-text-search document."""
|
||||
|
||||
ROW = "row"
|
||||
LIST_ELEMENT = "list_element"
|
||||
|
||||
|
||||
class Occur(str, Enum):
|
||||
SHOULD = "SHOULD"
|
||||
MUST = "MUST"
|
||||
@@ -485,10 +478,6 @@ class MatchQuery(FullTextQuery):
|
||||
prefix_length : int, optional
|
||||
The number of beginning characters being unchanged for fuzzy matching.
|
||||
This is useful to achieve prefix matching.
|
||||
document_granularity : DocumentGranularity, optional
|
||||
Explicitly select row or deepest-list-element documents. If omitted,
|
||||
the indexed granularity is inferred. When both granularities are indexed
|
||||
for the field, this must be specified. With no index, row granularity is used.
|
||||
"""
|
||||
|
||||
query: str
|
||||
@@ -498,9 +487,6 @@ class MatchQuery(FullTextQuery):
|
||||
max_expansions: int = pydantic.Field(50, kw_only=True)
|
||||
operator: FullTextOperator = pydantic.Field(FullTextOperator.OR, kw_only=True)
|
||||
prefix_length: int = pydantic.Field(0, kw_only=True)
|
||||
document_granularity: Optional[DocumentGranularity] = pydantic.Field(
|
||||
None, kw_only=True
|
||||
)
|
||||
|
||||
def query_type(self) -> FullTextQueryType:
|
||||
return FullTextQueryType.MATCH
|
||||
@@ -517,20 +503,11 @@ class PhraseQuery(FullTextQuery):
|
||||
The query string to match against.
|
||||
column : str
|
||||
The name of the column to match against.
|
||||
slop : int, default 0
|
||||
The maximum number of intervening positions permitted in the phrase.
|
||||
document_granularity : DocumentGranularity, optional
|
||||
Explicitly select row or deepest-list-element documents. If omitted,
|
||||
the indexed granularity is inferred. When both granularities are indexed
|
||||
for the field, this must be specified. With no index, row granularity is used.
|
||||
"""
|
||||
|
||||
query: str
|
||||
column: str
|
||||
slop: int = pydantic.Field(0, kw_only=True)
|
||||
document_granularity: Optional[DocumentGranularity] = pydantic.Field(
|
||||
None, kw_only=True
|
||||
)
|
||||
|
||||
def query_type(self) -> FullTextQueryType:
|
||||
return FullTextQueryType.MATCH_PHRASE
|
||||
@@ -3401,10 +3378,9 @@ class AsyncQuery(AsyncStandardQuery):
|
||||
pass in multiple vectors. When multiple vectors are passed in, if the vector
|
||||
column is with multivector type, then the vectors will be treated as a single
|
||||
query. Or the vectors will be treated as multiple queries, this can be useful
|
||||
if you want to find the nearest vectors to multiple query vectors. Flat
|
||||
searches share one table scan across the query vectors, avoiding the scan
|
||||
and memory amplification of making multiple queries concurrently. If
|
||||
multiple vectors are passed in then
|
||||
if you want to find the nearest vectors to multiple query vectors.
|
||||
This is not expected to be faster than making multiple queries concurrently;
|
||||
it is just a convenience method. If multiple vectors are passed in then
|
||||
an additional column `query_index` will be added to the results. This column
|
||||
will contain the index of the query vector that the result is nearest to.
|
||||
"""
|
||||
@@ -3533,8 +3509,8 @@ class AsyncFTSQuery(AsyncStandardQuery):
|
||||
|
||||
Typically, a single vector is passed in as the query. However, you can also
|
||||
pass in multiple vectors. This can be useful if you want to find the nearest
|
||||
vectors to multiple query vectors. Flat searches share one table scan across
|
||||
the query vectors instead of issuing concurrent full scans.
|
||||
vectors to multiple query vectors. This is not expected to be faster than
|
||||
making multiple queries concurrently; it is just a convenience method.
|
||||
If multiple vectors are passed in then an additional column `query_index`
|
||||
will be added to the results. This column will contain the index of the
|
||||
query vector that the result is nearest to.
|
||||
|
||||
@@ -61,7 +61,6 @@ from lancedb.table import _normalize_progress
|
||||
|
||||
from ..query import (
|
||||
AnalyzePlanDistributedMetrics,
|
||||
DocumentGranularity,
|
||||
LanceQueryBuilder,
|
||||
LanceTakeQueryBuilder,
|
||||
LanceVectorQueryBuilder,
|
||||
@@ -350,7 +349,6 @@ class RemoteTable(Table):
|
||||
ngram_max_length: int = 3,
|
||||
prefix_only: bool = False,
|
||||
block_size: int = 128,
|
||||
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
|
||||
name: Optional[str] = None,
|
||||
):
|
||||
"""Create a full-text search index on a column.
|
||||
@@ -373,7 +371,6 @@ class RemoteTable(Table):
|
||||
ngram_max_length=ngram_max_length,
|
||||
prefix_only=prefix_only,
|
||||
block_size=block_size,
|
||||
document_granularity=document_granularity,
|
||||
)
|
||||
LOOP.run(
|
||||
self._table.create_index(
|
||||
@@ -613,7 +610,6 @@ class RemoteTable(Table):
|
||||
fill_value: float = 0.0,
|
||||
progress: Optional[Union[bool, Callable, Any]] = None,
|
||||
write_parallelism: Optional[int] = None,
|
||||
allow_external_blob_outside_bases: bool = False,
|
||||
) -> AddResult:
|
||||
"""Add more data to the [Table][lancedb.table.Table].
|
||||
|
||||
@@ -646,8 +642,6 @@ class RemoteTable(Table):
|
||||
data in flight. Defaults to an estimate based on the data size,
|
||||
capped at the number of CPU cores. Lower this if bulk ingestion is
|
||||
using too much memory.
|
||||
allow_external_blob_outside_bases: bool, default False
|
||||
Not supported on LanceDB Cloud. Setting this raises.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -664,7 +658,6 @@ class RemoteTable(Table):
|
||||
fill_value=fill_value,
|
||||
progress=progress,
|
||||
write_parallelism=write_parallelism,
|
||||
allow_external_blob_outside_bases=allow_external_blob_outside_bases,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
|
||||
@@ -85,7 +85,6 @@ from .query import (
|
||||
AsyncQuery,
|
||||
AsyncTakeQuery,
|
||||
AsyncVectorQuery,
|
||||
DocumentGranularity,
|
||||
FullTextQuery,
|
||||
LanceEmptyQueryBuilder,
|
||||
LanceFtsQueryBuilder,
|
||||
@@ -1169,7 +1168,6 @@ class Table(ABC):
|
||||
ngram_max_length: int = 3,
|
||||
prefix_only: bool = False,
|
||||
block_size: int = 128,
|
||||
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
|
||||
wait_timeout: Optional[timedelta] = None,
|
||||
name: Optional[str] = None,
|
||||
):
|
||||
@@ -1248,11 +1246,6 @@ class Table(ABC):
|
||||
The number of documents per compressed posting block. Must be 128
|
||||
or 256. A value of 256 uses the experimental FTS V3 format and
|
||||
may introduce breaking changes.
|
||||
document_granularity: DocumentGranularity, default ROW
|
||||
``ROW`` treats the selected text in one table row as one document.
|
||||
``LIST_ELEMENT`` treats each element of the deepest list on the field
|
||||
path as one document and returns its physical coordinates in
|
||||
``_doc_index`` for matching queries.
|
||||
wait_timeout: timedelta, optional
|
||||
The timeout to wait if indexing is asynchronous.
|
||||
name: str, optional
|
||||
@@ -1276,7 +1269,6 @@ class Table(ABC):
|
||||
fill_value: float = 0.0,
|
||||
progress: Optional[Union[bool, Callable, Any]] = None,
|
||||
write_parallelism: Optional[int] = None,
|
||||
allow_external_blob_outside_bases: bool = False,
|
||||
) -> AddResult:
|
||||
"""Add more data to the [Table][lancedb.table.Table].
|
||||
|
||||
@@ -1328,10 +1320,6 @@ class Table(ABC):
|
||||
data in flight. Defaults to an estimate based on the data size,
|
||||
capped at the number of CPU cores. Lower this if bulk ingestion is
|
||||
using too much memory.
|
||||
allow_external_blob_outside_bases: bool, default False
|
||||
Store blob URIs that sit outside registered blob bases. The row
|
||||
keeps a reference, so the object has to stay readable. Local
|
||||
tables only.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -2127,25 +2115,12 @@ class Table(ABC):
|
||||
----------
|
||||
updates : dict
|
||||
One or more dicts, each with:
|
||||
|
||||
- "path": str — dot-path to the field (e.g. "embedding" or "a.b.c").
|
||||
- "metadata": dict[str, str | None] — keys to set; a value of ``None``
|
||||
deletes that key.
|
||||
- "replace": bool, optional — replace the field's whole metadata map
|
||||
instead of merging (default False).
|
||||
|
||||
The following keys are treated specially, by convention, and should
|
||||
be used when appropriate:
|
||||
|
||||
- "lancedb:description": for a human-readable description of a field.
|
||||
- ``"lancedb:tag:<name>"`` for a user-defined key-value tag, where the
|
||||
suffix names the tag category; e.g. "lancedb:tag:model": "clip".
|
||||
- "lancedb:logical-column" for a column grouping; e.g. "feature_v1"
|
||||
and "feature_v2" might be in the same logical column.
|
||||
- "lancedb:status" for status options ("production", "candidate",
|
||||
"deprecated", "archived") to designate the current life cycle
|
||||
state of this column.
|
||||
|
||||
Returns
|
||||
-------
|
||||
UpdateFieldMetadataResult
|
||||
@@ -3293,7 +3268,6 @@ class LanceTable(Table):
|
||||
ngram_max_length: int = 3,
|
||||
prefix_only: bool = False,
|
||||
block_size: int = 128,
|
||||
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
|
||||
name: Optional[str] = None,
|
||||
):
|
||||
"""Create a full-text search index on a column.
|
||||
@@ -3345,11 +3319,7 @@ class LanceTable(Table):
|
||||
tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)
|
||||
tokenizer_configs["custom_stop_words"] = custom_stop_words
|
||||
|
||||
config = FTS(
|
||||
block_size=block_size,
|
||||
document_granularity=document_granularity,
|
||||
**tokenizer_configs,
|
||||
)
|
||||
config = FTS(block_size=block_size, **tokenizer_configs)
|
||||
|
||||
try:
|
||||
LOOP.run(
|
||||
@@ -3439,7 +3409,6 @@ class LanceTable(Table):
|
||||
fill_value: float = 0.0,
|
||||
progress: Optional[Union[bool, Callable, Any]] = None,
|
||||
write_parallelism: Optional[int] = None,
|
||||
allow_external_blob_outside_bases: bool = False,
|
||||
) -> AddResult:
|
||||
"""Add data to the table.
|
||||
If vector columns are missing and the table
|
||||
@@ -3467,9 +3436,6 @@ class LanceTable(Table):
|
||||
data in flight. Defaults to an estimate based on the data size,
|
||||
capped at the number of CPU cores. Lower this if bulk ingestion is
|
||||
using too much memory.
|
||||
allow_external_blob_outside_bases: bool, default False
|
||||
Allow blob URIs outside registered bases. See :meth:`Table.add`.
|
||||
Local tables only.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -3486,7 +3452,6 @@ class LanceTable(Table):
|
||||
fill_value=fill_value,
|
||||
progress=progress,
|
||||
write_parallelism=write_parallelism,
|
||||
allow_external_blob_outside_bases=allow_external_blob_outside_bases,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
@@ -5400,7 +5365,6 @@ class AsyncTable:
|
||||
fill_value: Optional[float] = None,
|
||||
progress: Optional[Union[bool, Callable, Any]] = None,
|
||||
write_parallelism: Optional[int] = None,
|
||||
allow_external_blob_outside_bases: bool = False,
|
||||
) -> AddResult:
|
||||
"""Add more data to the [AsyncTable][lancedb.table.AsyncTable].
|
||||
|
||||
@@ -5431,9 +5395,6 @@ class AsyncTable:
|
||||
data in flight. Defaults to an estimate based on the data size,
|
||||
capped at the number of CPU cores. Lower this if bulk ingestion is
|
||||
using too much memory.
|
||||
allow_external_blob_outside_bases: bool, default False
|
||||
Allow blob URIs outside registered bases. See :meth:`Table.add`.
|
||||
Local tables only.
|
||||
|
||||
"""
|
||||
schema = await self.schema()
|
||||
@@ -5470,7 +5431,6 @@ class AsyncTable:
|
||||
mode or "append",
|
||||
progress=progress,
|
||||
write_parallelism=write_parallelism,
|
||||
allow_external_blob_outside_bases=allow_external_blob_outside_bases,
|
||||
)
|
||||
except RuntimeError as e:
|
||||
if "Cast error" in str(e):
|
||||
|
||||
@@ -105,7 +105,7 @@ def test_quickstart(tmp_path):
|
||||
tbl.create_index(num_sub_vectors=1)
|
||||
# --8<-- [end:create_index]
|
||||
# --8<-- [start:delete_rows]
|
||||
tbl.delete("item = 'fizz'")
|
||||
tbl.delete('item = "fizz"')
|
||||
# --8<-- [end:delete_rows]
|
||||
# --8<-- [start:drop_table]
|
||||
db.drop_table("my_table")
|
||||
@@ -201,7 +201,7 @@ async def test_quickstart_async(tmp_path):
|
||||
await tbl.create_index("vector")
|
||||
# --8<-- [end:create_index_async]
|
||||
# --8<-- [start:delete_rows_async]
|
||||
await tbl.delete("item = 'fizz'")
|
||||
await tbl.delete('item = "fizz"')
|
||||
# --8<-- [end:delete_rows_async]
|
||||
# --8<-- [start:drop_table_async]
|
||||
await db.drop_table("my_table_async")
|
||||
|
||||
@@ -266,7 +266,7 @@ def test_table():
|
||||
tbl.add(pydantic_model_items)
|
||||
# --8<-- [end:add_table_from_pydantic]
|
||||
# --8<-- [start:delete_row]
|
||||
tbl.delete("item = 'fizz'")
|
||||
tbl.delete('item = "fizz"')
|
||||
# --8<-- [end:delete_row]
|
||||
# --8<-- [start:delete_specific_row]
|
||||
data = [
|
||||
@@ -538,7 +538,7 @@ async def test_table_async():
|
||||
await async_tbl.add(pydantic_model_items)
|
||||
# --8<-- [end:add_table_async_from_pydantic]
|
||||
# --8<-- [start:delete_row_async]
|
||||
await async_tbl.delete("item = 'fizz'")
|
||||
await async_tbl.delete('item = "fizz"')
|
||||
# --8<-- [end:delete_row_async]
|
||||
# --8<-- [start:delete_specific_row_async]
|
||||
data = [
|
||||
|
||||
@@ -617,71 +617,3 @@ def test_fetch_blobs_nested_path_survives_sort_after_query():
|
||||
def _identifiable_payload(size: int) -> bytes:
|
||||
block = 256
|
||||
return b"".join(bytes([i % 256]) * block for i in range(size // block))
|
||||
|
||||
|
||||
def _external_uri_blob_array(uris):
|
||||
blob_type = lancedb.blob("image").type
|
||||
storage_type = blob_type.storage_type
|
||||
child_names = [field.name for field in storage_type]
|
||||
assert "uri" in child_names, "blob layout no longer has a uri child"
|
||||
children = [
|
||||
pa.array(uris if field.name == "uri" else [None] * len(uris), type=field.type)
|
||||
for field in storage_type
|
||||
]
|
||||
storage = pa.StructArray.from_arrays(children, fields=list(storage_type))
|
||||
return pa.ExtensionArray.from_storage(blob_type, storage)
|
||||
|
||||
|
||||
def _external_uri_table_and_rows(name, uris):
|
||||
db = lancedb.connect("memory:///")
|
||||
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
|
||||
table = db.create_table(name, schema=schema)
|
||||
rows = pa.Table.from_arrays(
|
||||
[
|
||||
pa.array(range(len(uris)), type=pa.int64()),
|
||||
_external_uri_blob_array(uris),
|
||||
],
|
||||
schema=schema,
|
||||
)
|
||||
return table, rows
|
||||
|
||||
|
||||
def test_add_external_uri_struct_round_trips_with_flag(tmp_path):
|
||||
payload = b"external-uri-bytes"
|
||||
blob_path = tmp_path / "payload.bin"
|
||||
blob_path.write_bytes(payload)
|
||||
|
||||
table, rows = _external_uri_table_and_rows("external_struct", [blob_path.as_uri()])
|
||||
table.add(rows, allow_external_blob_outside_bases=True)
|
||||
|
||||
hits = table.search().to_arrow()
|
||||
blobs = table.fetch_blobs("image", hits)
|
||||
assert blobs[0].as_py() == payload
|
||||
|
||||
|
||||
def test_add_external_uri_without_flag_raises(tmp_path):
|
||||
blob_path = tmp_path / "payload.bin"
|
||||
blob_path.write_bytes(b"unreachable")
|
||||
|
||||
table, rows = _external_uri_table_and_rows("external_no_flag", [blob_path.as_uri()])
|
||||
with pytest.raises(ValueError, match="allow_external_blob_outside_bases"):
|
||||
table.add(rows)
|
||||
assert table.count_rows() == 0
|
||||
|
||||
|
||||
def test_add_external_uri_string_round_trips_with_flag(tmp_path):
|
||||
payload = b"external-uri-bytes"
|
||||
blob_path = tmp_path / "payload.bin"
|
||||
blob_path.write_bytes(payload)
|
||||
|
||||
db = lancedb.connect("memory:///")
|
||||
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
|
||||
table = db.create_table("external_string", schema=schema)
|
||||
table.add(
|
||||
[{"id": 1, "image": blob_path.as_uri()}],
|
||||
allow_external_blob_outside_bases=True,
|
||||
)
|
||||
|
||||
hits = table.search().to_arrow()
|
||||
blobs = table.fetch_blobs("image", hits)
|
||||
assert blobs[0].as_py() == payload
|
||||
|
||||
@@ -69,26 +69,6 @@ def _run_packaged(definition, *args):
|
||||
return namespace[definition.registration_request.artifact.entrypoint](*args)
|
||||
|
||||
|
||||
def test_udf_conda_environment():
|
||||
@udf(conda=["scipy", "numpy"], conda_channels=["conda-forge", "defaults"])
|
||||
def halve(value: float) -> float:
|
||||
return value / 2
|
||||
|
||||
request = json.loads(halve.registration_request.to_canonical_json())
|
||||
assert request["runtime"]["environment"] == {
|
||||
"kind": "conda",
|
||||
"packages": ["numpy", "scipy"],
|
||||
"channels": ["conda-forge", "defaults"],
|
||||
}
|
||||
pip_request = json.loads(normalize_score.registration_request.to_canonical_json())
|
||||
assert "channels" not in pip_request["runtime"]["environment"]
|
||||
|
||||
with pytest.raises(ValueError, match="not both"):
|
||||
udf(name="both", pip=["numpy"], conda=["numpy"])(lambda value: value)
|
||||
with pytest.raises(ValueError, match="requires conda"):
|
||||
udf(name="channels", conda_channels=["conda-forge"])(lambda value: value)
|
||||
|
||||
|
||||
def test_udf_packages_attribute_access_and_body_imports():
|
||||
@udf
|
||||
def word_norm(body: str) -> float:
|
||||
|
||||
@@ -25,7 +25,6 @@ from lancedb.db import DBConnection
|
||||
from lancedb.index import FTS
|
||||
from lancedb.query import (
|
||||
BoostQuery,
|
||||
DocumentGranularity,
|
||||
MatchQuery,
|
||||
MultiMatchQuery,
|
||||
PhraseQuery,
|
||||
@@ -246,55 +245,6 @@ def test_create_inverted_index_rejects_invalid_block_size(table):
|
||||
table.create_index("text", config=FTS(block_size=129))
|
||||
|
||||
|
||||
def test_list_element_document_granularity(tmp_path):
|
||||
docs_type = pa.list_(pa.struct([pa.field("content", pa.string())]))
|
||||
docs = pa.array(
|
||||
[
|
||||
[
|
||||
{"content": "alpha beta"},
|
||||
None,
|
||||
{"content": ""},
|
||||
{"content": "the and"},
|
||||
{"content": "alpha beta"},
|
||||
]
|
||||
],
|
||||
type=docs_type,
|
||||
)
|
||||
table = ldb.connect(tmp_path).create_table(
|
||||
"list_element_docs", pa.table({"id": [0], "docs": docs})
|
||||
)
|
||||
row_table = ldb.connect(tmp_path).create_table(
|
||||
"row_docs", pa.table({"id": [0], "docs": docs})
|
||||
)
|
||||
row_table.create_index("docs.content", config=FTS())
|
||||
row_result = row_table.search(MatchQuery("alpha", "docs.content")).to_arrow()
|
||||
assert row_result.num_rows == 1
|
||||
assert "_doc_index" not in row_result.column_names
|
||||
|
||||
granularity = DocumentGranularity.LIST_ELEMENT
|
||||
table.create_index(
|
||||
"docs.content",
|
||||
config=FTS(with_position=True, document_granularity=granularity),
|
||||
)
|
||||
assert table.list_indices()[0].columns == ["docs.content"]
|
||||
|
||||
def coordinates(query):
|
||||
result = table.search(query).limit(10).to_arrow()
|
||||
doc_index_type = result.schema.field("_doc_index").type
|
||||
assert pa.types.is_list(doc_index_type)
|
||||
assert doc_index_type.value_type == pa.uint32()
|
||||
return sorted(result["_doc_index"].to_pylist())
|
||||
|
||||
assert coordinates(
|
||||
MatchQuery("alpha", "docs.content", document_granularity=granularity)
|
||||
) == [[0], [4]]
|
||||
assert coordinates(
|
||||
PhraseQuery("alpha beta", "docs.content", document_granularity=granularity)
|
||||
) == [[0], [4]]
|
||||
assert coordinates(MatchQuery("alpha", "docs.content")) == [[0], [4]]
|
||||
assert FTS().document_granularity is DocumentGranularity.ROW
|
||||
|
||||
|
||||
def test_create_inverted_index_respects_build_memory_limit(table):
|
||||
with pytest.raises(ValueError, match="exceeds worker memory limit"):
|
||||
table.create_index(
|
||||
@@ -1139,20 +1089,6 @@ def test_fts_query_to_json():
|
||||
)
|
||||
assert json_str == expected
|
||||
|
||||
# Test MatchQuery with list-element document granularity
|
||||
match_query = MatchQuery(
|
||||
"hello world",
|
||||
"text",
|
||||
document_granularity=DocumentGranularity.LIST_ELEMENT,
|
||||
)
|
||||
json_str = match_query.to_json()
|
||||
expected = (
|
||||
'{"match":{"column":"text","terms":"hello world","boost":1.0,'
|
||||
'"fuzziness":0,"max_expansions":50,"operator":"Or","prefix_length":0,'
|
||||
'"document_granularity":"list_element"}}'
|
||||
)
|
||||
assert json_str == expected
|
||||
|
||||
# Test MatchQuery with options
|
||||
match_query = MatchQuery("puppy", "text", fuzziness=2, boost=1.5, prefix_length=3)
|
||||
json_str = match_query.to_json()
|
||||
@@ -1162,19 +1098,6 @@ def test_fts_query_to_json():
|
||||
)
|
||||
assert json_str == expected
|
||||
|
||||
# Test PhraseQuery with list-element document granularity
|
||||
phrase_query = PhraseQuery(
|
||||
"quick brown fox",
|
||||
"title",
|
||||
document_granularity=DocumentGranularity.LIST_ELEMENT,
|
||||
)
|
||||
json_str = phrase_query.to_json()
|
||||
expected = (
|
||||
'{"phrase":{"column":"title","terms":"quick brown fox","slop":0,'
|
||||
'"document_granularity":"list_element"}}'
|
||||
)
|
||||
assert json_str == expected
|
||||
|
||||
# Test PhraseQuery
|
||||
phrase_query = PhraseQuery("quick brown fox", "title")
|
||||
json_str = phrase_query.to_json()
|
||||
|
||||
@@ -897,23 +897,6 @@ def test_query_builder_batches(table):
|
||||
assert rs_list["id"][1] == 2
|
||||
|
||||
|
||||
def test_batch_vector_query_shares_filtered_flat_scan(table):
|
||||
query = (
|
||||
table.search([[1.0, 2.0], [3.0, 4.0]])
|
||||
.where("id > 0", prefilter=True)
|
||||
.limit(1)
|
||||
.select(["id"])
|
||||
)
|
||||
|
||||
plan = query.explain_plan(verbose=True)
|
||||
assert "KNNVectorDistance: queries=2" in plan
|
||||
assert "UnionExec" not in plan
|
||||
|
||||
results = query.to_arrow()
|
||||
assert len(results) == 2
|
||||
assert results["query_index"].to_pylist() == [0, 1]
|
||||
|
||||
|
||||
def test_dynamic_projection(table):
|
||||
rs = (
|
||||
LanceVectorQueryBuilder(table, [0, 0], "vector")
|
||||
|
||||
@@ -1618,49 +1618,6 @@ def test_query_sync_fts():
|
||||
)
|
||||
|
||||
|
||||
def test_query_sync_fts_document_granularity():
|
||||
from lancedb.query import DocumentGranularity, MatchQuery
|
||||
|
||||
def handler(body):
|
||||
assert body == {
|
||||
"full_text_query": {
|
||||
"query": {
|
||||
"match": {
|
||||
"column": "docs.content",
|
||||
"terms": "alpha",
|
||||
"boost": 1.0,
|
||||
"fuzziness": 0,
|
||||
"max_expansions": 50,
|
||||
"operator": "Or",
|
||||
"prefix_length": 0,
|
||||
"document_granularity": "list_element",
|
||||
}
|
||||
}
|
||||
},
|
||||
"k": 10,
|
||||
"prefilter": True,
|
||||
"vector": [],
|
||||
"version": None,
|
||||
}
|
||||
return pa.table(
|
||||
{
|
||||
"id": [1, 1],
|
||||
"_doc_index": pa.array([[0], [4]], type=pa.list_(pa.uint32())),
|
||||
}
|
||||
)
|
||||
|
||||
with query_test_table(handler, server_version=Version("0.6.0")) as table:
|
||||
result = table.search(
|
||||
MatchQuery(
|
||||
"alpha",
|
||||
"docs.content",
|
||||
document_granularity=DocumentGranularity.LIST_ELEMENT,
|
||||
)
|
||||
).to_arrow()
|
||||
|
||||
assert result["_doc_index"].to_pylist() == [[0], [4]]
|
||||
|
||||
|
||||
def test_query_sync_hybrid():
|
||||
def handler(body):
|
||||
if "full_text_query" in body:
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import timedelta
|
||||
import threading
|
||||
|
||||
@@ -87,25 +86,6 @@ def test_s3_lifecycle(s3_bucket: str):
|
||||
asyncio.run(test())
|
||||
|
||||
|
||||
@pytest.mark.s3_test
|
||||
def test_concurrent_open_table(s3_bucket: str):
|
||||
uri = f"s3://{s3_bucket}/test_concurrent_open_table"
|
||||
db = lancedb.connect(uri, storage_options=copy.copy(CONFIG))
|
||||
db.create_table("test", pa.table({"x": [1, 2, 3]}))
|
||||
|
||||
num_workers = 32
|
||||
barrier = threading.Barrier(num_workers)
|
||||
|
||||
def open_and_count(_):
|
||||
barrier.wait()
|
||||
return db.open_table("test").count_rows()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=num_workers) as pool:
|
||||
row_counts = list(pool.map(open_and_count, range(num_workers)))
|
||||
|
||||
assert row_counts == [3] * num_workers
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def kms_key():
|
||||
kms = get_boto3_client("kms", endpoint_url=CONFIG["aws_endpoint"])
|
||||
|
||||
+2
-8
@@ -8,7 +8,7 @@ use lancedb::index::vector::{
|
||||
};
|
||||
use lancedb::index::{
|
||||
Index as LanceDbIndex,
|
||||
scalar::{BTreeIndexBuilder, DocumentGranularity, FmIndexBuilder, FtsIndexBuilder},
|
||||
scalar::{BTreeIndexBuilder, FmIndexBuilder, FtsIndexBuilder},
|
||||
};
|
||||
use pyo3::IntoPyObject;
|
||||
use pyo3::types::PyStringMethods;
|
||||
@@ -60,11 +60,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
|
||||
.ngram_min_length(params.ngram_min_length)
|
||||
.ngram_max_length(params.ngram_max_length)
|
||||
.ngram_prefix_only(params.prefix_only)
|
||||
.custom_stop_words(params.custom_stop_words)
|
||||
.document_granularity(
|
||||
DocumentGranularity::try_from(params.document_granularity.as_str())
|
||||
.map_err(|err| PyValueError::new_err(err.to_string()))?,
|
||||
);
|
||||
.custom_stop_words(params.custom_stop_words);
|
||||
if let Some(memory_limit) = params.memory_limit {
|
||||
inner_opts = inner_opts.memory_limit_mb(memory_limit);
|
||||
}
|
||||
@@ -225,7 +221,6 @@ struct FtsParams {
|
||||
block_size: usize,
|
||||
memory_limit: Option<u64>,
|
||||
num_workers: Option<usize>,
|
||||
document_granularity: String,
|
||||
}
|
||||
|
||||
#[derive(FromPyObject)]
|
||||
@@ -486,7 +481,6 @@ mod tests {
|
||||
block_size = 128
|
||||
memory_limit = 2048
|
||||
num_workers = 7
|
||||
document_granularity = 'row'
|
||||
|
||||
config = FTS()",
|
||||
None,
|
||||
|
||||
+12
-45
@@ -16,8 +16,8 @@ use arrow::pyarrow::FromPyArrow;
|
||||
use arrow::pyarrow::IntoPyArrow;
|
||||
use arrow::pyarrow::ToPyArrow;
|
||||
use lancedb::index::scalar::{
|
||||
BooleanQuery, BoostQuery, DocumentGranularity, FtsQuery, FullTextSearchQuery, MatchQuery,
|
||||
MultiMatchQuery, Occur, Operator, PhraseQuery,
|
||||
BooleanQuery, BoostQuery, FtsQuery, FullTextSearchQuery, MatchQuery, MultiMatchQuery, Occur,
|
||||
Operator, PhraseQuery,
|
||||
};
|
||||
use lancedb::query::AnalyzePlanDistributedMetrics;
|
||||
use lancedb::query::QueryBase;
|
||||
@@ -76,16 +76,8 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB<FtsQuery> {
|
||||
let max_expansions = ob.getattr("max_expansions")?.extract()?;
|
||||
let operator = ob.getattr("operator")?.extract::<String>()?;
|
||||
let prefix_length = ob.getattr("prefix_length")?.extract()?;
|
||||
let document_granularity = ob
|
||||
.getattr("document_granularity")?
|
||||
.extract::<Option<String>>()?
|
||||
.map(|value| {
|
||||
DocumentGranularity::try_from(value.as_str())
|
||||
.map_err(|err| PyValueError::new_err(err.to_string()))
|
||||
})
|
||||
.transpose()?;
|
||||
|
||||
let mut query =
|
||||
Ok(Self(
|
||||
MatchQuery::new(query)
|
||||
.with_column(Some(column))
|
||||
.with_boost(boost)
|
||||
@@ -94,32 +86,21 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB<FtsQuery> {
|
||||
.with_operator(Operator::try_from(operator.as_str()).map_err(|e| {
|
||||
PyValueError::new_err(format!("Invalid operator: {}", e))
|
||||
})?)
|
||||
.with_prefix_length(prefix_length);
|
||||
if let Some(document_granularity) = document_granularity {
|
||||
query = query.with_document_granularity(document_granularity);
|
||||
}
|
||||
Ok(Self(query.into()))
|
||||
.with_prefix_length(prefix_length)
|
||||
.into(),
|
||||
))
|
||||
}
|
||||
"PhraseQuery" => {
|
||||
let query = ob.getattr("query")?.extract()?;
|
||||
let column = ob.getattr("column")?.extract()?;
|
||||
let slop = ob.getattr("slop")?.extract()?;
|
||||
let document_granularity = ob
|
||||
.getattr("document_granularity")?
|
||||
.extract::<Option<String>>()?
|
||||
.map(|value| {
|
||||
DocumentGranularity::try_from(value.as_str())
|
||||
.map_err(|err| PyValueError::new_err(err.to_string()))
|
||||
})
|
||||
.transpose()?;
|
||||
|
||||
let mut query = PhraseQuery::new(query)
|
||||
.with_column(Some(column))
|
||||
.with_slop(slop);
|
||||
if let Some(document_granularity) = document_granularity {
|
||||
query = query.with_document_granularity(document_granularity);
|
||||
}
|
||||
Ok(Self(query.into()))
|
||||
Ok(Self(
|
||||
PhraseQuery::new(query)
|
||||
.with_column(Some(column))
|
||||
.with_slop(slop)
|
||||
.into(),
|
||||
))
|
||||
}
|
||||
"BoostQuery" => {
|
||||
let positive: Self = ob.getattr("positive")?.extract()?;
|
||||
@@ -186,13 +167,6 @@ impl<'py> IntoPyObject<'py> for PyLanceDB<FtsQuery> {
|
||||
kwargs.set_item("max_expansions", query.max_expansions)?;
|
||||
kwargs.set_item::<_, &str>("operator", query.operator.into())?;
|
||||
kwargs.set_item("prefix_length", query.prefix_length)?;
|
||||
if let Some(document_granularity) = query.document_granularity {
|
||||
let value = match document_granularity {
|
||||
DocumentGranularity::Row => "row",
|
||||
DocumentGranularity::ListElement => "list_element",
|
||||
};
|
||||
kwargs.set_item("document_granularity", value)?;
|
||||
}
|
||||
namespace
|
||||
.getattr(intern!(py, "MatchQuery"))?
|
||||
.call((query.terms, query.column.unwrap()), Some(&kwargs))
|
||||
@@ -200,13 +174,6 @@ impl<'py> IntoPyObject<'py> for PyLanceDB<FtsQuery> {
|
||||
FtsQuery::Phrase(query) => {
|
||||
let kwargs = PyDict::new(py);
|
||||
kwargs.set_item("slop", query.slop)?;
|
||||
if let Some(document_granularity) = query.document_granularity {
|
||||
let value = match document_granularity {
|
||||
DocumentGranularity::Row => "row",
|
||||
DocumentGranularity::ListElement => "list_element",
|
||||
};
|
||||
kwargs.set_item("document_granularity", value)?;
|
||||
}
|
||||
namespace
|
||||
.getattr(intern!(py, "PhraseQuery"))?
|
||||
.call((query.terms, query.column.unwrap()), Some(&kwargs))
|
||||
|
||||
+2
-6
@@ -780,19 +780,15 @@ impl Table {
|
||||
})
|
||||
}
|
||||
|
||||
#[pyo3(signature = (data, mode, progress=None, write_parallelism=None, allow_external_blob_outside_bases=false))]
|
||||
#[pyo3(signature = (data, mode, progress=None, write_parallelism=None))]
|
||||
pub fn add<'a>(
|
||||
self_: PyRef<'a, Self>,
|
||||
data: PyScannable,
|
||||
mode: String,
|
||||
progress: Option<Py<PyAny>>,
|
||||
write_parallelism: Option<usize>,
|
||||
allow_external_blob_outside_bases: bool,
|
||||
) -> PyResult<Bound<'a, PyAny>> {
|
||||
let mut op = self_
|
||||
.inner_ref()?
|
||||
.add(data)
|
||||
.allow_external_blob_outside_bases(allow_external_blob_outside_bases);
|
||||
let mut op = self_.inner_ref()?.add(data);
|
||||
if mode == "append" {
|
||||
op = op.mode(AddDataMode::Append);
|
||||
} else if mode == "overwrite" {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb"
|
||||
version = "0.38.0-beta.11"
|
||||
version = "0.38.0-beta.10"
|
||||
edition.workspace = true
|
||||
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
||||
license.workspace = true
|
||||
|
||||
@@ -1476,7 +1476,7 @@ mod tests {
|
||||
use crate::table::{AnyQuery, WriteOptions};
|
||||
use arrow_array::{Int32Array, RecordBatch, StringArray};
|
||||
use arrow_schema::{DataType, Field, Schema, SchemaRef};
|
||||
use futures::{TryStreamExt, future::try_join_all, stream::once};
|
||||
use futures::{TryStreamExt, stream::once};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
@@ -1614,59 +1614,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_concurrent_open_table_reuses_connection_object_store() {
|
||||
let tempdir = tempdir().unwrap();
|
||||
let uri = tempdir.path().to_str().unwrap();
|
||||
let session = Arc::new(lance::session::Session::default());
|
||||
let request = ConnectRequest {
|
||||
uri: uri.to_string(),
|
||||
#[cfg(feature = "remote")]
|
||||
client_config: Default::default(),
|
||||
options: Default::default(),
|
||||
namespace_client_properties: Default::default(),
|
||||
manifest_enabled: false,
|
||||
read_consistency_interval: None,
|
||||
session: Some(session.clone()),
|
||||
};
|
||||
let db = ListingDatabase::connect_with_options(&request)
|
||||
.await
|
||||
.unwrap();
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||
db.create_table(CreateTableRequest {
|
||||
name: "test".to_string(),
|
||||
namespace_path: vec![],
|
||||
data: Box::new(RecordBatch::new_empty(schema)) as Box<dyn Scannable>,
|
||||
mode: CreateTableMode::Create,
|
||||
write_options: Default::default(),
|
||||
location: None,
|
||||
namespace_client: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let before = session.store_registry().stats();
|
||||
let opened_tables = try_join_all((0..32).map(|_| {
|
||||
db.open_table(OpenTableRequest {
|
||||
name: "test".to_string(),
|
||||
namespace_path: vec![],
|
||||
index_cache_size: None,
|
||||
lance_read_params: None,
|
||||
location: None,
|
||||
namespace_client: None,
|
||||
managed_versioning: None,
|
||||
})
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
let after = session.store_registry().stats();
|
||||
|
||||
assert_eq!(opened_tables.len(), 32);
|
||||
assert_eq!(after.misses, before.misses);
|
||||
assert_eq!(after.active_stores, before.active_stores);
|
||||
assert!(after.hits >= before.hits + 32);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_listing_database_root_ops_do_not_create_manifest() {
|
||||
let tempdir = tempdir().unwrap();
|
||||
|
||||
@@ -19,7 +19,6 @@
|
||||
|
||||
mod sql;
|
||||
|
||||
pub(crate) use sql::canonicalize_sql_predicate;
|
||||
pub use sql::expr_to_sql_string;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -1,16 +1,10 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
use std::any::TypeId;
|
||||
|
||||
use datafusion_common::ScalarValue;
|
||||
use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
|
||||
use datafusion_expr::Expr;
|
||||
use datafusion_sql::sqlparser::{
|
||||
dialect::{Dialect as SqlParserDialect, GenericDialect},
|
||||
tokenizer::{Token, Tokenizer},
|
||||
};
|
||||
use datafusion_sql::unparser::{self, dialect::Dialect as UnparserDialect};
|
||||
use datafusion_sql::unparser::{self, dialect::Dialect};
|
||||
|
||||
/// Unparser dialect that matches the quoting style expected by the Lance SQL
|
||||
/// parser. Lance uses backtick (`` ` ``) as the only delimited-identifier
|
||||
@@ -25,7 +19,7 @@ use datafusion_sql::unparser::{self, dialect::Dialect as UnparserDialect};
|
||||
/// lower-case by the SQL parser, which would break case-sensitive schemas).
|
||||
struct LanceSqlDialect;
|
||||
|
||||
impl UnparserDialect for LanceSqlDialect {
|
||||
impl Dialect for LanceSqlDialect {
|
||||
fn identifier_quote_style(&self, identifier: &str) -> Option<char> {
|
||||
let needs_quote = identifier.chars().any(|c| c.is_ascii_uppercase())
|
||||
|| !identifier
|
||||
@@ -36,61 +30,6 @@ impl UnparserDialect for LanceSqlDialect {
|
||||
}
|
||||
}
|
||||
|
||||
/// Lance's tokenizer dialect with SQL-standard double-quoted identifiers added.
|
||||
///
|
||||
/// Keep this deliberately small: Lance's parser wraps `GenericDialect` and
|
||||
/// delegates only identifier recognition, leaving every other dialect option at
|
||||
/// its default. In particular, `/*! ... */` remains an ordinary block comment.
|
||||
#[derive(Debug, Default)]
|
||||
struct PredicateDialect(GenericDialect);
|
||||
|
||||
impl SqlParserDialect for PredicateDialect {
|
||||
fn dialect(&self) -> TypeId {
|
||||
self.0.dialect()
|
||||
}
|
||||
|
||||
fn is_identifier_start(&self, ch: char) -> bool {
|
||||
self.0.is_identifier_start(ch)
|
||||
}
|
||||
|
||||
fn is_identifier_part(&self, ch: char) -> bool {
|
||||
self.0.is_identifier_part(ch)
|
||||
}
|
||||
|
||||
fn is_delimited_identifier_start(&self, ch: char) -> bool {
|
||||
ch == '"' || ch == '`'
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonicalize a raw SQL predicate for Lance's parser.
|
||||
///
|
||||
/// Lance wraps [`GenericDialect`] for identifier recognition while retaining the
|
||||
/// default dialect behavior for every other lexical option. [`PredicateDialect`]
|
||||
/// mirrors that contract and additionally recognizes `"` as an identifier
|
||||
/// delimiter, allowing this function to rewrite only those identifier tokens.
|
||||
pub fn canonicalize_sql_predicate(predicate: &str) -> crate::Result<String> {
|
||||
let dialect = PredicateDialect::default();
|
||||
let tokens = Tokenizer::new(&dialect, predicate)
|
||||
.with_unescape(false)
|
||||
.tokenize()
|
||||
.map_err(|err| crate::Error::InvalidInput {
|
||||
message: format!("invalid SQL predicate: {err}"),
|
||||
})?;
|
||||
|
||||
Ok(tokens
|
||||
.into_iter()
|
||||
.map(|token| match token {
|
||||
Token::Word(word) if word.quote_style == Some('"') => {
|
||||
// with_unescape(false) retains doubled double quotes. Decode
|
||||
// those before escaping any backticks for Lance's delimiter.
|
||||
let identifier = word.value.replace("\"\"", "\"").replace('`', "``");
|
||||
format!("`{identifier}`")
|
||||
}
|
||||
other => other.to_string(),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Prefix for placeholder strings inserted in place of binary literals. Chosen
|
||||
/// to be extremely unlikely to occur in user data.
|
||||
const BINARY_PLACEHOLDER_PREFIX: &str = "__lancedb_binary_placeholder_";
|
||||
@@ -174,51 +113,3 @@ pub fn expr_to_sql_string(expr: &Expr) -> crate::Result<String> {
|
||||
}
|
||||
Ok(sql)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::canonicalize_sql_predicate;
|
||||
|
||||
#[test]
|
||||
fn normalizes_double_quoted_identifiers() {
|
||||
assert_eq!(
|
||||
canonicalize_sql_predicate(r#""PartyAbbrev" = 'D'"#).unwrap(),
|
||||
"`PartyAbbrev` = 'D'"
|
||||
);
|
||||
assert_eq!(
|
||||
canonicalize_sql_predicate(r#""MetaData"."userId" = 5"#).unwrap(),
|
||||
"`MetaData`.`userId` = 5"
|
||||
);
|
||||
assert_eq!(
|
||||
canonicalize_sql_predicate(r#""a""b" = 1"#).unwrap(),
|
||||
"`a\"b` = 1"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_quotes_inside_literals_and_backticks() {
|
||||
let filter = r#"name = 'Alice "Ace"' AND `quoted"field` = 1"#;
|
||||
assert_eq!(canonicalize_sql_predicate(filter).unwrap(), filter);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_literals_and_comments_using_lance_dialect_rules() {
|
||||
let predicate = r#"path = '\' AND "PartyAbbrev" = 'D' -- unmatched " in comment"#;
|
||||
assert_eq!(
|
||||
canonicalize_sql_predicate(predicate).unwrap(),
|
||||
r#"path = '\' AND `PartyAbbrev` = 'D' -- unmatched " in comment"#
|
||||
);
|
||||
|
||||
let predicate = r#"id = 1 /* unmatched " in block comment */"#;
|
||||
assert_eq!(canonicalize_sql_predicate(predicate).unwrap(), predicate);
|
||||
|
||||
let predicate = r#"id = 1 /*! OR "PartyAbbrev" = 'D' */"#;
|
||||
assert_eq!(canonicalize_sql_predicate(predicate).unwrap(), predicate);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_unterminated_double_quoted_identifier() {
|
||||
let error = canonicalize_sql_predicate(r#""PartyAbbrev = 'D'"#).unwrap_err();
|
||||
assert!(matches!(error, crate::Error::InvalidInput { .. }));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -186,9 +186,6 @@ pub struct PythonEnvironmentSpec {
|
||||
pub kind: String,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub packages: Vec<String>,
|
||||
/// Conda channels in priority order; conda environments only.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub channels: Vec<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub path: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
@@ -586,26 +583,3 @@ impl RefreshColumnResult {
|
||||
}
|
||||
|
||||
impl_json!(RefreshColumnResult);
|
||||
|
||||
#[cfg(test)]
|
||||
mod conda_environment_tests {
|
||||
use super::PythonEnvironmentSpec;
|
||||
|
||||
#[test]
|
||||
fn conda_channels_round_trip_and_pip_stays_bare() {
|
||||
let conda: PythonEnvironmentSpec = serde_json::from_str(
|
||||
r#"{"kind":"conda","packages":["numpy"],"channels":["conda-forge"]}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(conda.channels, ["conda-forge"]);
|
||||
assert!(
|
||||
serde_json::to_string(&conda)
|
||||
.unwrap()
|
||||
.contains(r#""channels":["conda-forge"]"#)
|
||||
);
|
||||
|
||||
let pip: PythonEnvironmentSpec =
|
||||
serde_json::from_str(r#"{"kind":"pip","packages":["numpy"]}"#).unwrap();
|
||||
assert!(!serde_json::to_string(&pip).unwrap().contains("channels"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,5 +63,4 @@ pub struct FmIndexBuilder {}
|
||||
pub use lance_index::scalar::FullTextSearchQuery;
|
||||
pub use lance_index::scalar::InvertedIndexParams as FtsIndexBuilder;
|
||||
pub use lance_index::scalar::InvertedIndexParams;
|
||||
pub use lance_index::scalar::inverted::DocumentGranularity;
|
||||
pub use lance_index::scalar::inverted::query::*;
|
||||
|
||||
@@ -10,7 +10,7 @@ use lance::io::WrappingObjectStore;
|
||||
use object_store::{
|
||||
CopyOptions, Error, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta,
|
||||
ObjectStore, ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result,
|
||||
UploadPart, list::PaginatedListStore, path::Path,
|
||||
UploadPart, path::Path,
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
@@ -187,14 +187,6 @@ impl WrappingObjectStore for MirroringObjectStoreWrapper {
|
||||
secondary: self.secondary.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
fn wrap_paginated(
|
||||
&self,
|
||||
_store_prefix: &str,
|
||||
original: Arc<dyn PaginatedListStore>,
|
||||
) -> Option<Arc<dyn PaginatedListStore>> {
|
||||
Some(original)
|
||||
}
|
||||
}
|
||||
|
||||
// windows pathing can't be simply concatenated
|
||||
|
||||
@@ -12,7 +12,7 @@ use lance::io::WrappingObjectStore;
|
||||
use object_store::{
|
||||
CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
|
||||
PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result as OSResult,
|
||||
UploadPart, list::PaginatedListStore, path::Path,
|
||||
UploadPart, path::Path,
|
||||
};
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
@@ -57,14 +57,6 @@ impl WrappingObjectStore for IoStatsHolder {
|
||||
stats: self.0.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
fn wrap_paginated(
|
||||
&self,
|
||||
_store_prefix: &str,
|
||||
original: Arc<dyn PaginatedListStore>,
|
||||
) -> Option<Arc<dyn PaginatedListStore>> {
|
||||
Some(original)
|
||||
}
|
||||
}
|
||||
|
||||
impl IoTrackingStore {
|
||||
|
||||
@@ -47,10 +47,6 @@ impl TerminalResult {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn value(&self) -> Option<&Value> {
|
||||
self.value.as_ref()
|
||||
}
|
||||
|
||||
fn decode<T: DeserializeOwned>(self) -> Result<T> {
|
||||
let value = self.value.ok_or_else(|| match &self.request_id {
|
||||
Some(request_id) => Error::Http {
|
||||
|
||||
@@ -170,15 +170,6 @@ pub(crate) fn plan(
|
||||
filter: Option<&str>,
|
||||
limit: Option<u64>,
|
||||
) -> Result<(MaterializedViewDefinition, Vec<ArrowField>, Lineage)> {
|
||||
let filter = filter
|
||||
.map(crate::expr::canonicalize_sql_predicate)
|
||||
.transpose()
|
||||
.map_err(|err| match err {
|
||||
Error::InvalidInput { message } => Error::InvalidInput {
|
||||
message: format!("invalid view filter: {message}"),
|
||||
},
|
||||
err => err,
|
||||
})?;
|
||||
let projections: Vec<(String, String)> = if projections.is_empty() {
|
||||
source_schema
|
||||
.fields()
|
||||
@@ -283,7 +274,7 @@ pub(crate) fn plan(
|
||||
declared.push(output);
|
||||
}
|
||||
|
||||
if let Some(filter) = filter.as_deref() {
|
||||
if let Some(filter) = filter {
|
||||
let expr = planner
|
||||
.parse_filter(filter)
|
||||
.map_err(|e| Error::InvalidInput {
|
||||
@@ -323,7 +314,7 @@ pub(crate) fn plan(
|
||||
.into_iter()
|
||||
.map(|(output, expression)| ViewProjection { output, expression })
|
||||
.collect(),
|
||||
filter,
|
||||
filter: filter.map(String::from),
|
||||
limit,
|
||||
inputs,
|
||||
};
|
||||
|
||||
@@ -46,9 +46,8 @@ use lance_table::format::Fragment;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{
|
||||
DEFINITION_META_KEY, INCARNATION_META_KEY, MaterializedViewDefinition,
|
||||
REFRESHED_AT_MS_META_KEY, SOURCE_ROW_ID_COLUMN, SOURCE_VERSION_META_KEY,
|
||||
definition_to_metadata,
|
||||
INCARNATION_META_KEY, MaterializedViewDefinition, REFRESHED_AT_MS_META_KEY,
|
||||
SOURCE_ROW_ID_COLUMN, SOURCE_VERSION_META_KEY,
|
||||
};
|
||||
use crate::database::OpenTableRequest;
|
||||
use crate::table::{NativeTable, NativeTableExt, Table};
|
||||
@@ -198,28 +197,8 @@ pub(crate) async fn execute_refresh(
|
||||
),
|
||||
});
|
||||
}
|
||||
let definition_changed =
|
||||
definition.filter != replanned.filter || definition.inputs != replanned.inputs;
|
||||
let definition = &replanned;
|
||||
|
||||
// A watermark written for a legacy raw filter certifies the rows that
|
||||
// filter produced, not the canonical predicate above. Rebuild instead of
|
||||
// accepting or advancing it, and persist the migrated definition in the
|
||||
// same metadata commit that certifies the replacement rows.
|
||||
if definition_changed {
|
||||
return rebuild(
|
||||
view_native,
|
||||
&view_ds,
|
||||
&source_ds,
|
||||
source_version,
|
||||
source_ts,
|
||||
definition,
|
||||
true,
|
||||
expected_incarnation,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
let metadata = &view_ds.schema().metadata;
|
||||
let watermark: Option<u64> = metadata
|
||||
.get(SOURCE_VERSION_META_KEY)
|
||||
@@ -278,7 +257,6 @@ pub(crate) async fn execute_refresh(
|
||||
source_version,
|
||||
source_ts,
|
||||
definition,
|
||||
false,
|
||||
expected_incarnation,
|
||||
)
|
||||
.await
|
||||
@@ -293,7 +271,6 @@ pub(crate) async fn execute_refresh(
|
||||
source_version,
|
||||
source_ts,
|
||||
definition,
|
||||
false,
|
||||
expected_incarnation,
|
||||
)
|
||||
.await
|
||||
@@ -706,7 +683,6 @@ async fn incremental(
|
||||
view_ds.clone(),
|
||||
source_version,
|
||||
source_ts,
|
||||
None,
|
||||
expected_incarnation,
|
||||
)
|
||||
.await?;
|
||||
@@ -728,7 +704,6 @@ async fn incremental(
|
||||
published,
|
||||
source_version,
|
||||
source_ts,
|
||||
None,
|
||||
expected_incarnation,
|
||||
)
|
||||
.await?;
|
||||
@@ -800,7 +775,6 @@ async fn incremental(
|
||||
published,
|
||||
source_version,
|
||||
source_ts,
|
||||
None,
|
||||
expected_incarnation,
|
||||
)
|
||||
.await?;
|
||||
@@ -850,14 +824,12 @@ async fn incremental(
|
||||
appended,
|
||||
source_version,
|
||||
source_ts,
|
||||
None,
|
||||
expected_incarnation,
|
||||
)
|
||||
.await?;
|
||||
Ok(Some(result))
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn rebuild(
|
||||
view_native: &NativeTable,
|
||||
view_ds: &Dataset,
|
||||
@@ -865,7 +837,6 @@ async fn rebuild(
|
||||
source_version: u64,
|
||||
source_ts: u128,
|
||||
definition: &MaterializedViewDefinition,
|
||||
persist_definition: bool,
|
||||
expected_incarnation: Option<&str>,
|
||||
) -> Result<RefreshMaterializedViewResult> {
|
||||
let rows_written = Arc::new(AtomicU64::new(0));
|
||||
@@ -896,7 +867,6 @@ async fn rebuild(
|
||||
replaced,
|
||||
source_version,
|
||||
source_ts,
|
||||
persist_definition.then_some(definition),
|
||||
expected_incarnation,
|
||||
)
|
||||
.await?;
|
||||
@@ -1011,7 +981,6 @@ async fn stamp_watermark(
|
||||
mut dataset: Dataset,
|
||||
source_version: u64,
|
||||
source_ts: u128,
|
||||
definition: Option<&MaterializedViewDefinition>,
|
||||
expected_incarnation: Option<&str>,
|
||||
) -> Result<u64> {
|
||||
ensure_incarnation(&dataset, expected_incarnation, dataset.uri()).await?;
|
||||
@@ -1024,32 +993,27 @@ async fn stamp_watermark(
|
||||
.get(INCARNATION_META_KEY)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
|
||||
let mut metadata = vec![(INCARNATION_META_KEY.to_string(), Some(incarnation))];
|
||||
if let Some(definition) = definition {
|
||||
metadata.push((
|
||||
DEFINITION_META_KEY.to_string(),
|
||||
Some(definition_to_metadata(definition)?),
|
||||
));
|
||||
}
|
||||
metadata.extend([
|
||||
(
|
||||
SOURCE_VERSION_META_KEY.to_string(),
|
||||
Some(source_version.to_string()),
|
||||
),
|
||||
(
|
||||
SOURCE_VERSION_TS_META_KEY.to_string(),
|
||||
Some(source_ts.to_string()),
|
||||
),
|
||||
(
|
||||
REFRESHED_AT_MS_META_KEY.to_string(),
|
||||
Some(now_ms().to_string()),
|
||||
),
|
||||
(
|
||||
VIEW_VERSION_META_KEY.to_string(),
|
||||
Some(predicted.to_string()),
|
||||
),
|
||||
]);
|
||||
dataset.update_schema_metadata(metadata).await?;
|
||||
dataset
|
||||
.update_schema_metadata([
|
||||
(INCARNATION_META_KEY.to_string(), Some(incarnation)),
|
||||
(
|
||||
SOURCE_VERSION_META_KEY.to_string(),
|
||||
Some(source_version.to_string()),
|
||||
),
|
||||
(
|
||||
SOURCE_VERSION_TS_META_KEY.to_string(),
|
||||
Some(source_ts.to_string()),
|
||||
),
|
||||
(
|
||||
REFRESHED_AT_MS_META_KEY.to_string(),
|
||||
Some(now_ms().to_string()),
|
||||
),
|
||||
(
|
||||
VIEW_VERSION_META_KEY.to_string(),
|
||||
Some(predicted.to_string()),
|
||||
),
|
||||
])
|
||||
.await?;
|
||||
let actual = dataset.version().version;
|
||||
if actual != predicted {
|
||||
return Err(Error::Runtime {
|
||||
@@ -1621,106 +1585,6 @@ mod tests {
|
||||
assert_eq!(read(view.table(), "x").await, vec![20, 40]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mixed_case_filter_is_canonicalized_for_lineage_and_refresh() {
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let batch = record_batch!(
|
||||
("id", Int32, [1, 2, 3]),
|
||||
("PartyAbbrev", Utf8, ["D", "R", "D"])
|
||||
)
|
||||
.unwrap();
|
||||
conn.create_table("src", batch)
|
||||
.write_options(crate::materialized_view::tests::stable_row_ids())
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
conn.create_materialized_view("democrats", "src")
|
||||
.select([("id", "id")])
|
||||
.only_if(r#""PartyAbbrev" = 'D'"#)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Reopen from schema metadata so these assertions cover the stored
|
||||
// predicate and lineage, not only the declaration-time handle.
|
||||
let view = conn.open_materialized_view("democrats").await.unwrap();
|
||||
assert_eq!(
|
||||
view.definition().filter.as_deref(),
|
||||
Some("`PartyAbbrev` = 'D'")
|
||||
);
|
||||
assert_eq!(view.definition().inputs, ["PartyAbbrev", "id"]);
|
||||
|
||||
let result = view.refresh().execute().await.unwrap();
|
||||
assert_eq!(result.rows_written, 2);
|
||||
assert_eq!(read(view.table(), "id").await, vec![1, 3]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_legacy_raw_filter_rebuilds_and_persists_canonical_definition() {
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let batch = record_batch!(
|
||||
("id", Int32, [1, 2, 3]),
|
||||
("PartyAbbrev", Utf8, ["D", "R", "D"])
|
||||
)
|
||||
.unwrap();
|
||||
conn.create_table("legacy_src", batch)
|
||||
.write_options(crate::materialized_view::tests::stable_row_ids())
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
let view = conn
|
||||
.create_materialized_view("legacy_view", "legacy_src")
|
||||
.select([("id", "id")])
|
||||
.only_if(r#""PartyAbbrev" = 'X'"#)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(view.refresh().execute().await.unwrap().rows_written, 0);
|
||||
|
||||
// Model a definition and up-to-date watermark written before filter
|
||||
// canonicalization was applied to materialized views.
|
||||
let mut legacy = view.definition().clone();
|
||||
legacy.filter = Some(r#""PartyAbbrev" = 'D'"#.into());
|
||||
legacy.inputs = vec!["id".into()];
|
||||
let native = view.table().as_native().unwrap();
|
||||
let mut dataset = native.dataset.get().await.unwrap().as_ref().clone();
|
||||
let predicted = dataset.version().version + 1;
|
||||
dataset
|
||||
.update_schema_metadata([
|
||||
(
|
||||
DEFINITION_META_KEY.to_string(),
|
||||
Some(definition_to_metadata(&legacy).unwrap()),
|
||||
),
|
||||
(
|
||||
VIEW_VERSION_META_KEY.to_string(),
|
||||
Some(predicted.to_string()),
|
||||
),
|
||||
])
|
||||
.await
|
||||
.unwrap();
|
||||
native.dataset.update(dataset);
|
||||
|
||||
let reopened = conn.open_materialized_view("legacy_view").await.unwrap();
|
||||
let result = reopened.refresh().execute().await.unwrap();
|
||||
assert_eq!(result.mode, RefreshMode::Rebuild);
|
||||
assert_eq!(result.rows_written, 2);
|
||||
assert_eq!(read(reopened.table(), "id").await, vec![1, 3]);
|
||||
|
||||
// A fresh handle proves the migration was stored alongside the new
|
||||
// watermark and therefore happens only once.
|
||||
let migrated = conn.open_materialized_view("legacy_view").await.unwrap();
|
||||
assert_eq!(
|
||||
migrated.definition().filter.as_deref(),
|
||||
Some("`PartyAbbrev` = 'D'")
|
||||
);
|
||||
assert_eq!(migrated.definition().inputs, ["PartyAbbrev", "id"]);
|
||||
assert_eq!(
|
||||
migrated.refresh().execute().await.unwrap().mode,
|
||||
RefreshMode::NoOp
|
||||
);
|
||||
assert_eq!(read(migrated.table(), "id").await, vec![1, 3]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_append_refreshes_incrementally() {
|
||||
let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await;
|
||||
@@ -2903,7 +2767,7 @@ mod tests {
|
||||
let stale = view_native.dataset.get().await.unwrap().as_ref().clone();
|
||||
view.table().delete("x = 1").await.unwrap();
|
||||
|
||||
let err = stamp_watermark(view_native, stale, 99, 99, None, None).await;
|
||||
let err = stamp_watermark(view_native, stale, 99, 99, None).await;
|
||||
assert!(err.is_err());
|
||||
|
||||
let result = view.refresh().execute().await.unwrap();
|
||||
|
||||
+9
-273
@@ -399,9 +399,6 @@ pub trait QueryBase {
|
||||
/// x > 5 OR y = 'test'
|
||||
/// ```
|
||||
///
|
||||
/// Identifiers may be delimited with SQL-standard double quotes or
|
||||
/// backticks. String literals must use single quotes.
|
||||
///
|
||||
/// Filtering performance can often be improved by creating a scalar index
|
||||
/// on the filter column(s).
|
||||
///
|
||||
@@ -916,17 +913,6 @@ impl QueryRequest {
|
||||
/// use different representations) the error is recorded and surfaced later
|
||||
/// by [`Self::check_filter`].
|
||||
pub(crate) fn add_filter(&mut self, new: QueryFilter) {
|
||||
let new = match new {
|
||||
QueryFilter::Sql(filter) => match crate::expr::canonicalize_sql_predicate(&filter) {
|
||||
Ok(filter) => QueryFilter::Sql(filter),
|
||||
Err(err) => {
|
||||
self.filter_error = Some(err.to_string());
|
||||
return;
|
||||
}
|
||||
},
|
||||
other => other,
|
||||
};
|
||||
|
||||
self.filter = Some(match self.filter.take() {
|
||||
None => new,
|
||||
Some(existing) => match and_filters(existing, new) {
|
||||
@@ -1188,12 +1174,12 @@ impl VectorQuery {
|
||||
|
||||
/// Add another query vector to the search.
|
||||
///
|
||||
/// Multiple searches will be dispatched as a batch. Flat searches share
|
||||
/// one table scan across the query vectors, avoiding the scan and memory
|
||||
/// amplification of issuing the searches concurrently. Indexed searches
|
||||
/// may still perform per-vector index work.
|
||||
/// Multiple searches will be dispatched as part of the query.
|
||||
/// This is a convenience method for adding multiple query vectors
|
||||
/// to the search. It is not expected to be faster than issuing
|
||||
/// multiple queries concurrently.
|
||||
///
|
||||
/// The output data will contain an additional column `query_index` which
|
||||
/// The output data will contain an additional columns `query_index` which
|
||||
/// will contain the index of the query vector that was used to generate the
|
||||
/// result.
|
||||
pub fn add_query_vector(mut self, vector: impl IntoQueryVector) -> Result<Self> {
|
||||
@@ -1660,14 +1646,10 @@ mod tests {
|
||||
use std::{collections::HashSet, sync::Arc};
|
||||
|
||||
use super::*;
|
||||
use arrow::{
|
||||
array::downcast_array,
|
||||
compute::concat_batches,
|
||||
datatypes::{Int32Type, UInt8Type},
|
||||
};
|
||||
use arrow::{array::downcast_array, compute::concat_batches, datatypes::Int32Type};
|
||||
use arrow_array::{
|
||||
FixedSizeListArray, Float32Array, Int32Array, RecordBatch, RecordBatchIterator,
|
||||
StringArray, cast::AsArray, types::Float32Type,
|
||||
FixedSizeListArray, Float32Array, Int32Array, RecordBatch, StringArray, cast::AsArray,
|
||||
types::Float32Type,
|
||||
};
|
||||
use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
|
||||
use futures::{StreamExt, TryStreamExt};
|
||||
@@ -1896,157 +1878,6 @@ mod tests {
|
||||
query.execute().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_double_quoted_predicates_across_table_operations() {
|
||||
let tmp_dir = tempdir().unwrap();
|
||||
let dataset_path = tmp_dir.path().join("test.lance");
|
||||
let uri = dataset_path.to_str().unwrap();
|
||||
let schema = Arc::new(ArrowSchema::new(vec![
|
||||
ArrowField::new("id", DataType::Int32, false),
|
||||
ArrowField::new("PartyAbbrev", DataType::Utf8, false),
|
||||
ArrowField::new("path", DataType::Utf8, false),
|
||||
]));
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![
|
||||
Arc::new(Int32Array::from(vec![1, 2, 3, 4])),
|
||||
Arc::new(StringArray::from(vec!["D", "R", "R", "D"])),
|
||||
Arc::new(StringArray::from(vec!["\\", "\\", "x", "x"])),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let conn = connect(uri).execute().await.unwrap();
|
||||
let table = conn.create_table("parties", batch).execute().await.unwrap();
|
||||
let batches = table
|
||||
.query()
|
||||
.only_if(r#""PartyAbbrev" = 'D'"#)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap()
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 2);
|
||||
assert_eq!(
|
||||
table
|
||||
.count_rows(Some(r#""PartyAbbrev" = 'D'"#.to_string()))
|
||||
.await
|
||||
.unwrap(),
|
||||
2
|
||||
);
|
||||
|
||||
// Public BaseTable dispatch cannot bypass canonicalization.
|
||||
let query = AnyQuery::Query(QueryRequest {
|
||||
filter: Some(QueryFilter::Sql(r#""PartyAbbrev" = 'D'"#.to_string())),
|
||||
..Default::default()
|
||||
});
|
||||
let batches = table
|
||||
.base_table()
|
||||
.query(&query, Default::default())
|
||||
.await
|
||||
.unwrap()
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 2);
|
||||
assert_eq!(
|
||||
table
|
||||
.base_table()
|
||||
.count_rows(Some(crate::table::Filter::Sql(
|
||||
r#""PartyAbbrev" = 'D'"#.to_string(),
|
||||
)))
|
||||
.await
|
||||
.unwrap(),
|
||||
2
|
||||
);
|
||||
|
||||
for predicate in [
|
||||
r#"id = 1 -- unmatched " in a valid SQL comment"#,
|
||||
r#"id = 1 /* unmatched " in a valid SQL comment */"#,
|
||||
r#"id = 1 /*! OR "PartyAbbrev" = 'D' */"#,
|
||||
r#"path = '\' AND "PartyAbbrev" = 'D'"#,
|
||||
] {
|
||||
let batches = table
|
||||
.query()
|
||||
.only_if(predicate)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap()
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 1);
|
||||
}
|
||||
|
||||
// The same canonical predicate contract applies to both merge filters.
|
||||
let source = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![
|
||||
Arc::new(Int32Array::from(vec![1, 2, 3])),
|
||||
Arc::new(StringArray::from(vec!["D", "R", "R"])),
|
||||
Arc::new(StringArray::from(vec!["\\", "\\", "x"])),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
let mut merge = table.merge_insert(&["id"]);
|
||||
merge.when_not_matched_by_source_delete(Some(r#""PartyAbbrev" = 'D'"#.to_string()));
|
||||
let result = table
|
||||
.base_table()
|
||||
.merge_insert(
|
||||
merge,
|
||||
Box::new(RecordBatchIterator::new(vec![Ok(source)], schema.clone())),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.num_deleted_rows, 1);
|
||||
|
||||
let source = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![
|
||||
Arc::new(Int32Array::from(vec![1, 2, 3])),
|
||||
Arc::new(StringArray::from(vec!["U", "U", "U"])),
|
||||
Arc::new(StringArray::from(vec!["\\", "\\", "x"])),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
let mut merge = table.merge_insert(&["id"]);
|
||||
merge.when_matched_update_all(Some(r#"target."PartyAbbrev" = 'D'"#.to_string()));
|
||||
merge
|
||||
.execute(Box::new(RecordBatchIterator::new(vec![Ok(source)], schema)))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
table
|
||||
.count_rows(Some(r#""PartyAbbrev" = 'U'"#.to_string()))
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
|
||||
let update = table
|
||||
.update()
|
||||
.only_if(r#""PartyAbbrev" = 'R'"#)
|
||||
.column("PartyAbbrev", "'X'");
|
||||
table.base_table().update(update).await.unwrap();
|
||||
assert_eq!(
|
||||
table
|
||||
.count_rows(Some(r#""PartyAbbrev" = 'X'"#.to_string()))
|
||||
.await
|
||||
.unwrap(),
|
||||
2
|
||||
);
|
||||
|
||||
let result = table
|
||||
.base_table()
|
||||
.delete(crate::table::Predicate::String(r#""PartyAbbrev" = 'X'"#))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.num_deleted_rows, 2);
|
||||
assert_eq!(table.count_rows(None).await.unwrap(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_select_with_transform() {
|
||||
let batches = make_non_empty_batches();
|
||||
@@ -2503,8 +2334,7 @@ mod tests {
|
||||
.limit(1);
|
||||
|
||||
let plan = query.explain_plan(true).await.unwrap();
|
||||
assert!(plan.contains("KNNVectorDistance: queries=2"));
|
||||
assert!(!plan.contains("UnionExec"));
|
||||
assert!(plan.contains("UnionExec"));
|
||||
|
||||
let results = query
|
||||
.execute()
|
||||
@@ -2519,100 +2349,6 @@ mod tests {
|
||||
// We don't guarantee order.
|
||||
assert!(query_index.values().contains(&0));
|
||||
assert!(query_index.values().contains(&1));
|
||||
|
||||
// Batch KNN does not support a per-query offset, so offset queries keep
|
||||
// the legacy per-vector plan to preserve their result semantics.
|
||||
let offset_query = table
|
||||
.query()
|
||||
.nearest_to(&[0.1, 0.2, 0.3, 0.4])
|
||||
.unwrap()
|
||||
.add_query_vector(&[0.5, 0.6, 0.7, 0.8])
|
||||
.unwrap()
|
||||
.limit(1)
|
||||
.offset(1);
|
||||
assert!(
|
||||
offset_query
|
||||
.explain_plan(true)
|
||||
.await
|
||||
.unwrap()
|
||||
.contains("UnionExec")
|
||||
);
|
||||
let offset_results = offset_query
|
||||
.execute()
|
||||
.await
|
||||
.unwrap()
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
offset_results
|
||||
.iter()
|
||||
.map(RecordBatch::num_rows)
|
||||
.sum::<usize>(),
|
||||
2
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_multiple_binary_query_vectors() {
|
||||
let vectors = FixedSizeListArray::from_iter_primitive::<UInt8Type, _, _>(
|
||||
vec![
|
||||
Some(vec![Some(0), Some(0)]),
|
||||
Some(vec![Some(255), Some(255)]),
|
||||
],
|
||||
2,
|
||||
);
|
||||
let schema = Arc::new(ArrowSchema::new(vec![
|
||||
ArrowField::new("id", DataType::Int32, false),
|
||||
ArrowField::new("vector", vectors.data_type().clone(), false),
|
||||
]));
|
||||
let batch = RecordBatch::try_new(
|
||||
schema,
|
||||
vec![Arc::new(Int32Array::from(vec![0, 1])), Arc::new(vectors)],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let table = conn
|
||||
.create_table("binary_batch", batch)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
let query = table
|
||||
.query()
|
||||
.nearest_to(&[0.0, 0.0])
|
||||
.unwrap()
|
||||
.add_query_vector(&[255.0, 255.0])
|
||||
.unwrap()
|
||||
.distance_type(DistanceType::Hamming)
|
||||
.limit(1);
|
||||
|
||||
// Binary queries retain the per-vector plan because Lance's binary
|
||||
// nearest path requires primitive UInt8 query arrays.
|
||||
assert!(
|
||||
query
|
||||
.explain_plan(true)
|
||||
.await
|
||||
.unwrap()
|
||||
.contains("UnionExec")
|
||||
);
|
||||
|
||||
let results = query
|
||||
.execute()
|
||||
.await
|
||||
.unwrap()
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.unwrap();
|
||||
let results = concat_batches(&results[0].schema(), &results).unwrap();
|
||||
assert_eq!(results.num_rows(), 2);
|
||||
|
||||
let ids = results["id"].as_primitive::<Int32Type>();
|
||||
assert!(ids.values().contains(&0));
|
||||
assert!(ids.values().contains(&1));
|
||||
let query_index = results["query_index"].as_primitive::<Int32Type>();
|
||||
assert!(query_index.values().contains(&0));
|
||||
assert!(query_index.values().contains(&1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -87,10 +87,6 @@ impl ServerVersion {
|
||||
pub fn support_blobs(&self) -> bool {
|
||||
self.0 >= semver::Version::new(0, 5, 0)
|
||||
}
|
||||
|
||||
pub fn support_fts_document_granularity(&self) -> bool {
|
||||
self.0 >= semver::Version::new(0, 6, 0)
|
||||
}
|
||||
}
|
||||
|
||||
pub const OPT_REMOTE_PREFIX: &str = "remote_database_";
|
||||
|
||||
+342
-1510
File diff suppressed because it is too large
Load Diff
@@ -6,7 +6,6 @@
|
||||
use std::ops::Range;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use arrow_array::{Array, LargeBinaryArray};
|
||||
use arrow_schema::DataType;
|
||||
@@ -21,7 +20,7 @@ use crate::error::Result;
|
||||
use crate::remote::client::{HttpSend, RequestResultExt, RestfulLanceDbClient};
|
||||
use crate::table::BaseTable;
|
||||
|
||||
use super::{FreshnessHeaders, FreshnessState, RemoteTable, freshness_headers_snapshot};
|
||||
use super::{FreshnessHeaders, RemoteTable};
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum RangeRequestMode {
|
||||
@@ -44,10 +43,7 @@ struct TableBlobRangeRequester<S: HttpSend> {
|
||||
path: String,
|
||||
version: Option<u64>,
|
||||
branch: Option<String>,
|
||||
freshness: Arc<std::sync::Mutex<FreshnessState>>,
|
||||
parent_freshness: Arc<std::sync::Mutex<FreshnessState>>,
|
||||
parent_freshness_request: FreshnessHeaders,
|
||||
read_consistency_interval: Option<Duration>,
|
||||
freshness: FreshnessHeaders,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -57,9 +53,8 @@ impl<S: HttpSend> BlobRangeRequester for TableBlobRangeRequester<S> {
|
||||
range_header: &str,
|
||||
mode: RangeRequestMode,
|
||||
) -> Result<(String, Response)> {
|
||||
let freshness_request =
|
||||
freshness_headers_snapshot(&self.freshness, self.read_consistency_interval);
|
||||
let mut request = freshness_request
|
||||
let mut request = self
|
||||
.freshness
|
||||
.apply(self.client.get(&self.path))
|
||||
.header(header::RANGE, range_header);
|
||||
if let Some(version) = self.version {
|
||||
@@ -76,9 +71,6 @@ impl<S: HttpSend> BlobRangeRequester for TableBlobRangeRequester<S> {
|
||||
return Ok((request_id, response));
|
||||
}
|
||||
let response = self.client.check_response(&request_id, response).await?;
|
||||
freshness_request.observe_headers(&self.freshness, response.headers());
|
||||
self.parent_freshness_request
|
||||
.observe_headers(&self.parent_freshness, response.headers());
|
||||
Ok((request_id, response))
|
||||
}
|
||||
}
|
||||
@@ -369,21 +361,18 @@ impl<S: HttpSend> RemoteTable<S> {
|
||||
message: "fetch_blobs is not supported on this LanceDB Cloud server".into(),
|
||||
});
|
||||
}
|
||||
let read_snapshot = self.snapshot_read_state().await;
|
||||
let version = self.current_version().await;
|
||||
let mut body = serde_json::json!({
|
||||
"version": read_snapshot.version,
|
||||
"version": version,
|
||||
"column": column,
|
||||
"row_ids": row_ids,
|
||||
});
|
||||
self.apply_branch_body(&mut body);
|
||||
|
||||
let request = self
|
||||
.client
|
||||
.post(&format!("/v1/table/{}/fetch_blobs/", self.identifier))
|
||||
.post_read(&format!("/v1/table/{}/fetch_blobs/", self.identifier))
|
||||
.json(&body);
|
||||
let (request_id, response) = self
|
||||
.send_with_freshness(request, true, read_snapshot.freshness)
|
||||
.await?;
|
||||
let (request_id, response) = self.send(request, true).await?;
|
||||
let mut stream = self.read_arrow_response(&request_id, response).await?;
|
||||
|
||||
let mut blob_chunks: Vec<Arc<dyn Array>> = Vec::new();
|
||||
@@ -459,7 +448,8 @@ impl<S: HttpSend> RemoteTable<S> {
|
||||
});
|
||||
}
|
||||
|
||||
let read_snapshot = self.snapshot_read_state().await;
|
||||
let version = self.current_version().await;
|
||||
let freshness = self.snapshot_freshness_headers();
|
||||
let encoded_column = urlencoding::encode(column);
|
||||
let requesters = row_ids
|
||||
.iter()
|
||||
@@ -471,12 +461,9 @@ impl<S: HttpSend> RemoteTable<S> {
|
||||
let requester: Arc<dyn BlobRangeRequester> = Arc::new(TableBlobRangeRequester {
|
||||
client: self.client.clone(),
|
||||
path,
|
||||
version: read_snapshot.version,
|
||||
version,
|
||||
branch: self.branch.clone(),
|
||||
freshness: Arc::new(std::sync::Mutex::new(read_snapshot.freshness_state)),
|
||||
parent_freshness: self.freshness.clone(),
|
||||
parent_freshness_request: read_snapshot.freshness,
|
||||
read_consistency_interval: self.client.read_consistency_interval,
|
||||
freshness,
|
||||
});
|
||||
requester
|
||||
})
|
||||
@@ -698,46 +685,6 @@ mod tests {
|
||||
assert!(requests.lock().unwrap().contains(&"bytes=5-11".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_blob_file_keeps_the_open_timeline_after_parent_checkout() {
|
||||
let range_requests = Arc::new(StdMutex::new(Vec::new()));
|
||||
let captured = range_requests.clone();
|
||||
let table = RemoteTable::new_mock(
|
||||
"my_table".to_string(),
|
||||
move |request| match request.url().path() {
|
||||
"/v1/table/my_table/describe/" => http::Response::builder()
|
||||
.status(200)
|
||||
.body(r#"{"version":5,"schema":{"fields":[]}}"#.as_bytes().to_vec())
|
||||
.unwrap(),
|
||||
"/v1/table/my_table/blob/image/10/bytes" => {
|
||||
captured.lock().unwrap().push((
|
||||
request.url().query().unwrap_or_default().to_string(),
|
||||
request.headers().clone(),
|
||||
));
|
||||
range_response(&request, PAYLOAD)
|
||||
}
|
||||
path => panic!("unexpected path: {path}"),
|
||||
},
|
||||
Some(Version::new(0, 5, 0)),
|
||||
);
|
||||
|
||||
table.checkout(5).await.unwrap();
|
||||
let file = table
|
||||
.fetch_blob_files_impl("image", &[10])
|
||||
.await
|
||||
.unwrap()
|
||||
.pop()
|
||||
.flatten()
|
||||
.unwrap();
|
||||
table.checkout_latest().await.unwrap();
|
||||
file.read_range(5..12).await.unwrap();
|
||||
|
||||
let requests = range_requests.lock().unwrap();
|
||||
let (query, headers) = requests.last().unwrap();
|
||||
assert!(query.contains("version=5"));
|
||||
assert!(!headers.contains_key("x-lancedb-min-timestamp"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_blob_file_reuses_sequential_response_until_seek() {
|
||||
let requests = Arc::new(StdMutex::new(Vec::new()));
|
||||
|
||||
@@ -24,10 +24,7 @@ use lance::io::exec::utils::InstrumentedRecordBatchStreamAdapter;
|
||||
use crate::Error;
|
||||
use crate::remote::ARROW_STREAM_CONTENT_TYPE;
|
||||
use crate::remote::client::{HttpSend, RestfulLanceDbClient, Sender};
|
||||
use crate::remote::table::{
|
||||
FreshnessHeaders, FreshnessState, MergeInsertRequest, REQUEST_TIMEOUT_HEADER, RemoteTable,
|
||||
freshness_headers_snapshot,
|
||||
};
|
||||
use crate::remote::table::{MergeInsertRequest, REQUEST_TIMEOUT_HEADER, RemoteTable};
|
||||
use crate::table::datafusion::insert::COUNT_SCHEMA;
|
||||
use crate::table::write_progress::WriteProgressTracker;
|
||||
use crate::table::{AddResult, MergeResult};
|
||||
@@ -57,38 +54,6 @@ pub enum WriteResult {
|
||||
Merge(MergeResult),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
struct WriteFreshness {
|
||||
state: Option<Arc<Mutex<FreshnessState>>>,
|
||||
read_consistency_interval: Option<Duration>,
|
||||
}
|
||||
|
||||
impl WriteFreshness {
|
||||
fn prepare(
|
||||
&self,
|
||||
request: reqwest::RequestBuilder,
|
||||
) -> (reqwest::RequestBuilder, Option<FreshnessHeaders>) {
|
||||
match &self.state {
|
||||
Some(state) => {
|
||||
let freshness_request =
|
||||
freshness_headers_snapshot(state, self.read_consistency_interval);
|
||||
(freshness_request.apply(request), Some(freshness_request))
|
||||
}
|
||||
None => (request, None),
|
||||
}
|
||||
}
|
||||
|
||||
fn observe(
|
||||
&self,
|
||||
freshness_request: Option<FreshnessHeaders>,
|
||||
headers: &reqwest::header::HeaderMap,
|
||||
) {
|
||||
if let (Some(state), Some(freshness_request)) = (&self.state, freshness_request) {
|
||||
freshness_request.observe_headers(state, headers);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// ExecutionPlan for streaming a write (add or merge_insert) to a remote
|
||||
/// LanceDB table.
|
||||
///
|
||||
@@ -106,7 +71,6 @@ pub struct RemoteWriteExec<S: HttpSend = Sender> {
|
||||
table_name: String,
|
||||
identifier: String,
|
||||
client: RestfulLanceDbClient<S>,
|
||||
freshness: WriteFreshness,
|
||||
input: Arc<dyn ExecutionPlan>,
|
||||
op: WriteOp,
|
||||
properties: Arc<PlanProperties>,
|
||||
@@ -206,7 +170,6 @@ impl<S: HttpSend + 'static> RemoteWriteExec<S> {
|
||||
table_name,
|
||||
identifier,
|
||||
client,
|
||||
freshness: WriteFreshness::default(),
|
||||
input,
|
||||
op,
|
||||
properties: Arc::new(properties),
|
||||
@@ -220,18 +183,6 @@ impl<S: HttpSend + 'static> RemoteWriteExec<S> {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn with_freshness(
|
||||
mut self,
|
||||
state: Arc<Mutex<FreshnessState>>,
|
||||
read_consistency_interval: Option<Duration>,
|
||||
) -> Self {
|
||||
self.freshness = WriteFreshness {
|
||||
state: Some(state),
|
||||
read_consistency_interval,
|
||||
};
|
||||
self
|
||||
}
|
||||
|
||||
/// Get the add result after execution, if this exec ran an insert.
|
||||
pub fn add_result(&self) -> Option<AddResult> {
|
||||
match self
|
||||
@@ -334,7 +285,6 @@ impl<S: HttpSend + 'static> RemoteWriteExec<S> {
|
||||
/// each threading the same handful of arguments.
|
||||
struct PartRequestCtx<'a, S: HttpSend> {
|
||||
client: &'a RestfulLanceDbClient<S>,
|
||||
freshness: &'a WriteFreshness,
|
||||
identifier: &'a str,
|
||||
table_name: &'a str,
|
||||
upload_id: &'a str,
|
||||
@@ -402,11 +352,7 @@ impl<S: HttpSend + 'static> PartRequestCtx<'_, S> {
|
||||
}
|
||||
|
||||
/// Build the `/insert` request for a single multipart part.
|
||||
fn build_part_request(
|
||||
&self,
|
||||
part_id: &str,
|
||||
body: reqwest::Body,
|
||||
) -> (reqwest::RequestBuilder, Option<FreshnessHeaders>) {
|
||||
fn build_part_request(&self, part_id: &str, body: reqwest::Body) -> reqwest::RequestBuilder {
|
||||
let mut request = self
|
||||
.client
|
||||
.post(&format!("/v1/table/{}/insert/", self.identifier))
|
||||
@@ -422,16 +368,12 @@ impl<S: HttpSend + 'static> PartRequestCtx<'_, S> {
|
||||
if let Some(b) = self.branch {
|
||||
request = request.query(&[("branch", b)]);
|
||||
}
|
||||
self.freshness.prepare(request.body(body))
|
||||
request.body(body)
|
||||
}
|
||||
|
||||
/// Send a single part's request and drain the response, mapping HTTP and
|
||||
/// table-not-found errors into `DataFusionError`.
|
||||
async fn send_part_request(
|
||||
&self,
|
||||
request: reqwest::RequestBuilder,
|
||||
freshness_request: Option<FreshnessHeaders>,
|
||||
) -> DataFusionResult<()> {
|
||||
async fn send_part_request(&self, request: reqwest::RequestBuilder) -> DataFusionResult<()> {
|
||||
let (request_id, response) = self
|
||||
.client
|
||||
.send(request)
|
||||
@@ -446,8 +388,6 @@ impl<S: HttpSend + 'static> PartRequestCtx<'_, S> {
|
||||
.check_response(&request_id, response)
|
||||
.await
|
||||
.map_err(|e| DataFusionError::External(Box::new(e)))?;
|
||||
self.freshness
|
||||
.observe(freshness_request, response.headers());
|
||||
response.bytes().await.map_err(|e| {
|
||||
DataFusionError::External(Box::new(Error::Http {
|
||||
source: Box::new(e),
|
||||
@@ -479,7 +419,7 @@ impl<S: HttpSend + 'static> PartRequestCtx<'_, S> {
|
||||
let body = reqwest::Body::wrap_stream(chunk_rx);
|
||||
|
||||
let part_id = uuid::Uuid::new_v4().to_string();
|
||||
let (request, freshness_request) = self.build_part_request(&part_id, body);
|
||||
let request = self.build_part_request(&part_id, body);
|
||||
|
||||
// Measured from just before the request is sent, matching the window the
|
||||
// client read timeout applies to the upload.
|
||||
@@ -555,7 +495,7 @@ impl<S: HttpSend + 'static> PartRequestCtx<'_, S> {
|
||||
Ok::<bool, DataFusionError>(input_ended)
|
||||
};
|
||||
|
||||
let send = self.send_part_request(request, freshness_request);
|
||||
let send = self.send_part_request(request);
|
||||
|
||||
// `join!` rather than `tokio::spawn`: the producer borrows `input` (and
|
||||
// `schema`), so it cannot satisfy the `'static` bound a spawned task
|
||||
@@ -629,7 +569,7 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
|
||||
// Building a fresh exec (with a new, empty `result`) is what makes the
|
||||
// outer rescannable retry loop work: `reset_state()` clears the captured
|
||||
// result so a re-execution starts clean.
|
||||
let mut exec = Self::new_inner(
|
||||
Ok(Arc::new(Self::new_inner(
|
||||
self.table_name.clone(),
|
||||
self.identifier.clone(),
|
||||
self.client.clone(),
|
||||
@@ -640,9 +580,7 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
|
||||
self.branch.clone(),
|
||||
self.max_bytes_per_request,
|
||||
self.max_request_duration,
|
||||
);
|
||||
exec.freshness = self.freshness.clone();
|
||||
Ok(Arc::new(exec))
|
||||
)))
|
||||
}
|
||||
|
||||
fn execute(
|
||||
@@ -675,7 +613,6 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
|
||||
&self.metrics,
|
||||
));
|
||||
let client = self.client.clone();
|
||||
let freshness = self.freshness.clone();
|
||||
let identifier = self.identifier.clone();
|
||||
let op = self.op.clone();
|
||||
let result_slot = self.result.clone();
|
||||
@@ -697,7 +634,6 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
|
||||
let overwrite = matches!(op, WriteOp::Insert { overwrite: true });
|
||||
let ctx = PartRequestCtx {
|
||||
client: &client,
|
||||
freshness: &freshness,
|
||||
identifier: &identifier,
|
||||
table_name: &table_name,
|
||||
upload_id,
|
||||
@@ -752,7 +688,7 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
|
||||
|
||||
let (error_tx, mut error_rx) = tokio::sync::oneshot::channel();
|
||||
let body = Self::stream_as_http_body(input_stream, error_tx, tracker)?;
|
||||
let (request, freshness_request) = freshness.prepare(request.body(body));
|
||||
let request = request.body(body);
|
||||
|
||||
let result: DataFusionResult<(String, _)> = async {
|
||||
let (request_id, response) = client
|
||||
@@ -772,7 +708,6 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
|
||||
.check_response(&request_id, response)
|
||||
.await
|
||||
.map_err(|e| DataFusionError::External(Box::new(e)))?;
|
||||
freshness.observe(freshness_request, response.headers());
|
||||
|
||||
Ok((request_id, response))
|
||||
}
|
||||
|
||||
+7
-123
@@ -59,9 +59,7 @@ use crate::index::{IndexConfig, IndexStatisticsImpl, IndexType};
|
||||
use crate::job::Job;
|
||||
use crate::query::{IntoQueryVector, Query, QueryExecutionOptions, TakeQuery, VectorQuery};
|
||||
use crate::table::datafusion::insert::InsertExec;
|
||||
use crate::utils::{
|
||||
PatchReadParam, PatchWriteParam, public_fts_field_path_by_id, resolve_arrow_field_path,
|
||||
};
|
||||
use crate::utils::{PatchReadParam, PatchWriteParam, resolve_arrow_field_path};
|
||||
|
||||
use self::dataset::DatasetConsistencyWrapper;
|
||||
use self::merge::MergeInsertBuilder;
|
||||
@@ -562,13 +560,6 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
|
||||
fn id(&self) -> &str;
|
||||
/// Get the arrow [Schema] of the table.
|
||||
async fn schema(&self) -> Result<SchemaRef>;
|
||||
/// Create a read-only handle pinned to the table's current active revision.
|
||||
///
|
||||
/// The returned handle is independent from later refreshes or checkouts on
|
||||
/// this handle. This is used by bindings that must prepare client-side
|
||||
/// query state from the same revision that the query will execute against.
|
||||
#[doc(hidden)]
|
||||
async fn query_snapshot(&self) -> Result<Arc<dyn BaseTable>>;
|
||||
/// Count the number of rows in this table.
|
||||
async fn count_rows(&self, filter: Option<Filter>) -> Result<usize>;
|
||||
/// Create a physical plan for the query.
|
||||
@@ -794,12 +785,6 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
|
||||
async fn drop_columns(&self, columns: &[&str]) -> Result<DropColumnsResult>;
|
||||
/// Get the version of the table.
|
||||
async fn version(&self) -> Result<u64>;
|
||||
/// Return a new table handle pinned to the exact revision currently visible.
|
||||
async fn checkout_current(&self) -> Result<Arc<dyn BaseTable>> {
|
||||
Err(Error::NotSupported {
|
||||
message: "checkout_current is not supported on this table type".into(),
|
||||
})
|
||||
}
|
||||
/// Checkout a specific version of the table.
|
||||
async fn checkout(&self, version: u64) -> Result<()>;
|
||||
/// Checkout a table version referenced by a tag.
|
||||
@@ -1148,26 +1133,13 @@ impl Table {
|
||||
self.inner.schema().await
|
||||
}
|
||||
|
||||
/// Create a read-only handle pinned to the current active revision.
|
||||
#[doc(hidden)]
|
||||
pub async fn query_snapshot(&self) -> Result<Self> {
|
||||
Ok(Self {
|
||||
inner: self.inner.query_snapshot().await?,
|
||||
database: self.database.clone(),
|
||||
embedding_registry: self.embedding_registry.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Count the number of rows in this dataset.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `filter` if present, only count rows matching the filter
|
||||
pub async fn count_rows(&self, filter: Option<String>) -> Result<usize> {
|
||||
let filter = filter
|
||||
.map(|predicate| crate::expr::canonicalize_sql_predicate(&predicate).map(Filter::Sql))
|
||||
.transpose()?;
|
||||
self.inner.count_rows(filter).await
|
||||
self.inner.count_rows(filter.map(Filter::Sql)).await
|
||||
}
|
||||
|
||||
/// Names of the blob v2 columns in this table, in declaration order.
|
||||
@@ -1367,13 +1339,7 @@ impl Table {
|
||||
/// # });
|
||||
/// ```
|
||||
pub async fn delete(&self, predicate: impl Into<Predicate<'_>>) -> Result<DeleteResult> {
|
||||
match predicate.into() {
|
||||
Predicate::String(predicate) => {
|
||||
let predicate = crate::expr::canonicalize_sql_predicate(predicate)?;
|
||||
self.inner.delete(Predicate::String(&predicate)).await
|
||||
}
|
||||
predicate @ Predicate::Expr(_) => self.inner.delete(predicate).await,
|
||||
}
|
||||
self.inner.delete(predicate.into()).await
|
||||
}
|
||||
|
||||
/// Create an index on the provided column(s).
|
||||
@@ -1786,23 +1752,7 @@ impl Table {
|
||||
self.inner.alter_columns(alterations).await
|
||||
}
|
||||
|
||||
/// Update per-field (column) metadata.
|
||||
///
|
||||
/// Each [`FieldMetadataUpdate`] is merged into the field's existing metadata
|
||||
/// by default; use [`FieldMetadataUpdate::remove`] to delete a key, or
|
||||
/// [`FieldMetadataUpdate::replace`] to swap the field's entire metadata map.
|
||||
///
|
||||
/// The following keys are treated specially, by convention, and should be
|
||||
/// used when appropriate:
|
||||
///
|
||||
/// - `lancedb:description`: for a human-readable description of a field.
|
||||
/// - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
|
||||
/// names the tag category; e.g. `lancedb:tag:model: "clip"`.
|
||||
/// - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
|
||||
/// `feature_v2` might be in the same logical column.
|
||||
/// - `lancedb:status`: for status options (`production`, `candidate`,
|
||||
/// `deprecated`, `archived`) to designate the current life cycle state of
|
||||
/// this column.
|
||||
/// Update per-field metadata (merges by default).
|
||||
pub async fn update_field_metadata(
|
||||
&self,
|
||||
updates: &[FieldMetadataUpdate],
|
||||
@@ -1994,20 +1944,6 @@ impl Table {
|
||||
self.inner.version().await
|
||||
}
|
||||
|
||||
/// Return a new table handle pinned to the exact revision currently visible.
|
||||
///
|
||||
/// This is used when asynchronous preparation must remain consistent with
|
||||
/// the revision used for a later read.
|
||||
#[doc(hidden)]
|
||||
pub async fn checkout_current(&self) -> Result<Self> {
|
||||
let inner = self.inner.checkout_current().await?;
|
||||
Ok(Self {
|
||||
inner,
|
||||
database: self.database.clone(),
|
||||
embedding_registry: self.embedding_registry.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Checks out a specific version of the Table
|
||||
///
|
||||
/// Any read operation on the table will now access the data at the checked out version.
|
||||
@@ -3103,33 +3039,10 @@ impl BaseTable for NativeTable {
|
||||
&self.id
|
||||
}
|
||||
|
||||
async fn query_snapshot(&self) -> Result<Arc<dyn BaseTable>> {
|
||||
let snapshot = self.dataset.new_query_snapshot().await?;
|
||||
let mut table = self.with_dataset(snapshot);
|
||||
// QueryTable requests do not carry a revision. A pinned snapshot must
|
||||
// execute locally until the namespace API can accept that revision.
|
||||
table
|
||||
.pushdown_operations
|
||||
.remove(&NamespaceClientPushdownOperation::QueryTable);
|
||||
Ok(Arc::new(table))
|
||||
}
|
||||
|
||||
async fn version(&self) -> Result<u64> {
|
||||
Ok(self.dataset.get().await?.version().version)
|
||||
}
|
||||
|
||||
async fn checkout_current(&self) -> Result<Arc<dyn BaseTable>> {
|
||||
let current = self.dataset.get().await?;
|
||||
let dataset = dataset::DatasetConsistencyWrapper::new_time_travel(
|
||||
current.as_ref().clone(),
|
||||
self.read_consistency_interval,
|
||||
);
|
||||
Ok(Arc::new(Self {
|
||||
dataset,
|
||||
..self.clone()
|
||||
}))
|
||||
}
|
||||
|
||||
async fn checkout(&self, version: u64) -> Result<()> {
|
||||
self.dataset.as_time_travel(version).await
|
||||
}
|
||||
@@ -3248,10 +3161,7 @@ impl BaseTable for NativeTable {
|
||||
let dataset = self.dataset.get().await?;
|
||||
match filter {
|
||||
None => Ok(dataset.count_rows(None).await?),
|
||||
Some(Filter::Sql(sql)) => {
|
||||
let sql = crate::expr::canonicalize_sql_predicate(&sql)?;
|
||||
Ok(dataset.count_rows(Some(sql)).await?)
|
||||
}
|
||||
Some(Filter::Sql(sql)) => Ok(dataset.count_rows(Some(sql)).await?),
|
||||
Some(Filter::Datafusion(_)) => Err(Error::NotSupported {
|
||||
message: "Datafusion filters are not yet supported".to_string(),
|
||||
}),
|
||||
@@ -3298,7 +3208,7 @@ impl BaseTable for NativeTable {
|
||||
|
||||
let output = add.into_plan(&table_schema, &table_def)?;
|
||||
|
||||
let mut lance_params = output
|
||||
let lance_params = output
|
||||
.write_options
|
||||
.lance_write_params
|
||||
.unwrap_or(WriteParams {
|
||||
@@ -3308,9 +3218,6 @@ impl BaseTable for NativeTable {
|
||||
},
|
||||
..Default::default()
|
||||
});
|
||||
if output.allow_external_blob_outside_bases {
|
||||
lance_params.allow_external_blob_outside_bases = true;
|
||||
}
|
||||
|
||||
// Repartition for write parallelism if beneficial.
|
||||
let plan = if num_partitions > 1 {
|
||||
@@ -3589,14 +3496,7 @@ impl BaseTable for NativeTable {
|
||||
let field_ids = idx_desc.field_ids();
|
||||
let mut columns = Vec::with_capacity(field_ids.len());
|
||||
for field_id in field_ids {
|
||||
let field_path = match if index_type == crate::index::IndexType::FTS {
|
||||
public_fts_field_path_by_id(dataset.schema(), *field_id as i32)
|
||||
} else {
|
||||
dataset
|
||||
.schema()
|
||||
.field_path(*field_id as i32)
|
||||
.map_err(Into::into)
|
||||
} {
|
||||
let field_path = match dataset.schema().field_path(*field_id as i32) {
|
||||
Ok(field_path) => field_path,
|
||||
Err(e) => {
|
||||
log::warn!(
|
||||
@@ -4183,14 +4083,6 @@ mod tests {
|
||||
parent_list_calls: self.parent_list_calls.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
fn wrap_paginated(
|
||||
&self,
|
||||
_store_prefix: &str,
|
||||
_original: Arc<dyn object_store::list::PaginatedListStore>,
|
||||
) -> Option<Arc<dyn object_store::list::PaginatedListStore>> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -4294,14 +4186,6 @@ mod tests {
|
||||
self.called.store(true, Ordering::Relaxed);
|
||||
original
|
||||
}
|
||||
|
||||
fn wrap_paginated(
|
||||
&self,
|
||||
_store_prefix: &str,
|
||||
original: Arc<dyn object_store::list::PaginatedListStore>,
|
||||
) -> Option<Arc<dyn object_store::list::PaginatedListStore>> {
|
||||
Some(original)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -60,7 +60,6 @@ pub struct AddDataBuilder {
|
||||
pub(crate) embedding_registry: Option<Arc<dyn EmbeddingRegistry>>,
|
||||
pub(crate) progress_callback: Option<ProgressCallback>,
|
||||
pub(crate) write_parallelism: Option<usize>,
|
||||
pub(crate) allow_external_blob_outside_bases: bool,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for AddDataBuilder {
|
||||
@@ -88,7 +87,6 @@ impl AddDataBuilder {
|
||||
embedding_registry,
|
||||
progress_callback: None,
|
||||
write_parallelism: None,
|
||||
allow_external_blob_outside_bases: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -143,16 +141,6 @@ impl AddDataBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Store blob URIs that sit outside registered blob bases.
|
||||
///
|
||||
/// The row keeps a reference, so the object has to stay readable.
|
||||
/// [`crate::table::Table::fetch_blobs`] reads from that location.
|
||||
/// Defaults to `false`. Local tables only.
|
||||
pub fn allow_external_blob_outside_bases(mut self, allow: bool) -> Self {
|
||||
self.allow_external_blob_outside_bases = allow;
|
||||
self
|
||||
}
|
||||
|
||||
pub async fn execute(self) -> Result<AddResult> {
|
||||
if self.write_parallelism.map(|p| p == 0).unwrap_or(false) {
|
||||
return Err(Error::InvalidInput {
|
||||
@@ -211,7 +199,6 @@ impl AddDataBuilder {
|
||||
write_options: self.write_options,
|
||||
mode: self.mode,
|
||||
tracker,
|
||||
allow_external_blob_outside_bases: self.allow_external_blob_outside_bases,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -225,7 +212,6 @@ pub struct PreprocessingOutput {
|
||||
pub write_options: WriteOptions,
|
||||
pub mode: AddDataMode,
|
||||
pub tracker: Option<Arc<WriteProgressTracker>>,
|
||||
pub allow_external_blob_outside_bases: bool,
|
||||
}
|
||||
|
||||
/// Check that the input schema is valid for insert.
|
||||
|
||||
@@ -28,9 +28,8 @@ pub(super) type PreparedIndex = (String, Box<dyn lance::index::IndexParams>, Ind
|
||||
use crate::index::Index;
|
||||
use crate::index::vector::{VectorIndex, suggested_num_sub_vectors};
|
||||
use crate::utils::{
|
||||
resolve_lance_fts_field_path, supported_bitmap_data_type, supported_btree_data_type,
|
||||
supported_fm_data_type, supported_fts_data_type, supported_label_list_data_type,
|
||||
supported_vector_data_type,
|
||||
supported_bitmap_data_type, supported_btree_data_type, supported_fm_data_type,
|
||||
supported_fts_data_type, supported_label_list_data_type, supported_vector_data_type,
|
||||
};
|
||||
|
||||
use super::NativeTable;
|
||||
@@ -123,20 +122,7 @@ impl NativeTable {
|
||||
}
|
||||
self.dataset.ensure_mutable()?;
|
||||
let dataset = self.dataset.get().await?;
|
||||
let (column, field) = if let Index::FTS(params) = &opts.index {
|
||||
let resolved = resolve_lance_fts_field_path(dataset.schema(), &opts.columns[0])?;
|
||||
if params.get_document_granularity().is_list_element() && resolved.list_depth == 0 {
|
||||
return Err(Error::InvalidInput {
|
||||
message: format!(
|
||||
"FTS field path '{}' has no List layer and cannot use ListElement document granularity",
|
||||
resolved.canonical_path
|
||||
),
|
||||
});
|
||||
}
|
||||
(resolved.canonical_path, resolved.field)
|
||||
} else {
|
||||
Self::resolve_index_field(dataset.schema(), &opts.columns[0])?
|
||||
};
|
||||
let (column, field) = Self::resolve_index_field(dataset.schema(), &opts.columns[0])?;
|
||||
let params = self.make_index_params(&field, opts.index.clone()).await?;
|
||||
let index_type = self.get_index_type_for_field(&field, &opts.index);
|
||||
Ok((column, params, index_type))
|
||||
@@ -450,7 +436,7 @@ mod tests {
|
||||
use crate::connection::ConnectBuilder;
|
||||
use crate::index::Index;
|
||||
use crate::index::scalar::{
|
||||
BTreeIndexBuilder, BitmapIndexBuilder, DocumentGranularity, FmIndexBuilder, FtsIndexBuilder,
|
||||
BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder, FtsIndexBuilder,
|
||||
};
|
||||
use crate::index::vector::{
|
||||
IvfHnswFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder,
|
||||
@@ -567,38 +553,6 @@ mod tests {
|
||||
job.cancel().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_async_validates_fts_input_before_starting_job() {
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let batch =
|
||||
record_batch!(("id", Int32, [1, 2]), ("text", Utf8, ["alpha", "beta"])).unwrap();
|
||||
let table = conn.create_table("t", batch).execute().await.unwrap();
|
||||
|
||||
let missing = table
|
||||
.create_index(&["missing"], Index::FTS(FtsIndexBuilder::default()))
|
||||
.execute_async()
|
||||
.await;
|
||||
assert!(missing.is_err());
|
||||
|
||||
let invalid_type = table
|
||||
.create_index(&["id"], Index::FTS(FtsIndexBuilder::default()))
|
||||
.execute_async()
|
||||
.await;
|
||||
assert!(invalid_type.is_err());
|
||||
|
||||
let invalid_granularity = table
|
||||
.create_index(
|
||||
&["text"],
|
||||
Index::FTS(
|
||||
FtsIndexBuilder::default()
|
||||
.document_granularity(DocumentGranularity::ListElement),
|
||||
),
|
||||
)
|
||||
.execute_async()
|
||||
.await;
|
||||
assert!(invalid_granularity.is_err());
|
||||
}
|
||||
|
||||
/// Concurrent waiters, and a wait issued after the job settled, all
|
||||
/// succeed once the build does.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_schema::{DataType, Field, FieldRef, Fields};
|
||||
use arrow_schema::{DataType, Field, FieldRef};
|
||||
use datafusion::functions::core::{get_field, named_struct};
|
||||
use datafusion_common::ScalarValue;
|
||||
use datafusion_common::config::ConfigOptions;
|
||||
@@ -35,9 +35,8 @@ pub(super) fn coerce_blob_expr(
|
||||
});
|
||||
};
|
||||
|
||||
let input_shape = match input_field.data_type() {
|
||||
DataType::Binary | DataType::LargeBinary | DataType::BinaryView => BlobInputShape::Bytes,
|
||||
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => BlobInputShape::String,
|
||||
let input_struct_children = match input_field.data_type() {
|
||||
DataType::Binary | DataType::LargeBinary | DataType::BinaryView => None,
|
||||
DataType::Struct(children) => {
|
||||
if !children
|
||||
.iter()
|
||||
@@ -50,15 +49,13 @@ pub(super) fn coerce_blob_expr(
|
||||
),
|
||||
});
|
||||
}
|
||||
BlobInputShape::Struct(children)
|
||||
Some(children)
|
||||
}
|
||||
other => {
|
||||
return Err(Error::InvalidInput {
|
||||
message: format!(
|
||||
"cannot coerce column '{}' with type {} into a blob v2 struct. \
|
||||
expected binary bytes (Binary, LargeBinary, BinaryView), \
|
||||
strings (Utf8, LargeUtf8, Utf8View), \
|
||||
or a Struct with a 'data' or 'uri' child",
|
||||
expected Binary, LargeBinary, BinaryView, or a Struct with a 'data' or 'uri' child",
|
||||
table_field.name(),
|
||||
other,
|
||||
),
|
||||
@@ -72,8 +69,9 @@ pub(super) fn coerce_blob_expr(
|
||||
declared.name().as_str(),
|
||||
))));
|
||||
|
||||
let value: Arc<dyn PhysicalExpr> = match &input_shape {
|
||||
BlobInputShape::Bytes => {
|
||||
let value: Arc<dyn PhysicalExpr> = match input_struct_children {
|
||||
// Raw binary lands in `data` and everything else is a typed null.
|
||||
None => {
|
||||
if declared.name() == "data" {
|
||||
Arc::new(CastExpr::new(
|
||||
input_expr.clone(),
|
||||
@@ -84,43 +82,30 @@ pub(super) fn coerce_blob_expr(
|
||||
typed_null(declared.data_type())?
|
||||
}
|
||||
}
|
||||
BlobInputShape::String => {
|
||||
if declared.name() == "uri" {
|
||||
Arc::new(CastExpr::new(
|
||||
input_expr.clone(),
|
||||
declared.data_type().clone(),
|
||||
None,
|
||||
))
|
||||
} else {
|
||||
typed_null(declared.data_type())?
|
||||
}
|
||||
}
|
||||
BlobInputShape::Struct(children) => {
|
||||
match children.iter().find(|c| c.name() == declared.name()) {
|
||||
Some(child) => {
|
||||
let field_expr: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::new(
|
||||
&format!("get_field({})", declared.name()),
|
||||
get_field(),
|
||||
vec![
|
||||
input_expr.clone(),
|
||||
Arc::new(Literal::new(ScalarValue::from(declared.name().as_str()))),
|
||||
],
|
||||
Arc::new(child.as_ref().clone()),
|
||||
config.clone(),
|
||||
));
|
||||
if child.data_type() == declared.data_type() {
|
||||
field_expr
|
||||
} else {
|
||||
Arc::new(CastExpr::new(
|
||||
field_expr,
|
||||
declared.data_type().clone(),
|
||||
None,
|
||||
))
|
||||
}
|
||||
Some(children) => match children.iter().find(|c| c.name() == declared.name()) {
|
||||
Some(child) => {
|
||||
let field_expr: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::new(
|
||||
&format!("get_field({})", declared.name()),
|
||||
get_field(),
|
||||
vec![
|
||||
input_expr.clone(),
|
||||
Arc::new(Literal::new(ScalarValue::from(declared.name().as_str()))),
|
||||
],
|
||||
Arc::new(child.as_ref().clone()),
|
||||
config.clone(),
|
||||
));
|
||||
if child.data_type() == declared.data_type() {
|
||||
field_expr
|
||||
} else {
|
||||
Arc::new(CastExpr::new(
|
||||
field_expr,
|
||||
declared.data_type().clone(),
|
||||
None,
|
||||
))
|
||||
}
|
||||
None => typed_null(declared.data_type())?,
|
||||
}
|
||||
}
|
||||
None => typed_null(declared.data_type())?,
|
||||
},
|
||||
};
|
||||
ns_args.push(value);
|
||||
}
|
||||
@@ -135,12 +120,6 @@ pub(super) fn coerce_blob_expr(
|
||||
Ok((expr, table_field.clone()))
|
||||
}
|
||||
|
||||
enum BlobInputShape<'a> {
|
||||
Bytes,
|
||||
String,
|
||||
Struct(&'a Fields),
|
||||
}
|
||||
|
||||
fn typed_null(data_type: &DataType) -> Result<Arc<dyn PhysicalExpr>> {
|
||||
let scalar = ScalarValue::try_from(data_type).map_err(|e| Error::InvalidInput {
|
||||
message: format!("cannot build null literal for blob child type {data_type}: {e}"),
|
||||
@@ -155,7 +134,7 @@ mod tests {
|
||||
use crate::blob::blob;
|
||||
use arrow_array::{
|
||||
Array, ArrayRef, BinaryArray, BinaryViewArray, Int32Array, Int64Array, LargeBinaryArray,
|
||||
RecordBatch, StringArray, StringViewArray, StructArray, UInt8Array, UInt64Array,
|
||||
RecordBatch, StringArray, StructArray, UInt8Array, UInt64Array,
|
||||
};
|
||||
use arrow_schema::Schema;
|
||||
use datafusion::prelude::SessionContext;
|
||||
@@ -457,78 +436,14 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn unsupported_input_type_is_rejected_with_column_name() {
|
||||
let batch = batch_with_image(
|
||||
Field::new("image", DataType::Int64, true),
|
||||
Arc::new(Int64Array::from(vec![42])),
|
||||
Field::new("image", DataType::Utf8, true),
|
||||
Arc::new(StringArray::from(vec!["not bytes"])),
|
||||
);
|
||||
let err = coerce_err(batch, &blob_table_schema()).await;
|
||||
assert!(matches!(err, Error::InvalidInput { .. }), "got {err:?}");
|
||||
assert!(err.to_string().contains("image"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn utf8_string_coerces_to_uri_child() {
|
||||
let batch = batch_with_image(
|
||||
Field::new("image", DataType::Utf8, true),
|
||||
Arc::new(StringArray::from(vec![Some("s3://bucket/key"), None])),
|
||||
);
|
||||
let coerced = coerce(batch, &blob_table_schema()).await;
|
||||
let image = image_struct(&coerced);
|
||||
let uri: &StringArray = image
|
||||
.column_by_name("uri")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref()
|
||||
.unwrap();
|
||||
assert_eq!(uri.value(0), "s3://bucket/key");
|
||||
assert!(image.column_by_name("data").unwrap().is_null(0));
|
||||
assert!(uri.is_null(1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn large_utf8_string_coerces_into_four_child_blob_layout() {
|
||||
use arrow_array::LargeStringArray;
|
||||
|
||||
let table_schema = Schema::new(vec![
|
||||
Field::new("id", DataType::Int64, false),
|
||||
wide_blob_field("image"),
|
||||
]);
|
||||
let batch = batch_with_image(
|
||||
Field::new("image", DataType::LargeUtf8, true),
|
||||
Arc::new(LargeStringArray::from(vec!["file:///tmp/blob.bin"])),
|
||||
);
|
||||
let coerced = coerce(batch, &table_schema).await;
|
||||
let image = image_struct(&coerced);
|
||||
assert_eq!(image.num_columns(), 4);
|
||||
let uri: &StringArray = image
|
||||
.column_by_name("uri")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref()
|
||||
.unwrap();
|
||||
assert_eq!(uri.value(0), "file:///tmp/blob.bin");
|
||||
assert!(image.column_by_name("data").unwrap().is_null(0));
|
||||
assert!(image.column_by_name("position").unwrap().is_null(0));
|
||||
assert!(image.column_by_name("size").unwrap().is_null(0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn utf8_view_string_coerces_to_uri_child() {
|
||||
let batch = batch_with_image(
|
||||
Field::new("image", DataType::Utf8View, true),
|
||||
Arc::new(StringViewArray::from(vec![Some("s3://bucket/view-key")])),
|
||||
);
|
||||
let coerced = coerce(batch, &blob_table_schema()).await;
|
||||
let image = image_struct(&coerced);
|
||||
let uri: &StringArray = image
|
||||
.column_by_name("uri")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref()
|
||||
.unwrap();
|
||||
assert_eq!(uri.value(0), "s3://bucket/view-key");
|
||||
assert!(image.column_by_name("data").unwrap().is_null(0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn blob_metadata_survives_cast_of_sibling_column() {
|
||||
let batch = RecordBatch::try_new(
|
||||
|
||||
@@ -32,10 +32,6 @@ struct DatasetState {
|
||||
/// `Some(version)` = pinned to a specific version (time travel),
|
||||
/// `None` = tracking latest.
|
||||
pinned_version: Option<u64>,
|
||||
/// Whether the pin is an internal query snapshot rather than user-visible
|
||||
/// time travel. Query snapshots remain read-only but preserve MemWAL read
|
||||
/// semantics.
|
||||
query_snapshot: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -74,7 +70,6 @@ impl DatasetConsistencyWrapper {
|
||||
state: Arc::new(Mutex::new(DatasetState {
|
||||
dataset,
|
||||
pinned_version: None,
|
||||
query_snapshot: false,
|
||||
})),
|
||||
consistency,
|
||||
shard_writer: Arc::new(ShardWriterCache::default()),
|
||||
@@ -98,36 +93,6 @@ impl DatasetConsistencyWrapper {
|
||||
wrapper
|
||||
}
|
||||
|
||||
/// Create an independent read-only wrapper pinned to the current dataset
|
||||
/// while retaining this wrapper's live MemWAL read context.
|
||||
pub async fn new_query_snapshot(&self) -> Result<Self> {
|
||||
// Apply the configured consistency policy before taking the snapshot.
|
||||
// The returned dataset is intentionally discarded: a checkout may race
|
||||
// after this await, so the dataset and its pin provenance must instead
|
||||
// be cloned together from one authoritative state sample below.
|
||||
self.get().await?;
|
||||
|
||||
let (dataset, query_snapshot) = {
|
||||
let state = self.state.lock()?;
|
||||
// Preserve user time travel so the MemWAL safety guard still sees
|
||||
// it. Latest and already-internal snapshots remain internal pins.
|
||||
(
|
||||
state.dataset.clone(),
|
||||
state.query_snapshot || state.pinned_version.is_none(),
|
||||
)
|
||||
};
|
||||
let version = dataset.version().version;
|
||||
Ok(Self {
|
||||
state: Arc::new(Mutex::new(DatasetState {
|
||||
dataset,
|
||||
pinned_version: Some(version),
|
||||
query_snapshot,
|
||||
})),
|
||||
consistency: ConsistencyMode::Lazy,
|
||||
shard_writer: self.shard_writer.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// The MemWAL `ShardWriter` cache co-located with this dataset.
|
||||
pub(crate) fn shard_writer(&self) -> &Arc<ShardWriterCache> {
|
||||
&self.shard_writer
|
||||
@@ -204,7 +169,6 @@ impl DatasetConsistencyWrapper {
|
||||
let mut state = self.state.lock()?;
|
||||
state.dataset = Arc::new(new_dataset);
|
||||
state.pinned_version = None;
|
||||
state.query_snapshot = false;
|
||||
drop(state);
|
||||
if let ConsistencyMode::Eventual(bg_cache) = &self.consistency {
|
||||
bg_cache.invalidate();
|
||||
@@ -238,10 +202,10 @@ impl DatasetConsistencyWrapper {
|
||||
|
||||
/// Returns the version, if in time travel mode, or None otherwise.
|
||||
pub fn time_travel_version(&self) -> Option<u64> {
|
||||
let state = self.state.lock().unwrap_or_else(|e| e.into_inner());
|
||||
(!state.query_snapshot)
|
||||
.then_some(state.pinned_version)
|
||||
.flatten()
|
||||
self.state
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.pinned_version
|
||||
}
|
||||
|
||||
/// Convert into a wrapper in latest version mode.
|
||||
@@ -261,7 +225,6 @@ impl DatasetConsistencyWrapper {
|
||||
if state.pinned_version.is_some() {
|
||||
state.dataset = Arc::new(new_dataset);
|
||||
state.pinned_version = None;
|
||||
state.query_snapshot = false;
|
||||
}
|
||||
drop(state);
|
||||
if let ConsistencyMode::Eventual(bg_cache) = &self.consistency {
|
||||
@@ -297,7 +260,6 @@ impl DatasetConsistencyWrapper {
|
||||
let mut state = self.state.lock()?;
|
||||
state.dataset = Arc::new(new_dataset);
|
||||
state.pinned_version = Some(version_value);
|
||||
state.query_snapshot = false;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -499,29 +461,6 @@ mod tests {
|
||||
assert_eq!(wrapper.time_travel_version(), Some(1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_query_snapshot_samples_dataset_and_pin_together() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let uri = dir.path().to_str().unwrap();
|
||||
let ds = create_test_dataset(uri).await;
|
||||
|
||||
let wrapper = DatasetConsistencyWrapper::new_latest(ds, None);
|
||||
wrapper.as_time_travel(1u64).await.unwrap();
|
||||
let stale_time_travel_dataset = wrapper.get().await.unwrap();
|
||||
|
||||
append_to_dataset(uri).await;
|
||||
wrapper.as_latest().await.unwrap();
|
||||
|
||||
let snapshot = wrapper.new_query_snapshot().await.unwrap();
|
||||
let snapshot_dataset = snapshot.get().await.unwrap();
|
||||
assert_eq!(snapshot_dataset.version().version, 2);
|
||||
assert_ne!(
|
||||
snapshot_dataset.version().version,
|
||||
stale_time_travel_dataset.version().version
|
||||
);
|
||||
assert_eq!(snapshot.time_travel_version(), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_as_latest_from_time_travel() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -31,9 +31,8 @@ pub(crate) async fn execute_delete(
|
||||
table.dataset.ensure_mutable()?;
|
||||
match predicate {
|
||||
Predicate::String(s) => {
|
||||
let predicate = crate::expr::canonicalize_sql_predicate(s)?;
|
||||
let mut dataset = (*table.dataset.get().await?).clone();
|
||||
let delete_result = dataset.delete(&predicate).boxed().await?;
|
||||
let delete_result = dataset.delete(s).boxed().await?;
|
||||
let num_deleted_rows = delete_result.num_deleted_rows;
|
||||
let version = dataset.version().version;
|
||||
table.dataset.update(dataset);
|
||||
|
||||
@@ -220,32 +220,9 @@ impl MergeInsertBuilder {
|
||||
///
|
||||
/// Returns version and statistics about the merge operation including the number of rows
|
||||
/// inserted, updated, and deleted.
|
||||
pub async fn execute(
|
||||
mut self,
|
||||
new_data: Box<dyn RecordBatchReader + Send>,
|
||||
) -> Result<MergeResult> {
|
||||
self.canonicalize_filters()?;
|
||||
pub async fn execute(self, new_data: Box<dyn RecordBatchReader + Send>) -> Result<MergeResult> {
|
||||
self.table.clone().merge_insert(self, new_data).await
|
||||
}
|
||||
|
||||
pub(crate) fn canonicalize_filters(&mut self) -> Result<()> {
|
||||
self.when_matched_update_all_filt =
|
||||
canonicalize_merge_filter(self.when_matched_update_all_filt.take())?;
|
||||
self.when_not_matched_by_source_delete_filt =
|
||||
canonicalize_merge_filter(self.when_not_matched_by_source_delete_filt.take())?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn canonicalize_merge_filter(filter: Option<MergeFilter>) -> Result<Option<MergeFilter>> {
|
||||
filter
|
||||
.map(|filter| match filter {
|
||||
MergeFilter::Sql(predicate) => {
|
||||
crate::expr::canonicalize_sql_predicate(&predicate).map(MergeFilter::Sql)
|
||||
}
|
||||
filter @ MergeFilter::Expr(_) => Ok(filter),
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
/// Internal implementation of the merge insert logic
|
||||
@@ -253,10 +230,9 @@ fn canonicalize_merge_filter(filter: Option<MergeFilter>) -> Result<Option<Merge
|
||||
/// This logic was moved from NativeTable::merge_insert to keep table.rs clean.
|
||||
pub(crate) async fn execute_merge_insert(
|
||||
table: &NativeTable,
|
||||
mut params: MergeInsertBuilder,
|
||||
params: MergeInsertBuilder,
|
||||
new_data: Box<dyn RecordBatchReader + Send>,
|
||||
) -> Result<MergeResult> {
|
||||
params.canonicalize_filters()?;
|
||||
super::computed_columns::ensure_no_function_bindings_for_mutation(
|
||||
table.schema().await?.as_ref(),
|
||||
"merge_insert",
|
||||
@@ -1080,44 +1056,6 @@ mod lsm_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn query_snapshot_preserves_lsm_read_semantics() {
|
||||
let dir = tempdir().unwrap();
|
||||
let table = id_value_table(&dir).await;
|
||||
table
|
||||
.set_lsm_write_spec(LsmWriteSpec::unsharded())
|
||||
.await
|
||||
.unwrap();
|
||||
lsm_upsert(&table, vec![4, 5]).await;
|
||||
|
||||
let snapshot = table.query_snapshot().await.unwrap();
|
||||
let rows = collect_id_value(snapshot.query().execute().await.unwrap()).await;
|
||||
assert_eq!(
|
||||
rows.iter().map(|(id, _)| *id).collect::<Vec<_>>(),
|
||||
vec![1, 2, 3, 4, 5]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn query_snapshot_preserves_time_travel_lsm_guard() {
|
||||
let dir = tempdir().unwrap();
|
||||
let table = id_value_table(&dir).await;
|
||||
table
|
||||
.set_lsm_write_spec(LsmWriteSpec::unsharded())
|
||||
.await
|
||||
.unwrap();
|
||||
lsm_upsert(&table, vec![4]).await;
|
||||
|
||||
let version = table.version().await.unwrap();
|
||||
table.checkout(version).await.unwrap();
|
||||
let direct_error = table.query().execute().await.err().unwrap();
|
||||
assert!(matches!(direct_error, Error::NotSupported { .. }));
|
||||
|
||||
let snapshot = table.query_snapshot().await.unwrap();
|
||||
let snapshot_error = snapshot.query().execute().await.err().unwrap();
|
||||
assert!(matches!(snapshot_error, Error::NotSupported { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lsm_read_dedup_newest_wins() {
|
||||
let dir = tempdir().unwrap();
|
||||
|
||||
+31
-412
@@ -17,11 +17,11 @@ use arrow::array::{AsArray, FixedSizeListBuilder, Float32Builder};
|
||||
use arrow::datatypes::{Float32Type, UInt8Type};
|
||||
use arrow_array::Array;
|
||||
use arrow_schema::{DataType, Schema};
|
||||
use datafusion_common::{Column, DataFusionError, SchemaError};
|
||||
use datafusion_physical_plan::ExecutionPlan;
|
||||
use datafusion_physical_plan::projection::ProjectionExec;
|
||||
use datafusion_physical_plan::repartition::RepartitionExec;
|
||||
use datafusion_physical_plan::union::UnionExec;
|
||||
use futures::future::try_join_all;
|
||||
use lance::dataset::mem_wal::DatasetMemWalExt;
|
||||
use lance::dataset::scanner::DatasetRecordBatchStream;
|
||||
use lance::dataset::scanner::Scanner;
|
||||
@@ -45,22 +45,6 @@ impl AnyQuery {
|
||||
Self::VectorQuery(query) => &query.base,
|
||||
}
|
||||
}
|
||||
|
||||
fn base_mut(&mut self) -> &mut QueryRequest {
|
||||
match self {
|
||||
Self::Query(query) => query,
|
||||
Self::VectorQuery(query) => &mut query.base,
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonicalize any raw SQL filter immediately before backend dispatch.
|
||||
pub(crate) fn canonicalized(&self) -> Result<Self> {
|
||||
let mut query = self.clone();
|
||||
if let Some(QueryFilter::Sql(predicate)) = &mut query.base_mut().filter {
|
||||
*predicate = crate::expr::canonicalize_sql_predicate(predicate)?;
|
||||
}
|
||||
Ok(query)
|
||||
}
|
||||
}
|
||||
|
||||
//Decide between namespace or local
|
||||
@@ -69,16 +53,15 @@ pub async fn execute_query(
|
||||
query: &AnyQuery,
|
||||
options: QueryExecutionOptions,
|
||||
) -> Result<DatasetRecordBatchStream> {
|
||||
let query = query.canonicalized()?;
|
||||
// QueryTable pushdown runs the query server-side, but only on the main
|
||||
// branch: the namespace request carries no branch yet, so a branch handle
|
||||
// must fall through to local execution.
|
||||
if can_execute_namespace_query(table, &query).await?
|
||||
if can_execute_namespace_query(table, query).await?
|
||||
&& let Some(ref namespace_client) = table.namespace_client
|
||||
{
|
||||
return execute_namespace_query(table, namespace_client.clone(), &query, options).await;
|
||||
return execute_namespace_query(table, namespace_client.clone(), query, options).await;
|
||||
}
|
||||
execute_generic_query(table, &query, options).await
|
||||
execute_generic_query(table, query, options).await
|
||||
}
|
||||
|
||||
async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> Result<bool> {
|
||||
@@ -153,10 +136,9 @@ pub async fn create_plan(
|
||||
query: &AnyQuery,
|
||||
options: QueryExecutionOptions,
|
||||
) -> Result<Arc<dyn ExecutionPlan>> {
|
||||
let query = query.canonicalized()?;
|
||||
let query = match query {
|
||||
AnyQuery::VectorQuery(query) => query,
|
||||
AnyQuery::Query(query) => VectorQueryRequest::from_plain_query(query),
|
||||
AnyQuery::VectorQuery(query) => query.clone(),
|
||||
AnyQuery::Query(query) => VectorQueryRequest::from_plain_query(query.clone()),
|
||||
};
|
||||
query.base.check_filter()?;
|
||||
|
||||
@@ -188,48 +170,26 @@ pub async fn create_plan(
|
||||
let mut column = query.column.clone();
|
||||
|
||||
let mut query_vector = query.query_vector.first().cloned();
|
||||
let mut is_batch_query = false;
|
||||
if query.query_vector.len() > 1 {
|
||||
if column.is_none() {
|
||||
// Infer a vector column with the same dimension of the query vector.
|
||||
let arrow_schema = Schema::from(schema);
|
||||
let arrow_schema = Schema::from(ds_ref.schema());
|
||||
column = Some(default_vector_column(
|
||||
&arrow_schema,
|
||||
Some(query.query_vector[0].len() as i32),
|
||||
)?);
|
||||
}
|
||||
let vector_field = schema.field(column.as_ref().unwrap()).unwrap();
|
||||
let (_, element_type) =
|
||||
lance::index::vector::utils::get_vector_type(schema, column.as_ref().unwrap())?;
|
||||
let is_binary = matches!(element_type, DataType::UInt8);
|
||||
if matches!(vector_field.data_type(), DataType::List(_))
|
||||
|| (query.base.offset.unwrap_or(0) == 0 && !is_binary)
|
||||
{
|
||||
// Lance distinguishes these cases from the vector column type: a
|
||||
// list-like query against a List column is one multivector query,
|
||||
// while the same query against a FixedSizeList column is a batch of
|
||||
// independent queries. The batch path shares a single flat scan and
|
||||
// bounds retained candidate data instead of running one scan per
|
||||
// query vector.
|
||||
if let DataType::List(_) = vector_field.data_type() {
|
||||
// Multivector handling: concatenate into FixedSizeList<FixedSizeList<_>>
|
||||
let vectors = query
|
||||
.query_vector
|
||||
.iter()
|
||||
.map(|arr| arr.as_ref())
|
||||
.collect::<Vec<_>>();
|
||||
let dim = vectors[0].len();
|
||||
if let Some((query_index, actual_dim)) = vectors
|
||||
.iter()
|
||||
.enumerate()
|
||||
.find_map(|(index, vector)| (vector.len() != dim).then_some((index, vector.len())))
|
||||
{
|
||||
return Err(Error::InvalidInput {
|
||||
message: format!(
|
||||
"query vector at index {query_index} has dimension {actual_dim}, expected {dim}"
|
||||
),
|
||||
});
|
||||
}
|
||||
let mut fsl_builder = FixedSizeListBuilder::with_capacity(
|
||||
Float32Builder::with_capacity(dim * vectors.len()),
|
||||
Float32Builder::with_capacity(dim),
|
||||
dim as i32,
|
||||
vectors.len(),
|
||||
);
|
||||
@@ -240,12 +200,8 @@ pub async fn create_plan(
|
||||
fsl_builder.append(true);
|
||||
}
|
||||
query_vector = Some(Arc::new(fsl_builder.finish()));
|
||||
is_batch_query = !matches!(vector_field.data_type(), DataType::List(_));
|
||||
} else {
|
||||
// Lance's batch path has no per-query offset, and its binary path
|
||||
// requires primitive UInt8 queries rather than a fixed-size list.
|
||||
// Keep the prior plan shape for these cases so offsets are applied
|
||||
// per query and binary query vectors retain their primitive shape.
|
||||
// Multiple query vectors: create a plan for each and union them
|
||||
let query_vecs = query.query_vector.clone();
|
||||
let plan_futures = query_vecs
|
||||
.into_iter()
|
||||
@@ -258,7 +214,7 @@ pub async fn create_plan(
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let plans = futures::future::try_join_all(plan_futures).await?;
|
||||
let plans = try_join_all(plan_futures).await?;
|
||||
return create_multi_vector_plan(plans);
|
||||
}
|
||||
}
|
||||
@@ -269,7 +225,7 @@ pub async fn create_plan(
|
||||
let column = if let Some(col) = column {
|
||||
col
|
||||
} else {
|
||||
let arrow_schema = Schema::from(schema);
|
||||
let arrow_schema = Schema::from(ds_ref.schema());
|
||||
default_vector_column(&arrow_schema, Some(query_vector.len() as i32))?
|
||||
};
|
||||
|
||||
@@ -295,14 +251,10 @@ pub async fn create_plan(
|
||||
}
|
||||
}
|
||||
|
||||
// For a batch query, `nearest` already applies k to each query vector.
|
||||
// Adding Scanner's global limit would truncate the combined result to k rows.
|
||||
if !is_batch_query {
|
||||
scanner.limit(
|
||||
query.base.limit.map(|limit| limit as i64),
|
||||
query.base.offset.map(|offset| offset as i64),
|
||||
)?;
|
||||
}
|
||||
scanner.limit(
|
||||
query.base.limit.map(|limit| limit as i64),
|
||||
query.base.offset.map(|offset| offset as i64),
|
||||
)?;
|
||||
|
||||
if let Some(ef) = query.ef {
|
||||
scanner.ef(ef);
|
||||
@@ -375,97 +327,7 @@ pub async fn create_plan(
|
||||
scanner.order_by(Some(order_by.clone()))?;
|
||||
}
|
||||
|
||||
scanner
|
||||
.create_plan()
|
||||
.await
|
||||
.map_err(|error| enrich_lance_field_not_found(error, schema))
|
||||
}
|
||||
|
||||
/// Replace DataFusion's top-level field candidates with qualified leaf paths.
|
||||
///
|
||||
/// DataFusion resolves nested fields but its `FieldNotFound` error only lists the
|
||||
/// top-level Arrow fields. This makes a missing leaf look unavailable even when it
|
||||
/// exists below a struct. Keep every other Lance/DataFusion error unchanged and
|
||||
/// enrich only this one schema error at the LanceDB query boundary.
|
||||
fn enrich_lance_field_not_found(
|
||||
error: lance::Error,
|
||||
schema: &lance_core::datatypes::Schema,
|
||||
) -> Error {
|
||||
let Some(field) = find_missing_field(&error) else {
|
||||
return error.into();
|
||||
};
|
||||
field_not_found_error(field, &Schema::from(schema))
|
||||
}
|
||||
|
||||
fn field_not_found_diagnostic(
|
||||
error: &(dyn std::error::Error + 'static),
|
||||
schema: &Schema,
|
||||
) -> Option<Error> {
|
||||
let field = find_missing_field(error)?;
|
||||
Some(field_not_found_error(field, schema))
|
||||
}
|
||||
|
||||
fn field_not_found_error(field: &Column, schema: &Schema) -> Error {
|
||||
let valid_fields = leaf_field_paths(schema);
|
||||
let mut message = format!("Schema error: No field named {}", field.quoted_flat_name());
|
||||
if !valid_fields.is_empty() {
|
||||
message.push_str(". Valid fields are ");
|
||||
message.push_str(&valid_fields.join(", "));
|
||||
}
|
||||
message.push('.');
|
||||
|
||||
Error::InvalidInput { message }
|
||||
}
|
||||
|
||||
fn find_missing_field<'a>(error: &'a (dyn std::error::Error + 'static)) -> Option<&'a Column> {
|
||||
if let Some(DataFusionError::SchemaError(schema_error, _)) =
|
||||
error.downcast_ref::<DataFusionError>()
|
||||
&& let SchemaError::FieldNotFound { field, .. } = schema_error.as_ref()
|
||||
{
|
||||
return Some(field);
|
||||
}
|
||||
|
||||
error.source().and_then(find_missing_field)
|
||||
}
|
||||
|
||||
fn leaf_field_paths(schema: &Schema) -> Vec<String> {
|
||||
fn format_segment(segment: &str) -> String {
|
||||
// Quote every segment instead of maintaining a SQL keyword list. Bare
|
||||
// lowercase names such as `true` can be parsed as expressions rather
|
||||
// than identifiers, while backticks preserve all field names in both
|
||||
// local SQL parsers.
|
||||
format!("`{}`", segment.replace('`', "``"))
|
||||
}
|
||||
|
||||
fn visit(fields: &arrow_schema::Fields, path: &mut Vec<String>, paths: &mut Vec<String>) {
|
||||
for field in fields {
|
||||
// Neither local planner can address an empty field-path segment,
|
||||
// even when it is backtick-quoted. Do not advertise leaves beneath
|
||||
// such a segment as valid filter fields.
|
||||
if field.name().is_empty() {
|
||||
continue;
|
||||
}
|
||||
path.push(field.name().clone());
|
||||
match field.data_type() {
|
||||
DataType::Struct(children) if !children.is_empty() => {
|
||||
visit(children, path, paths);
|
||||
}
|
||||
_ => {
|
||||
paths.push(
|
||||
path.iter()
|
||||
.map(|segment| format_segment(segment))
|
||||
.collect::<Vec<_>>()
|
||||
.join("."),
|
||||
);
|
||||
}
|
||||
}
|
||||
path.pop();
|
||||
}
|
||||
}
|
||||
|
||||
let mut paths = Vec::new();
|
||||
visit(schema.fields(), &mut Vec::new(), &mut paths);
|
||||
paths
|
||||
Ok(scanner.create_plan().await?)
|
||||
}
|
||||
|
||||
//Helper functions below
|
||||
@@ -825,10 +687,7 @@ async fn parse_arrow_ipc_response(bytes: bytes::Bytes) -> Result<DatasetRecordBa
|
||||
#[cfg(test)]
|
||||
#[allow(deprecated)]
|
||||
mod tests {
|
||||
use arrow_array::{
|
||||
ArrayRef, FixedSizeListArray, Float32Array, Int32Array, RecordBatch, StringArray,
|
||||
StructArray,
|
||||
};
|
||||
use arrow_array::{ArrayRef, FixedSizeListArray, Float32Array};
|
||||
use futures::TryStreamExt;
|
||||
use lance_arrow::FixedSizeListArrayExt;
|
||||
use std::sync::{
|
||||
@@ -837,8 +696,7 @@ mod tests {
|
||||
};
|
||||
|
||||
use super::*;
|
||||
use crate::query::{ExecutableQuery, QueryBase, QueryExecutionOptions, QueryRequest};
|
||||
use crate::table::BaseTable;
|
||||
use crate::query::{QueryExecutionOptions, QueryRequest};
|
||||
|
||||
fn fixed_size_list_array(values: Vec<f32>, dimension: i32) -> FixedSizeListArray {
|
||||
FixedSizeListArray::try_new_from_values(Float32Array::from(values), dimension).unwrap()
|
||||
@@ -978,6 +836,7 @@ mod tests {
|
||||
async fn test_execute_query_local_routing() {
|
||||
use crate::connect;
|
||||
use crate::table::query::execute_query;
|
||||
use arrow_array::{Int32Array, RecordBatch};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
@@ -1017,164 +876,6 @@ mod tests {
|
||||
assert_eq!(count, 2); // 4 and 5
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_missing_filter_field_lists_nested_fields_in_local_planners() {
|
||||
use crate::connect;
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let metadata = Arc::new(StructArray::from(vec![
|
||||
(
|
||||
Arc::new(Field::new("year", DataType::Int32, false)),
|
||||
Arc::new(Int32Array::from(vec![2024])) as ArrayRef,
|
||||
),
|
||||
(
|
||||
Arc::new(Field::new("genre", DataType::Utf8, false)),
|
||||
Arc::new(StringArray::from(vec!["fiction"])) as ArrayRef,
|
||||
),
|
||||
(
|
||||
Arc::new(Field::new("Title", DataType::Int32, false)),
|
||||
Arc::new(Int32Array::from(vec![7])) as ArrayRef,
|
||||
),
|
||||
(
|
||||
Arc::new(Field::new("true", DataType::Int32, false)),
|
||||
Arc::new(Int32Array::from(vec![8])) as ArrayRef,
|
||||
),
|
||||
(
|
||||
Arc::new(Field::new("", DataType::Int32, false)),
|
||||
Arc::new(Int32Array::from(vec![10])) as ArrayRef,
|
||||
),
|
||||
]));
|
||||
let vector = Arc::new(fixed_size_list_array(vec![0.0, 1.0], 2));
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("id", DataType::Int32, false),
|
||||
Field::new("vector", vector.data_type().clone(), false),
|
||||
Field::new("content", DataType::Utf8, false),
|
||||
Field::new("metadata", metadata.data_type().clone(), false),
|
||||
]));
|
||||
let batch = RecordBatch::try_new(
|
||||
schema,
|
||||
vec![
|
||||
Arc::new(Int32Array::from(vec![1])),
|
||||
vector,
|
||||
Arc::new(StringArray::from(vec!["example"])),
|
||||
metadata,
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
let table = conn
|
||||
.create_table("nested_error", batch)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let error = table
|
||||
.query()
|
||||
.only_if("year = 2024")
|
||||
.execute()
|
||||
.await
|
||||
.err()
|
||||
.expect("query should reject the unqualified nested field");
|
||||
let case_sensitive_path = "`metadata`.`Title`";
|
||||
let keyword_path = "`metadata`.`true`";
|
||||
let expected = format!(
|
||||
"No field named year. Valid fields are `id`, `vector`, `content`, `metadata`.`year`, `metadata`.`genre`, {case_sensitive_path}, {keyword_path}."
|
||||
);
|
||||
|
||||
assert!(
|
||||
error.to_string().contains(&expected),
|
||||
"unexpected error: {error}"
|
||||
);
|
||||
for (path, value) in [(case_sensitive_path, 7), (keyword_path, 8)] {
|
||||
table
|
||||
.query()
|
||||
.only_if(format!("{path} = {value}"))
|
||||
.execute()
|
||||
.await
|
||||
.expect("the path advertised by the diagnostic should be reusable");
|
||||
}
|
||||
|
||||
table.set_unenforced_primary_key(["id"]).await.unwrap();
|
||||
table
|
||||
.set_lsm_write_spec(crate::table::LsmWriteSpec::unsharded())
|
||||
.await
|
||||
.unwrap();
|
||||
let lsm_error = table
|
||||
.query()
|
||||
.only_if("year = 2024")
|
||||
.execute()
|
||||
.await
|
||||
.err()
|
||||
.expect("LSM query should reject the unqualified nested field");
|
||||
|
||||
assert!(
|
||||
lsm_error.to_string().contains(&expected),
|
||||
"unexpected LSM error: {lsm_error}"
|
||||
);
|
||||
for (path, value) in [(case_sensitive_path, 7), (keyword_path, 8)] {
|
||||
table
|
||||
.query()
|
||||
.only_if(format!("{path} = {value}"))
|
||||
.execute()
|
||||
.await
|
||||
.expect("the path advertised by the diagnostic should be reusable in LSM queries");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_leaf_field_paths_preserve_arbitrary_depth() {
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
|
||||
fn nested_field(path: &[&str]) -> Field {
|
||||
let mut segments = path.iter().rev();
|
||||
let mut field = Field::new(
|
||||
*segments.next().expect("path must have a leaf"),
|
||||
DataType::Int32,
|
||||
false,
|
||||
);
|
||||
for segment in segments {
|
||||
field = Field::new(*segment, DataType::Struct(vec![field].into()), false);
|
||||
}
|
||||
field
|
||||
}
|
||||
|
||||
let schema = Schema::new(vec![
|
||||
nested_field(&["a", "b", "c", "d", "e"]),
|
||||
nested_field(&["metadata", "child.with.dot"]),
|
||||
nested_field(&["metadata", "Title"]),
|
||||
nested_field(&["metadata", "123child"]),
|
||||
nested_field(&["metadata", "child`tick"]),
|
||||
nested_field(&["metadata", ""]),
|
||||
nested_field(&["", "child"]),
|
||||
]);
|
||||
|
||||
assert_eq!(
|
||||
leaf_field_paths(&schema),
|
||||
vec![
|
||||
"`a`.`b`.`c`.`d`.`e`",
|
||||
"`metadata`.`child.with.dot`",
|
||||
"`metadata`.`Title`",
|
||||
"`metadata`.`123child`",
|
||||
"`metadata`.`child``tick`",
|
||||
]
|
||||
);
|
||||
|
||||
let source = DataFusionError::SchemaError(
|
||||
Box::new(SchemaError::FieldNotFound {
|
||||
field: Box::new(Column::from_name("missing")),
|
||||
valid_fields: Vec::new(),
|
||||
}),
|
||||
Box::new(None),
|
||||
);
|
||||
let error = field_not_found_diagnostic(&source, &schema).unwrap();
|
||||
assert!(
|
||||
error.to_string().contains(
|
||||
"Valid fields are `a`.`b`.`c`.`d`.`e`, `metadata`.`child.with.dot`, `metadata`.`Title`, `metadata`.`123child`, `metadata`.`child``tick`"
|
||||
),
|
||||
"unexpected error: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct CountingNamespaceClient {
|
||||
query_table_calls: AtomicUsize,
|
||||
@@ -1188,56 +889,10 @@ mod tests {
|
||||
|
||||
async fn query_table(&self, _request: NsQueryTableRequest) -> lance::Result<bytes::Bytes> {
|
||||
self.query_table_calls.fetch_add(1, Ordering::SeqCst);
|
||||
panic!("query must not be pushed down to namespace query_table");
|
||||
panic!("approx_mode queries must not be pushed down to namespace query_table");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_query_pinned_snapshot_with_namespace_pushdown_runs_locally() {
|
||||
use crate::connect;
|
||||
use arrow_array::{Int32Array, RecordBatch};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||
let batch = RecordBatch::try_new(
|
||||
schema,
|
||||
vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))],
|
||||
)
|
||||
.unwrap();
|
||||
let table = conn
|
||||
.create_table("test_pinned_namespace_fallback", vec![batch])
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let namespace_client = Arc::new(CountingNamespaceClient::default());
|
||||
let mut native_table = table.as_native().unwrap().clone();
|
||||
native_table.namespace_client = Some(namespace_client.clone());
|
||||
native_table
|
||||
.pushdown_operations
|
||||
.insert(NamespaceClientPushdownOperation::QueryTable);
|
||||
|
||||
let snapshot = native_table.checkout_current().await.unwrap();
|
||||
let snapshot = snapshot.as_any().downcast_ref::<NativeTable>().unwrap();
|
||||
assert!(snapshot.dataset.time_travel_version().is_some());
|
||||
|
||||
let query = AnyQuery::Query(QueryRequest {
|
||||
filter: Some(QueryFilter::Sql("id > 3".to_string())),
|
||||
..Default::default()
|
||||
});
|
||||
let stream = execute_query(snapshot, &query, QueryExecutionOptions::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let batches = stream.try_collect::<Vec<_>>().await.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
batches.iter().map(|batch| batch.num_rows()).sum::<usize>(),
|
||||
2
|
||||
);
|
||||
assert_eq!(namespace_client.query_table_calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_query_approx_mode_with_namespace_pushdown_runs_locally() {
|
||||
use crate::connect;
|
||||
@@ -1355,38 +1010,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_query_snapshot_disables_namespace_pushdown() {
|
||||
use crate::connect;
|
||||
use crate::table::BaseTable;
|
||||
use arrow_array::{Int32Array, RecordBatch};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||
let batch =
|
||||
RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]).unwrap();
|
||||
let table = conn
|
||||
.create_table("test_snapshot_namespace_fallback", vec![batch])
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
let mut native_table = table.as_native().unwrap().clone();
|
||||
native_table.namespace_client = Some(Arc::new(CountingNamespaceClient::default()));
|
||||
native_table
|
||||
.pushdown_operations
|
||||
.insert(NamespaceClientPushdownOperation::QueryTable);
|
||||
|
||||
let snapshot = BaseTable::query_snapshot(&native_table).await.unwrap();
|
||||
let snapshot = snapshot.as_any().downcast_ref::<NativeTable>().unwrap();
|
||||
assert!(
|
||||
!can_execute_namespace_query(snapshot, &AnyQuery::Query(QueryRequest::default()),)
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_plan_batch_vector_uses_shared_scan() {
|
||||
async fn test_create_plan_multivector_structure() {
|
||||
use arrow_array::{Float32Array, RecordBatch};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use datafusion_physical_plan::display::DisplayableExecutionPlan;
|
||||
@@ -1413,18 +1037,11 @@ mod tests {
|
||||
.unwrap();
|
||||
let native_table = table.as_native().unwrap();
|
||||
|
||||
// A batch of vectors against a fixed-size vector column should use
|
||||
// Lance's native batch KNN path instead of independent scan plans.
|
||||
// This triggers the "create_multi_vector_plan" logic branch
|
||||
let q1 = Arc::new(Float32Array::from(vec![1.0, 2.0]));
|
||||
let q2 = Arc::new(Float32Array::from(vec![3.0, 4.0]));
|
||||
|
||||
let req = VectorQueryRequest {
|
||||
base: QueryRequest {
|
||||
filter: Some(QueryFilter::Sql("id >= 0".to_string())),
|
||||
limit: Some(1),
|
||||
select: Select::Columns(vec!["id".to_string()]),
|
||||
..Default::default()
|
||||
},
|
||||
column: Some("vector".to_string()),
|
||||
query_vector: vec![q1, q2],
|
||||
..Default::default()
|
||||
@@ -1441,17 +1058,19 @@ mod tests {
|
||||
.indent(true)
|
||||
.to_string();
|
||||
|
||||
// We expect a RepartitionExec wrapping a UnionExec
|
||||
assert!(
|
||||
display.contains("KNNVectorDistance: queries=2"),
|
||||
"plan should use native batch KNN, got:\n{display}"
|
||||
display.contains("RepartitionExec"),
|
||||
"Plan should include Repartitioning"
|
||||
);
|
||||
assert!(
|
||||
!display.contains("UnionExec"),
|
||||
"flat batch KNN should share one scan, got:\n{display}"
|
||||
display.contains("UnionExec"),
|
||||
"Plan should include a Union of multiple searches"
|
||||
);
|
||||
// We expect the projection to add the 'query_index' column (logic inside multi_vector_plan)
|
||||
assert!(
|
||||
display.contains("query_index"),
|
||||
"plan should add query_index column, got:\n{display}"
|
||||
"Plan should add query_index column"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -27,8 +27,6 @@ use std::sync::Arc;
|
||||
|
||||
use arrow_array::Array;
|
||||
use arrow_schema::{DataType, Schema as ArrowSchema};
|
||||
use datafusion::common::{DataFusionError, ToDFSchema};
|
||||
use datafusion::prelude::SessionContext;
|
||||
use datafusion_physical_plan::expressions::Column;
|
||||
use datafusion_physical_plan::projection::ProjectionExec;
|
||||
use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr};
|
||||
@@ -300,7 +298,7 @@ async fn build_read_context(
|
||||
for shard_id in shard_ids {
|
||||
let manifest_store =
|
||||
ShardManifestStore::new(store.clone(), &base_path, shard_id, scan_batch_size);
|
||||
if let Some(manifest) = manifest_store.latest().await? {
|
||||
if let Some(manifest) = manifest_store.read_latest().await? {
|
||||
snapshots.push(snapshot_from_manifest(shard_id, &manifest, &exclude));
|
||||
}
|
||||
}
|
||||
@@ -393,21 +391,7 @@ fn base_scanner(
|
||||
}
|
||||
if let Some(filter) = &query.base.filter {
|
||||
scanner = match filter {
|
||||
QueryFilter::Sql(sql) => {
|
||||
// Parse here instead of inside `LsmScanner::filter` so the typed
|
||||
// DataFusion `FieldNotFound` error is still available for the
|
||||
// same nested-field enrichment used by the ordinary scanner.
|
||||
let schema = ArrowSchema::from(dataset.schema());
|
||||
let df_schema = schema.clone().to_dfschema().map_err(|error| {
|
||||
enrich_filter_error(error, &schema, "Failed to create DFSchema")
|
||||
})?;
|
||||
let expr = SessionContext::new()
|
||||
.parse_sql_expr(sql, &df_schema)
|
||||
.map_err(|error| {
|
||||
enrich_filter_error(error, &schema, "Failed to parse filter expression")
|
||||
})?;
|
||||
scanner.filter_expr(expr)
|
||||
}
|
||||
QueryFilter::Sql(sql) => scanner.filter(sql)?,
|
||||
QueryFilter::Datafusion(expr) => scanner.filter_expr(expr.clone()),
|
||||
QueryFilter::Substrait(_) => {
|
||||
return Err(Error::NotSupported {
|
||||
@@ -419,12 +403,6 @@ fn base_scanner(
|
||||
Ok(scanner)
|
||||
}
|
||||
|
||||
fn enrich_filter_error(error: DataFusionError, schema: &ArrowSchema, context: &str) -> Error {
|
||||
super::field_not_found_diagnostic(&error, schema).unwrap_or_else(|| Error::InvalidInput {
|
||||
message: format!("{context}: {error}"),
|
||||
})
|
||||
}
|
||||
|
||||
/// Plain scan: filter / projection / limit over base ∪ SSTables ∪ in-memory.
|
||||
/// The plain scan applies limit and offset inside the planner.
|
||||
async fn plain_plan(
|
||||
|
||||
@@ -55,9 +55,7 @@ pub struct DropColumnsResult {
|
||||
pub struct FieldMetadataUpdate {
|
||||
/// Dot-separated path to the field (e.g. `"embedding"` or `"address.zip"`).
|
||||
pub path: String,
|
||||
/// Keys to set (`Some`) or delete (`None`). See
|
||||
/// [`Table::update_field_metadata`](crate::Table::update_field_metadata) for
|
||||
/// the conventional `lancedb:*` keys.
|
||||
/// Keys to set (`Some`) or delete (`None`).
|
||||
pub metadata: HashMap<String, Option<String>>,
|
||||
/// If `true`, replace the field's entire metadata map instead of merging.
|
||||
pub replace: bool,
|
||||
|
||||
@@ -62,33 +62,22 @@ impl UpdateBuilder {
|
||||
}
|
||||
|
||||
/// Executes the update operation.
|
||||
pub async fn execute(mut self) -> Result<UpdateResult> {
|
||||
pub async fn execute(self) -> Result<UpdateResult> {
|
||||
if self.columns.is_empty() {
|
||||
Err(Error::InvalidInput {
|
||||
message: "at least one column must be specified in an update operation".to_string(),
|
||||
})
|
||||
} else {
|
||||
self.canonicalize_filter()?;
|
||||
self.parent.clone().update(self).await
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn canonicalize_filter(&mut self) -> Result<()> {
|
||||
self.filter = self
|
||||
.filter
|
||||
.take()
|
||||
.map(|predicate| crate::expr::canonicalize_sql_predicate(&predicate))
|
||||
.transpose()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Internal implementation of the update logic
|
||||
pub(crate) async fn execute_update(
|
||||
table: &NativeTable,
|
||||
mut update: UpdateBuilder,
|
||||
update: UpdateBuilder,
|
||||
) -> Result<UpdateResult> {
|
||||
update.canonicalize_filter()?;
|
||||
table.dataset.ensure_mutable()?;
|
||||
|
||||
// 1. Snapshot the current dataset
|
||||
|
||||
@@ -225,159 +225,6 @@ pub(crate) fn resolve_arrow_field_path(schema: &Schema, column: &str) -> Result<
|
||||
Ok((canonical_path, Field::from(*field)))
|
||||
}
|
||||
|
||||
pub(crate) struct ResolvedFtsField {
|
||||
pub canonical_path: String,
|
||||
pub field: Field,
|
||||
pub list_depth: usize,
|
||||
}
|
||||
|
||||
/// Canonicalize a public FTS field path while keeping Arrow list item names hidden.
|
||||
pub(crate) fn resolve_lance_fts_field_path(
|
||||
schema: &lance_core::datatypes::Schema,
|
||||
column: &str,
|
||||
) -> Result<ResolvedFtsField> {
|
||||
let names =
|
||||
lance_core::datatypes::parse_field_path(column).map_err(|e| Error::InvalidInput {
|
||||
message: format!("Invalid field path `{}`: {}", column, e),
|
||||
})?;
|
||||
let (root_name, remaining_names) = names.split_first().ok_or_else(|| Error::InvalidInput {
|
||||
message: "FTS field path cannot be empty".to_string(),
|
||||
})?;
|
||||
let mut field = schema
|
||||
.fields
|
||||
.iter()
|
||||
.find(|field| field.name == *root_name)
|
||||
.or_else(|| {
|
||||
schema
|
||||
.fields
|
||||
.iter()
|
||||
.find(|field| field.name.eq_ignore_ascii_case(root_name))
|
||||
})
|
||||
.ok_or_else(|| fts_field_not_found(schema, column))?;
|
||||
let mut canonical_names = vec![field.name.clone()];
|
||||
let mut list_depth = 0;
|
||||
|
||||
for name in remaining_names {
|
||||
while matches!(
|
||||
field.data_type(),
|
||||
DataType::List(_) | DataType::LargeList(_)
|
||||
) {
|
||||
list_depth += 1;
|
||||
field = field.children.first().ok_or_else(|| Error::Schema {
|
||||
message: format!(
|
||||
"FTS field path `{}` has a list without an item field",
|
||||
column
|
||||
),
|
||||
})?;
|
||||
}
|
||||
if !matches!(field.data_type(), DataType::Struct(_)) {
|
||||
return Err(fts_field_not_found(schema, column));
|
||||
}
|
||||
field = field
|
||||
.children
|
||||
.iter()
|
||||
.find(|field| field.name == *name)
|
||||
.or_else(|| {
|
||||
field
|
||||
.children
|
||||
.iter()
|
||||
.find(|field| field.name.eq_ignore_ascii_case(name))
|
||||
})
|
||||
.ok_or_else(|| fts_field_not_found(schema, column))?;
|
||||
canonical_names.push(field.name.clone());
|
||||
}
|
||||
|
||||
let mut terminal = field;
|
||||
while matches!(
|
||||
terminal.data_type(),
|
||||
DataType::List(_) | DataType::LargeList(_)
|
||||
) {
|
||||
list_depth += 1;
|
||||
terminal = terminal.children.first().ok_or_else(|| Error::Schema {
|
||||
message: format!(
|
||||
"FTS field path `{}` has a list without an item field",
|
||||
column
|
||||
),
|
||||
})?;
|
||||
}
|
||||
|
||||
let canonical_path = lance_core::datatypes::format_field_path(
|
||||
&canonical_names
|
||||
.iter()
|
||||
.map(String::as_str)
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
Ok(ResolvedFtsField {
|
||||
canonical_path,
|
||||
field: Field::from(field),
|
||||
list_depth,
|
||||
})
|
||||
}
|
||||
|
||||
fn fts_field_not_found(schema: &lance_core::datatypes::Schema, column: &str) -> Error {
|
||||
Error::Schema {
|
||||
message: format!(
|
||||
"Field path `{}` not found in schema. Available field paths: {}",
|
||||
column,
|
||||
schema.field_paths().join(", ")
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn find_public_fts_field_path_by_id(
|
||||
field: &lance_core::datatypes::Field,
|
||||
field_id: i32,
|
||||
path: &mut Vec<String>,
|
||||
) -> bool {
|
||||
if field.id == field_id {
|
||||
return true;
|
||||
}
|
||||
match field.data_type() {
|
||||
DataType::List(_) | DataType::LargeList(_) => field
|
||||
.children
|
||||
.first()
|
||||
.is_some_and(|child| find_public_fts_field_path_by_id(child, field_id, path)),
|
||||
DataType::Struct(_) => field.children.iter().any(|child| {
|
||||
path.push(child.name.clone());
|
||||
let found = find_public_fts_field_path_by_id(child, field_id, path);
|
||||
if !found {
|
||||
path.pop();
|
||||
}
|
||||
found
|
||||
}),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn public_fts_field_path_by_id(
|
||||
schema: &lance_core::datatypes::Schema,
|
||||
field_id: i32,
|
||||
) -> Result<String> {
|
||||
for root in &schema.fields {
|
||||
let mut path = vec![root.name.clone()];
|
||||
if find_public_fts_field_path_by_id(root, field_id, &mut path) {
|
||||
return Ok(lance_core::datatypes::format_field_path(
|
||||
&path.iter().map(String::as_str).collect::<Vec<_>>(),
|
||||
));
|
||||
}
|
||||
}
|
||||
Err(Error::Schema {
|
||||
message: format!("Field id `{}` not found in schema", field_id),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn resolve_arrow_fts_field_path(
|
||||
schema: &Schema,
|
||||
column: &str,
|
||||
) -> Result<(String, Field)> {
|
||||
let lance_schema =
|
||||
lance_core::datatypes::Schema::try_from(schema).map_err(|e| Error::Schema {
|
||||
message: format!("Invalid schema: {}", e),
|
||||
})?;
|
||||
let resolved = resolve_lance_fts_field_path(&lance_schema, column)?;
|
||||
Ok((resolved.canonical_path, resolved.field))
|
||||
}
|
||||
|
||||
pub fn supported_btree_data_type(dtype: &DataType) -> bool {
|
||||
dtype.is_integer()
|
||||
|| dtype.is_floating()
|
||||
@@ -633,36 +480,6 @@ mod tests {
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_public_fts_field_path_prefers_exact_case() {
|
||||
let text_list = || {
|
||||
DataType::List(Arc::new(Field::new(
|
||||
"item",
|
||||
DataType::Struct(vec![Field::new("content", DataType::Utf8, true)].into()),
|
||||
true,
|
||||
)))
|
||||
};
|
||||
let schema = Schema::new(vec![
|
||||
Field::new("Docs", text_list(), true),
|
||||
Field::new("docs", text_list(), true),
|
||||
]);
|
||||
|
||||
let (path, _) = resolve_arrow_fts_field_path(&schema, "docs.content").unwrap();
|
||||
assert_eq!(path, "docs.content");
|
||||
|
||||
let lance_schema = lance_core::datatypes::Schema::try_from(&schema).unwrap();
|
||||
let field_id = lance_schema
|
||||
.resolve_case_insensitive("docs.item.content")
|
||||
.unwrap()
|
||||
.last()
|
||||
.unwrap()
|
||||
.id;
|
||||
assert_eq!(
|
||||
public_fts_field_path_by_id(&lance_schema, field_id).unwrap(),
|
||||
"docs.content"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_guess_default_column() {
|
||||
let schema_no_vector = Schema::new(vec![
|
||||
|
||||
@@ -5,14 +5,12 @@ use std::sync::Arc;
|
||||
|
||||
use arrow_array::{
|
||||
Array, ArrayRef, BinaryArray, Int64Array, LargeBinaryArray, RecordBatch, StringArray,
|
||||
StructArray, UInt64Array, new_null_array,
|
||||
StructArray, UInt64Array,
|
||||
};
|
||||
use arrow_schema::{DataType, Field, Fields, Schema};
|
||||
use futures::TryStreamExt;
|
||||
use lance::Dataset;
|
||||
use lance::dataset::WriteParams;
|
||||
use lance_file::version::{ConcreteFileVersion, LanceFileVersion};
|
||||
use lance_table::format::BasePath;
|
||||
use lancedb::{
|
||||
Connection, Error, Result, Table,
|
||||
blob::{BlobRangeRequest, blob},
|
||||
@@ -21,7 +19,7 @@ use lancedb::{
|
||||
ListingDatabaseOptions, NewTableConfig, OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS,
|
||||
},
|
||||
query::{ExecutableQuery, QueryBase},
|
||||
table::{AddDataMode, CompactionOptions, OptimizeAction, OptimizeStats, WriteOptions},
|
||||
table::{AddDataMode, CompactionOptions, OptimizeAction, OptimizeStats},
|
||||
};
|
||||
use tempfile::tempdir;
|
||||
|
||||
@@ -263,11 +261,11 @@ async fn add_rejects_uncoercible_blob_input() -> Result<()> {
|
||||
let batch = RecordBatch::try_new(
|
||||
Arc::new(Schema::new(vec![
|
||||
Field::new("id", DataType::Int64, false),
|
||||
Field::new("image", DataType::Int64, true),
|
||||
Field::new("image", DataType::Utf8, true),
|
||||
])),
|
||||
vec![
|
||||
Arc::new(Int64Array::from(vec![1])),
|
||||
Arc::new(Int64Array::from(vec![42])),
|
||||
Arc::new(StringArray::from(vec!["not bytes"])),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
@@ -1334,223 +1332,3 @@ async fn optimize_preserves_blob_v2_null_and_empty_distinction() -> Result<()> {
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn uri_struct_batch(id: i64, uri: &str) -> RecordBatch {
|
||||
let image_field = blob("image", true);
|
||||
let DataType::Struct(child_fields) = image_field.data_type().clone() else {
|
||||
unreachable!("blob field is a struct");
|
||||
};
|
||||
let children: Vec<ArrayRef> = child_fields
|
||||
.iter()
|
||||
.map(|field| match field.name().as_str() {
|
||||
"uri" => Arc::new(StringArray::from(vec![Some(uri)])) as ArrayRef,
|
||||
_ => new_null_array(field.data_type(), 1),
|
||||
})
|
||||
.collect();
|
||||
let image = StructArray::new(child_fields, children, None);
|
||||
RecordBatch::try_new(
|
||||
Arc::new(Schema::new(vec![
|
||||
Field::new("id", DataType::Int64, false),
|
||||
image_field,
|
||||
])),
|
||||
vec![Arc::new(Int64Array::from(vec![id])), Arc::new(image)],
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn uri_string_batch(id: i64, uri: &str) -> RecordBatch {
|
||||
RecordBatch::try_new(
|
||||
Arc::new(Schema::new(vec![
|
||||
Field::new("id", DataType::Int64, false),
|
||||
Field::new("image", DataType::Utf8, true),
|
||||
])),
|
||||
vec![
|
||||
Arc::new(Int64Array::from(vec![id])),
|
||||
Arc::new(StringArray::from(vec![Some(uri)])),
|
||||
],
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn write_payload_file_uri(dir: &std::path::Path, name: &str, payload: &[u8]) -> String {
|
||||
let path = dir.join(name);
|
||||
std::fs::write(&path, payload).unwrap();
|
||||
url::Url::from_file_path(&path).unwrap().to_string()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn external_uri_struct_round_trips_with_flag() -> Result<()> {
|
||||
let tmp = tempdir().unwrap();
|
||||
let db = connect(tmp.path().join("db").to_str().unwrap())
|
||||
.execute()
|
||||
.await?;
|
||||
let payload: &[u8] = b"external-struct-payload";
|
||||
let uri = write_payload_file_uri(tmp.path(), "payload.bin", payload);
|
||||
let table = db
|
||||
.create_empty_table("t", blob_table_schema())
|
||||
.execute()
|
||||
.await?;
|
||||
|
||||
table
|
||||
.add(uri_struct_batch(1, &uri))
|
||||
.allow_external_blob_outside_bases(true)
|
||||
.execute()
|
||||
.await?;
|
||||
|
||||
let ids = collect_row_ids(&table).await?;
|
||||
let bytes = table.fetch_blobs("image", &ids).await?;
|
||||
assert_eq!(bytes.value(0), payload);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn external_uri_add_requires_opt_in() -> Result<()> {
|
||||
let tmp = tempdir().unwrap();
|
||||
let db = connect(tmp.path().join("db").to_str().unwrap())
|
||||
.execute()
|
||||
.await?;
|
||||
let uri = write_payload_file_uri(tmp.path(), "payload.bin", b"unreachable");
|
||||
let table = db
|
||||
.create_empty_table("t", blob_table_schema())
|
||||
.execute()
|
||||
.await?;
|
||||
|
||||
let err = table
|
||||
.add(uri_struct_batch(1, &uri))
|
||||
.execute()
|
||||
.await
|
||||
.unwrap_err();
|
||||
|
||||
assert!(
|
||||
err.to_string()
|
||||
.contains("allow_external_blob_outside_bases"),
|
||||
"got: {err}"
|
||||
);
|
||||
assert_eq!(table.count_rows(None).await?, 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn string_uri_input_round_trips_as_external_reference() -> Result<()> {
|
||||
let tmp = tempdir().unwrap();
|
||||
let db = connect(tmp.path().join("db").to_str().unwrap())
|
||||
.execute()
|
||||
.await?;
|
||||
let payload: &[u8] = b"external-string-payload";
|
||||
let uri = write_payload_file_uri(tmp.path(), "payload.bin", payload);
|
||||
let table = db
|
||||
.create_empty_table("t", blob_table_schema())
|
||||
.execute()
|
||||
.await?;
|
||||
|
||||
table
|
||||
.add(uri_string_batch(1, &uri))
|
||||
.allow_external_blob_outside_bases(true)
|
||||
.execute()
|
||||
.await?;
|
||||
|
||||
let ids = collect_row_ids(&table).await?;
|
||||
let bytes = table.fetch_blobs("image", &ids).await?;
|
||||
assert_eq!(bytes.value(0), payload);
|
||||
|
||||
let files = table.fetch_blob_files("image", &ids).await?;
|
||||
let file = files[0].as_ref().expect("missing blob file");
|
||||
assert_eq!(file.uri(), Some(uri.as_str()));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn string_uri_inside_registered_base_does_not_need_the_flag() -> Result<()> {
|
||||
let tmp = tempdir().unwrap();
|
||||
let db_path = tmp.path().join("db");
|
||||
let external_base = tmp.path().join("external_base");
|
||||
let object_dir = external_base.join("objects");
|
||||
std::fs::create_dir_all(&object_dir).unwrap();
|
||||
let payload: &[u8] = b"mapped-in-base";
|
||||
let object_path = object_dir.join("mapped.bin");
|
||||
std::fs::write(&object_path, payload).unwrap();
|
||||
let object_uri = url::Url::from_file_path(&object_path).unwrap().to_string();
|
||||
let base_uri = url::Url::from_file_path(&external_base)
|
||||
.unwrap()
|
||||
.to_string();
|
||||
|
||||
let db = connect(db_path.to_str().unwrap()).execute().await?;
|
||||
let table = db
|
||||
.create_empty_table("t", blob_table_schema())
|
||||
.write_options(WriteOptions {
|
||||
lance_write_params: Some(WriteParams {
|
||||
initial_bases: Some(vec![BasePath {
|
||||
id: 1,
|
||||
name: Some("external".to_string()),
|
||||
path: base_uri,
|
||||
is_dataset_root: false,
|
||||
}]),
|
||||
..Default::default()
|
||||
}),
|
||||
})
|
||||
.execute()
|
||||
.await?;
|
||||
|
||||
table
|
||||
.add(uri_string_batch(1, &object_uri))
|
||||
.execute()
|
||||
.await?;
|
||||
|
||||
let ids = collect_row_ids(&table).await?;
|
||||
let bytes = table.fetch_blobs("image", &ids).await?;
|
||||
assert_eq!(bytes.value(0), payload);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn external_uri_rows_mix_with_inline_rows() -> Result<()> {
|
||||
let tmp = tempdir().unwrap();
|
||||
let db = connect(tmp.path().join("db").to_str().unwrap())
|
||||
.execute()
|
||||
.await?;
|
||||
let external_payload: &[u8] = b"external-bytes";
|
||||
let uri = write_payload_file_uri(tmp.path(), "payload.bin", external_payload);
|
||||
let table =
|
||||
create_inline_blob_table(&db, "t", &[1], &[Some(b"inline-bytes".as_slice())]).await?;
|
||||
|
||||
table
|
||||
.add(uri_string_batch(2, &uri))
|
||||
.allow_external_blob_outside_bases(true)
|
||||
.execute()
|
||||
.await?;
|
||||
|
||||
let pairs = collect_id_rowid(&table).await?;
|
||||
let row_ids: Vec<u64> = pairs.iter().map(|(_, r)| *r).collect();
|
||||
let bytes = table.fetch_blobs("image", &row_ids).await?;
|
||||
for (i, (id, _)) in pairs.iter().enumerate() {
|
||||
match id {
|
||||
1 => assert_eq!(bytes.value(i), b"inline-bytes"),
|
||||
2 => assert_eq!(bytes.value(i), external_payload),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_string_uri_is_rejected_at_write() -> Result<()> {
|
||||
let tmp = tempdir().unwrap();
|
||||
let db = connect(tmp.path().join("db").to_str().unwrap())
|
||||
.execute()
|
||||
.await?;
|
||||
let table = db
|
||||
.create_empty_table("t", blob_table_schema())
|
||||
.execute()
|
||||
.await?;
|
||||
|
||||
let err = table
|
||||
.add(uri_string_batch(1, "not a uri"))
|
||||
.allow_external_blob_outside_bases(true)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap_err();
|
||||
|
||||
assert!(err.to_string().contains("not a uri"), "got: {err}");
|
||||
assert_eq!(table.count_rows(None).await?, 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user