Compare commits

..

1 Commits

Author SHA1 Message Date
Lance Release 2bab493265 Bump version: 0.38.0-beta.9 → 0.38.0-beta.10 2026-08-25 10:36:33 +00:00
86 changed files with 744 additions and 5856 deletions
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.38.0-beta.11"
current_version = "0.38.0-beta.10"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
Generated
+45 -45
View File
@@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arc-swap",
"arrow",
@@ -4888,8 +4888,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4911,7 +4911,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4925,7 +4925,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -4934,8 +4934,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrayref",
"crunchy",
@@ -4945,8 +4945,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4983,8 +4983,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5013,8 +5013,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5031,8 +5031,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"proc-macro2",
"quote",
@@ -5041,8 +5041,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5075,8 +5075,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5107,8 +5107,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arc-swap",
"arrow",
@@ -5172,8 +5172,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5195,8 +5195,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5236,8 +5236,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5251,8 +5251,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"async-trait",
@@ -5264,8 +5264,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5318,8 +5318,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5333,8 +5333,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5374,8 +5374,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5388,8 +5388,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "12.0.0-beta.2"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"frostem",
"icu_segmenter",
@@ -5402,7 +5402,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.38.0-beta.11"
version = "0.38.0-beta.9"
dependencies = [
"ahash",
"anyhow",
@@ -5490,7 +5490,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.38.0-beta.11"
version = "0.38.0-beta.9"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5515,7 +5515,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.38.0-beta.11"
version = "0.38.0-beta.9"
dependencies = [
"arrow",
"async-trait",
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.38.0-beta.11</version>
<version>0.38.0-beta.10</version>
</dependency>
```
-518
View File
@@ -1,518 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / AutoQuery
# Class: AutoQuery
A builder for automatic string searches.
Automatic search determines whether to use full-text or vector search from
the table revision selected for each execution. This builder exposes the
common operations supported by both query families.
## Extends
- `StandardQueryBase`&lt;`NativeQuery` \| `NativeVectorQuery`&gt;
## Properties
### inner
```ts
protected inner: Query | VectorQuery | Promise<Query | VectorQuery>;
```
#### Inherited from
`StandardQueryBase.inner`
## Methods
### analyzePlan()
```ts
analyzePlan(distributedMetrics?): Promise<string>
```
Executes the query and returns the physical query plan annotated with runtime metrics.
This is useful for debugging and performance analysis, as it shows how the query was executed
and includes metrics such as elapsed time, rows processed, and I/O statistics.
#### Parameters
* **distributedMetrics?**: [`AnalyzePlanDistributedMetrics`](../type-aliases/AnalyzePlanDistributedMetrics.md)
How distributed worker metrics are displayed for remote query plans.
Defaults to `"aggregate"`.
#### Returns
`Promise`&lt;`string`&gt;
A query execution plan with runtime metrics for each step.
#### Example
```ts
import * as lancedb from "@lancedb/lancedb"
const db = await lancedb.connect("./.lancedb");
const table = await db.createTable("my_table", [
{ vector: [1.1, 0.9], id: "1" },
]);
const plan = await table.query().nearestTo([0.5, 0.2]).analyzePlan();
Example output (with runtime metrics inlined):
AnalyzeExec verbose=true, metrics=[]
ProjectionExec: expr=[id@3 as id, vector@0 as vector, _distance@2 as _distance], metrics=[output_rows=1, elapsed_compute=3.292µs]
Take: columns="vector, _rowid, _distance, (id)", metrics=[output_rows=1, elapsed_compute=66.001µs, batches_processed=1, bytes_read=8, iops=1, requests=1]
CoalesceBatchesExec: target_batch_size=1024, metrics=[output_rows=1, elapsed_compute=3.333µs]
GlobalLimitExec: skip=0, fetch=10, metrics=[output_rows=1, elapsed_compute=167ns]
FilterExec: _distance@2 IS NOT NULL, metrics=[output_rows=1, elapsed_compute=8.542µs]
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=63.25µs, row_replacements=1]
KNNVectorDistance: metric=l2, metrics=[output_rows=1, elapsed_compute=114.333µs, output_batches=1]
LanceScan: uri=/path/to/data, projection=[vector], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1, elapsed_compute=103.626µs, bytes_read=549, iops=2, requests=2]
```
#### Inherited from
`StandardQueryBase.analyzePlan`
***
### execute()
```ts
protected execute(options?): AsyncGenerator<RecordBatch<any>, void, unknown>
```
Execute the query and return the results as an
#### Parameters
* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
#### Returns
`AsyncGenerator`&lt;`RecordBatch`&lt;`any`&gt;, `void`, `unknown`&gt;
#### See
- AsyncIterator
of
- RecordBatch.
By default, LanceDb will use many threads to calculate results and, when
the result set is large, multiple batches will be processed at one time.
This readahead is limited however and backpressure will be applied if this
stream is consumed slowly (this constrains the maximum memory used by a
single query)
#### Inherited from
`StandardQueryBase.execute`
***
### explainPlan()
```ts
explainPlan(verbose): Promise<string>
```
Generates an explanation of the query execution plan.
#### Parameters
* **verbose**: `boolean` = `false`
If true, provides a more detailed explanation. Defaults to false.
#### Returns
`Promise`&lt;`string`&gt;
A Promise that resolves to a string containing the query execution plan explanation.
#### Example
```ts
import * as lancedb from "@lancedb/lancedb"
const db = await lancedb.connect("./.lancedb");
const table = await db.createTable("my_table", [
{ vector: [1.1, 0.9], id: "1" },
]);
const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
```
#### Inherited from
`StandardQueryBase.explainPlan`
***
### fastSearch()
```ts
fastSearch(): this
```
Skip searching un-indexed data. This can make search faster, but will miss
any data that is not yet indexed.
Use [Table#optimize](Table.md#optimize) to index all un-indexed data.
#### Returns
`this`
#### Inherited from
`StandardQueryBase.fastSearch`
***
### ~~filter()~~
```ts
filter(predicate): this
```
A filter statement to be applied to this query.
#### Parameters
* **predicate**: `string`
#### Returns
`this`
#### See
where
#### Deprecated
Use `where` instead
#### Inherited from
`StandardQueryBase.filter`
***
### fullTextSearch()
```ts
fullTextSearch(query, options?): this
```
#### Parameters
* **query**: `string` \| [`FullTextQuery`](../interfaces/FullTextQuery.md)
* **options?**: `Partial`&lt;[`FullTextSearchOptions`](../interfaces/FullTextSearchOptions.md)&gt;
#### Returns
`this`
#### Inherited from
`StandardQueryBase.fullTextSearch`
***
### limit()
```ts
limit(limit): this
```
Set the maximum number of results to return.
By default, a plain search has no limit. If this method is not
called then every valid row from the table will be returned.
#### Parameters
* **limit**: `number`
#### Returns
`this`
#### Inherited from
`StandardQueryBase.limit`
***
### offset()
```ts
offset(offset): this
```
Set the number of rows to skip before returning results.
This is useful for pagination.
#### Parameters
* **offset**: `number`
#### Returns
`this`
#### Inherited from
`StandardQueryBase.offset`
***
### orderBy()
```ts
orderBy(ordering): this
```
Sort the results by the specified column(s).
#### Parameters
* **ordering**: [`ColumnOrdering`](../interfaces/ColumnOrdering.md) \| [`ColumnOrdering`](../interfaces/ColumnOrdering.md)[]
#### Returns
`this`
This query builder.
#### Inherited from
`StandardQueryBase.orderBy`
***
### outputSchema()
```ts
outputSchema(): Promise<Schema<any>>
```
Returns the schema of the output that will be returned by this query.
This can be used to inspect the types and names of the columns that will be
returned by the query before executing it.
#### Returns
`Promise`&lt;`Schema`&lt;`any`&gt;&gt;
An Arrow Schema describing the output columns.
#### Inherited from
`StandardQueryBase.outputSchema`
***
### select()
```ts
select(columns): this
```
Return only the specified columns.
By default a query will return all columns from the table. However, this can have
a very significant impact on latency. LanceDb stores data in a columnar fashion. This
means we can finely tune our I/O to select exactly the columns we need.
As a best practice you should always limit queries to the columns that you need. If you
pass in an array of column names then only those columns will be returned.
You can also use this method to create new "dynamic" columns based on your existing columns.
For example, you may not care about "a" or "b" but instead simply want "a + b". This is often
seen in the SELECT clause of an SQL query (e.g. `SELECT a+b FROM my_table`).
To create dynamic columns you can pass in a Map<string, string>. A column will be returned
for each entry in the map. The key provides the name of the column. The value is
an SQL string used to specify how the column is calculated.
For example, an SQL query might state `SELECT a + b AS combined, c`. The equivalent
input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
#### Returns
`this`
#### Example
```ts
new Map([["combined", "a + b"], ["c", "c"]])
Columns will always be returned in the order given, even if that order is different than
the order used when adding the data.
Note that you can pass in a `Record<string, string>` (e.g. an object literal). This method
uses `Object.entries` which should preserve the insertion order of the object. However,
object insertion order is easy to get wrong and `Map` is more foolproof.
```
#### Inherited from
`StandardQueryBase.select`
***
### toArray()
```ts
toArray(options?): Promise<any[]>
```
Collect the results as an array of objects.
#### Parameters
* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
#### Returns
`Promise`&lt;`any`[]&gt;
#### Inherited from
`StandardQueryBase.toArray`
***
### toArrow()
```ts
toArrow(options?): Promise<Table<any>>
```
Collect the results as an Arrow
#### Parameters
* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
#### Returns
`Promise`&lt;`Table`&lt;`any`&gt;&gt;
#### See
ArrowTable.
#### Inherited from
`StandardQueryBase.toArrow`
***
### useLsm()
```ts
useLsm(enable): this
```
Control MemWAL read routing for this query.
By default (unset), when the table carries a MemWAL write spec (see
[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so
they also return data written via the `mergeInsert` LSM path that has not yet
been compacted into the base table (the active/frozen in-memory memtables and
the flushed generations), deduplicated by primary key; a table without a spec
reads the base table.
#### Parameters
* **enable**: `boolean`
`true` forces the LSM scanner and errors if the table has no
MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
even when a spec is present.
Note: the LSM scanner does not support every query shape (e.g. reranking,
hybrid search, `orderBy`). On a MemWAL table those shapes error unless
`useLsm(false)` is set, because a base-only read would silently exclude
un-compacted MemWAL data.
#### Returns
`this`
#### Inherited from
`StandardQueryBase.useLsm`
***
### where()
```ts
where(predicate): this
```
A filter statement to be applied to this query.
The filter should be supplied as an SQL query string. For example:
#### Parameters
* **predicate**: `string`
#### Returns
`this`
#### Example
```ts
x > 10
y > 0 AND y < 100
x > 5 OR y = 'test'
Filtering performance can often be improved by creating a scalar index
on the filter column(s).
Calling this multiple times combines the filters with a logical AND rather
than replacing the previous filter.
```
#### Inherited from
`StandardQueryBase.where`
***
### withRowId()
```ts
withRowId(): this
```
Whether to return the row id in the results.
This column can be used to match results between different queries. For
example, to match results from a full text search and a vector search in
order to perform hybrid search.
#### Returns
`this`
#### Inherited from
`StandardQueryBase.withRowId`
+2 -14
View File
@@ -942,7 +942,7 @@ Get the schema of the table.
abstract search(
query,
queryType?,
ftsColumns?): Query | VectorQuery | AutoQuery
ftsColumns?): Query | VectorQuery
```
Create a search query to find the nearest neighbors
@@ -964,7 +964,7 @@ of the given query
#### Returns
[`Query`](Query.md) \| [`VectorQuery`](VectorQuery.md) \| [`AutoQuery`](AutoQuery.md)
[`Query`](Query.md) \| [`VectorQuery`](VectorQuery.md)
***
@@ -1292,18 +1292,6 @@ abstract updateFieldMetadata(updates): Promise<UpdateFieldMetadataResult>
Update per-field (column) metadata.
The following keys are treated specially, by convention, and should be
used when appropriate:
- `lancedb:description`: for a human-readable description of a field.
- `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
names the tag category; e.g. `lancedb:tag:model: "clip"`.
- `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
`feature_v2` might be in the same logical column.
- `lancedb:status`: for status options (`production`, `candidate`,
`deprecated`, `archived`) to designate the current life cycle state of
this column.
#### Parameters
* **updates**: [`FieldMetadataUpdate`](../interfaces/FieldMetadataUpdate.md)[]
-1
View File
@@ -18,7 +18,6 @@
## Classes
- [AutoQuery](classes/AutoQuery.md)
- [BooleanQuery](classes/BooleanQuery.md)
- [BoostQuery](classes/BoostQuery.md)
- [BranchContents](classes/BranchContents.md)
@@ -17,8 +17,7 @@ metadata: Record<string, null | string>;
```
Metadata key/value pairs. Merged into the field's existing metadata by
default; a value of `null` deletes that key. See
[Table.updateFieldMetadata](../classes/Table.md#updatefieldmetadata) for the conventional `lancedb:*` keys.
default; a value of `null` deletes that key.
***
@@ -10,12 +10,16 @@
function getRegistry(): EmbeddingFunctionRegistry
```
Get the global embedding function registry.
LanceDB built-in providers are initialized when this public API is first
used, so importing the root package does not change automatic search
selection for tables without embedding metadata.
Utility function to get the global instance of the registry
## Returns
[`EmbeddingFunctionRegistry`](../classes/EmbeddingFunctionRegistry.md)
`EmbeddingFunctionRegistry` The global instance of the registry
## Example
```ts
const registry = getRegistry();
const openai = registry.get("openai").create();
-2
View File
@@ -159,8 +159,6 @@ and combined with [BooleanQuery][lancedb.query.BooleanQuery].
::: lancedb.query.FullTextOperator
::: lancedb.query.DocumentGranularity
::: lancedb.query.Occur
## Embeddings
+1 -1
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.38.0-beta.11</version>
<version>0.38.0-beta.10</version>
<relativePath>../pom.xml</relativePath>
</parent>
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.38.0-beta.11</version>
<version>0.38.0-beta.10</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>12.0.0-beta.2</lance-core.version>
<lance-core.version>11.0.0-beta.22</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.38.0-beta.11"
version = "0.38.0-beta.10"
publish = false
license.workspace = true
description.workspace = true
-21
View File
@@ -6,9 +6,6 @@ import * as arrow17 from "apache-arrow-17";
import * as arrow18 from "apache-arrow-18";
import {
Field as CurrentField,
LargeBinary as CurrentLargeBinary,
Schema as CurrentSchema,
Vector as CurrentVector,
convertToTable,
tableFromIPC as currentTableFromIPC,
@@ -39,24 +36,6 @@ function sampleRecords(): Array<Record<string, any>> {
},
];
}
it("preserves field metadata from a provided schema", async function () {
const jsonMetadata = new Map([["ARROW:extension:name", "lance.json"]]);
const schema = new CurrentSchema([
new CurrentField("meta", new CurrentLargeBinary(), true, jsonMetadata),
]);
const table = makeArrowTable(
[{ meta: Buffer.from(JSON.stringify({ source: "test" })) }],
{ schema },
);
expect(table.schema.fields[0].metadata).toEqual(jsonMetadata);
const roundTripped = currentTableFromIPC(await fromTableToBuffer(table));
expect(roundTripped.schema.fields[0].metadata).toEqual(jsonMetadata);
});
describe.each([arrow15, arrow16, arrow17, arrow18])(
"Arrow",
(
-52
View File
@@ -187,58 +187,6 @@ describe("embedding functions", () => {
const vector0 = JSON.parse(JSON.stringify(arr[0].vector));
expect(vector0).toEqual([1, 2, 3]);
});
it("should append multiple Python embeddings with the same alias", async () => {
@register("python-mock")
// biome-ignore lint/correctness/noUnusedVariables: the decorator registers this class
class MockEmbeddingFunction extends EmbeddingFunction<string> {
ndims() {
return 3;
}
embeddingDataType(): Float {
return new Float32();
}
async computeQueryEmbeddings(_data: string) {
return [1, 2, 3];
}
async computeSourceEmbeddings(data: string[]) {
return data.map((value) =>
value === "hello world" ? [1, 2, 3] : [4, 5, 6],
);
}
}
const metadata = new Map([
[
"embedding_functions",
'[{"source_column":"text1","vector_column":"vector1","name":"python-mock","model":{}},{"source_column":"text2","vector_column":"vector2","name":"python-mock","model":{}}]',
],
]);
const schema = new Schema(
[
new Field("text1", new Utf8(), true),
new Field("text2", new Utf8(), true),
new Field(
"vector1",
new FixedSizeList(3, new Field("item", new Float32(), true)),
true,
),
new Field(
"vector2",
new FixedSizeList(3, new Field("item", new Float32(), true)),
true,
),
],
metadata,
);
const db = await connect(tmpDir.name);
const table = await db.createEmptyTable("test", schema);
await table.add([{ text1: "hello world", text2: "goodbye world" }]);
const rows = await table.query().toArray();
expect(JSON.parse(JSON.stringify(rows[0].vector1))).toEqual([1, 2, 3]);
expect(JSON.parse(JSON.stringify(rows[0].vector2))).toEqual([4, 5, 6]);
});
it("should append generated vectors to a non-nullable schema", async () => {
@register("non_nullable_schema_test")
@@ -1,95 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { execFileSync } from "node:child_process";
import { resolve } from "node:path";
import type { OpenAIEmbeddingFunction } from "../lancedb/embedding/openai";
import type { EmbeddingFunctionRegistry } from "../lancedb/embedding/registry";
type EmbeddingModule = typeof import("../lancedb/embedding");
type OpenAIModule = typeof import("../lancedb/embedding/openai");
type RegistryModule = typeof import("../lancedb/embedding/registry");
describe("embedding function registry", () => {
const registries: EmbeddingFunctionRegistry[] = [];
afterEach(() => {
for (const registry of registries) {
registry.reset();
}
registries.length = 0;
});
it("defers built-in providers until the public registry API is used", () => {
jest.isolateModules(() => {
const embedding = require("../lancedb/embedding") as EmbeddingModule;
const { getRegistry: getInternalRegistry } =
require("../lancedb/embedding/registry") as RegistryModule;
const registry = getInternalRegistry();
registries.push(registry);
expect(registry.length()).toBe(0);
expect(embedding.getRegistry()).toBe(registry);
expect(registry.get("openai")).toBeDefined();
expect(registry.get("huggingface")).toBeDefined();
});
});
it("preserves automatic FTS search in a fresh process", () => {
execFileSync(
process.execPath,
[resolve(__dirname, "fixtures", "auto_fts_search.cjs")],
{ stdio: "pipe" },
);
});
it("shares registrations across duplicated provider module graphs", () => {
let registeringRegistry: EmbeddingFunctionRegistry | undefined;
let latestOpenAIConstructor: typeof OpenAIEmbeddingFunction | undefined;
jest.isolateModules(() => {
require("../lancedb/embedding/openai");
const { getRegistry } =
require("../lancedb/embedding/registry") as RegistryModule;
registeringRegistry = getRegistry();
registries.push(registeringRegistry);
expect(registeringRegistry.get("openai")).toBeDefined();
});
expect(() => {
jest.isolateModules(() => {
const { OpenAIEmbeddingFunction } =
require("../lancedb/embedding/openai") as OpenAIModule;
latestOpenAIConstructor = OpenAIEmbeddingFunction;
const { getRegistry } =
require("../lancedb/embedding/registry") as RegistryModule;
registries.push(getRegistry());
});
}).not.toThrow();
const previousApiKey = process.env.OPENAI_API_KEY;
process.env.OPENAI_API_KEY = "test";
try {
const latestOpenAI = registeringRegistry!
.get<OpenAIEmbeddingFunction>("openai")!
.create();
expect(latestOpenAI).toBeInstanceOf(latestOpenAIConstructor!);
} finally {
if (previousApiKey === undefined) {
delete process.env.OPENAI_API_KEY;
} else {
process.env.OPENAI_API_KEY = previousApiKey;
}
}
jest.isolateModules(() => {
const { getRegistry } =
require("../lancedb/embedding") as EmbeddingModule;
const publicRegistry = getRegistry();
registries.push(publicRegistry);
expect(publicRegistry).toBe(registeringRegistry);
expect(publicRegistry.get("openai")).toBeDefined();
});
});
});
@@ -1,33 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
const assert = require("node:assert/strict");
const tmp = require("tmp");
const { connect, embedding, Index } = require("../../dist");
const { getRegistry } = require("../../dist/embedding/registry");
async function main() {
assert.equal(typeof embedding.getRegistry, "function");
assert.equal(getRegistry().length(), 0);
assert.equal(embedding.getRegistry(), getRegistry());
assert.equal(getRegistry().length(), 2);
const dir = tmp.dirSync({ unsafeCleanup: true });
let db;
try {
db = await connect(dir.name);
const table = await db.createTable("docs", [{ text: "hello world" }]);
await table.createIndex("text", { config: Index.fts() });
const rows = await table.search("hello").toArray();
assert.equal(rows[0].text, "hello world");
} finally {
db?.close();
dir.removeCallback();
}
}
main().catch((error) => {
console.error(error);
process.exitCode = 1;
});
+1 -604
View File
@@ -11,13 +11,10 @@ import * as arrow17 from "apache-arrow-17";
import * as arrow18 from "apache-arrow-18";
import {
AutoQuery,
Connection,
MatchQuery,
PhraseQuery,
Query,
Table,
VectorQuery,
connect,
tokenize,
} from "../lancedb";
@@ -685,56 +682,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
},
);
// https://github.com/lancedb/lancedb/issues/1963
it("should query documents with LangChain PDF metadata", async () => {
const tmpDir = tmp.dirSync({ unsafeCleanup: true });
try {
const db = await connect(tmpDir.name);
const documents = [
{
text: "first page",
vector: [1, 0],
source: "first.pdf",
loc: { pageNumber: 1, lines: { from: 1, to: 12 } },
pdf: {
version: "1.10.100",
info: {
format: "PDF 1.7",
producer: "pdf.js",
creator: "Writer",
},
totalPages: 2,
},
},
{
text: "second page",
vector: [0, 1],
source: "second.pdf",
loc: { pageNumber: 2, lines: { from: 13, to: 24 } },
pdf: {
version: "1.10.100",
info: {
format: "PDF 1.7",
producer: "pdf.js",
creator: "Writer",
},
totalPages: 2,
},
},
];
const documentsTable = await db.createTable("documents", documents);
const results = await documentsTable.query().toArray();
expect(results).toHaveLength(2);
expect(results[0].source).toBe("first.pdf");
expect(results[0].pdf.info.producer).toBe("pdf.js");
expect(results[1].loc.pageNumber).toBe(2);
} finally {
tmpDir.removeCallback();
}
});
describe("merge insert", () => {
let tmpDir: tmp.DirResult;
let table: Table;
@@ -1830,194 +1777,6 @@ describe("Read consistency interval", () => {
});
});
describe("automatic search schema consistency", () => {
let tmpDir: tmp.DirResult;
class SchemaRefreshEmbedding extends EmbeddingFunction<string> {
ndims() {
return 2;
}
embeddingDataType() {
return new Float32();
}
async computeSourceEmbeddings(data: string[]) {
return data.map((value) => [value.length, 1]);
}
async computeQueryEmbeddings(value: string) {
return [value.length, 1];
}
}
function embeddingSchema() {
const func = new SchemaRefreshEmbedding();
return LanceSchema({
text: func.sourceField(new Utf8()),
vector: func.vectorField(),
});
}
beforeEach(() => {
getRegistry().reset();
register("schema-refresh")(SchemaRefreshEmbedding);
tmpDir = tmp.dirSync({ unsafeCleanup: true });
});
afterEach(() => {
getRegistry().reset();
tmpDir.removeCallback();
});
it("uses the schema refreshed from another connection", async () => {
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
try {
const stale = await first.createTable("docs", [{ text: "before" }], {
schema: embeddingSchema(),
});
const replacement = await second.createTable(
"docs",
[{ text: "after hello" }],
{ mode: "overwrite" },
);
await replacement.createIndex("text", { config: Index.fts() });
const search = stale.search("hello");
expect(search).toBeInstanceOf(AutoQuery);
expect(search).not.toBeInstanceOf(Query);
expect(search).not.toBeInstanceOf(VectorQuery);
expect("nprobes" in search).toBe(false);
const rows = await search.toArray();
expect(rows[0].text).toBe("after hello");
expect((await stale.schema()).metadata.has("embedding_functions")).toBe(
false,
);
} finally {
first.close();
second.close();
}
});
it("tracks embedding metadata across checkout and restore", async () => {
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
try {
await first.createTable("docs", [{ text: "before" }], {
schema: embeddingSchema(),
});
const table = await second.createTable(
"docs",
[{ text: "after hello" }],
{ mode: "overwrite" },
);
await table.createIndex("text", { config: Index.fts() });
await table.checkout(1);
expect((await table.search("before").toArray())[0].text).toBe("before");
await table.checkoutLatest();
expect((await table.search("hello").toArray())[0].text).toBe(
"after hello",
);
await table.checkout(1);
await table.restore();
expect((await table.search("before").toArray())[0].text).toBe("before");
} finally {
first.close();
second.close();
}
});
it("pins automatic search while computing an embedding", async () => {
let markStarted!: () => void;
let releaseEmbedding!: () => void;
const started = new Promise<void>((resolve) => {
markStarted = resolve;
});
const released = new Promise<void>((resolve) => {
releaseEmbedding = resolve;
});
class BlockingEmbedding extends SchemaRefreshEmbedding {
async computeQueryEmbeddings(value: string) {
markStarted();
await released;
return [value.length, 1];
}
}
register("schema-refresh-blocking")(BlockingEmbedding);
const func = new BlockingEmbedding();
const schema = LanceSchema({
text: func.sourceField(new Utf8()),
vector: func.vectorField(),
});
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
try {
const table = await first.createTable(
"docs",
[{ text: "hello before" }],
{ schema },
);
const pending = table.search("hello").toArray();
await started;
const replacement = await second.createTable(
"docs",
[{ text: "hello after" }],
{ mode: "overwrite" },
);
await replacement.createIndex("text", { config: Index.fts() });
releaseEmbedding();
expect((await pending)[0].text).toBe("hello before");
} finally {
releaseEmbedding();
first.close();
second.close();
}
});
it("refreshes a reused automatic search for every execution", async () => {
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
try {
const table = await first.createTable("docs", [
{ text: "hello before", marker: "before" },
]);
await table.createIndex("text", { config: Index.fts() });
const search = table.search("hello").select(["text"]);
const before = (await search.toArray())[0];
expect(before.text).toBe("hello before");
expect(before.marker).toBeUndefined();
const replacement = await second.createTable(
"docs",
[{ text: "hello after", marker: "after" }],
{ mode: "overwrite" },
);
await replacement.createIndex("text", { config: Index.fts() });
const after = (await search.toArray())[0];
expect(after.text).toBe("hello after");
expect(after.marker).toBeUndefined();
} finally {
first.close();
second.close();
}
});
});
describe("schema evolution", function () {
let tmpDir: tmp.DirResult;
beforeEach(() => {
@@ -2585,24 +2344,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
);
});
test("full text search if only an unrelated embedding function is registered", async () => {
register("unused")(
class extends EmbeddingFunction<string> {
ndims() {
return 3;
}
embeddingDataType() {
return new Float32();
}
async computeQueryEmbeddings(_data: string) {
return [1, 2, 3];
}
async computeSourceEmbeddings(data: string[]) {
return data.map(() => [1, 2, 3]);
}
},
);
test("full text search if no embedding function provided", async () => {
const db = await connect(tmpDir.name);
const data = [
{ text: "hello world", vector: [0.1, 0.2, 0.3] },
@@ -2624,306 +2366,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
expect(results2[0].text).toBe(data[1].text);
});
test("auto search stays consistent with the active revision", async () => {
let initCalls = 0;
let queryCalls = 0;
let markStarted!: () => void;
const started = new Promise<void>((resolve) => {
markStarted = resolve;
});
let releaseEmbedding!: () => void;
const embeddingReleased = new Promise<void>((resolve) => {
releaseEmbedding = resolve;
});
@register("refresh-test")
class TestEmbedding extends EmbeddingFunction<string> {
async init() {
initCalls += 1;
}
ndims() {
return 1;
}
embeddingDataType() {
return new arrow.Float32();
}
async computeQueryEmbeddings(value: string) {
queryCalls += 1;
if (value === "blocked") {
markStarted();
await embeddingReleased;
}
return value === "greetings" ? [0.1] : [0.2];
}
async computeSourceEmbeddings(values: string[]) {
return values.map((value) =>
value === "hello world" ? [0.1] : [0.2],
);
}
}
const writer = await connect(tmpDir.name);
await writer.createTable("test", [{ text: "plain", vector: [0.0] }]);
const reader = await connect(tmpDir.name, {
readConsistencyInterval: 0,
});
const tracked = await reader.openTable("test");
type SnapshotCountingNative = {
querySnapshot: () => Promise<unknown>;
};
const native = (tracked as unknown as { inner: SnapshotCountingNative })
.inner;
const querySnapshot = native.querySnapshot.bind(native);
let snapshotCalls = 0;
native.querySnapshot = async () => {
snapshotCalls += 1;
return await querySnapshot();
};
const autoQuery = tracked.search("greetings").select(["text"]).limit(1);
const func = new TestEmbedding();
const schema = LanceSchema({
text: func.sourceField(new arrow.Utf8()),
vector: func.vectorField(),
});
const data = [{ text: "hello world" }, { text: "goodbye world" }];
await writer.createTable("test", data, { mode: "overwrite", schema });
const baselineInitCalls = initCalls;
expect(
(await tracked.schema()).metadata.get("embedding_functions"),
).toBeDefined();
const results = await autoQuery.toArray();
expect(results[0].text).toBe(data[0].text);
expect(initCalls).toBe(baselineInitCalls + 1);
expect(queryCalls).toBe(1);
expect(snapshotCalls).toBe(1);
const repeatedResults = await autoQuery.toArray();
expect(repeatedResults[0].text).toBe(data[0].text);
expect(initCalls).toBe(baselineInitCalls + 1);
expect(queryCalls).toBe(1);
expect(snapshotCalls).toBe(2);
const pending = tracked
.search("blocked")
.select(["text"])
.limit(1)
.toArray();
await started;
const ftsData = [
{ text: "greetings from full text", vector: [0.0] },
{ text: "blocked from full text", vector: [0.0] },
];
const ftsTable = await writer.createTable("test", ftsData, {
mode: "overwrite",
});
await ftsTable.createIndex("text", { config: Index.fts() });
releaseEmbedding();
const pendingResults = await pending;
expect(pendingResults[0].text).toBe(data[1].text);
expect(
(await tracked.schema()).metadata.get("embedding_functions"),
).toBeUndefined();
const ftsResults = await autoQuery.toArray();
expect(ftsResults[0].text).toBe(ftsData[0].text);
});
test("auto search keeps newer preparation during a revision race", async () => {
let aCalls = 0;
let bCalls = 0;
let markAStarted!: () => void;
const aStarted = new Promise<void>((resolve) => {
markAStarted = resolve;
});
let releaseA!: () => void;
const aReleased = new Promise<void>((resolve) => {
releaseA = resolve;
});
let markBStarted!: () => void;
const bStarted = new Promise<void>((resolve) => {
markBStarted = resolve;
});
let releaseB!: () => void;
const bReleased = new Promise<void>((resolve) => {
releaseB = resolve;
});
@register("race-a")
class EmbeddingA extends EmbeddingFunction<string> {
ndims() {
return 1;
}
embeddingDataType() {
return new arrow.Float32();
}
async computeQueryEmbeddings() {
aCalls += 1;
markAStarted();
await aReleased;
return [0.1];
}
async computeSourceEmbeddings(values: string[]) {
return values.map(() => [0.1]);
}
}
@register("race-b")
class EmbeddingB extends EmbeddingFunction<string> {
ndims() {
return 1;
}
embeddingDataType() {
return new arrow.Float32();
}
async computeQueryEmbeddings() {
bCalls += 1;
markBStarted();
await bReleased;
return [0.2];
}
async computeSourceEmbeddings(values: string[]) {
return values.map(() => [0.2]);
}
}
const writer = await connect(tmpDir.name);
const embeddingA = new EmbeddingA();
const schemaA = LanceSchema({
text: embeddingA.sourceField(new arrow.Utf8()),
vector: embeddingA.vectorField(),
});
await writer.createTable("race", [{ text: "revision a" }], {
schema: schemaA,
});
const reader = await connect(tmpDir.name, {
readConsistencyInterval: 0,
});
const tracked = await reader.openTable("race");
const query = tracked.search("query");
const first = query.toArray();
await aStarted;
const embeddingB = new EmbeddingB();
const schemaB = LanceSchema({
text: embeddingB.sourceField(new arrow.Utf8()),
vector: embeddingB.vectorField(),
});
await writer.createTable("race", [{ text: "revision b" }], {
mode: "overwrite",
schema: schemaB,
});
const second = query.toArray();
await bStarted;
releaseA();
releaseB();
await Promise.all([first, second]);
expect(aCalls).toBe(1);
expect(bCalls).toBe(1);
});
test("stale FTS routing keeps newer vector preparation", async () => {
let vectorCalls = 0;
let markVectorStarted!: () => void;
const vectorStarted = new Promise<void>((resolve) => {
markVectorStarted = resolve;
});
let releaseVector!: () => void;
const vectorReleased = new Promise<void>((resolve) => {
releaseVector = resolve;
});
@register("stale-fts-race")
class RaceEmbedding extends EmbeddingFunction<string> {
ndims() {
return 1;
}
embeddingDataType() {
return new arrow.Float32();
}
async computeQueryEmbeddings() {
vectorCalls += 1;
markVectorStarted();
await vectorReleased;
return [0.1];
}
async computeSourceEmbeddings(values: string[]) {
return values.map(() => [0.1]);
}
}
const writer = await connect(tmpDir.name);
const ftsTable = await writer.createTable("stale_fts", [
{ text: "hello", vector: [0.0] },
]);
await ftsTable.createIndex("text", { config: Index.fts() });
const reader = await connect(tmpDir.name, {
readConsistencyInterval: 0,
});
const tracked = await reader.openTable("stale_fts");
type Snapshot = {
schema: () => Promise<Buffer>;
};
type NativeWithSnapshot = {
querySnapshot: () => Promise<Snapshot>;
};
const native = (tracked as unknown as { inner: NativeWithSnapshot })
.inner;
const querySnapshot = native.querySnapshot.bind(native);
let snapshotCalls = 0;
let markStaleSchemaStarted!: () => void;
const staleSchemaStarted = new Promise<void>((resolve) => {
markStaleSchemaStarted = resolve;
});
let releaseStaleSchema!: () => void;
const staleSchemaReleased = new Promise<void>((resolve) => {
releaseStaleSchema = resolve;
});
native.querySnapshot = async () => {
const snapshot = await querySnapshot();
snapshotCalls += 1;
if (snapshotCalls === 1) {
const schema = snapshot.schema.bind(snapshot);
snapshot.schema = async () => {
markStaleSchemaStarted();
await staleSchemaReleased;
return await schema();
};
}
return snapshot;
};
const query = tracked.search("hello");
const staleFtsExecution = query.toArray();
await staleSchemaStarted;
const embedding = new RaceEmbedding();
const vectorSchema = LanceSchema({
text: embedding.sourceField(new arrow.Utf8()),
vector: embedding.vectorField(),
});
await writer.createTable("stale_fts", [{ text: "hello" }], {
mode: "overwrite",
schema: vectorSchema,
});
const vectorExecution = query.toArray();
await vectorStarted;
releaseStaleSchema();
await staleFtsExecution;
releaseVector();
await vectorExecution;
await query.toArray();
expect(vectorCalls).toBe(1);
});
test("tokenizes FTS queries by column or index name", async () => {
const db = await connect(tmpDir.name);
const data = [
@@ -3474,30 +2916,6 @@ describe("column name options", () => {
expect(results[1].query_index).toBe(1);
});
test("observes promised additional vectors while the query is pending", async () => {
const initialVector = new Promise<number[]>(() => undefined);
const query = table.query().nearestTo(initialVector);
const unhandled: unknown[] = [];
const onUnhandled = (reason: unknown) => unhandled.push(reason);
process.on("unhandledRejection", onUnhandled);
try {
query.addQueryVector(Promise.reject(new Error("extra vector failed")));
await new Promise<void>((resolve) => setImmediate(resolve));
expect(unhandled).toEqual([]);
const rejectedQuery = table
.query()
.nearestTo([0.1, 0.2])
.addQueryVector(Promise.reject(new Error("consumed vector failed")));
await expect(rejectedQuery.toArray()).rejects.toThrow(
"consumed vector failed",
);
} finally {
process.off("unhandledRejection", onUnhandled);
}
});
test("index and search multivectors", async () => {
const db = await connect(tmpDir.name);
const data = [];
@@ -3561,27 +2979,6 @@ describe("when creating an empty table", () => {
expect((actualSchema.fields[1].type as Float64).precision).toBe(2);
});
it("can add and query JSON data", async () => {
const schema = new Schema([
new Field("id", new Int32(), true),
new Field(
"meta",
new Utf8(),
true,
new Map([["ARROW:extension:name", "arrow.json"]]),
),
]);
const table = await con.createEmptyTable("json", schema);
const meta = JSON.stringify({ x: 1 });
await table.add([{ id: 1, meta }]);
const rows = await table.query().toArray();
expect(rows).toHaveLength(1);
expect(rows[0].id).toBe(1);
expect(rows[0].meta).toBe(meta);
});
it("can create an empty table from schema that specifies field types by name", async () => {
const schemaLike = {
fields: [
+1 -1
View File
@@ -170,7 +170,7 @@ test("basic table examples", async () => {
// --8<-- [end:create_index]
// --8<-- [start:delete_rows]
await tbl.delete("item = 'fizz'");
await tbl.delete('item = "fizz"');
// --8<-- [end:delete_rows]
// --8<-- [start:drop_table]
+2 -42
View File
@@ -4,15 +4,7 @@
import { Field, Schema } from "../arrow";
import { sanitizeType } from "../sanitize";
import { EmbeddingFunction } from "./embedding_function";
import {
EmbeddingFunctionConfig,
EmbeddingFunctionRegistry,
getRegistry as getGlobalRegistry,
registerBuiltIn,
} from "./registry";
type OpenAIModule = typeof import("./openai");
type TransformersModule = typeof import("./transformers");
import { EmbeddingFunctionConfig, getRegistry } from "./registry";
export {
FieldOptions,
@@ -22,39 +14,7 @@ export {
EmbeddingFunctionConstructor,
} from "./embedding_function";
export {
EmbeddingFunctionRegistry,
parseEmbeddingMetadata,
register,
} from "./registry";
export type {
CreateReturnType,
EmbeddingFunctionConfig,
EmbeddingFunctionCreate,
EmbeddingMetadataEntry,
ResolvedEmbeddingFunctionConfig,
} from "./registry";
function initializeBuiltInProviders() {
const { OpenAIEmbeddingFunction } = require("./openai") as OpenAIModule;
const { TransformersEmbeddingFunction } =
require("./transformers") as TransformersModule;
registerBuiltIn("openai", OpenAIEmbeddingFunction);
registerBuiltIn("huggingface", TransformersEmbeddingFunction);
}
/**
* Get the global embedding function registry.
*
* LanceDB built-in providers are initialized when this public API is first
* used, so importing the root package does not change automatic search
* selection for tables without embedding metadata.
*/
export function getRegistry(): EmbeddingFunctionRegistry {
initializeBuiltInProviders();
return getGlobalRegistry();
}
export * from "./registry";
/**
* Create a schema with embedding functions.
+2 -3
View File
@@ -5,13 +5,14 @@ import type OpenAI from "openai";
import type { EmbeddingCreateParams } from "openai/resources/index";
import { Float, Float32 } from "../arrow";
import { EmbeddingFunction } from "./embedding_function";
import { registerBuiltIn } from "./registry";
import { register } from "./registry";
export type OpenAIOptions = {
apiKey: string;
model: EmbeddingCreateParams["model"];
};
@register("openai")
export class OpenAIEmbeddingFunction extends EmbeddingFunction<
string,
Partial<OpenAIOptions>
@@ -99,5 +100,3 @@ export class OpenAIEmbeddingFunction extends EmbeddingFunction<
return response.data[0].embedding;
}
}
registerBuiltIn("openai", OpenAIEmbeddingFunction);
+1 -59
View File
@@ -7,10 +7,6 @@ import {
} from "./embedding_function";
import "reflect-metadata";
const builtInFunctionsKey = Symbol.for(
"@lancedb/lancedb::embedding-built-in-functions::v1",
);
export type CreateReturnType<T> = T extends { init: () => Promise<void> }
? Promise<T>
: T;
@@ -63,15 +59,6 @@ export class EmbeddingFunctionRegistry {
};
}
/** @ignore */
setBuiltIn<
T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
>(name: string, ctor: T): T {
this.#functions.set(name, ctor);
Reflect.defineMetadata("lancedb::embedding::name", name, ctor);
return ctor;
}
get<T extends EmbeddingFunction<unknown>>(
name: string,
): EmbeddingFunctionCreate<T> | undefined;
@@ -109,7 +96,6 @@ export class EmbeddingFunctionRegistry {
*/
reset(this: EmbeddingFunctionRegistry) {
this.#functions.clear();
getBuiltInFunctions(this).clear();
}
/**
@@ -197,56 +183,12 @@ export class EmbeddingFunctionRegistry {
}
}
function getBuiltInFunctions(registry: EmbeddingFunctionRegistry): Set<string> {
const registryWithBuiltIns = registry as EmbeddingFunctionRegistry & {
[key: symbol]: Set<string> | undefined;
};
let builtInFunctions = registryWithBuiltIns[builtInFunctionsKey];
if (builtInFunctions === undefined) {
builtInFunctions = new Set<string>();
registryWithBuiltIns[builtInFunctionsKey] = builtInFunctions;
}
return builtInFunctions;
}
// Server bundlers can load the side-effect embedding entry points and the public
// embedding API from separate module graphs. Keep their registry shared.
const registryKey = Symbol.for(
"@lancedb/lancedb::embedding-function-registry::v1",
);
const registryGlobal = globalThis as typeof globalThis & {
[key: symbol]: EmbeddingFunctionRegistry | undefined;
};
function getGlobalRegistry(): EmbeddingFunctionRegistry {
const existingRegistry = registryGlobal[registryKey];
if (existingRegistry !== undefined) {
return existingRegistry;
}
const registry = new EmbeddingFunctionRegistry();
registryGlobal[registryKey] = registry;
return registry;
}
const _REGISTRY = getGlobalRegistry();
const _REGISTRY = new EmbeddingFunctionRegistry();
export function register(name?: string) {
return _REGISTRY.register(name);
}
/** @ignore */
export function registerBuiltIn<
T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
>(name: string, ctor: T): T {
const builtInFunctions = getBuiltInFunctions(_REGISTRY);
if (builtInFunctions.has(name)) {
return _REGISTRY.setBuiltIn(name, ctor);
}
_REGISTRY.register(name)(ctor);
builtInFunctions.add(name);
return ctor;
}
/**
* Utility function to get the global instance of the registry
* @returns `EmbeddingFunctionRegistry` The global instance of the registry
+2 -3
View File
@@ -3,7 +3,7 @@
import { Float, Float32 } from "../arrow";
import { EmbeddingFunction } from "./embedding_function";
import { registerBuiltIn } from "./registry";
import { register } from "./registry";
export type XenovaTransformerOptions = {
/** The wasm compatible model to use */
@@ -31,6 +31,7 @@ export type XenovaTransformerOptions = {
};
};
@register("huggingface")
export class TransformersEmbeddingFunction extends EmbeddingFunction<
string,
Partial<XenovaTransformerOptions>
@@ -157,8 +158,6 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
}
}
registerBuiltIn("huggingface", TransformersEmbeddingFunction);
const tensorDiv = (
src: import("@huggingface/transformers").Tensor,
divBy: number,
-1
View File
@@ -103,7 +103,6 @@ export {
} from "./native.js";
export {
AutoQuery,
ExecutableQuery,
Query,
QueryBase,
+106 -205
View File
@@ -100,29 +100,6 @@ export interface FullTextSearchOptions {
columns?: string | string[];
}
function nearestToNative(
inner: NativeQuery,
vector: Awaited<IntoVector>,
): NativeVectorQuery {
const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector);
if (raw) {
return inner.nearestToRaw(raw.data, raw.dtype);
}
return inner.nearestTo(Float32Array.from(vector as number[]));
}
function addQueryVectorToNative(
inner: NativeVectorQuery,
vector: Awaited<IntoVector>,
) {
const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector);
if (raw) {
inner.addQueryVectorRaw(raw.data, raw.dtype);
} else {
inner.addQueryVector(Float32Array.from(vector as number[]));
}
}
/** Common methods supported by all query types
*
* @see {@link Query}
@@ -134,15 +111,13 @@ export class QueryBase<
NativeQueryType extends NativeQuery | NativeVectorQuery | NativeTakeQuery,
> implements AsyncIterable<RecordBatch>
{
protected inner!: NativeQueryType | Promise<NativeQueryType>;
/**
* @hidden
*/
protected constructor(inner?: NativeQueryType | Promise<NativeQueryType>) {
if (inner !== undefined) {
this.inner = inner;
}
protected constructor(
protected inner: NativeQueryType | Promise<NativeQueryType>,
) {
// intentionally empty
}
// call a function on the inner (either a promise or the actual object)
@@ -160,15 +135,6 @@ export class QueryBase<
}
}
/**
* Return the native query used by the next terminal operation.
*
* @hidden
*/
protected async getInner(): Promise<NativeQueryType> {
return this.inner;
}
/**
* Return only the specified columns.
*
@@ -241,11 +207,16 @@ export class QueryBase<
/**
* @hidden
*/
protected async nativeExecute(
protected nativeExecute(
options?: Partial<QueryExecutionOptions>,
): Promise<NativeBatchIterator> {
const inner = await this.getInner();
return inner.execute(options?.maxBatchLength, options?.timeoutMs);
if (this.inner instanceof Promise) {
return this.inner.then((inner) =>
inner.execute(options?.maxBatchLength, options?.timeoutMs),
);
} else {
return this.inner.execute(options?.maxBatchLength, options?.timeoutMs);
}
}
/**
@@ -274,7 +245,12 @@ export class QueryBase<
/** Collect the results as an Arrow @see {@link ArrowTable}. */
async toArrow(options?: Partial<QueryExecutionOptions>): Promise<ArrowTable> {
const batches = [];
const inner = await this.getInner();
let inner;
if (this.inner instanceof Promise) {
inner = await this.inner;
} else {
inner = this.inner;
}
for await (const batch of new RecordBatchIterable(inner, options)) {
batches.push(batch);
}
@@ -303,8 +279,11 @@ export class QueryBase<
* @returns A Promise that resolves to a string containing the query execution plan explanation.
*/
async explainPlan(verbose = false): Promise<string> {
const inner = await this.getInner();
return inner.explainPlan(verbose);
if (this.inner instanceof Promise) {
return this.inner.then((inner) => inner.explainPlan(verbose));
} else {
return this.inner.explainPlan(verbose);
}
}
/**
@@ -342,8 +321,13 @@ export class QueryBase<
distributedMetrics?: AnalyzePlanDistributedMetrics,
): Promise<string> {
const distributedMetricsMode = distributedMetrics ?? "aggregate";
const inner = await this.getInner();
return inner.analyzePlan(distributedMetricsMode);
if (this.inner instanceof Promise) {
return this.inner.then((inner) =>
inner.analyzePlan(distributedMetricsMode),
);
} else {
return this.inner.analyzePlan(distributedMetricsMode);
}
}
/**
@@ -355,8 +339,12 @@ export class QueryBase<
* @returns An Arrow Schema describing the output columns.
*/
async outputSchema(): Promise<import("./arrow").Schema> {
const inner = await this.getInner();
const schemaBuffer = await inner.outputSchema();
let schemaBuffer: Buffer;
if (this.inner instanceof Promise) {
schemaBuffer = await this.inner.then((inner) => inner.outputSchema());
} else {
schemaBuffer = await this.inner.outputSchema();
}
const schema = tableFromIPC(schemaBuffer).schema;
return schema;
}
@@ -368,7 +356,7 @@ export class StandardQueryBase<
extends QueryBase<NativeQueryType>
implements ExecutableQuery
{
constructor(inner?: NativeQueryType | Promise<NativeQueryType>) {
constructor(inner: NativeQueryType | Promise<NativeQueryType>) {
super(inner);
}
@@ -522,13 +510,6 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
super(inner);
}
/**
* @hidden
*/
protected doVectorCall(fn: (inner: NativeVectorQuery) => void) {
super.doCall(fn);
}
/**
* Set the number of partitions to search (probe)
*
@@ -556,7 +537,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* the minimum and maximum to the same value.
*/
nprobes(nprobes: number): VectorQuery {
this.doVectorCall((inner) => inner.nprobes(nprobes));
super.doCall((inner) => inner.nprobes(nprobes));
return this;
}
@@ -570,7 +551,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* but will also increase latency.
*/
minimumNprobes(minimumNprobes: number): VectorQuery {
this.doVectorCall((inner) => inner.minimumNprobes(minimumNprobes));
super.doCall((inner) => inner.minimumNprobes(minimumNprobes));
return this;
}
@@ -584,7 +565,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* potential false negatives.
*/
maximumNprobes(maximumNprobes: number): VectorQuery {
this.doVectorCall((inner) => inner.maximumNprobes(maximumNprobes));
super.doCall((inner) => inner.maximumNprobes(maximumNprobes));
return this;
}
@@ -597,7 +578,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* `undefined` means no lower or upper bound.
*/
distanceRange(lowerBound?: number, upperBound?: number): VectorQuery {
this.doVectorCall((inner) => inner.distanceRange(lowerBound, upperBound));
super.doCall((inner) => inner.distanceRange(lowerBound, upperBound));
return this;
}
@@ -611,7 +592,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* also increase the latency of your query. The default value is 1.5*limit.
*/
ef(ef: number): VectorQuery {
this.doVectorCall((inner) => inner.ef(ef));
super.doCall((inner) => inner.ef(ef));
return this;
}
@@ -625,7 +606,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* whose data type is a fixed-size-list of floats.
*/
column(column: string): VectorQuery {
this.doVectorCall((inner) => inner.column(column));
super.doCall((inner) => inner.column(column));
return this;
}
@@ -646,7 +627,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
distanceType(
distanceType: Required<IvfPqOptions>["distanceType"],
): VectorQuery {
this.doVectorCall((inner) => inner.distanceType(distanceType));
super.doCall((inner) => inner.distanceType(distanceType));
return this;
}
@@ -680,7 +661,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* distance between the query vector and the actual uncompressed vector.
*/
refineFactor(refineFactor: number): VectorQuery {
this.doVectorCall((inner) => inner.refineFactor(refineFactor));
super.doCall((inner) => inner.refineFactor(refineFactor));
return this;
}
@@ -705,7 +686,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* factor can often help restore some of the results lost by post filtering.
*/
postfilter(): VectorQuery {
this.doVectorCall((inner) => inner.postfilter());
super.doCall((inner) => inner.postfilter());
return this;
}
@@ -719,7 +700,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* calculate your recall to select an appropriate value for nprobes.
*/
bypassVectorIndex(): VectorQuery {
this.doVectorCall((inner) => inner.bypassVectorIndex());
super.doCall((inner) => inner.bypassVectorIndex());
return this;
}
@@ -727,39 +708,43 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* Add a query vector to the search
*
* This method can be called multiple times to add multiple query vectors
* to the search. A column called `query_index` will be added to indicate the index
* of the query vector that produced the result. Flat searches share one table scan
* across the query vectors, avoiding the scan and memory amplification of running
* multiple queries concurrently. Indexed searches may still perform per-vector
* index work.
* to the search. If multiple query vectors are added, then they will be searched
* in parallel, and the results will be concatenated. A column called `query_index`
* will be added to indicate the index of the query vector that produced the result.
*
* Performance wise, this is equivalent to running multiple queries concurrently.
*/
addQueryVector(vector: IntoVector): VectorQuery {
if (vector instanceof Promise) {
// Observe the promise as soon as it is accepted. The existing native
// query may still be pending, and delaying observation until it resolves
// can otherwise surface a fast rejection as unhandled.
const settledVector = vector.then(
(value) => ({ status: "fulfilled" as const, value }),
(reason) => ({ status: "rejected" as const, reason }),
);
const res = (async () => {
const inner = await this.getInner();
const outcome = await settledVector;
if (outcome.status === "rejected") {
throw outcome.reason;
try {
const v = await vector;
// biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
const value: any = this.addQueryVector(v);
const inner = value.inner as
| NativeVectorQuery
| Promise<NativeVectorQuery>;
return inner;
} catch (e) {
return Promise.reject(e);
}
addQueryVectorToNative(inner, outcome.value);
return inner;
})();
return new VectorQuery(res);
} else {
this.doVectorCall((inner) => addQueryVectorToNative(inner, vector));
super.doCall((inner) => {
const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector);
if (raw) {
inner.addQueryVectorRaw(raw.data, raw.dtype);
} else {
inner.addQueryVector(Float32Array.from(vector as number[]));
}
});
return this;
}
}
rerank(reranker: Reranker): VectorQuery {
this.doVectorCall((inner) =>
super.doCall((inner) =>
inner.rerank(async (args) => {
const vecResults = await fromBufferToRecordBatch(args.vecResults);
const ftsResults = await fromBufferToRecordBatch(args.ftsResults);
@@ -778,71 +763,6 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
}
}
/**
* Create a string query whose vector/FTS routing is resolved against the active
* table schema when the query executes.
*
* @hidden
*/
export function createAutoQuery(
table: NativeTable,
query: string,
columns: string[] | null,
getVector: (metadata: string) => Promise<Awaited<IntoVector>>,
): AutoQuery {
type RouteSnapshot = {
table: NativeTable;
embeddingMetadata: string | undefined;
};
type CachedPreparation = {
metadata: string;
vector: Promise<Awaited<IntoVector>>;
};
let cachedPreparation: CachedPreparation | undefined;
const snapshotRoute = async (): Promise<RouteSnapshot> => {
const snapshot = await table.querySnapshot();
const schema = tableFromIPC(await snapshot.schema()).schema;
return {
table: snapshot,
embeddingMetadata: schema.metadata.get("embedding_functions"),
};
};
const createInner = async (): Promise<NativeQuery | NativeVectorQuery> => {
const route = await snapshotRoute();
if (route.embeddingMetadata === undefined) {
const inner = route.table.query();
inner.fullTextSearch({ query, columns });
return inner;
}
const metadata = route.embeddingMetadata;
if (cachedPreparation?.metadata !== metadata) {
cachedPreparation = {
metadata,
vector: Promise.resolve().then(() => getVector(metadata)),
};
}
const preparation = cachedPreparation;
let vector: Awaited<IntoVector>;
try {
vector = await preparation.vector;
} catch (error) {
if (cachedPreparation === preparation) {
cachedPreparation = undefined;
}
throw error;
}
return nearestToNative(route.table.query(), vector);
};
return new AutoQuery(createInner);
}
/**
* A query that returns a subset of the rows in the table.
*
@@ -868,51 +788,6 @@ export class TakeQuery extends QueryBase<NativeTakeQuery> {
}
}
/**
* A builder for automatic string searches.
*
* Automatic search determines whether to use full-text or vector search from
* the table revision selected for each execution. This builder exposes the
* common operations supported by both query families.
*
* @hideconstructor
*/
export class AutoQuery extends StandardQueryBase<
NativeQuery | NativeVectorQuery
> {
private readonly calls: Array<
(inner: NativeQuery | NativeVectorQuery) => void
> = [];
/** @hidden */
constructor(
private readonly createInner: () => Promise<
NativeQuery | NativeVectorQuery
>,
) {
super();
}
/** @hidden */
protected override doCall(
fn: (inner: NativeQuery | NativeVectorQuery) => void,
) {
this.calls.push(fn);
}
/** @hidden */
protected override async getInner(): Promise<
NativeQuery | NativeVectorQuery
> {
const calls = [...this.calls];
const inner = await this.createInner();
for (const call of calls) {
call(inner);
}
return inner;
}
}
/** A builder for LanceDB queries.
*
* @see {@link Table#query}, {@link Table#search}
@@ -965,19 +840,45 @@ export class Query extends StandardQueryBase<NativeQuery> {
* a default `limit` of 10 will be used. @see {@link Query#limit}
*/
nearestTo(vector: IntoVector): VectorQuery {
const inner = this.inner;
if (inner instanceof Promise) {
const nativeQuery = inner.then(async (resolvedInner) =>
nearestToNative(resolvedInner, await vector),
);
const callNearestTo = (
inner: NativeQuery,
resolved: Float32Array | Float64Array | Uint8Array | number[],
): NativeVectorQuery => {
const raw = Array.isArray(resolved)
? null
: extractVectorBuffer(resolved);
if (raw) {
return inner.nearestToRaw(raw.data, raw.dtype);
}
return inner.nearestTo(Float32Array.from(resolved as number[]));
};
if (this.inner instanceof Promise) {
const nativeQuery = this.inner.then(async (inner) => {
const resolved = vector instanceof Promise ? await vector : vector;
return callNearestTo(inner, resolved);
});
return new VectorQuery(nativeQuery);
}
if (vector instanceof Promise) {
return new VectorQuery(
vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)),
);
const res = (async () => {
try {
const v = await vector;
// biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
const value: any = this.nearestTo(v);
const inner = value.inner as
| NativeVectorQuery
| Promise<NativeVectorQuery>;
return inner;
} catch (e) {
return Promise.reject(e);
}
})();
return new VectorQuery(res);
} else {
const vectorQuery = callNearestTo(this.inner, vector);
return new VectorQuery(vectorQuery);
}
return new VectorQuery(nearestToNative(inner, vector));
}
nearestToText(query: string | FullTextQuery, columns?: string[]): Query {
+1 -2
View File
@@ -406,11 +406,10 @@ function matchingFields(fields: Field[], tree: FieldTree): Field[] {
field.name,
new Struct(matchingFields(struct.children, value)),
field.nullable,
field.metadata,
),
);
} else {
matches.push(field);
matches.push(new Field(field.name, value as DataType, field.nullable));
}
}
return matches;
+15 -45
View File
@@ -43,12 +43,10 @@ import {
Table as _NativeTable,
} from "./native";
import {
AutoQuery,
FullTextQuery,
Query,
TakeQuery,
VectorQuery,
createAutoQuery,
instanceOfFullTextQuery,
} from "./query";
import { sanitizeType } from "./sanitize";
@@ -525,7 +523,7 @@ export abstract class Table {
query: string | IntoVector | MultiVector | FullTextQuery,
queryType?: string,
ftsColumns?: string | string[],
): VectorQuery | Query | AutoQuery;
): VectorQuery | Query;
/**
* Search the table with a given query vector.
*
@@ -630,18 +628,6 @@ export abstract class Table {
/**
* Update per-field (column) metadata.
*
* The following keys are treated specially, by convention, and should be
* used when appropriate:
*
* - `lancedb:description`: for a human-readable description of a field.
* - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
* names the tag category; e.g. `lancedb:tag:model: "clip"`.
* - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
* `feature_v2` might be in the same logical column.
* - `lancedb:status`: for status options (`production`, `candidate`,
* `deprecated`, `archived`) to designate the current life cycle state of
* this column.
* @param {FieldMetadataUpdate[]} updates One or more per-field updates. Each
* update's metadata is merged into the field's existing metadata by default;
* a value of `null` deletes that key, and `replace: true` swaps the whole map.
@@ -989,11 +975,10 @@ export class LocalTable extends Table {
return this.inner.display();
}
private async getEmbeddingFunctions(
inner: _NativeTable = this.inner,
): Promise<Map<string, EmbeddingFunctionConfig>> {
const schemaBuf = await inner.schema();
const schema = tableFromIPC(schemaBuf).schema;
private async getEmbeddingFunctions(): Promise<
Map<string, EmbeddingFunctionConfig>
> {
const schema = await this.schema();
const registry = getRegistry();
return registry.parseFunctions(schema.metadata);
}
@@ -1175,7 +1160,7 @@ export class LocalTable extends Table {
query: string | IntoVector | MultiVector | FullTextQuery,
queryType: string = "auto",
ftsColumns?: string | string[],
): VectorQuery | Query | AutoQuery {
): VectorQuery | Query {
if (typeof query !== "string" && !instanceOfFullTextQuery(query)) {
if (queryType === "fts") {
throw new Error("Cannot perform full text search on a vector query");
@@ -1190,28 +1175,14 @@ export class LocalTable extends Table {
});
}
if (queryType === "auto") {
if (instanceOfFullTextQuery(query)) {
return this.query().fullTextSearch(query, {
columns: ftsColumns,
});
}
const columns =
typeof ftsColumns === "string" ? [ftsColumns] : (ftsColumns ?? null);
return createAutoQuery(this.inner, query, columns, async (metadata) => {
const functions = await getRegistry().parseFunctions(
new Map([["embedding_functions", metadata]]),
);
// TODO: Support multiple embedding functions
const embeddingFunc: EmbeddingFunctionConfig | undefined = functions
.values()
.next().value;
// The route only calls this callback when embedding metadata exists.
// parseFunctions either yields a provider or reports malformed metadata.
if (!embeddingFunc)
throw new Error("Invalid embedding function metadata");
return await embeddingFunc.function.computeQueryEmbeddings(query);
// The query type is auto or vector
// fall back to full text search if no embedding functions are defined and the query is a string
if (
queryType === "auto" &&
(getRegistry().length() === 0 || instanceOfFullTextQuery(query))
) {
return this.query().fullTextSearch(query, {
columns: ftsColumns,
});
}
@@ -1567,8 +1538,7 @@ export interface FieldMetadataUpdate {
path: string;
/**
* Metadata key/value pairs. Merged into the field's existing metadata by
* default; a value of `null` deletes that key. See
* {@link Table.updateFieldMetadata} for the conventional `lancedb:*` keys.
* default; a value of `null` deletes that key.
*/
metadata: Record<string, string | null>;
/** If true, replace the field's entire metadata map instead of merging. */
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.38.0-beta.11",
"version": "0.38.0-beta.10",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.38.0-beta.11",
"version": "0.38.0-beta.10",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.38.0-beta.11",
"version": "0.38.0-beta.10",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.38.0-beta.11",
"version": "0.38.0-beta.10",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.38.0-beta.11",
"version": "0.38.0-beta.10",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.38.0-beta.11",
"version": "0.38.0-beta.10",
"os": [
"win32"
],
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.38.0-beta.11",
"version": "0.38.0-beta.10",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "@lancedb/lancedb",
"version": "0.38.0-beta.11",
"version": "0.38.0-beta.9",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@lancedb/lancedb",
"version": "0.38.0-beta.11",
"version": "0.38.0-beta.9",
"cpu": [
"x64",
"arm64"
+1 -1
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.38.0-beta.11",
"version": "0.38.0-beta.10",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
-13
View File
@@ -278,13 +278,6 @@ impl Table {
Ok(Query::new(self.inner_ref()?.query()))
}
/// Return a read-only table handle pinned to the current query revision.
#[napi(catch_unwind)]
pub async fn query_snapshot(&self) -> napi::Result<Self> {
let snapshot = self.inner_ref()?.query_snapshot().await.default_error()?;
Ok(Self::new(snapshot))
}
#[napi(catch_unwind)]
pub fn take_offsets(&self, offsets: Vec<i64>) -> napi::Result<TakeQuery> {
Ok(TakeQuery::new(
@@ -561,12 +554,6 @@ impl Table {
.default_error()
}
#[napi(catch_unwind)]
pub async fn checkout_current(&self) -> napi::Result<Self> {
let table = self.inner_ref()?.checkout_current().await.default_error()?;
Ok(Self::new(table))
}
#[napi(catch_unwind)]
pub async fn checkout(&self, version: i64) -> napi::Result<()> {
self.inner_ref()?
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.38.0-beta.11"
version = "0.38.0-beta.10"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
+1 -4
View File
@@ -101,12 +101,9 @@ azure = ["adlfs>=2024.2.0"]
[tool.maturin]
python-source = "python"
module-name = "lancedb._lancedb"
# uv installs the project as an editable package before `uv run`, so keep that
# bootstrap build consistent with `maturin develop`.
editable-profile = "dev"
[build-system]
requires = ["maturin>=1.10"]
requires = ["maturin>=1.9.4"]
build-backend = "maturin"
[tool.ruff.lint]
-21
View File
@@ -179,18 +179,6 @@ def connect(
... },
... )
For Azure Blob Storage, credentials can be passed directly without setting
environment variables:
>>> azure_storage_options = {
... "account_name": "some-account",
... "account_key": "some-key",
... }
>>> db = lancedb.connect( # doctest: +SKIP
... "az://my-container/my-database",
... storage_options=azure_storage_options,
... )
For tests and temporary data, use an in-memory database:
>>> db = lancedb.connect("memory://")
@@ -477,10 +465,6 @@ async def connect_async(
--------
>>> import lancedb
>>> azure_storage_options = {
... "account_name": "some-account",
... "account_key": "some-key",
... }
>>> async def doctest_example():
... # For a local directory, provide a path to the database
... db = await lancedb.connect_async("~/.lancedb")
@@ -488,11 +472,6 @@ async def connect_async(
... db = await lancedb.connect_async("s3://my-bucket/lancedb",
... storage_options={
... "aws_access_key_id": "***"})
... # Azure credentials can also be passed directly
... db = await lancedb.connect_async(
... "az://my-container/my-database",
... storage_options=azure_storage_options,
... )
... # For tests and temporary data, use an in-memory database
... db = await lancedb.connect_async("memory://")
... # Connect to LanceDB cloud
-1
View File
@@ -283,7 +283,6 @@ class Table:
mode: Literal["append", "overwrite"],
progress: Optional[Any] = None,
write_parallelism: Optional[int] = None,
allow_external_blob_outside_bases: bool = False,
) -> AddResult: ...
async def update(
self, updates: Dict[str, str], where: Optional[str]
+3 -26
View File
@@ -222,7 +222,6 @@ class PythonEnvironmentSpec(_RemoteValue):
kind: str
packages: tuple[str, ...] = ()
channels: tuple[str, ...] = ()
path: Optional[str] = None
modules: tuple[str, ...] = ()
image: Optional[str] = None
@@ -910,25 +909,13 @@ class UdfDefinition:
pip: tuple[str, ...],
env: Mapping[str, str],
python_version: Optional[str],
conda: tuple[str, ...] = (),
conda_channels: tuple[str, ...] = (),
):
function_name = name or function.__name__
if not _FUNCTION_NAME.fullmatch(function_name):
raise ValueError(f"invalid Function name: {function_name!r}")
if pip and conda:
raise ValueError("a Function environment is pip or conda, not both")
if conda_channels and not conda:
raise ValueError("conda_channels requires conda packages")
packages = tuple(sorted(set(conda if conda else pip)))
packages = tuple(sorted(set(pip)))
if any(not package or package != package.strip() for package in packages):
raise ValueError("package requirements must be non-empty and trimmed")
if conda:
environment_spec = PythonEnvironmentSpec(
kind="conda", packages=packages, channels=tuple(conda_channels)
)
else:
environment_spec = PythonEnvironmentSpec(kind="pip", packages=packages)
raise ValueError("pip requirements must be non-empty and trimmed")
environment = dict(env)
if any(
not isinstance(key, str) or not isinstance(value, str)
@@ -942,7 +929,7 @@ class UdfDefinition:
kind="python",
python_version=python_version
or f"{sys.version_info.major}.{sys.version_info.minor}",
environment=environment_spec,
environment=PythonEnvironmentSpec(kind="pip", packages=packages),
env=environment,
)
self._function = function
@@ -989,8 +976,6 @@ def udf(
pip: tuple[str, ...] | list[str] = (),
env: Optional[Mapping[str, str]] = None,
python_version: Optional[str] = None,
conda: tuple[str, ...] | list[str] = (),
conda_channels: tuple[str, ...] | list[str] = (),
) -> Callable[[Callable[..., Any]], UdfDefinition]: ...
@@ -1003,8 +988,6 @@ def udf(
pip: tuple[str, ...] | list[str] = (),
env: Optional[Mapping[str, str]] = None,
python_version: Optional[str] = None,
conda: tuple[str, ...] | list[str] = (),
conda_channels: tuple[str, ...] | list[str] = (),
):
"""Prepare a scalar Python callable for remote Function registration.
@@ -1027,10 +1010,6 @@ def udf(
provided together with ``input_schema``.
pip : sequence of str, optional
Pip requirements for the remote environment.
conda : sequence of str, optional
Conda packages for the remote environment, instead of ``pip``.
conda_channels : sequence of str, optional
Conda channels in priority order; requires ``conda``.
env : mapping of str to str, optional
Environment variables included in the Function definition.
python_version : str, optional
@@ -1070,8 +1049,6 @@ def udf(
pip=tuple(pip),
env={} if env is None else env,
python_version=python_version,
conda=tuple(conda),
conda_channels=tuple(conda_channels),
)
if function is None:
-12
View File
@@ -7,7 +7,6 @@ from typing import List, Literal, Optional
from ._lancedb import (
IndexConfig,
)
from .query import DocumentGranularity
from .types import BaseTokenizerType
lang_mapping = {
@@ -122,11 +121,6 @@ class FTS:
>>> config = FTS(block_size=256)
Create an index that treats each deepest-list element as one document:
>>> from lancedb.query import DocumentGranularity
>>> config = FTS(document_granularity=DocumentGranularity.LIST_ELEMENT)
Attributes
----------
with_position : bool, default False
@@ -178,11 +172,6 @@ class FTS:
roughly half of the available CPU cores. The effective value is
limited by the available compute capacity. This build-only setting is
not persisted with the index and does not apply to remote tables.
document_granularity : DocumentGranularity, default ROW
``ROW`` treats the selected text in one table row as one document.
``LIST_ELEMENT`` treats each element of the deepest list on the indexed
field path as one document and returns its physical coordinates in
``_doc_index`` for matching queries.
Notes
-----
@@ -207,7 +196,6 @@ class FTS:
custom_stop_words: Optional[List[str]] = None
memory_limit: Optional[int] = None
num_workers: Optional[int] = None
document_granularity: DocumentGranularity = DocumentGranularity.ROW
@dataclass
+5 -29
View File
@@ -375,13 +375,6 @@ class FullTextOperator(str, Enum):
OR = "OR"
class DocumentGranularity(str, Enum):
"""The unit treated as one full-text-search document."""
ROW = "row"
LIST_ELEMENT = "list_element"
class Occur(str, Enum):
SHOULD = "SHOULD"
MUST = "MUST"
@@ -485,10 +478,6 @@ class MatchQuery(FullTextQuery):
prefix_length : int, optional
The number of beginning characters being unchanged for fuzzy matching.
This is useful to achieve prefix matching.
document_granularity : DocumentGranularity, optional
Explicitly select row or deepest-list-element documents. If omitted,
the indexed granularity is inferred. When both granularities are indexed
for the field, this must be specified. With no index, row granularity is used.
"""
query: str
@@ -498,9 +487,6 @@ class MatchQuery(FullTextQuery):
max_expansions: int = pydantic.Field(50, kw_only=True)
operator: FullTextOperator = pydantic.Field(FullTextOperator.OR, kw_only=True)
prefix_length: int = pydantic.Field(0, kw_only=True)
document_granularity: Optional[DocumentGranularity] = pydantic.Field(
None, kw_only=True
)
def query_type(self) -> FullTextQueryType:
return FullTextQueryType.MATCH
@@ -517,20 +503,11 @@ class PhraseQuery(FullTextQuery):
The query string to match against.
column : str
The name of the column to match against.
slop : int, default 0
The maximum number of intervening positions permitted in the phrase.
document_granularity : DocumentGranularity, optional
Explicitly select row or deepest-list-element documents. If omitted,
the indexed granularity is inferred. When both granularities are indexed
for the field, this must be specified. With no index, row granularity is used.
"""
query: str
column: str
slop: int = pydantic.Field(0, kw_only=True)
document_granularity: Optional[DocumentGranularity] = pydantic.Field(
None, kw_only=True
)
def query_type(self) -> FullTextQueryType:
return FullTextQueryType.MATCH_PHRASE
@@ -3401,10 +3378,9 @@ class AsyncQuery(AsyncStandardQuery):
pass in multiple vectors. When multiple vectors are passed in, if the vector
column is with multivector type, then the vectors will be treated as a single
query. Or the vectors will be treated as multiple queries, this can be useful
if you want to find the nearest vectors to multiple query vectors. Flat
searches share one table scan across the query vectors, avoiding the scan
and memory amplification of making multiple queries concurrently. If
multiple vectors are passed in then
if you want to find the nearest vectors to multiple query vectors.
This is not expected to be faster than making multiple queries concurrently;
it is just a convenience method. If multiple vectors are passed in then
an additional column `query_index` will be added to the results. This column
will contain the index of the query vector that the result is nearest to.
"""
@@ -3533,8 +3509,8 @@ class AsyncFTSQuery(AsyncStandardQuery):
Typically, a single vector is passed in as the query. However, you can also
pass in multiple vectors. This can be useful if you want to find the nearest
vectors to multiple query vectors. Flat searches share one table scan across
the query vectors instead of issuing concurrent full scans.
vectors to multiple query vectors. This is not expected to be faster than
making multiple queries concurrently; it is just a convenience method.
If multiple vectors are passed in then an additional column `query_index`
will be added to the results. This column will contain the index of the
query vector that the result is nearest to.
-7
View File
@@ -61,7 +61,6 @@ from lancedb.table import _normalize_progress
from ..query import (
AnalyzePlanDistributedMetrics,
DocumentGranularity,
LanceQueryBuilder,
LanceTakeQueryBuilder,
LanceVectorQueryBuilder,
@@ -350,7 +349,6 @@ class RemoteTable(Table):
ngram_max_length: int = 3,
prefix_only: bool = False,
block_size: int = 128,
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
name: Optional[str] = None,
):
"""Create a full-text search index on a column.
@@ -373,7 +371,6 @@ class RemoteTable(Table):
ngram_max_length=ngram_max_length,
prefix_only=prefix_only,
block_size=block_size,
document_granularity=document_granularity,
)
LOOP.run(
self._table.create_index(
@@ -613,7 +610,6 @@ class RemoteTable(Table):
fill_value: float = 0.0,
progress: Optional[Union[bool, Callable, Any]] = None,
write_parallelism: Optional[int] = None,
allow_external_blob_outside_bases: bool = False,
) -> AddResult:
"""Add more data to the [Table][lancedb.table.Table].
@@ -646,8 +642,6 @@ class RemoteTable(Table):
data in flight. Defaults to an estimate based on the data size,
capped at the number of CPU cores. Lower this if bulk ingestion is
using too much memory.
allow_external_blob_outside_bases: bool, default False
Not supported on LanceDB Cloud. Setting this raises.
Returns
-------
@@ -664,7 +658,6 @@ class RemoteTable(Table):
fill_value=fill_value,
progress=progress,
write_parallelism=write_parallelism,
allow_external_blob_outside_bases=allow_external_blob_outside_bases,
)
)
finally:
+1 -41
View File
@@ -85,7 +85,6 @@ from .query import (
AsyncQuery,
AsyncTakeQuery,
AsyncVectorQuery,
DocumentGranularity,
FullTextQuery,
LanceEmptyQueryBuilder,
LanceFtsQueryBuilder,
@@ -1169,7 +1168,6 @@ class Table(ABC):
ngram_max_length: int = 3,
prefix_only: bool = False,
block_size: int = 128,
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
):
@@ -1248,11 +1246,6 @@ class Table(ABC):
The number of documents per compressed posting block. Must be 128
or 256. A value of 256 uses the experimental FTS V3 format and
may introduce breaking changes.
document_granularity: DocumentGranularity, default ROW
``ROW`` treats the selected text in one table row as one document.
``LIST_ELEMENT`` treats each element of the deepest list on the field
path as one document and returns its physical coordinates in
``_doc_index`` for matching queries.
wait_timeout: timedelta, optional
The timeout to wait if indexing is asynchronous.
name: str, optional
@@ -1276,7 +1269,6 @@ class Table(ABC):
fill_value: float = 0.0,
progress: Optional[Union[bool, Callable, Any]] = None,
write_parallelism: Optional[int] = None,
allow_external_blob_outside_bases: bool = False,
) -> AddResult:
"""Add more data to the [Table][lancedb.table.Table].
@@ -1328,10 +1320,6 @@ class Table(ABC):
data in flight. Defaults to an estimate based on the data size,
capped at the number of CPU cores. Lower this if bulk ingestion is
using too much memory.
allow_external_blob_outside_bases: bool, default False
Store blob URIs that sit outside registered blob bases. The row
keeps a reference, so the object has to stay readable. Local
tables only.
Returns
-------
@@ -2127,25 +2115,12 @@ class Table(ABC):
----------
updates : dict
One or more dicts, each with:
- "path": str dot-path to the field (e.g. "embedding" or "a.b.c").
- "metadata": dict[str, str | None] keys to set; a value of ``None``
deletes that key.
- "replace": bool, optional replace the field's whole metadata map
instead of merging (default False).
The following keys are treated specially, by convention, and should
be used when appropriate:
- "lancedb:description": for a human-readable description of a field.
- ``"lancedb:tag:<name>"`` for a user-defined key-value tag, where the
suffix names the tag category; e.g. "lancedb:tag:model": "clip".
- "lancedb:logical-column" for a column grouping; e.g. "feature_v1"
and "feature_v2" might be in the same logical column.
- "lancedb:status" for status options ("production", "candidate",
"deprecated", "archived") to designate the current life cycle
state of this column.
Returns
-------
UpdateFieldMetadataResult
@@ -3293,7 +3268,6 @@ class LanceTable(Table):
ngram_max_length: int = 3,
prefix_only: bool = False,
block_size: int = 128,
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
name: Optional[str] = None,
):
"""Create a full-text search index on a column.
@@ -3345,11 +3319,7 @@ class LanceTable(Table):
tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)
tokenizer_configs["custom_stop_words"] = custom_stop_words
config = FTS(
block_size=block_size,
document_granularity=document_granularity,
**tokenizer_configs,
)
config = FTS(block_size=block_size, **tokenizer_configs)
try:
LOOP.run(
@@ -3439,7 +3409,6 @@ class LanceTable(Table):
fill_value: float = 0.0,
progress: Optional[Union[bool, Callable, Any]] = None,
write_parallelism: Optional[int] = None,
allow_external_blob_outside_bases: bool = False,
) -> AddResult:
"""Add data to the table.
If vector columns are missing and the table
@@ -3467,9 +3436,6 @@ class LanceTable(Table):
data in flight. Defaults to an estimate based on the data size,
capped at the number of CPU cores. Lower this if bulk ingestion is
using too much memory.
allow_external_blob_outside_bases: bool, default False
Allow blob URIs outside registered bases. See :meth:`Table.add`.
Local tables only.
Returns
-------
@@ -3486,7 +3452,6 @@ class LanceTable(Table):
fill_value=fill_value,
progress=progress,
write_parallelism=write_parallelism,
allow_external_blob_outside_bases=allow_external_blob_outside_bases,
)
)
finally:
@@ -5400,7 +5365,6 @@ class AsyncTable:
fill_value: Optional[float] = None,
progress: Optional[Union[bool, Callable, Any]] = None,
write_parallelism: Optional[int] = None,
allow_external_blob_outside_bases: bool = False,
) -> AddResult:
"""Add more data to the [AsyncTable][lancedb.table.AsyncTable].
@@ -5431,9 +5395,6 @@ class AsyncTable:
data in flight. Defaults to an estimate based on the data size,
capped at the number of CPU cores. Lower this if bulk ingestion is
using too much memory.
allow_external_blob_outside_bases: bool, default False
Allow blob URIs outside registered bases. See :meth:`Table.add`.
Local tables only.
"""
schema = await self.schema()
@@ -5470,7 +5431,6 @@ class AsyncTable:
mode or "append",
progress=progress,
write_parallelism=write_parallelism,
allow_external_blob_outside_bases=allow_external_blob_outside_bases,
)
except RuntimeError as e:
if "Cast error" in str(e):
+2 -2
View File
@@ -105,7 +105,7 @@ def test_quickstart(tmp_path):
tbl.create_index(num_sub_vectors=1)
# --8<-- [end:create_index]
# --8<-- [start:delete_rows]
tbl.delete("item = 'fizz'")
tbl.delete('item = "fizz"')
# --8<-- [end:delete_rows]
# --8<-- [start:drop_table]
db.drop_table("my_table")
@@ -201,7 +201,7 @@ async def test_quickstart_async(tmp_path):
await tbl.create_index("vector")
# --8<-- [end:create_index_async]
# --8<-- [start:delete_rows_async]
await tbl.delete("item = 'fizz'")
await tbl.delete('item = "fizz"')
# --8<-- [end:delete_rows_async]
# --8<-- [start:drop_table_async]
await db.drop_table("my_table_async")
@@ -266,7 +266,7 @@ def test_table():
tbl.add(pydantic_model_items)
# --8<-- [end:add_table_from_pydantic]
# --8<-- [start:delete_row]
tbl.delete("item = 'fizz'")
tbl.delete('item = "fizz"')
# --8<-- [end:delete_row]
# --8<-- [start:delete_specific_row]
data = [
@@ -538,7 +538,7 @@ async def test_table_async():
await async_tbl.add(pydantic_model_items)
# --8<-- [end:add_table_async_from_pydantic]
# --8<-- [start:delete_row_async]
await async_tbl.delete("item = 'fizz'")
await async_tbl.delete('item = "fizz"')
# --8<-- [end:delete_row_async]
# --8<-- [start:delete_specific_row_async]
data = [
-68
View File
@@ -617,71 +617,3 @@ def test_fetch_blobs_nested_path_survives_sort_after_query():
def _identifiable_payload(size: int) -> bytes:
block = 256
return b"".join(bytes([i % 256]) * block for i in range(size // block))
def _external_uri_blob_array(uris):
blob_type = lancedb.blob("image").type
storage_type = blob_type.storage_type
child_names = [field.name for field in storage_type]
assert "uri" in child_names, "blob layout no longer has a uri child"
children = [
pa.array(uris if field.name == "uri" else [None] * len(uris), type=field.type)
for field in storage_type
]
storage = pa.StructArray.from_arrays(children, fields=list(storage_type))
return pa.ExtensionArray.from_storage(blob_type, storage)
def _external_uri_table_and_rows(name, uris):
db = lancedb.connect("memory:///")
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
table = db.create_table(name, schema=schema)
rows = pa.Table.from_arrays(
[
pa.array(range(len(uris)), type=pa.int64()),
_external_uri_blob_array(uris),
],
schema=schema,
)
return table, rows
def test_add_external_uri_struct_round_trips_with_flag(tmp_path):
payload = b"external-uri-bytes"
blob_path = tmp_path / "payload.bin"
blob_path.write_bytes(payload)
table, rows = _external_uri_table_and_rows("external_struct", [blob_path.as_uri()])
table.add(rows, allow_external_blob_outside_bases=True)
hits = table.search().to_arrow()
blobs = table.fetch_blobs("image", hits)
assert blobs[0].as_py() == payload
def test_add_external_uri_without_flag_raises(tmp_path):
blob_path = tmp_path / "payload.bin"
blob_path.write_bytes(b"unreachable")
table, rows = _external_uri_table_and_rows("external_no_flag", [blob_path.as_uri()])
with pytest.raises(ValueError, match="allow_external_blob_outside_bases"):
table.add(rows)
assert table.count_rows() == 0
def test_add_external_uri_string_round_trips_with_flag(tmp_path):
payload = b"external-uri-bytes"
blob_path = tmp_path / "payload.bin"
blob_path.write_bytes(payload)
db = lancedb.connect("memory:///")
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
table = db.create_table("external_string", schema=schema)
table.add(
[{"id": 1, "image": blob_path.as_uri()}],
allow_external_blob_outside_bases=True,
)
hits = table.search().to_arrow()
blobs = table.fetch_blobs("image", hits)
assert blobs[0].as_py() == payload
@@ -69,26 +69,6 @@ def _run_packaged(definition, *args):
return namespace[definition.registration_request.artifact.entrypoint](*args)
def test_udf_conda_environment():
@udf(conda=["scipy", "numpy"], conda_channels=["conda-forge", "defaults"])
def halve(value: float) -> float:
return value / 2
request = json.loads(halve.registration_request.to_canonical_json())
assert request["runtime"]["environment"] == {
"kind": "conda",
"packages": ["numpy", "scipy"],
"channels": ["conda-forge", "defaults"],
}
pip_request = json.loads(normalize_score.registration_request.to_canonical_json())
assert "channels" not in pip_request["runtime"]["environment"]
with pytest.raises(ValueError, match="not both"):
udf(name="both", pip=["numpy"], conda=["numpy"])(lambda value: value)
with pytest.raises(ValueError, match="requires conda"):
udf(name="channels", conda_channels=["conda-forge"])(lambda value: value)
def test_udf_packages_attribute_access_and_body_imports():
@udf
def word_norm(body: str) -> float:
-77
View File
@@ -25,7 +25,6 @@ from lancedb.db import DBConnection
from lancedb.index import FTS
from lancedb.query import (
BoostQuery,
DocumentGranularity,
MatchQuery,
MultiMatchQuery,
PhraseQuery,
@@ -246,55 +245,6 @@ def test_create_inverted_index_rejects_invalid_block_size(table):
table.create_index("text", config=FTS(block_size=129))
def test_list_element_document_granularity(tmp_path):
docs_type = pa.list_(pa.struct([pa.field("content", pa.string())]))
docs = pa.array(
[
[
{"content": "alpha beta"},
None,
{"content": ""},
{"content": "the and"},
{"content": "alpha beta"},
]
],
type=docs_type,
)
table = ldb.connect(tmp_path).create_table(
"list_element_docs", pa.table({"id": [0], "docs": docs})
)
row_table = ldb.connect(tmp_path).create_table(
"row_docs", pa.table({"id": [0], "docs": docs})
)
row_table.create_index("docs.content", config=FTS())
row_result = row_table.search(MatchQuery("alpha", "docs.content")).to_arrow()
assert row_result.num_rows == 1
assert "_doc_index" not in row_result.column_names
granularity = DocumentGranularity.LIST_ELEMENT
table.create_index(
"docs.content",
config=FTS(with_position=True, document_granularity=granularity),
)
assert table.list_indices()[0].columns == ["docs.content"]
def coordinates(query):
result = table.search(query).limit(10).to_arrow()
doc_index_type = result.schema.field("_doc_index").type
assert pa.types.is_list(doc_index_type)
assert doc_index_type.value_type == pa.uint32()
return sorted(result["_doc_index"].to_pylist())
assert coordinates(
MatchQuery("alpha", "docs.content", document_granularity=granularity)
) == [[0], [4]]
assert coordinates(
PhraseQuery("alpha beta", "docs.content", document_granularity=granularity)
) == [[0], [4]]
assert coordinates(MatchQuery("alpha", "docs.content")) == [[0], [4]]
assert FTS().document_granularity is DocumentGranularity.ROW
def test_create_inverted_index_respects_build_memory_limit(table):
with pytest.raises(ValueError, match="exceeds worker memory limit"):
table.create_index(
@@ -1139,20 +1089,6 @@ def test_fts_query_to_json():
)
assert json_str == expected
# Test MatchQuery with list-element document granularity
match_query = MatchQuery(
"hello world",
"text",
document_granularity=DocumentGranularity.LIST_ELEMENT,
)
json_str = match_query.to_json()
expected = (
'{"match":{"column":"text","terms":"hello world","boost":1.0,'
'"fuzziness":0,"max_expansions":50,"operator":"Or","prefix_length":0,'
'"document_granularity":"list_element"}}'
)
assert json_str == expected
# Test MatchQuery with options
match_query = MatchQuery("puppy", "text", fuzziness=2, boost=1.5, prefix_length=3)
json_str = match_query.to_json()
@@ -1162,19 +1098,6 @@ def test_fts_query_to_json():
)
assert json_str == expected
# Test PhraseQuery with list-element document granularity
phrase_query = PhraseQuery(
"quick brown fox",
"title",
document_granularity=DocumentGranularity.LIST_ELEMENT,
)
json_str = phrase_query.to_json()
expected = (
'{"phrase":{"column":"title","terms":"quick brown fox","slop":0,'
'"document_granularity":"list_element"}}'
)
assert json_str == expected
# Test PhraseQuery
phrase_query = PhraseQuery("quick brown fox", "title")
json_str = phrase_query.to_json()
-17
View File
@@ -897,23 +897,6 @@ def test_query_builder_batches(table):
assert rs_list["id"][1] == 2
def test_batch_vector_query_shares_filtered_flat_scan(table):
query = (
table.search([[1.0, 2.0], [3.0, 4.0]])
.where("id > 0", prefilter=True)
.limit(1)
.select(["id"])
)
plan = query.explain_plan(verbose=True)
assert "KNNVectorDistance: queries=2" in plan
assert "UnionExec" not in plan
results = query.to_arrow()
assert len(results) == 2
assert results["query_index"].to_pylist() == [0, 1]
def test_dynamic_projection(table):
rs = (
LanceVectorQueryBuilder(table, [0, 0], "vector")
-43
View File
@@ -1618,49 +1618,6 @@ def test_query_sync_fts():
)
def test_query_sync_fts_document_granularity():
from lancedb.query import DocumentGranularity, MatchQuery
def handler(body):
assert body == {
"full_text_query": {
"query": {
"match": {
"column": "docs.content",
"terms": "alpha",
"boost": 1.0,
"fuzziness": 0,
"max_expansions": 50,
"operator": "Or",
"prefix_length": 0,
"document_granularity": "list_element",
}
}
},
"k": 10,
"prefilter": True,
"vector": [],
"version": None,
}
return pa.table(
{
"id": [1, 1],
"_doc_index": pa.array([[0], [4]], type=pa.list_(pa.uint32())),
}
)
with query_test_table(handler, server_version=Version("0.6.0")) as table:
result = table.search(
MatchQuery(
"alpha",
"docs.content",
document_granularity=DocumentGranularity.LIST_ELEMENT,
)
).to_arrow()
assert result["_doc_index"].to_pylist() == [[0], [4]]
def test_query_sync_hybrid():
def handler(body):
if "full_text_query" in body:
-20
View File
@@ -4,7 +4,6 @@
import asyncio
import copy
from concurrent.futures import ThreadPoolExecutor
from datetime import timedelta
import threading
@@ -87,25 +86,6 @@ def test_s3_lifecycle(s3_bucket: str):
asyncio.run(test())
@pytest.mark.s3_test
def test_concurrent_open_table(s3_bucket: str):
uri = f"s3://{s3_bucket}/test_concurrent_open_table"
db = lancedb.connect(uri, storage_options=copy.copy(CONFIG))
db.create_table("test", pa.table({"x": [1, 2, 3]}))
num_workers = 32
barrier = threading.Barrier(num_workers)
def open_and_count(_):
barrier.wait()
return db.open_table("test").count_rows()
with ThreadPoolExecutor(max_workers=num_workers) as pool:
row_counts = list(pool.map(open_and_count, range(num_workers)))
assert row_counts == [3] * num_workers
@pytest.fixture()
def kms_key():
kms = get_boto3_client("kms", endpoint_url=CONFIG["aws_endpoint"])
+2 -8
View File
@@ -8,7 +8,7 @@ use lancedb::index::vector::{
};
use lancedb::index::{
Index as LanceDbIndex,
scalar::{BTreeIndexBuilder, DocumentGranularity, FmIndexBuilder, FtsIndexBuilder},
scalar::{BTreeIndexBuilder, FmIndexBuilder, FtsIndexBuilder},
};
use pyo3::IntoPyObject;
use pyo3::types::PyStringMethods;
@@ -60,11 +60,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
.ngram_min_length(params.ngram_min_length)
.ngram_max_length(params.ngram_max_length)
.ngram_prefix_only(params.prefix_only)
.custom_stop_words(params.custom_stop_words)
.document_granularity(
DocumentGranularity::try_from(params.document_granularity.as_str())
.map_err(|err| PyValueError::new_err(err.to_string()))?,
);
.custom_stop_words(params.custom_stop_words);
if let Some(memory_limit) = params.memory_limit {
inner_opts = inner_opts.memory_limit_mb(memory_limit);
}
@@ -225,7 +221,6 @@ struct FtsParams {
block_size: usize,
memory_limit: Option<u64>,
num_workers: Option<usize>,
document_granularity: String,
}
#[derive(FromPyObject)]
@@ -486,7 +481,6 @@ mod tests {
block_size = 128
memory_limit = 2048
num_workers = 7
document_granularity = 'row'
config = FTS()",
None,
+12 -45
View File
@@ -16,8 +16,8 @@ use arrow::pyarrow::FromPyArrow;
use arrow::pyarrow::IntoPyArrow;
use arrow::pyarrow::ToPyArrow;
use lancedb::index::scalar::{
BooleanQuery, BoostQuery, DocumentGranularity, FtsQuery, FullTextSearchQuery, MatchQuery,
MultiMatchQuery, Occur, Operator, PhraseQuery,
BooleanQuery, BoostQuery, FtsQuery, FullTextSearchQuery, MatchQuery, MultiMatchQuery, Occur,
Operator, PhraseQuery,
};
use lancedb::query::AnalyzePlanDistributedMetrics;
use lancedb::query::QueryBase;
@@ -76,16 +76,8 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB<FtsQuery> {
let max_expansions = ob.getattr("max_expansions")?.extract()?;
let operator = ob.getattr("operator")?.extract::<String>()?;
let prefix_length = ob.getattr("prefix_length")?.extract()?;
let document_granularity = ob
.getattr("document_granularity")?
.extract::<Option<String>>()?
.map(|value| {
DocumentGranularity::try_from(value.as_str())
.map_err(|err| PyValueError::new_err(err.to_string()))
})
.transpose()?;
let mut query =
Ok(Self(
MatchQuery::new(query)
.with_column(Some(column))
.with_boost(boost)
@@ -94,32 +86,21 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB<FtsQuery> {
.with_operator(Operator::try_from(operator.as_str()).map_err(|e| {
PyValueError::new_err(format!("Invalid operator: {}", e))
})?)
.with_prefix_length(prefix_length);
if let Some(document_granularity) = document_granularity {
query = query.with_document_granularity(document_granularity);
}
Ok(Self(query.into()))
.with_prefix_length(prefix_length)
.into(),
))
}
"PhraseQuery" => {
let query = ob.getattr("query")?.extract()?;
let column = ob.getattr("column")?.extract()?;
let slop = ob.getattr("slop")?.extract()?;
let document_granularity = ob
.getattr("document_granularity")?
.extract::<Option<String>>()?
.map(|value| {
DocumentGranularity::try_from(value.as_str())
.map_err(|err| PyValueError::new_err(err.to_string()))
})
.transpose()?;
let mut query = PhraseQuery::new(query)
.with_column(Some(column))
.with_slop(slop);
if let Some(document_granularity) = document_granularity {
query = query.with_document_granularity(document_granularity);
}
Ok(Self(query.into()))
Ok(Self(
PhraseQuery::new(query)
.with_column(Some(column))
.with_slop(slop)
.into(),
))
}
"BoostQuery" => {
let positive: Self = ob.getattr("positive")?.extract()?;
@@ -186,13 +167,6 @@ impl<'py> IntoPyObject<'py> for PyLanceDB<FtsQuery> {
kwargs.set_item("max_expansions", query.max_expansions)?;
kwargs.set_item::<_, &str>("operator", query.operator.into())?;
kwargs.set_item("prefix_length", query.prefix_length)?;
if let Some(document_granularity) = query.document_granularity {
let value = match document_granularity {
DocumentGranularity::Row => "row",
DocumentGranularity::ListElement => "list_element",
};
kwargs.set_item("document_granularity", value)?;
}
namespace
.getattr(intern!(py, "MatchQuery"))?
.call((query.terms, query.column.unwrap()), Some(&kwargs))
@@ -200,13 +174,6 @@ impl<'py> IntoPyObject<'py> for PyLanceDB<FtsQuery> {
FtsQuery::Phrase(query) => {
let kwargs = PyDict::new(py);
kwargs.set_item("slop", query.slop)?;
if let Some(document_granularity) = query.document_granularity {
let value = match document_granularity {
DocumentGranularity::Row => "row",
DocumentGranularity::ListElement => "list_element",
};
kwargs.set_item("document_granularity", value)?;
}
namespace
.getattr(intern!(py, "PhraseQuery"))?
.call((query.terms, query.column.unwrap()), Some(&kwargs))
+2 -6
View File
@@ -780,19 +780,15 @@ impl Table {
})
}
#[pyo3(signature = (data, mode, progress=None, write_parallelism=None, allow_external_blob_outside_bases=false))]
#[pyo3(signature = (data, mode, progress=None, write_parallelism=None))]
pub fn add<'a>(
self_: PyRef<'a, Self>,
data: PyScannable,
mode: String,
progress: Option<Py<PyAny>>,
write_parallelism: Option<usize>,
allow_external_blob_outside_bases: bool,
) -> PyResult<Bound<'a, PyAny>> {
let mut op = self_
.inner_ref()?
.add(data)
.allow_external_blob_outside_bases(allow_external_blob_outside_bases);
let mut op = self_.inner_ref()?.add(data);
if mode == "append" {
op = op.mode(AddDataMode::Append);
} else if mode == "overwrite" {
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb"
version = "0.38.0-beta.11"
version = "0.38.0-beta.10"
edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true
+1 -54
View File
@@ -1476,7 +1476,7 @@ mod tests {
use crate::table::{AnyQuery, WriteOptions};
use arrow_array::{Int32Array, RecordBatch, StringArray};
use arrow_schema::{DataType, Field, Schema, SchemaRef};
use futures::{TryStreamExt, future::try_join_all, stream::once};
use futures::{TryStreamExt, stream::once};
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
@@ -1614,59 +1614,6 @@ mod tests {
);
}
#[tokio::test]
async fn test_concurrent_open_table_reuses_connection_object_store() {
let tempdir = tempdir().unwrap();
let uri = tempdir.path().to_str().unwrap();
let session = Arc::new(lance::session::Session::default());
let request = ConnectRequest {
uri: uri.to_string(),
#[cfg(feature = "remote")]
client_config: Default::default(),
options: Default::default(),
namespace_client_properties: Default::default(),
manifest_enabled: false,
read_consistency_interval: None,
session: Some(session.clone()),
};
let db = ListingDatabase::connect_with_options(&request)
.await
.unwrap();
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
db.create_table(CreateTableRequest {
name: "test".to_string(),
namespace_path: vec![],
data: Box::new(RecordBatch::new_empty(schema)) as Box<dyn Scannable>,
mode: CreateTableMode::Create,
write_options: Default::default(),
location: None,
namespace_client: None,
})
.await
.unwrap();
let before = session.store_registry().stats();
let opened_tables = try_join_all((0..32).map(|_| {
db.open_table(OpenTableRequest {
name: "test".to_string(),
namespace_path: vec![],
index_cache_size: None,
lance_read_params: None,
location: None,
namespace_client: None,
managed_versioning: None,
})
}))
.await
.unwrap();
let after = session.store_registry().stats();
assert_eq!(opened_tables.len(), 32);
assert_eq!(after.misses, before.misses);
assert_eq!(after.active_stores, before.active_stores);
assert!(after.hits >= before.hits + 32);
}
#[tokio::test]
async fn test_listing_database_root_ops_do_not_create_manifest() {
let tempdir = tempdir().unwrap();
-1
View File
@@ -19,7 +19,6 @@
mod sql;
pub(crate) use sql::canonicalize_sql_predicate;
pub use sql::expr_to_sql_string;
use std::sync::Arc;
+2 -111
View File
@@ -1,16 +1,10 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::any::TypeId;
use datafusion_common::ScalarValue;
use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
use datafusion_expr::Expr;
use datafusion_sql::sqlparser::{
dialect::{Dialect as SqlParserDialect, GenericDialect},
tokenizer::{Token, Tokenizer},
};
use datafusion_sql::unparser::{self, dialect::Dialect as UnparserDialect};
use datafusion_sql::unparser::{self, dialect::Dialect};
/// Unparser dialect that matches the quoting style expected by the Lance SQL
/// parser. Lance uses backtick (`` ` ``) as the only delimited-identifier
@@ -25,7 +19,7 @@ use datafusion_sql::unparser::{self, dialect::Dialect as UnparserDialect};
/// lower-case by the SQL parser, which would break case-sensitive schemas).
struct LanceSqlDialect;
impl UnparserDialect for LanceSqlDialect {
impl Dialect for LanceSqlDialect {
fn identifier_quote_style(&self, identifier: &str) -> Option<char> {
let needs_quote = identifier.chars().any(|c| c.is_ascii_uppercase())
|| !identifier
@@ -36,61 +30,6 @@ impl UnparserDialect for LanceSqlDialect {
}
}
/// Lance's tokenizer dialect with SQL-standard double-quoted identifiers added.
///
/// Keep this deliberately small: Lance's parser wraps `GenericDialect` and
/// delegates only identifier recognition, leaving every other dialect option at
/// its default. In particular, `/*! ... */` remains an ordinary block comment.
#[derive(Debug, Default)]
struct PredicateDialect(GenericDialect);
impl SqlParserDialect for PredicateDialect {
fn dialect(&self) -> TypeId {
self.0.dialect()
}
fn is_identifier_start(&self, ch: char) -> bool {
self.0.is_identifier_start(ch)
}
fn is_identifier_part(&self, ch: char) -> bool {
self.0.is_identifier_part(ch)
}
fn is_delimited_identifier_start(&self, ch: char) -> bool {
ch == '"' || ch == '`'
}
}
/// Canonicalize a raw SQL predicate for Lance's parser.
///
/// Lance wraps [`GenericDialect`] for identifier recognition while retaining the
/// default dialect behavior for every other lexical option. [`PredicateDialect`]
/// mirrors that contract and additionally recognizes `"` as an identifier
/// delimiter, allowing this function to rewrite only those identifier tokens.
pub fn canonicalize_sql_predicate(predicate: &str) -> crate::Result<String> {
let dialect = PredicateDialect::default();
let tokens = Tokenizer::new(&dialect, predicate)
.with_unescape(false)
.tokenize()
.map_err(|err| crate::Error::InvalidInput {
message: format!("invalid SQL predicate: {err}"),
})?;
Ok(tokens
.into_iter()
.map(|token| match token {
Token::Word(word) if word.quote_style == Some('"') => {
// with_unescape(false) retains doubled double quotes. Decode
// those before escaping any backticks for Lance's delimiter.
let identifier = word.value.replace("\"\"", "\"").replace('`', "``");
format!("`{identifier}`")
}
other => other.to_string(),
})
.collect())
}
/// Prefix for placeholder strings inserted in place of binary literals. Chosen
/// to be extremely unlikely to occur in user data.
const BINARY_PLACEHOLDER_PREFIX: &str = "__lancedb_binary_placeholder_";
@@ -174,51 +113,3 @@ pub fn expr_to_sql_string(expr: &Expr) -> crate::Result<String> {
}
Ok(sql)
}
#[cfg(test)]
mod tests {
use super::canonicalize_sql_predicate;
#[test]
fn normalizes_double_quoted_identifiers() {
assert_eq!(
canonicalize_sql_predicate(r#""PartyAbbrev" = 'D'"#).unwrap(),
"`PartyAbbrev` = 'D'"
);
assert_eq!(
canonicalize_sql_predicate(r#""MetaData"."userId" = 5"#).unwrap(),
"`MetaData`.`userId` = 5"
);
assert_eq!(
canonicalize_sql_predicate(r#""a""b" = 1"#).unwrap(),
"`a\"b` = 1"
);
}
#[test]
fn preserves_quotes_inside_literals_and_backticks() {
let filter = r#"name = 'Alice "Ace"' AND `quoted"field` = 1"#;
assert_eq!(canonicalize_sql_predicate(filter).unwrap(), filter);
}
#[test]
fn preserves_literals_and_comments_using_lance_dialect_rules() {
let predicate = r#"path = '\' AND "PartyAbbrev" = 'D' -- unmatched " in comment"#;
assert_eq!(
canonicalize_sql_predicate(predicate).unwrap(),
r#"path = '\' AND `PartyAbbrev` = 'D' -- unmatched " in comment"#
);
let predicate = r#"id = 1 /* unmatched " in block comment */"#;
assert_eq!(canonicalize_sql_predicate(predicate).unwrap(), predicate);
let predicate = r#"id = 1 /*! OR "PartyAbbrev" = 'D' */"#;
assert_eq!(canonicalize_sql_predicate(predicate).unwrap(), predicate);
}
#[test]
fn rejects_unterminated_double_quoted_identifier() {
let error = canonicalize_sql_predicate(r#""PartyAbbrev = 'D'"#).unwrap_err();
assert!(matches!(error, crate::Error::InvalidInput { .. }));
}
}
-26
View File
@@ -186,9 +186,6 @@ pub struct PythonEnvironmentSpec {
pub kind: String,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub packages: Vec<String>,
/// Conda channels in priority order; conda environments only.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub channels: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub path: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
@@ -586,26 +583,3 @@ impl RefreshColumnResult {
}
impl_json!(RefreshColumnResult);
#[cfg(test)]
mod conda_environment_tests {
use super::PythonEnvironmentSpec;
#[test]
fn conda_channels_round_trip_and_pip_stays_bare() {
let conda: PythonEnvironmentSpec = serde_json::from_str(
r#"{"kind":"conda","packages":["numpy"],"channels":["conda-forge"]}"#,
)
.unwrap();
assert_eq!(conda.channels, ["conda-forge"]);
assert!(
serde_json::to_string(&conda)
.unwrap()
.contains(r#""channels":["conda-forge"]"#)
);
let pip: PythonEnvironmentSpec =
serde_json::from_str(r#"{"kind":"pip","packages":["numpy"]}"#).unwrap();
assert!(!serde_json::to_string(&pip).unwrap().contains("channels"));
}
}
-1
View File
@@ -63,5 +63,4 @@ pub struct FmIndexBuilder {}
pub use lance_index::scalar::FullTextSearchQuery;
pub use lance_index::scalar::InvertedIndexParams as FtsIndexBuilder;
pub use lance_index::scalar::InvertedIndexParams;
pub use lance_index::scalar::inverted::DocumentGranularity;
pub use lance_index::scalar::inverted::query::*;
+1 -9
View File
@@ -10,7 +10,7 @@ use lance::io::WrappingObjectStore;
use object_store::{
CopyOptions, Error, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta,
ObjectStore, ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result,
UploadPart, list::PaginatedListStore, path::Path,
UploadPart, path::Path,
};
use async_trait::async_trait;
@@ -187,14 +187,6 @@ impl WrappingObjectStore for MirroringObjectStoreWrapper {
secondary: self.secondary.clone(),
})
}
fn wrap_paginated(
&self,
_store_prefix: &str,
original: Arc<dyn PaginatedListStore>,
) -> Option<Arc<dyn PaginatedListStore>> {
Some(original)
}
}
// windows pathing can't be simply concatenated
@@ -12,7 +12,7 @@ use lance::io::WrappingObjectStore;
use object_store::{
CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result as OSResult,
UploadPart, list::PaginatedListStore, path::Path,
UploadPart, path::Path,
};
#[derive(Debug, Default)]
@@ -57,14 +57,6 @@ impl WrappingObjectStore for IoStatsHolder {
stats: self.0.clone(),
})
}
fn wrap_paginated(
&self,
_store_prefix: &str,
original: Arc<dyn PaginatedListStore>,
) -> Option<Arc<dyn PaginatedListStore>> {
Some(original)
}
}
impl IoTrackingStore {
-4
View File
@@ -47,10 +47,6 @@ impl TerminalResult {
}
}
pub(crate) fn value(&self) -> Option<&Value> {
self.value.as_ref()
}
fn decode<T: DeserializeOwned>(self) -> Result<T> {
let value = self.value.ok_or_else(|| match &self.request_id {
Some(request_id) => Error::Http {
+2 -11
View File
@@ -170,15 +170,6 @@ pub(crate) fn plan(
filter: Option<&str>,
limit: Option<u64>,
) -> Result<(MaterializedViewDefinition, Vec<ArrowField>, Lineage)> {
let filter = filter
.map(crate::expr::canonicalize_sql_predicate)
.transpose()
.map_err(|err| match err {
Error::InvalidInput { message } => Error::InvalidInput {
message: format!("invalid view filter: {message}"),
},
err => err,
})?;
let projections: Vec<(String, String)> = if projections.is_empty() {
source_schema
.fields()
@@ -283,7 +274,7 @@ pub(crate) fn plan(
declared.push(output);
}
if let Some(filter) = filter.as_deref() {
if let Some(filter) = filter {
let expr = planner
.parse_filter(filter)
.map_err(|e| Error::InvalidInput {
@@ -323,7 +314,7 @@ pub(crate) fn plan(
.into_iter()
.map(|(output, expression)| ViewProjection { output, expression })
.collect(),
filter,
filter: filter.map(String::from),
limit,
inputs,
};
+24 -160
View File
@@ -46,9 +46,8 @@ use lance_table::format::Fragment;
use serde::{Deserialize, Serialize};
use super::{
DEFINITION_META_KEY, INCARNATION_META_KEY, MaterializedViewDefinition,
REFRESHED_AT_MS_META_KEY, SOURCE_ROW_ID_COLUMN, SOURCE_VERSION_META_KEY,
definition_to_metadata,
INCARNATION_META_KEY, MaterializedViewDefinition, REFRESHED_AT_MS_META_KEY,
SOURCE_ROW_ID_COLUMN, SOURCE_VERSION_META_KEY,
};
use crate::database::OpenTableRequest;
use crate::table::{NativeTable, NativeTableExt, Table};
@@ -198,28 +197,8 @@ pub(crate) async fn execute_refresh(
),
});
}
let definition_changed =
definition.filter != replanned.filter || definition.inputs != replanned.inputs;
let definition = &replanned;
// A watermark written for a legacy raw filter certifies the rows that
// filter produced, not the canonical predicate above. Rebuild instead of
// accepting or advancing it, and persist the migrated definition in the
// same metadata commit that certifies the replacement rows.
if definition_changed {
return rebuild(
view_native,
&view_ds,
&source_ds,
source_version,
source_ts,
definition,
true,
expected_incarnation,
)
.await;
}
let metadata = &view_ds.schema().metadata;
let watermark: Option<u64> = metadata
.get(SOURCE_VERSION_META_KEY)
@@ -278,7 +257,6 @@ pub(crate) async fn execute_refresh(
source_version,
source_ts,
definition,
false,
expected_incarnation,
)
.await
@@ -293,7 +271,6 @@ pub(crate) async fn execute_refresh(
source_version,
source_ts,
definition,
false,
expected_incarnation,
)
.await
@@ -706,7 +683,6 @@ async fn incremental(
view_ds.clone(),
source_version,
source_ts,
None,
expected_incarnation,
)
.await?;
@@ -728,7 +704,6 @@ async fn incremental(
published,
source_version,
source_ts,
None,
expected_incarnation,
)
.await?;
@@ -800,7 +775,6 @@ async fn incremental(
published,
source_version,
source_ts,
None,
expected_incarnation,
)
.await?;
@@ -850,14 +824,12 @@ async fn incremental(
appended,
source_version,
source_ts,
None,
expected_incarnation,
)
.await?;
Ok(Some(result))
}
#[allow(clippy::too_many_arguments)]
async fn rebuild(
view_native: &NativeTable,
view_ds: &Dataset,
@@ -865,7 +837,6 @@ async fn rebuild(
source_version: u64,
source_ts: u128,
definition: &MaterializedViewDefinition,
persist_definition: bool,
expected_incarnation: Option<&str>,
) -> Result<RefreshMaterializedViewResult> {
let rows_written = Arc::new(AtomicU64::new(0));
@@ -896,7 +867,6 @@ async fn rebuild(
replaced,
source_version,
source_ts,
persist_definition.then_some(definition),
expected_incarnation,
)
.await?;
@@ -1011,7 +981,6 @@ async fn stamp_watermark(
mut dataset: Dataset,
source_version: u64,
source_ts: u128,
definition: Option<&MaterializedViewDefinition>,
expected_incarnation: Option<&str>,
) -> Result<u64> {
ensure_incarnation(&dataset, expected_incarnation, dataset.uri()).await?;
@@ -1024,32 +993,27 @@ async fn stamp_watermark(
.get(INCARNATION_META_KEY)
.cloned()
.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
let mut metadata = vec![(INCARNATION_META_KEY.to_string(), Some(incarnation))];
if let Some(definition) = definition {
metadata.push((
DEFINITION_META_KEY.to_string(),
Some(definition_to_metadata(definition)?),
));
}
metadata.extend([
(
SOURCE_VERSION_META_KEY.to_string(),
Some(source_version.to_string()),
),
(
SOURCE_VERSION_TS_META_KEY.to_string(),
Some(source_ts.to_string()),
),
(
REFRESHED_AT_MS_META_KEY.to_string(),
Some(now_ms().to_string()),
),
(
VIEW_VERSION_META_KEY.to_string(),
Some(predicted.to_string()),
),
]);
dataset.update_schema_metadata(metadata).await?;
dataset
.update_schema_metadata([
(INCARNATION_META_KEY.to_string(), Some(incarnation)),
(
SOURCE_VERSION_META_KEY.to_string(),
Some(source_version.to_string()),
),
(
SOURCE_VERSION_TS_META_KEY.to_string(),
Some(source_ts.to_string()),
),
(
REFRESHED_AT_MS_META_KEY.to_string(),
Some(now_ms().to_string()),
),
(
VIEW_VERSION_META_KEY.to_string(),
Some(predicted.to_string()),
),
])
.await?;
let actual = dataset.version().version;
if actual != predicted {
return Err(Error::Runtime {
@@ -1621,106 +1585,6 @@ mod tests {
assert_eq!(read(view.table(), "x").await, vec![20, 40]);
}
#[tokio::test]
async fn test_mixed_case_filter_is_canonicalized_for_lineage_and_refresh() {
let conn = connect("memory://").execute().await.unwrap();
let batch = record_batch!(
("id", Int32, [1, 2, 3]),
("PartyAbbrev", Utf8, ["D", "R", "D"])
)
.unwrap();
conn.create_table("src", batch)
.write_options(crate::materialized_view::tests::stable_row_ids())
.execute()
.await
.unwrap();
conn.create_materialized_view("democrats", "src")
.select([("id", "id")])
.only_if(r#""PartyAbbrev" = 'D'"#)
.execute()
.await
.unwrap();
// Reopen from schema metadata so these assertions cover the stored
// predicate and lineage, not only the declaration-time handle.
let view = conn.open_materialized_view("democrats").await.unwrap();
assert_eq!(
view.definition().filter.as_deref(),
Some("`PartyAbbrev` = 'D'")
);
assert_eq!(view.definition().inputs, ["PartyAbbrev", "id"]);
let result = view.refresh().execute().await.unwrap();
assert_eq!(result.rows_written, 2);
assert_eq!(read(view.table(), "id").await, vec![1, 3]);
}
#[tokio::test]
async fn test_legacy_raw_filter_rebuilds_and_persists_canonical_definition() {
let conn = connect("memory://").execute().await.unwrap();
let batch = record_batch!(
("id", Int32, [1, 2, 3]),
("PartyAbbrev", Utf8, ["D", "R", "D"])
)
.unwrap();
conn.create_table("legacy_src", batch)
.write_options(crate::materialized_view::tests::stable_row_ids())
.execute()
.await
.unwrap();
let view = conn
.create_materialized_view("legacy_view", "legacy_src")
.select([("id", "id")])
.only_if(r#""PartyAbbrev" = 'X'"#)
.execute()
.await
.unwrap();
assert_eq!(view.refresh().execute().await.unwrap().rows_written, 0);
// Model a definition and up-to-date watermark written before filter
// canonicalization was applied to materialized views.
let mut legacy = view.definition().clone();
legacy.filter = Some(r#""PartyAbbrev" = 'D'"#.into());
legacy.inputs = vec!["id".into()];
let native = view.table().as_native().unwrap();
let mut dataset = native.dataset.get().await.unwrap().as_ref().clone();
let predicted = dataset.version().version + 1;
dataset
.update_schema_metadata([
(
DEFINITION_META_KEY.to_string(),
Some(definition_to_metadata(&legacy).unwrap()),
),
(
VIEW_VERSION_META_KEY.to_string(),
Some(predicted.to_string()),
),
])
.await
.unwrap();
native.dataset.update(dataset);
let reopened = conn.open_materialized_view("legacy_view").await.unwrap();
let result = reopened.refresh().execute().await.unwrap();
assert_eq!(result.mode, RefreshMode::Rebuild);
assert_eq!(result.rows_written, 2);
assert_eq!(read(reopened.table(), "id").await, vec![1, 3]);
// A fresh handle proves the migration was stored alongside the new
// watermark and therefore happens only once.
let migrated = conn.open_materialized_view("legacy_view").await.unwrap();
assert_eq!(
migrated.definition().filter.as_deref(),
Some("`PartyAbbrev` = 'D'")
);
assert_eq!(migrated.definition().inputs, ["PartyAbbrev", "id"]);
assert_eq!(
migrated.refresh().execute().await.unwrap().mode,
RefreshMode::NoOp
);
assert_eq!(read(migrated.table(), "id").await, vec![1, 3]);
}
#[tokio::test]
async fn test_append_refreshes_incrementally() {
let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await;
@@ -2903,7 +2767,7 @@ mod tests {
let stale = view_native.dataset.get().await.unwrap().as_ref().clone();
view.table().delete("x = 1").await.unwrap();
let err = stamp_watermark(view_native, stale, 99, 99, None, None).await;
let err = stamp_watermark(view_native, stale, 99, 99, None).await;
assert!(err.is_err());
let result = view.refresh().execute().await.unwrap();
+9 -273
View File
@@ -399,9 +399,6 @@ pub trait QueryBase {
/// x > 5 OR y = 'test'
/// ```
///
/// Identifiers may be delimited with SQL-standard double quotes or
/// backticks. String literals must use single quotes.
///
/// Filtering performance can often be improved by creating a scalar index
/// on the filter column(s).
///
@@ -916,17 +913,6 @@ impl QueryRequest {
/// use different representations) the error is recorded and surfaced later
/// by [`Self::check_filter`].
pub(crate) fn add_filter(&mut self, new: QueryFilter) {
let new = match new {
QueryFilter::Sql(filter) => match crate::expr::canonicalize_sql_predicate(&filter) {
Ok(filter) => QueryFilter::Sql(filter),
Err(err) => {
self.filter_error = Some(err.to_string());
return;
}
},
other => other,
};
self.filter = Some(match self.filter.take() {
None => new,
Some(existing) => match and_filters(existing, new) {
@@ -1188,12 +1174,12 @@ impl VectorQuery {
/// Add another query vector to the search.
///
/// Multiple searches will be dispatched as a batch. Flat searches share
/// one table scan across the query vectors, avoiding the scan and memory
/// amplification of issuing the searches concurrently. Indexed searches
/// may still perform per-vector index work.
/// Multiple searches will be dispatched as part of the query.
/// This is a convenience method for adding multiple query vectors
/// to the search. It is not expected to be faster than issuing
/// multiple queries concurrently.
///
/// The output data will contain an additional column `query_index` which
/// The output data will contain an additional columns `query_index` which
/// will contain the index of the query vector that was used to generate the
/// result.
pub fn add_query_vector(mut self, vector: impl IntoQueryVector) -> Result<Self> {
@@ -1660,14 +1646,10 @@ mod tests {
use std::{collections::HashSet, sync::Arc};
use super::*;
use arrow::{
array::downcast_array,
compute::concat_batches,
datatypes::{Int32Type, UInt8Type},
};
use arrow::{array::downcast_array, compute::concat_batches, datatypes::Int32Type};
use arrow_array::{
FixedSizeListArray, Float32Array, Int32Array, RecordBatch, RecordBatchIterator,
StringArray, cast::AsArray, types::Float32Type,
FixedSizeListArray, Float32Array, Int32Array, RecordBatch, StringArray, cast::AsArray,
types::Float32Type,
};
use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
use futures::{StreamExt, TryStreamExt};
@@ -1896,157 +1878,6 @@ mod tests {
query.execute().await.unwrap();
}
#[tokio::test]
async fn test_double_quoted_predicates_across_table_operations() {
let tmp_dir = tempdir().unwrap();
let dataset_path = tmp_dir.path().join("test.lance");
let uri = dataset_path.to_str().unwrap();
let schema = Arc::new(ArrowSchema::new(vec![
ArrowField::new("id", DataType::Int32, false),
ArrowField::new("PartyAbbrev", DataType::Utf8, false),
ArrowField::new("path", DataType::Utf8, false),
]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int32Array::from(vec![1, 2, 3, 4])),
Arc::new(StringArray::from(vec!["D", "R", "R", "D"])),
Arc::new(StringArray::from(vec!["\\", "\\", "x", "x"])),
],
)
.unwrap();
let conn = connect(uri).execute().await.unwrap();
let table = conn.create_table("parties", batch).execute().await.unwrap();
let batches = table
.query()
.only_if(r#""PartyAbbrev" = 'D'"#)
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 2);
assert_eq!(
table
.count_rows(Some(r#""PartyAbbrev" = 'D'"#.to_string()))
.await
.unwrap(),
2
);
// Public BaseTable dispatch cannot bypass canonicalization.
let query = AnyQuery::Query(QueryRequest {
filter: Some(QueryFilter::Sql(r#""PartyAbbrev" = 'D'"#.to_string())),
..Default::default()
});
let batches = table
.base_table()
.query(&query, Default::default())
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 2);
assert_eq!(
table
.base_table()
.count_rows(Some(crate::table::Filter::Sql(
r#""PartyAbbrev" = 'D'"#.to_string(),
)))
.await
.unwrap(),
2
);
for predicate in [
r#"id = 1 -- unmatched " in a valid SQL comment"#,
r#"id = 1 /* unmatched " in a valid SQL comment */"#,
r#"id = 1 /*! OR "PartyAbbrev" = 'D' */"#,
r#"path = '\' AND "PartyAbbrev" = 'D'"#,
] {
let batches = table
.query()
.only_if(predicate)
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 1);
}
// The same canonical predicate contract applies to both merge filters.
let source = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int32Array::from(vec![1, 2, 3])),
Arc::new(StringArray::from(vec!["D", "R", "R"])),
Arc::new(StringArray::from(vec!["\\", "\\", "x"])),
],
)
.unwrap();
let mut merge = table.merge_insert(&["id"]);
merge.when_not_matched_by_source_delete(Some(r#""PartyAbbrev" = 'D'"#.to_string()));
let result = table
.base_table()
.merge_insert(
merge,
Box::new(RecordBatchIterator::new(vec![Ok(source)], schema.clone())),
)
.await
.unwrap();
assert_eq!(result.num_deleted_rows, 1);
let source = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int32Array::from(vec![1, 2, 3])),
Arc::new(StringArray::from(vec!["U", "U", "U"])),
Arc::new(StringArray::from(vec!["\\", "\\", "x"])),
],
)
.unwrap();
let mut merge = table.merge_insert(&["id"]);
merge.when_matched_update_all(Some(r#"target."PartyAbbrev" = 'D'"#.to_string()));
merge
.execute(Box::new(RecordBatchIterator::new(vec![Ok(source)], schema)))
.await
.unwrap();
assert_eq!(
table
.count_rows(Some(r#""PartyAbbrev" = 'U'"#.to_string()))
.await
.unwrap(),
1
);
let update = table
.update()
.only_if(r#""PartyAbbrev" = 'R'"#)
.column("PartyAbbrev", "'X'");
table.base_table().update(update).await.unwrap();
assert_eq!(
table
.count_rows(Some(r#""PartyAbbrev" = 'X'"#.to_string()))
.await
.unwrap(),
2
);
let result = table
.base_table()
.delete(crate::table::Predicate::String(r#""PartyAbbrev" = 'X'"#))
.await
.unwrap();
assert_eq!(result.num_deleted_rows, 2);
assert_eq!(table.count_rows(None).await.unwrap(), 1);
}
#[tokio::test]
async fn test_select_with_transform() {
let batches = make_non_empty_batches();
@@ -2503,8 +2334,7 @@ mod tests {
.limit(1);
let plan = query.explain_plan(true).await.unwrap();
assert!(plan.contains("KNNVectorDistance: queries=2"));
assert!(!plan.contains("UnionExec"));
assert!(plan.contains("UnionExec"));
let results = query
.execute()
@@ -2519,100 +2349,6 @@ mod tests {
// We don't guarantee order.
assert!(query_index.values().contains(&0));
assert!(query_index.values().contains(&1));
// Batch KNN does not support a per-query offset, so offset queries keep
// the legacy per-vector plan to preserve their result semantics.
let offset_query = table
.query()
.nearest_to(&[0.1, 0.2, 0.3, 0.4])
.unwrap()
.add_query_vector(&[0.5, 0.6, 0.7, 0.8])
.unwrap()
.limit(1)
.offset(1);
assert!(
offset_query
.explain_plan(true)
.await
.unwrap()
.contains("UnionExec")
);
let offset_results = offset_query
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
assert_eq!(
offset_results
.iter()
.map(RecordBatch::num_rows)
.sum::<usize>(),
2
);
}
#[tokio::test]
async fn test_multiple_binary_query_vectors() {
let vectors = FixedSizeListArray::from_iter_primitive::<UInt8Type, _, _>(
vec![
Some(vec![Some(0), Some(0)]),
Some(vec![Some(255), Some(255)]),
],
2,
);
let schema = Arc::new(ArrowSchema::new(vec![
ArrowField::new("id", DataType::Int32, false),
ArrowField::new("vector", vectors.data_type().clone(), false),
]));
let batch = RecordBatch::try_new(
schema,
vec![Arc::new(Int32Array::from(vec![0, 1])), Arc::new(vectors)],
)
.unwrap();
let conn = connect("memory://").execute().await.unwrap();
let table = conn
.create_table("binary_batch", batch)
.execute()
.await
.unwrap();
let query = table
.query()
.nearest_to(&[0.0, 0.0])
.unwrap()
.add_query_vector(&[255.0, 255.0])
.unwrap()
.distance_type(DistanceType::Hamming)
.limit(1);
// Binary queries retain the per-vector plan because Lance's binary
// nearest path requires primitive UInt8 query arrays.
assert!(
query
.explain_plan(true)
.await
.unwrap()
.contains("UnionExec")
);
let results = query
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
let results = concat_batches(&results[0].schema(), &results).unwrap();
assert_eq!(results.num_rows(), 2);
let ids = results["id"].as_primitive::<Int32Type>();
assert!(ids.values().contains(&0));
assert!(ids.values().contains(&1));
let query_index = results["query_index"].as_primitive::<Int32Type>();
assert!(query_index.values().contains(&0));
assert!(query_index.values().contains(&1));
}
#[tokio::test]
-4
View File
@@ -87,10 +87,6 @@ impl ServerVersion {
pub fn support_blobs(&self) -> bool {
self.0 >= semver::Version::new(0, 5, 0)
}
pub fn support_fts_document_granularity(&self) -> bool {
self.0 >= semver::Version::new(0, 6, 0)
}
}
pub const OPT_REMOTE_PREFIX: &str = "remote_database_";
File diff suppressed because it is too large Load Diff
+12 -65
View File
@@ -6,7 +6,6 @@
use std::ops::Range;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use arrow_array::{Array, LargeBinaryArray};
use arrow_schema::DataType;
@@ -21,7 +20,7 @@ use crate::error::Result;
use crate::remote::client::{HttpSend, RequestResultExt, RestfulLanceDbClient};
use crate::table::BaseTable;
use super::{FreshnessHeaders, FreshnessState, RemoteTable, freshness_headers_snapshot};
use super::{FreshnessHeaders, RemoteTable};
#[derive(Debug, Clone, Copy)]
enum RangeRequestMode {
@@ -44,10 +43,7 @@ struct TableBlobRangeRequester<S: HttpSend> {
path: String,
version: Option<u64>,
branch: Option<String>,
freshness: Arc<std::sync::Mutex<FreshnessState>>,
parent_freshness: Arc<std::sync::Mutex<FreshnessState>>,
parent_freshness_request: FreshnessHeaders,
read_consistency_interval: Option<Duration>,
freshness: FreshnessHeaders,
}
#[async_trait::async_trait]
@@ -57,9 +53,8 @@ impl<S: HttpSend> BlobRangeRequester for TableBlobRangeRequester<S> {
range_header: &str,
mode: RangeRequestMode,
) -> Result<(String, Response)> {
let freshness_request =
freshness_headers_snapshot(&self.freshness, self.read_consistency_interval);
let mut request = freshness_request
let mut request = self
.freshness
.apply(self.client.get(&self.path))
.header(header::RANGE, range_header);
if let Some(version) = self.version {
@@ -76,9 +71,6 @@ impl<S: HttpSend> BlobRangeRequester for TableBlobRangeRequester<S> {
return Ok((request_id, response));
}
let response = self.client.check_response(&request_id, response).await?;
freshness_request.observe_headers(&self.freshness, response.headers());
self.parent_freshness_request
.observe_headers(&self.parent_freshness, response.headers());
Ok((request_id, response))
}
}
@@ -369,21 +361,18 @@ impl<S: HttpSend> RemoteTable<S> {
message: "fetch_blobs is not supported on this LanceDB Cloud server".into(),
});
}
let read_snapshot = self.snapshot_read_state().await;
let version = self.current_version().await;
let mut body = serde_json::json!({
"version": read_snapshot.version,
"version": version,
"column": column,
"row_ids": row_ids,
});
self.apply_branch_body(&mut body);
let request = self
.client
.post(&format!("/v1/table/{}/fetch_blobs/", self.identifier))
.post_read(&format!("/v1/table/{}/fetch_blobs/", self.identifier))
.json(&body);
let (request_id, response) = self
.send_with_freshness(request, true, read_snapshot.freshness)
.await?;
let (request_id, response) = self.send(request, true).await?;
let mut stream = self.read_arrow_response(&request_id, response).await?;
let mut blob_chunks: Vec<Arc<dyn Array>> = Vec::new();
@@ -459,7 +448,8 @@ impl<S: HttpSend> RemoteTable<S> {
});
}
let read_snapshot = self.snapshot_read_state().await;
let version = self.current_version().await;
let freshness = self.snapshot_freshness_headers();
let encoded_column = urlencoding::encode(column);
let requesters = row_ids
.iter()
@@ -471,12 +461,9 @@ impl<S: HttpSend> RemoteTable<S> {
let requester: Arc<dyn BlobRangeRequester> = Arc::new(TableBlobRangeRequester {
client: self.client.clone(),
path,
version: read_snapshot.version,
version,
branch: self.branch.clone(),
freshness: Arc::new(std::sync::Mutex::new(read_snapshot.freshness_state)),
parent_freshness: self.freshness.clone(),
parent_freshness_request: read_snapshot.freshness,
read_consistency_interval: self.client.read_consistency_interval,
freshness,
});
requester
})
@@ -698,46 +685,6 @@ mod tests {
assert!(requests.lock().unwrap().contains(&"bytes=5-11".to_string()));
}
#[tokio::test]
async fn remote_blob_file_keeps_the_open_timeline_after_parent_checkout() {
let range_requests = Arc::new(StdMutex::new(Vec::new()));
let captured = range_requests.clone();
let table = RemoteTable::new_mock(
"my_table".to_string(),
move |request| match request.url().path() {
"/v1/table/my_table/describe/" => http::Response::builder()
.status(200)
.body(r#"{"version":5,"schema":{"fields":[]}}"#.as_bytes().to_vec())
.unwrap(),
"/v1/table/my_table/blob/image/10/bytes" => {
captured.lock().unwrap().push((
request.url().query().unwrap_or_default().to_string(),
request.headers().clone(),
));
range_response(&request, PAYLOAD)
}
path => panic!("unexpected path: {path}"),
},
Some(Version::new(0, 5, 0)),
);
table.checkout(5).await.unwrap();
let file = table
.fetch_blob_files_impl("image", &[10])
.await
.unwrap()
.pop()
.flatten()
.unwrap();
table.checkout_latest().await.unwrap();
file.read_range(5..12).await.unwrap();
let requests = range_requests.lock().unwrap();
let (query, headers) = requests.last().unwrap();
assert!(query.contains("version=5"));
assert!(!headers.contains_key("x-lancedb-min-timestamp"));
}
#[tokio::test]
async fn remote_blob_file_reuses_sequential_response_until_seek() {
let requests = Arc::new(StdMutex::new(Vec::new()));
+9 -74
View File
@@ -24,10 +24,7 @@ use lance::io::exec::utils::InstrumentedRecordBatchStreamAdapter;
use crate::Error;
use crate::remote::ARROW_STREAM_CONTENT_TYPE;
use crate::remote::client::{HttpSend, RestfulLanceDbClient, Sender};
use crate::remote::table::{
FreshnessHeaders, FreshnessState, MergeInsertRequest, REQUEST_TIMEOUT_HEADER, RemoteTable,
freshness_headers_snapshot,
};
use crate::remote::table::{MergeInsertRequest, REQUEST_TIMEOUT_HEADER, RemoteTable};
use crate::table::datafusion::insert::COUNT_SCHEMA;
use crate::table::write_progress::WriteProgressTracker;
use crate::table::{AddResult, MergeResult};
@@ -57,38 +54,6 @@ pub enum WriteResult {
Merge(MergeResult),
}
#[derive(Debug, Clone, Default)]
struct WriteFreshness {
state: Option<Arc<Mutex<FreshnessState>>>,
read_consistency_interval: Option<Duration>,
}
impl WriteFreshness {
fn prepare(
&self,
request: reqwest::RequestBuilder,
) -> (reqwest::RequestBuilder, Option<FreshnessHeaders>) {
match &self.state {
Some(state) => {
let freshness_request =
freshness_headers_snapshot(state, self.read_consistency_interval);
(freshness_request.apply(request), Some(freshness_request))
}
None => (request, None),
}
}
fn observe(
&self,
freshness_request: Option<FreshnessHeaders>,
headers: &reqwest::header::HeaderMap,
) {
if let (Some(state), Some(freshness_request)) = (&self.state, freshness_request) {
freshness_request.observe_headers(state, headers);
}
}
}
/// ExecutionPlan for streaming a write (add or merge_insert) to a remote
/// LanceDB table.
///
@@ -106,7 +71,6 @@ pub struct RemoteWriteExec<S: HttpSend = Sender> {
table_name: String,
identifier: String,
client: RestfulLanceDbClient<S>,
freshness: WriteFreshness,
input: Arc<dyn ExecutionPlan>,
op: WriteOp,
properties: Arc<PlanProperties>,
@@ -206,7 +170,6 @@ impl<S: HttpSend + 'static> RemoteWriteExec<S> {
table_name,
identifier,
client,
freshness: WriteFreshness::default(),
input,
op,
properties: Arc::new(properties),
@@ -220,18 +183,6 @@ impl<S: HttpSend + 'static> RemoteWriteExec<S> {
}
}
pub(super) fn with_freshness(
mut self,
state: Arc<Mutex<FreshnessState>>,
read_consistency_interval: Option<Duration>,
) -> Self {
self.freshness = WriteFreshness {
state: Some(state),
read_consistency_interval,
};
self
}
/// Get the add result after execution, if this exec ran an insert.
pub fn add_result(&self) -> Option<AddResult> {
match self
@@ -334,7 +285,6 @@ impl<S: HttpSend + 'static> RemoteWriteExec<S> {
/// each threading the same handful of arguments.
struct PartRequestCtx<'a, S: HttpSend> {
client: &'a RestfulLanceDbClient<S>,
freshness: &'a WriteFreshness,
identifier: &'a str,
table_name: &'a str,
upload_id: &'a str,
@@ -402,11 +352,7 @@ impl<S: HttpSend + 'static> PartRequestCtx<'_, S> {
}
/// Build the `/insert` request for a single multipart part.
fn build_part_request(
&self,
part_id: &str,
body: reqwest::Body,
) -> (reqwest::RequestBuilder, Option<FreshnessHeaders>) {
fn build_part_request(&self, part_id: &str, body: reqwest::Body) -> reqwest::RequestBuilder {
let mut request = self
.client
.post(&format!("/v1/table/{}/insert/", self.identifier))
@@ -422,16 +368,12 @@ impl<S: HttpSend + 'static> PartRequestCtx<'_, S> {
if let Some(b) = self.branch {
request = request.query(&[("branch", b)]);
}
self.freshness.prepare(request.body(body))
request.body(body)
}
/// Send a single part's request and drain the response, mapping HTTP and
/// table-not-found errors into `DataFusionError`.
async fn send_part_request(
&self,
request: reqwest::RequestBuilder,
freshness_request: Option<FreshnessHeaders>,
) -> DataFusionResult<()> {
async fn send_part_request(&self, request: reqwest::RequestBuilder) -> DataFusionResult<()> {
let (request_id, response) = self
.client
.send(request)
@@ -446,8 +388,6 @@ impl<S: HttpSend + 'static> PartRequestCtx<'_, S> {
.check_response(&request_id, response)
.await
.map_err(|e| DataFusionError::External(Box::new(e)))?;
self.freshness
.observe(freshness_request, response.headers());
response.bytes().await.map_err(|e| {
DataFusionError::External(Box::new(Error::Http {
source: Box::new(e),
@@ -479,7 +419,7 @@ impl<S: HttpSend + 'static> PartRequestCtx<'_, S> {
let body = reqwest::Body::wrap_stream(chunk_rx);
let part_id = uuid::Uuid::new_v4().to_string();
let (request, freshness_request) = self.build_part_request(&part_id, body);
let request = self.build_part_request(&part_id, body);
// Measured from just before the request is sent, matching the window the
// client read timeout applies to the upload.
@@ -555,7 +495,7 @@ impl<S: HttpSend + 'static> PartRequestCtx<'_, S> {
Ok::<bool, DataFusionError>(input_ended)
};
let send = self.send_part_request(request, freshness_request);
let send = self.send_part_request(request);
// `join!` rather than `tokio::spawn`: the producer borrows `input` (and
// `schema`), so it cannot satisfy the `'static` bound a spawned task
@@ -629,7 +569,7 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
// Building a fresh exec (with a new, empty `result`) is what makes the
// outer rescannable retry loop work: `reset_state()` clears the captured
// result so a re-execution starts clean.
let mut exec = Self::new_inner(
Ok(Arc::new(Self::new_inner(
self.table_name.clone(),
self.identifier.clone(),
self.client.clone(),
@@ -640,9 +580,7 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
self.branch.clone(),
self.max_bytes_per_request,
self.max_request_duration,
);
exec.freshness = self.freshness.clone();
Ok(Arc::new(exec))
)))
}
fn execute(
@@ -675,7 +613,6 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
&self.metrics,
));
let client = self.client.clone();
let freshness = self.freshness.clone();
let identifier = self.identifier.clone();
let op = self.op.clone();
let result_slot = self.result.clone();
@@ -697,7 +634,6 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
let overwrite = matches!(op, WriteOp::Insert { overwrite: true });
let ctx = PartRequestCtx {
client: &client,
freshness: &freshness,
identifier: &identifier,
table_name: &table_name,
upload_id,
@@ -752,7 +688,7 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
let (error_tx, mut error_rx) = tokio::sync::oneshot::channel();
let body = Self::stream_as_http_body(input_stream, error_tx, tracker)?;
let (request, freshness_request) = freshness.prepare(request.body(body));
let request = request.body(body);
let result: DataFusionResult<(String, _)> = async {
let (request_id, response) = client
@@ -772,7 +708,6 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
.check_response(&request_id, response)
.await
.map_err(|e| DataFusionError::External(Box::new(e)))?;
freshness.observe(freshness_request, response.headers());
Ok((request_id, response))
}
+7 -123
View File
@@ -59,9 +59,7 @@ use crate::index::{IndexConfig, IndexStatisticsImpl, IndexType};
use crate::job::Job;
use crate::query::{IntoQueryVector, Query, QueryExecutionOptions, TakeQuery, VectorQuery};
use crate::table::datafusion::insert::InsertExec;
use crate::utils::{
PatchReadParam, PatchWriteParam, public_fts_field_path_by_id, resolve_arrow_field_path,
};
use crate::utils::{PatchReadParam, PatchWriteParam, resolve_arrow_field_path};
use self::dataset::DatasetConsistencyWrapper;
use self::merge::MergeInsertBuilder;
@@ -562,13 +560,6 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
fn id(&self) -> &str;
/// Get the arrow [Schema] of the table.
async fn schema(&self) -> Result<SchemaRef>;
/// Create a read-only handle pinned to the table's current active revision.
///
/// The returned handle is independent from later refreshes or checkouts on
/// this handle. This is used by bindings that must prepare client-side
/// query state from the same revision that the query will execute against.
#[doc(hidden)]
async fn query_snapshot(&self) -> Result<Arc<dyn BaseTable>>;
/// Count the number of rows in this table.
async fn count_rows(&self, filter: Option<Filter>) -> Result<usize>;
/// Create a physical plan for the query.
@@ -794,12 +785,6 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
async fn drop_columns(&self, columns: &[&str]) -> Result<DropColumnsResult>;
/// Get the version of the table.
async fn version(&self) -> Result<u64>;
/// Return a new table handle pinned to the exact revision currently visible.
async fn checkout_current(&self) -> Result<Arc<dyn BaseTable>> {
Err(Error::NotSupported {
message: "checkout_current is not supported on this table type".into(),
})
}
/// Checkout a specific version of the table.
async fn checkout(&self, version: u64) -> Result<()>;
/// Checkout a table version referenced by a tag.
@@ -1148,26 +1133,13 @@ impl Table {
self.inner.schema().await
}
/// Create a read-only handle pinned to the current active revision.
#[doc(hidden)]
pub async fn query_snapshot(&self) -> Result<Self> {
Ok(Self {
inner: self.inner.query_snapshot().await?,
database: self.database.clone(),
embedding_registry: self.embedding_registry.clone(),
})
}
/// Count the number of rows in this dataset.
///
/// # Arguments
///
/// * `filter` if present, only count rows matching the filter
pub async fn count_rows(&self, filter: Option<String>) -> Result<usize> {
let filter = filter
.map(|predicate| crate::expr::canonicalize_sql_predicate(&predicate).map(Filter::Sql))
.transpose()?;
self.inner.count_rows(filter).await
self.inner.count_rows(filter.map(Filter::Sql)).await
}
/// Names of the blob v2 columns in this table, in declaration order.
@@ -1367,13 +1339,7 @@ impl Table {
/// # });
/// ```
pub async fn delete(&self, predicate: impl Into<Predicate<'_>>) -> Result<DeleteResult> {
match predicate.into() {
Predicate::String(predicate) => {
let predicate = crate::expr::canonicalize_sql_predicate(predicate)?;
self.inner.delete(Predicate::String(&predicate)).await
}
predicate @ Predicate::Expr(_) => self.inner.delete(predicate).await,
}
self.inner.delete(predicate.into()).await
}
/// Create an index on the provided column(s).
@@ -1786,23 +1752,7 @@ impl Table {
self.inner.alter_columns(alterations).await
}
/// Update per-field (column) metadata.
///
/// Each [`FieldMetadataUpdate`] is merged into the field's existing metadata
/// by default; use [`FieldMetadataUpdate::remove`] to delete a key, or
/// [`FieldMetadataUpdate::replace`] to swap the field's entire metadata map.
///
/// The following keys are treated specially, by convention, and should be
/// used when appropriate:
///
/// - `lancedb:description`: for a human-readable description of a field.
/// - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
/// names the tag category; e.g. `lancedb:tag:model: "clip"`.
/// - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
/// `feature_v2` might be in the same logical column.
/// - `lancedb:status`: for status options (`production`, `candidate`,
/// `deprecated`, `archived`) to designate the current life cycle state of
/// this column.
/// Update per-field metadata (merges by default).
pub async fn update_field_metadata(
&self,
updates: &[FieldMetadataUpdate],
@@ -1994,20 +1944,6 @@ impl Table {
self.inner.version().await
}
/// Return a new table handle pinned to the exact revision currently visible.
///
/// This is used when asynchronous preparation must remain consistent with
/// the revision used for a later read.
#[doc(hidden)]
pub async fn checkout_current(&self) -> Result<Self> {
let inner = self.inner.checkout_current().await?;
Ok(Self {
inner,
database: self.database.clone(),
embedding_registry: self.embedding_registry.clone(),
})
}
/// Checks out a specific version of the Table
///
/// Any read operation on the table will now access the data at the checked out version.
@@ -3103,33 +3039,10 @@ impl BaseTable for NativeTable {
&self.id
}
async fn query_snapshot(&self) -> Result<Arc<dyn BaseTable>> {
let snapshot = self.dataset.new_query_snapshot().await?;
let mut table = self.with_dataset(snapshot);
// QueryTable requests do not carry a revision. A pinned snapshot must
// execute locally until the namespace API can accept that revision.
table
.pushdown_operations
.remove(&NamespaceClientPushdownOperation::QueryTable);
Ok(Arc::new(table))
}
async fn version(&self) -> Result<u64> {
Ok(self.dataset.get().await?.version().version)
}
async fn checkout_current(&self) -> Result<Arc<dyn BaseTable>> {
let current = self.dataset.get().await?;
let dataset = dataset::DatasetConsistencyWrapper::new_time_travel(
current.as_ref().clone(),
self.read_consistency_interval,
);
Ok(Arc::new(Self {
dataset,
..self.clone()
}))
}
async fn checkout(&self, version: u64) -> Result<()> {
self.dataset.as_time_travel(version).await
}
@@ -3248,10 +3161,7 @@ impl BaseTable for NativeTable {
let dataset = self.dataset.get().await?;
match filter {
None => Ok(dataset.count_rows(None).await?),
Some(Filter::Sql(sql)) => {
let sql = crate::expr::canonicalize_sql_predicate(&sql)?;
Ok(dataset.count_rows(Some(sql)).await?)
}
Some(Filter::Sql(sql)) => Ok(dataset.count_rows(Some(sql)).await?),
Some(Filter::Datafusion(_)) => Err(Error::NotSupported {
message: "Datafusion filters are not yet supported".to_string(),
}),
@@ -3298,7 +3208,7 @@ impl BaseTable for NativeTable {
let output = add.into_plan(&table_schema, &table_def)?;
let mut lance_params = output
let lance_params = output
.write_options
.lance_write_params
.unwrap_or(WriteParams {
@@ -3308,9 +3218,6 @@ impl BaseTable for NativeTable {
},
..Default::default()
});
if output.allow_external_blob_outside_bases {
lance_params.allow_external_blob_outside_bases = true;
}
// Repartition for write parallelism if beneficial.
let plan = if num_partitions > 1 {
@@ -3589,14 +3496,7 @@ impl BaseTable for NativeTable {
let field_ids = idx_desc.field_ids();
let mut columns = Vec::with_capacity(field_ids.len());
for field_id in field_ids {
let field_path = match if index_type == crate::index::IndexType::FTS {
public_fts_field_path_by_id(dataset.schema(), *field_id as i32)
} else {
dataset
.schema()
.field_path(*field_id as i32)
.map_err(Into::into)
} {
let field_path = match dataset.schema().field_path(*field_id as i32) {
Ok(field_path) => field_path,
Err(e) => {
log::warn!(
@@ -4183,14 +4083,6 @@ mod tests {
parent_list_calls: self.parent_list_calls.clone(),
})
}
fn wrap_paginated(
&self,
_store_prefix: &str,
_original: Arc<dyn object_store::list::PaginatedListStore>,
) -> Option<Arc<dyn object_store::list::PaginatedListStore>> {
None
}
}
#[tokio::test]
@@ -4294,14 +4186,6 @@ mod tests {
self.called.store(true, Ordering::Relaxed);
original
}
fn wrap_paginated(
&self,
_store_prefix: &str,
original: Arc<dyn object_store::list::PaginatedListStore>,
) -> Option<Arc<dyn object_store::list::PaginatedListStore>> {
Some(original)
}
}
#[tokio::test]
-14
View File
@@ -60,7 +60,6 @@ pub struct AddDataBuilder {
pub(crate) embedding_registry: Option<Arc<dyn EmbeddingRegistry>>,
pub(crate) progress_callback: Option<ProgressCallback>,
pub(crate) write_parallelism: Option<usize>,
pub(crate) allow_external_blob_outside_bases: bool,
}
impl std::fmt::Debug for AddDataBuilder {
@@ -88,7 +87,6 @@ impl AddDataBuilder {
embedding_registry,
progress_callback: None,
write_parallelism: None,
allow_external_blob_outside_bases: false,
}
}
@@ -143,16 +141,6 @@ impl AddDataBuilder {
self
}
/// Store blob URIs that sit outside registered blob bases.
///
/// The row keeps a reference, so the object has to stay readable.
/// [`crate::table::Table::fetch_blobs`] reads from that location.
/// Defaults to `false`. Local tables only.
pub fn allow_external_blob_outside_bases(mut self, allow: bool) -> Self {
self.allow_external_blob_outside_bases = allow;
self
}
pub async fn execute(self) -> Result<AddResult> {
if self.write_parallelism.map(|p| p == 0).unwrap_or(false) {
return Err(Error::InvalidInput {
@@ -211,7 +199,6 @@ impl AddDataBuilder {
write_options: self.write_options,
mode: self.mode,
tracker,
allow_external_blob_outside_bases: self.allow_external_blob_outside_bases,
})
}
}
@@ -225,7 +212,6 @@ pub struct PreprocessingOutput {
pub write_options: WriteOptions,
pub mode: AddDataMode,
pub tracker: Option<Arc<WriteProgressTracker>>,
pub allow_external_blob_outside_bases: bool,
}
/// Check that the input schema is valid for insert.
+4 -50
View File
@@ -28,9 +28,8 @@ pub(super) type PreparedIndex = (String, Box<dyn lance::index::IndexParams>, Ind
use crate::index::Index;
use crate::index::vector::{VectorIndex, suggested_num_sub_vectors};
use crate::utils::{
resolve_lance_fts_field_path, supported_bitmap_data_type, supported_btree_data_type,
supported_fm_data_type, supported_fts_data_type, supported_label_list_data_type,
supported_vector_data_type,
supported_bitmap_data_type, supported_btree_data_type, supported_fm_data_type,
supported_fts_data_type, supported_label_list_data_type, supported_vector_data_type,
};
use super::NativeTable;
@@ -123,20 +122,7 @@ impl NativeTable {
}
self.dataset.ensure_mutable()?;
let dataset = self.dataset.get().await?;
let (column, field) = if let Index::FTS(params) = &opts.index {
let resolved = resolve_lance_fts_field_path(dataset.schema(), &opts.columns[0])?;
if params.get_document_granularity().is_list_element() && resolved.list_depth == 0 {
return Err(Error::InvalidInput {
message: format!(
"FTS field path '{}' has no List layer and cannot use ListElement document granularity",
resolved.canonical_path
),
});
}
(resolved.canonical_path, resolved.field)
} else {
Self::resolve_index_field(dataset.schema(), &opts.columns[0])?
};
let (column, field) = Self::resolve_index_field(dataset.schema(), &opts.columns[0])?;
let params = self.make_index_params(&field, opts.index.clone()).await?;
let index_type = self.get_index_type_for_field(&field, &opts.index);
Ok((column, params, index_type))
@@ -450,7 +436,7 @@ mod tests {
use crate::connection::ConnectBuilder;
use crate::index::Index;
use crate::index::scalar::{
BTreeIndexBuilder, BitmapIndexBuilder, DocumentGranularity, FmIndexBuilder, FtsIndexBuilder,
BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder, FtsIndexBuilder,
};
use crate::index::vector::{
IvfHnswFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder,
@@ -567,38 +553,6 @@ mod tests {
job.cancel().await.unwrap();
}
#[tokio::test]
async fn test_execute_async_validates_fts_input_before_starting_job() {
let conn = connect("memory://").execute().await.unwrap();
let batch =
record_batch!(("id", Int32, [1, 2]), ("text", Utf8, ["alpha", "beta"])).unwrap();
let table = conn.create_table("t", batch).execute().await.unwrap();
let missing = table
.create_index(&["missing"], Index::FTS(FtsIndexBuilder::default()))
.execute_async()
.await;
assert!(missing.is_err());
let invalid_type = table
.create_index(&["id"], Index::FTS(FtsIndexBuilder::default()))
.execute_async()
.await;
assert!(invalid_type.is_err());
let invalid_granularity = table
.create_index(
&["text"],
Index::FTS(
FtsIndexBuilder::default()
.document_granularity(DocumentGranularity::ListElement),
),
)
.execute_async()
.await;
assert!(invalid_granularity.is_err());
}
/// Concurrent waiters, and a wait issued after the job settled, all
/// succeed once the build does.
#[tokio::test]
+33 -118
View File
@@ -7,7 +7,7 @@
use std::sync::Arc;
use arrow_schema::{DataType, Field, FieldRef, Fields};
use arrow_schema::{DataType, Field, FieldRef};
use datafusion::functions::core::{get_field, named_struct};
use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
@@ -35,9 +35,8 @@ pub(super) fn coerce_blob_expr(
});
};
let input_shape = match input_field.data_type() {
DataType::Binary | DataType::LargeBinary | DataType::BinaryView => BlobInputShape::Bytes,
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => BlobInputShape::String,
let input_struct_children = match input_field.data_type() {
DataType::Binary | DataType::LargeBinary | DataType::BinaryView => None,
DataType::Struct(children) => {
if !children
.iter()
@@ -50,15 +49,13 @@ pub(super) fn coerce_blob_expr(
),
});
}
BlobInputShape::Struct(children)
Some(children)
}
other => {
return Err(Error::InvalidInput {
message: format!(
"cannot coerce column '{}' with type {} into a blob v2 struct. \
expected binary bytes (Binary, LargeBinary, BinaryView), \
strings (Utf8, LargeUtf8, Utf8View), \
or a Struct with a 'data' or 'uri' child",
expected Binary, LargeBinary, BinaryView, or a Struct with a 'data' or 'uri' child",
table_field.name(),
other,
),
@@ -72,8 +69,9 @@ pub(super) fn coerce_blob_expr(
declared.name().as_str(),
))));
let value: Arc<dyn PhysicalExpr> = match &input_shape {
BlobInputShape::Bytes => {
let value: Arc<dyn PhysicalExpr> = match input_struct_children {
// Raw binary lands in `data` and everything else is a typed null.
None => {
if declared.name() == "data" {
Arc::new(CastExpr::new(
input_expr.clone(),
@@ -84,43 +82,30 @@ pub(super) fn coerce_blob_expr(
typed_null(declared.data_type())?
}
}
BlobInputShape::String => {
if declared.name() == "uri" {
Arc::new(CastExpr::new(
input_expr.clone(),
declared.data_type().clone(),
None,
))
} else {
typed_null(declared.data_type())?
}
}
BlobInputShape::Struct(children) => {
match children.iter().find(|c| c.name() == declared.name()) {
Some(child) => {
let field_expr: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::new(
&format!("get_field({})", declared.name()),
get_field(),
vec![
input_expr.clone(),
Arc::new(Literal::new(ScalarValue::from(declared.name().as_str()))),
],
Arc::new(child.as_ref().clone()),
config.clone(),
));
if child.data_type() == declared.data_type() {
field_expr
} else {
Arc::new(CastExpr::new(
field_expr,
declared.data_type().clone(),
None,
))
}
Some(children) => match children.iter().find(|c| c.name() == declared.name()) {
Some(child) => {
let field_expr: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::new(
&format!("get_field({})", declared.name()),
get_field(),
vec![
input_expr.clone(),
Arc::new(Literal::new(ScalarValue::from(declared.name().as_str()))),
],
Arc::new(child.as_ref().clone()),
config.clone(),
));
if child.data_type() == declared.data_type() {
field_expr
} else {
Arc::new(CastExpr::new(
field_expr,
declared.data_type().clone(),
None,
))
}
None => typed_null(declared.data_type())?,
}
}
None => typed_null(declared.data_type())?,
},
};
ns_args.push(value);
}
@@ -135,12 +120,6 @@ pub(super) fn coerce_blob_expr(
Ok((expr, table_field.clone()))
}
enum BlobInputShape<'a> {
Bytes,
String,
Struct(&'a Fields),
}
fn typed_null(data_type: &DataType) -> Result<Arc<dyn PhysicalExpr>> {
let scalar = ScalarValue::try_from(data_type).map_err(|e| Error::InvalidInput {
message: format!("cannot build null literal for blob child type {data_type}: {e}"),
@@ -155,7 +134,7 @@ mod tests {
use crate::blob::blob;
use arrow_array::{
Array, ArrayRef, BinaryArray, BinaryViewArray, Int32Array, Int64Array, LargeBinaryArray,
RecordBatch, StringArray, StringViewArray, StructArray, UInt8Array, UInt64Array,
RecordBatch, StringArray, StructArray, UInt8Array, UInt64Array,
};
use arrow_schema::Schema;
use datafusion::prelude::SessionContext;
@@ -457,78 +436,14 @@ mod tests {
#[tokio::test]
async fn unsupported_input_type_is_rejected_with_column_name() {
let batch = batch_with_image(
Field::new("image", DataType::Int64, true),
Arc::new(Int64Array::from(vec![42])),
Field::new("image", DataType::Utf8, true),
Arc::new(StringArray::from(vec!["not bytes"])),
);
let err = coerce_err(batch, &blob_table_schema()).await;
assert!(matches!(err, Error::InvalidInput { .. }), "got {err:?}");
assert!(err.to_string().contains("image"));
}
#[tokio::test]
async fn utf8_string_coerces_to_uri_child() {
let batch = batch_with_image(
Field::new("image", DataType::Utf8, true),
Arc::new(StringArray::from(vec![Some("s3://bucket/key"), None])),
);
let coerced = coerce(batch, &blob_table_schema()).await;
let image = image_struct(&coerced);
let uri: &StringArray = image
.column_by_name("uri")
.unwrap()
.as_any()
.downcast_ref()
.unwrap();
assert_eq!(uri.value(0), "s3://bucket/key");
assert!(image.column_by_name("data").unwrap().is_null(0));
assert!(uri.is_null(1));
}
#[tokio::test]
async fn large_utf8_string_coerces_into_four_child_blob_layout() {
use arrow_array::LargeStringArray;
let table_schema = Schema::new(vec![
Field::new("id", DataType::Int64, false),
wide_blob_field("image"),
]);
let batch = batch_with_image(
Field::new("image", DataType::LargeUtf8, true),
Arc::new(LargeStringArray::from(vec!["file:///tmp/blob.bin"])),
);
let coerced = coerce(batch, &table_schema).await;
let image = image_struct(&coerced);
assert_eq!(image.num_columns(), 4);
let uri: &StringArray = image
.column_by_name("uri")
.unwrap()
.as_any()
.downcast_ref()
.unwrap();
assert_eq!(uri.value(0), "file:///tmp/blob.bin");
assert!(image.column_by_name("data").unwrap().is_null(0));
assert!(image.column_by_name("position").unwrap().is_null(0));
assert!(image.column_by_name("size").unwrap().is_null(0));
}
#[tokio::test]
async fn utf8_view_string_coerces_to_uri_child() {
let batch = batch_with_image(
Field::new("image", DataType::Utf8View, true),
Arc::new(StringViewArray::from(vec![Some("s3://bucket/view-key")])),
);
let coerced = coerce(batch, &blob_table_schema()).await;
let image = image_struct(&coerced);
let uri: &StringArray = image
.column_by_name("uri")
.unwrap()
.as_any()
.downcast_ref()
.unwrap();
assert_eq!(uri.value(0), "s3://bucket/view-key");
assert!(image.column_by_name("data").unwrap().is_null(0));
}
#[tokio::test]
async fn blob_metadata_survives_cast_of_sibling_column() {
let batch = RecordBatch::try_new(
+4 -65
View File
@@ -32,10 +32,6 @@ struct DatasetState {
/// `Some(version)` = pinned to a specific version (time travel),
/// `None` = tracking latest.
pinned_version: Option<u64>,
/// Whether the pin is an internal query snapshot rather than user-visible
/// time travel. Query snapshots remain read-only but preserve MemWAL read
/// semantics.
query_snapshot: bool,
}
#[derive(Debug, Clone)]
@@ -74,7 +70,6 @@ impl DatasetConsistencyWrapper {
state: Arc::new(Mutex::new(DatasetState {
dataset,
pinned_version: None,
query_snapshot: false,
})),
consistency,
shard_writer: Arc::new(ShardWriterCache::default()),
@@ -98,36 +93,6 @@ impl DatasetConsistencyWrapper {
wrapper
}
/// Create an independent read-only wrapper pinned to the current dataset
/// while retaining this wrapper's live MemWAL read context.
pub async fn new_query_snapshot(&self) -> Result<Self> {
// Apply the configured consistency policy before taking the snapshot.
// The returned dataset is intentionally discarded: a checkout may race
// after this await, so the dataset and its pin provenance must instead
// be cloned together from one authoritative state sample below.
self.get().await?;
let (dataset, query_snapshot) = {
let state = self.state.lock()?;
// Preserve user time travel so the MemWAL safety guard still sees
// it. Latest and already-internal snapshots remain internal pins.
(
state.dataset.clone(),
state.query_snapshot || state.pinned_version.is_none(),
)
};
let version = dataset.version().version;
Ok(Self {
state: Arc::new(Mutex::new(DatasetState {
dataset,
pinned_version: Some(version),
query_snapshot,
})),
consistency: ConsistencyMode::Lazy,
shard_writer: self.shard_writer.clone(),
})
}
/// The MemWAL `ShardWriter` cache co-located with this dataset.
pub(crate) fn shard_writer(&self) -> &Arc<ShardWriterCache> {
&self.shard_writer
@@ -204,7 +169,6 @@ impl DatasetConsistencyWrapper {
let mut state = self.state.lock()?;
state.dataset = Arc::new(new_dataset);
state.pinned_version = None;
state.query_snapshot = false;
drop(state);
if let ConsistencyMode::Eventual(bg_cache) = &self.consistency {
bg_cache.invalidate();
@@ -238,10 +202,10 @@ impl DatasetConsistencyWrapper {
/// Returns the version, if in time travel mode, or None otherwise.
pub fn time_travel_version(&self) -> Option<u64> {
let state = self.state.lock().unwrap_or_else(|e| e.into_inner());
(!state.query_snapshot)
.then_some(state.pinned_version)
.flatten()
self.state
.lock()
.unwrap_or_else(|e| e.into_inner())
.pinned_version
}
/// Convert into a wrapper in latest version mode.
@@ -261,7 +225,6 @@ impl DatasetConsistencyWrapper {
if state.pinned_version.is_some() {
state.dataset = Arc::new(new_dataset);
state.pinned_version = None;
state.query_snapshot = false;
}
drop(state);
if let ConsistencyMode::Eventual(bg_cache) = &self.consistency {
@@ -297,7 +260,6 @@ impl DatasetConsistencyWrapper {
let mut state = self.state.lock()?;
state.dataset = Arc::new(new_dataset);
state.pinned_version = Some(version_value);
state.query_snapshot = false;
Ok(())
}
@@ -499,29 +461,6 @@ mod tests {
assert_eq!(wrapper.time_travel_version(), Some(1));
}
#[tokio::test]
async fn test_query_snapshot_samples_dataset_and_pin_together() {
let dir = tempfile::tempdir().unwrap();
let uri = dir.path().to_str().unwrap();
let ds = create_test_dataset(uri).await;
let wrapper = DatasetConsistencyWrapper::new_latest(ds, None);
wrapper.as_time_travel(1u64).await.unwrap();
let stale_time_travel_dataset = wrapper.get().await.unwrap();
append_to_dataset(uri).await;
wrapper.as_latest().await.unwrap();
let snapshot = wrapper.new_query_snapshot().await.unwrap();
let snapshot_dataset = snapshot.get().await.unwrap();
assert_eq!(snapshot_dataset.version().version, 2);
assert_ne!(
snapshot_dataset.version().version,
stale_time_travel_dataset.version().version
);
assert_eq!(snapshot.time_travel_version(), None);
}
#[tokio::test]
async fn test_as_latest_from_time_travel() {
let dir = tempfile::tempdir().unwrap();
+1 -2
View File
@@ -31,9 +31,8 @@ pub(crate) async fn execute_delete(
table.dataset.ensure_mutable()?;
match predicate {
Predicate::String(s) => {
let predicate = crate::expr::canonicalize_sql_predicate(s)?;
let mut dataset = (*table.dataset.get().await?).clone();
let delete_result = dataset.delete(&predicate).boxed().await?;
let delete_result = dataset.delete(s).boxed().await?;
let num_deleted_rows = delete_result.num_deleted_rows;
let version = dataset.version().version;
table.dataset.update(dataset);
+2 -64
View File
@@ -220,32 +220,9 @@ impl MergeInsertBuilder {
///
/// Returns version and statistics about the merge operation including the number of rows
/// inserted, updated, and deleted.
pub async fn execute(
mut self,
new_data: Box<dyn RecordBatchReader + Send>,
) -> Result<MergeResult> {
self.canonicalize_filters()?;
pub async fn execute(self, new_data: Box<dyn RecordBatchReader + Send>) -> Result<MergeResult> {
self.table.clone().merge_insert(self, new_data).await
}
pub(crate) fn canonicalize_filters(&mut self) -> Result<()> {
self.when_matched_update_all_filt =
canonicalize_merge_filter(self.when_matched_update_all_filt.take())?;
self.when_not_matched_by_source_delete_filt =
canonicalize_merge_filter(self.when_not_matched_by_source_delete_filt.take())?;
Ok(())
}
}
fn canonicalize_merge_filter(filter: Option<MergeFilter>) -> Result<Option<MergeFilter>> {
filter
.map(|filter| match filter {
MergeFilter::Sql(predicate) => {
crate::expr::canonicalize_sql_predicate(&predicate).map(MergeFilter::Sql)
}
filter @ MergeFilter::Expr(_) => Ok(filter),
})
.transpose()
}
/// Internal implementation of the merge insert logic
@@ -253,10 +230,9 @@ fn canonicalize_merge_filter(filter: Option<MergeFilter>) -> Result<Option<Merge
/// This logic was moved from NativeTable::merge_insert to keep table.rs clean.
pub(crate) async fn execute_merge_insert(
table: &NativeTable,
mut params: MergeInsertBuilder,
params: MergeInsertBuilder,
new_data: Box<dyn RecordBatchReader + Send>,
) -> Result<MergeResult> {
params.canonicalize_filters()?;
super::computed_columns::ensure_no_function_bindings_for_mutation(
table.schema().await?.as_ref(),
"merge_insert",
@@ -1080,44 +1056,6 @@ mod lsm_tests {
);
}
#[tokio::test]
async fn query_snapshot_preserves_lsm_read_semantics() {
let dir = tempdir().unwrap();
let table = id_value_table(&dir).await;
table
.set_lsm_write_spec(LsmWriteSpec::unsharded())
.await
.unwrap();
lsm_upsert(&table, vec![4, 5]).await;
let snapshot = table.query_snapshot().await.unwrap();
let rows = collect_id_value(snapshot.query().execute().await.unwrap()).await;
assert_eq!(
rows.iter().map(|(id, _)| *id).collect::<Vec<_>>(),
vec![1, 2, 3, 4, 5]
);
}
#[tokio::test]
async fn query_snapshot_preserves_time_travel_lsm_guard() {
let dir = tempdir().unwrap();
let table = id_value_table(&dir).await;
table
.set_lsm_write_spec(LsmWriteSpec::unsharded())
.await
.unwrap();
lsm_upsert(&table, vec![4]).await;
let version = table.version().await.unwrap();
table.checkout(version).await.unwrap();
let direct_error = table.query().execute().await.err().unwrap();
assert!(matches!(direct_error, Error::NotSupported { .. }));
let snapshot = table.query_snapshot().await.unwrap();
let snapshot_error = snapshot.query().execute().await.err().unwrap();
assert!(matches!(snapshot_error, Error::NotSupported { .. }));
}
#[tokio::test]
async fn lsm_read_dedup_newest_wins() {
let dir = tempdir().unwrap();
+31 -412
View File
@@ -17,11 +17,11 @@ use arrow::array::{AsArray, FixedSizeListBuilder, Float32Builder};
use arrow::datatypes::{Float32Type, UInt8Type};
use arrow_array::Array;
use arrow_schema::{DataType, Schema};
use datafusion_common::{Column, DataFusionError, SchemaError};
use datafusion_physical_plan::ExecutionPlan;
use datafusion_physical_plan::projection::ProjectionExec;
use datafusion_physical_plan::repartition::RepartitionExec;
use datafusion_physical_plan::union::UnionExec;
use futures::future::try_join_all;
use lance::dataset::mem_wal::DatasetMemWalExt;
use lance::dataset::scanner::DatasetRecordBatchStream;
use lance::dataset::scanner::Scanner;
@@ -45,22 +45,6 @@ impl AnyQuery {
Self::VectorQuery(query) => &query.base,
}
}
fn base_mut(&mut self) -> &mut QueryRequest {
match self {
Self::Query(query) => query,
Self::VectorQuery(query) => &mut query.base,
}
}
/// Canonicalize any raw SQL filter immediately before backend dispatch.
pub(crate) fn canonicalized(&self) -> Result<Self> {
let mut query = self.clone();
if let Some(QueryFilter::Sql(predicate)) = &mut query.base_mut().filter {
*predicate = crate::expr::canonicalize_sql_predicate(predicate)?;
}
Ok(query)
}
}
//Decide between namespace or local
@@ -69,16 +53,15 @@ pub async fn execute_query(
query: &AnyQuery,
options: QueryExecutionOptions,
) -> Result<DatasetRecordBatchStream> {
let query = query.canonicalized()?;
// QueryTable pushdown runs the query server-side, but only on the main
// branch: the namespace request carries no branch yet, so a branch handle
// must fall through to local execution.
if can_execute_namespace_query(table, &query).await?
if can_execute_namespace_query(table, query).await?
&& let Some(ref namespace_client) = table.namespace_client
{
return execute_namespace_query(table, namespace_client.clone(), &query, options).await;
return execute_namespace_query(table, namespace_client.clone(), query, options).await;
}
execute_generic_query(table, &query, options).await
execute_generic_query(table, query, options).await
}
async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> Result<bool> {
@@ -153,10 +136,9 @@ pub async fn create_plan(
query: &AnyQuery,
options: QueryExecutionOptions,
) -> Result<Arc<dyn ExecutionPlan>> {
let query = query.canonicalized()?;
let query = match query {
AnyQuery::VectorQuery(query) => query,
AnyQuery::Query(query) => VectorQueryRequest::from_plain_query(query),
AnyQuery::VectorQuery(query) => query.clone(),
AnyQuery::Query(query) => VectorQueryRequest::from_plain_query(query.clone()),
};
query.base.check_filter()?;
@@ -188,48 +170,26 @@ pub async fn create_plan(
let mut column = query.column.clone();
let mut query_vector = query.query_vector.first().cloned();
let mut is_batch_query = false;
if query.query_vector.len() > 1 {
if column.is_none() {
// Infer a vector column with the same dimension of the query vector.
let arrow_schema = Schema::from(schema);
let arrow_schema = Schema::from(ds_ref.schema());
column = Some(default_vector_column(
&arrow_schema,
Some(query.query_vector[0].len() as i32),
)?);
}
let vector_field = schema.field(column.as_ref().unwrap()).unwrap();
let (_, element_type) =
lance::index::vector::utils::get_vector_type(schema, column.as_ref().unwrap())?;
let is_binary = matches!(element_type, DataType::UInt8);
if matches!(vector_field.data_type(), DataType::List(_))
|| (query.base.offset.unwrap_or(0) == 0 && !is_binary)
{
// Lance distinguishes these cases from the vector column type: a
// list-like query against a List column is one multivector query,
// while the same query against a FixedSizeList column is a batch of
// independent queries. The batch path shares a single flat scan and
// bounds retained candidate data instead of running one scan per
// query vector.
if let DataType::List(_) = vector_field.data_type() {
// Multivector handling: concatenate into FixedSizeList<FixedSizeList<_>>
let vectors = query
.query_vector
.iter()
.map(|arr| arr.as_ref())
.collect::<Vec<_>>();
let dim = vectors[0].len();
if let Some((query_index, actual_dim)) = vectors
.iter()
.enumerate()
.find_map(|(index, vector)| (vector.len() != dim).then_some((index, vector.len())))
{
return Err(Error::InvalidInput {
message: format!(
"query vector at index {query_index} has dimension {actual_dim}, expected {dim}"
),
});
}
let mut fsl_builder = FixedSizeListBuilder::with_capacity(
Float32Builder::with_capacity(dim * vectors.len()),
Float32Builder::with_capacity(dim),
dim as i32,
vectors.len(),
);
@@ -240,12 +200,8 @@ pub async fn create_plan(
fsl_builder.append(true);
}
query_vector = Some(Arc::new(fsl_builder.finish()));
is_batch_query = !matches!(vector_field.data_type(), DataType::List(_));
} else {
// Lance's batch path has no per-query offset, and its binary path
// requires primitive UInt8 queries rather than a fixed-size list.
// Keep the prior plan shape for these cases so offsets are applied
// per query and binary query vectors retain their primitive shape.
// Multiple query vectors: create a plan for each and union them
let query_vecs = query.query_vector.clone();
let plan_futures = query_vecs
.into_iter()
@@ -258,7 +214,7 @@ pub async fn create_plan(
}
})
.collect::<Vec<_>>();
let plans = futures::future::try_join_all(plan_futures).await?;
let plans = try_join_all(plan_futures).await?;
return create_multi_vector_plan(plans);
}
}
@@ -269,7 +225,7 @@ pub async fn create_plan(
let column = if let Some(col) = column {
col
} else {
let arrow_schema = Schema::from(schema);
let arrow_schema = Schema::from(ds_ref.schema());
default_vector_column(&arrow_schema, Some(query_vector.len() as i32))?
};
@@ -295,14 +251,10 @@ pub async fn create_plan(
}
}
// For a batch query, `nearest` already applies k to each query vector.
// Adding Scanner's global limit would truncate the combined result to k rows.
if !is_batch_query {
scanner.limit(
query.base.limit.map(|limit| limit as i64),
query.base.offset.map(|offset| offset as i64),
)?;
}
scanner.limit(
query.base.limit.map(|limit| limit as i64),
query.base.offset.map(|offset| offset as i64),
)?;
if let Some(ef) = query.ef {
scanner.ef(ef);
@@ -375,97 +327,7 @@ pub async fn create_plan(
scanner.order_by(Some(order_by.clone()))?;
}
scanner
.create_plan()
.await
.map_err(|error| enrich_lance_field_not_found(error, schema))
}
/// Replace DataFusion's top-level field candidates with qualified leaf paths.
///
/// DataFusion resolves nested fields but its `FieldNotFound` error only lists the
/// top-level Arrow fields. This makes a missing leaf look unavailable even when it
/// exists below a struct. Keep every other Lance/DataFusion error unchanged and
/// enrich only this one schema error at the LanceDB query boundary.
fn enrich_lance_field_not_found(
error: lance::Error,
schema: &lance_core::datatypes::Schema,
) -> Error {
let Some(field) = find_missing_field(&error) else {
return error.into();
};
field_not_found_error(field, &Schema::from(schema))
}
fn field_not_found_diagnostic(
error: &(dyn std::error::Error + 'static),
schema: &Schema,
) -> Option<Error> {
let field = find_missing_field(error)?;
Some(field_not_found_error(field, schema))
}
fn field_not_found_error(field: &Column, schema: &Schema) -> Error {
let valid_fields = leaf_field_paths(schema);
let mut message = format!("Schema error: No field named {}", field.quoted_flat_name());
if !valid_fields.is_empty() {
message.push_str(". Valid fields are ");
message.push_str(&valid_fields.join(", "));
}
message.push('.');
Error::InvalidInput { message }
}
fn find_missing_field<'a>(error: &'a (dyn std::error::Error + 'static)) -> Option<&'a Column> {
if let Some(DataFusionError::SchemaError(schema_error, _)) =
error.downcast_ref::<DataFusionError>()
&& let SchemaError::FieldNotFound { field, .. } = schema_error.as_ref()
{
return Some(field);
}
error.source().and_then(find_missing_field)
}
fn leaf_field_paths(schema: &Schema) -> Vec<String> {
fn format_segment(segment: &str) -> String {
// Quote every segment instead of maintaining a SQL keyword list. Bare
// lowercase names such as `true` can be parsed as expressions rather
// than identifiers, while backticks preserve all field names in both
// local SQL parsers.
format!("`{}`", segment.replace('`', "``"))
}
fn visit(fields: &arrow_schema::Fields, path: &mut Vec<String>, paths: &mut Vec<String>) {
for field in fields {
// Neither local planner can address an empty field-path segment,
// even when it is backtick-quoted. Do not advertise leaves beneath
// such a segment as valid filter fields.
if field.name().is_empty() {
continue;
}
path.push(field.name().clone());
match field.data_type() {
DataType::Struct(children) if !children.is_empty() => {
visit(children, path, paths);
}
_ => {
paths.push(
path.iter()
.map(|segment| format_segment(segment))
.collect::<Vec<_>>()
.join("."),
);
}
}
path.pop();
}
}
let mut paths = Vec::new();
visit(schema.fields(), &mut Vec::new(), &mut paths);
paths
Ok(scanner.create_plan().await?)
}
//Helper functions below
@@ -825,10 +687,7 @@ async fn parse_arrow_ipc_response(bytes: bytes::Bytes) -> Result<DatasetRecordBa
#[cfg(test)]
#[allow(deprecated)]
mod tests {
use arrow_array::{
ArrayRef, FixedSizeListArray, Float32Array, Int32Array, RecordBatch, StringArray,
StructArray,
};
use arrow_array::{ArrayRef, FixedSizeListArray, Float32Array};
use futures::TryStreamExt;
use lance_arrow::FixedSizeListArrayExt;
use std::sync::{
@@ -837,8 +696,7 @@ mod tests {
};
use super::*;
use crate::query::{ExecutableQuery, QueryBase, QueryExecutionOptions, QueryRequest};
use crate::table::BaseTable;
use crate::query::{QueryExecutionOptions, QueryRequest};
fn fixed_size_list_array(values: Vec<f32>, dimension: i32) -> FixedSizeListArray {
FixedSizeListArray::try_new_from_values(Float32Array::from(values), dimension).unwrap()
@@ -978,6 +836,7 @@ mod tests {
async fn test_execute_query_local_routing() {
use crate::connect;
use crate::table::query::execute_query;
use arrow_array::{Int32Array, RecordBatch};
use arrow_schema::{DataType, Field, Schema};
let conn = connect("memory://").execute().await.unwrap();
@@ -1017,164 +876,6 @@ mod tests {
assert_eq!(count, 2); // 4 and 5
}
#[tokio::test]
async fn test_missing_filter_field_lists_nested_fields_in_local_planners() {
use crate::connect;
use arrow_schema::{DataType, Field, Schema};
let conn = connect("memory://").execute().await.unwrap();
let metadata = Arc::new(StructArray::from(vec![
(
Arc::new(Field::new("year", DataType::Int32, false)),
Arc::new(Int32Array::from(vec![2024])) as ArrayRef,
),
(
Arc::new(Field::new("genre", DataType::Utf8, false)),
Arc::new(StringArray::from(vec!["fiction"])) as ArrayRef,
),
(
Arc::new(Field::new("Title", DataType::Int32, false)),
Arc::new(Int32Array::from(vec![7])) as ArrayRef,
),
(
Arc::new(Field::new("true", DataType::Int32, false)),
Arc::new(Int32Array::from(vec![8])) as ArrayRef,
),
(
Arc::new(Field::new("", DataType::Int32, false)),
Arc::new(Int32Array::from(vec![10])) as ArrayRef,
),
]));
let vector = Arc::new(fixed_size_list_array(vec![0.0, 1.0], 2));
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new("vector", vector.data_type().clone(), false),
Field::new("content", DataType::Utf8, false),
Field::new("metadata", metadata.data_type().clone(), false),
]));
let batch = RecordBatch::try_new(
schema,
vec![
Arc::new(Int32Array::from(vec![1])),
vector,
Arc::new(StringArray::from(vec!["example"])),
metadata,
],
)
.unwrap();
let table = conn
.create_table("nested_error", batch)
.execute()
.await
.unwrap();
let error = table
.query()
.only_if("year = 2024")
.execute()
.await
.err()
.expect("query should reject the unqualified nested field");
let case_sensitive_path = "`metadata`.`Title`";
let keyword_path = "`metadata`.`true`";
let expected = format!(
"No field named year. Valid fields are `id`, `vector`, `content`, `metadata`.`year`, `metadata`.`genre`, {case_sensitive_path}, {keyword_path}."
);
assert!(
error.to_string().contains(&expected),
"unexpected error: {error}"
);
for (path, value) in [(case_sensitive_path, 7), (keyword_path, 8)] {
table
.query()
.only_if(format!("{path} = {value}"))
.execute()
.await
.expect("the path advertised by the diagnostic should be reusable");
}
table.set_unenforced_primary_key(["id"]).await.unwrap();
table
.set_lsm_write_spec(crate::table::LsmWriteSpec::unsharded())
.await
.unwrap();
let lsm_error = table
.query()
.only_if("year = 2024")
.execute()
.await
.err()
.expect("LSM query should reject the unqualified nested field");
assert!(
lsm_error.to_string().contains(&expected),
"unexpected LSM error: {lsm_error}"
);
for (path, value) in [(case_sensitive_path, 7), (keyword_path, 8)] {
table
.query()
.only_if(format!("{path} = {value}"))
.execute()
.await
.expect("the path advertised by the diagnostic should be reusable in LSM queries");
}
}
#[test]
fn test_leaf_field_paths_preserve_arbitrary_depth() {
use arrow_schema::{DataType, Field, Schema};
fn nested_field(path: &[&str]) -> Field {
let mut segments = path.iter().rev();
let mut field = Field::new(
*segments.next().expect("path must have a leaf"),
DataType::Int32,
false,
);
for segment in segments {
field = Field::new(*segment, DataType::Struct(vec![field].into()), false);
}
field
}
let schema = Schema::new(vec![
nested_field(&["a", "b", "c", "d", "e"]),
nested_field(&["metadata", "child.with.dot"]),
nested_field(&["metadata", "Title"]),
nested_field(&["metadata", "123child"]),
nested_field(&["metadata", "child`tick"]),
nested_field(&["metadata", ""]),
nested_field(&["", "child"]),
]);
assert_eq!(
leaf_field_paths(&schema),
vec![
"`a`.`b`.`c`.`d`.`e`",
"`metadata`.`child.with.dot`",
"`metadata`.`Title`",
"`metadata`.`123child`",
"`metadata`.`child``tick`",
]
);
let source = DataFusionError::SchemaError(
Box::new(SchemaError::FieldNotFound {
field: Box::new(Column::from_name("missing")),
valid_fields: Vec::new(),
}),
Box::new(None),
);
let error = field_not_found_diagnostic(&source, &schema).unwrap();
assert!(
error.to_string().contains(
"Valid fields are `a`.`b`.`c`.`d`.`e`, `metadata`.`child.with.dot`, `metadata`.`Title`, `metadata`.`123child`, `metadata`.`child``tick`"
),
"unexpected error: {error}"
);
}
#[derive(Debug, Default)]
struct CountingNamespaceClient {
query_table_calls: AtomicUsize,
@@ -1188,56 +889,10 @@ mod tests {
async fn query_table(&self, _request: NsQueryTableRequest) -> lance::Result<bytes::Bytes> {
self.query_table_calls.fetch_add(1, Ordering::SeqCst);
panic!("query must not be pushed down to namespace query_table");
panic!("approx_mode queries must not be pushed down to namespace query_table");
}
}
#[tokio::test]
async fn test_execute_query_pinned_snapshot_with_namespace_pushdown_runs_locally() {
use crate::connect;
use arrow_array::{Int32Array, RecordBatch};
use arrow_schema::{DataType, Field, Schema};
let conn = connect("memory://").execute().await.unwrap();
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
let batch = RecordBatch::try_new(
schema,
vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))],
)
.unwrap();
let table = conn
.create_table("test_pinned_namespace_fallback", vec![batch])
.execute()
.await
.unwrap();
let namespace_client = Arc::new(CountingNamespaceClient::default());
let mut native_table = table.as_native().unwrap().clone();
native_table.namespace_client = Some(namespace_client.clone());
native_table
.pushdown_operations
.insert(NamespaceClientPushdownOperation::QueryTable);
let snapshot = native_table.checkout_current().await.unwrap();
let snapshot = snapshot.as_any().downcast_ref::<NativeTable>().unwrap();
assert!(snapshot.dataset.time_travel_version().is_some());
let query = AnyQuery::Query(QueryRequest {
filter: Some(QueryFilter::Sql("id > 3".to_string())),
..Default::default()
});
let stream = execute_query(snapshot, &query, QueryExecutionOptions::default())
.await
.unwrap();
let batches = stream.try_collect::<Vec<_>>().await.unwrap();
assert_eq!(
batches.iter().map(|batch| batch.num_rows()).sum::<usize>(),
2
);
assert_eq!(namespace_client.query_table_calls.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn test_execute_query_approx_mode_with_namespace_pushdown_runs_locally() {
use crate::connect;
@@ -1355,38 +1010,7 @@ mod tests {
}
#[tokio::test]
async fn test_query_snapshot_disables_namespace_pushdown() {
use crate::connect;
use crate::table::BaseTable;
use arrow_array::{Int32Array, RecordBatch};
use arrow_schema::{DataType, Field, Schema};
let conn = connect("memory://").execute().await.unwrap();
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
let batch =
RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]).unwrap();
let table = conn
.create_table("test_snapshot_namespace_fallback", vec![batch])
.execute()
.await
.unwrap();
let mut native_table = table.as_native().unwrap().clone();
native_table.namespace_client = Some(Arc::new(CountingNamespaceClient::default()));
native_table
.pushdown_operations
.insert(NamespaceClientPushdownOperation::QueryTable);
let snapshot = BaseTable::query_snapshot(&native_table).await.unwrap();
let snapshot = snapshot.as_any().downcast_ref::<NativeTable>().unwrap();
assert!(
!can_execute_namespace_query(snapshot, &AnyQuery::Query(QueryRequest::default()),)
.await
.unwrap()
);
}
#[tokio::test]
async fn test_create_plan_batch_vector_uses_shared_scan() {
async fn test_create_plan_multivector_structure() {
use arrow_array::{Float32Array, RecordBatch};
use arrow_schema::{DataType, Field, Schema};
use datafusion_physical_plan::display::DisplayableExecutionPlan;
@@ -1413,18 +1037,11 @@ mod tests {
.unwrap();
let native_table = table.as_native().unwrap();
// A batch of vectors against a fixed-size vector column should use
// Lance's native batch KNN path instead of independent scan plans.
// This triggers the "create_multi_vector_plan" logic branch
let q1 = Arc::new(Float32Array::from(vec![1.0, 2.0]));
let q2 = Arc::new(Float32Array::from(vec![3.0, 4.0]));
let req = VectorQueryRequest {
base: QueryRequest {
filter: Some(QueryFilter::Sql("id >= 0".to_string())),
limit: Some(1),
select: Select::Columns(vec!["id".to_string()]),
..Default::default()
},
column: Some("vector".to_string()),
query_vector: vec![q1, q2],
..Default::default()
@@ -1441,17 +1058,19 @@ mod tests {
.indent(true)
.to_string();
// We expect a RepartitionExec wrapping a UnionExec
assert!(
display.contains("KNNVectorDistance: queries=2"),
"plan should use native batch KNN, got:\n{display}"
display.contains("RepartitionExec"),
"Plan should include Repartitioning"
);
assert!(
!display.contains("UnionExec"),
"flat batch KNN should share one scan, got:\n{display}"
display.contains("UnionExec"),
"Plan should include a Union of multiple searches"
);
// We expect the projection to add the 'query_index' column (logic inside multi_vector_plan)
assert!(
display.contains("query_index"),
"plan should add query_index column, got:\n{display}"
"Plan should add query_index column"
);
}
+2 -24
View File
@@ -27,8 +27,6 @@ use std::sync::Arc;
use arrow_array::Array;
use arrow_schema::{DataType, Schema as ArrowSchema};
use datafusion::common::{DataFusionError, ToDFSchema};
use datafusion::prelude::SessionContext;
use datafusion_physical_plan::expressions::Column;
use datafusion_physical_plan::projection::ProjectionExec;
use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr};
@@ -300,7 +298,7 @@ async fn build_read_context(
for shard_id in shard_ids {
let manifest_store =
ShardManifestStore::new(store.clone(), &base_path, shard_id, scan_batch_size);
if let Some(manifest) = manifest_store.latest().await? {
if let Some(manifest) = manifest_store.read_latest().await? {
snapshots.push(snapshot_from_manifest(shard_id, &manifest, &exclude));
}
}
@@ -393,21 +391,7 @@ fn base_scanner(
}
if let Some(filter) = &query.base.filter {
scanner = match filter {
QueryFilter::Sql(sql) => {
// Parse here instead of inside `LsmScanner::filter` so the typed
// DataFusion `FieldNotFound` error is still available for the
// same nested-field enrichment used by the ordinary scanner.
let schema = ArrowSchema::from(dataset.schema());
let df_schema = schema.clone().to_dfschema().map_err(|error| {
enrich_filter_error(error, &schema, "Failed to create DFSchema")
})?;
let expr = SessionContext::new()
.parse_sql_expr(sql, &df_schema)
.map_err(|error| {
enrich_filter_error(error, &schema, "Failed to parse filter expression")
})?;
scanner.filter_expr(expr)
}
QueryFilter::Sql(sql) => scanner.filter(sql)?,
QueryFilter::Datafusion(expr) => scanner.filter_expr(expr.clone()),
QueryFilter::Substrait(_) => {
return Err(Error::NotSupported {
@@ -419,12 +403,6 @@ fn base_scanner(
Ok(scanner)
}
fn enrich_filter_error(error: DataFusionError, schema: &ArrowSchema, context: &str) -> Error {
super::field_not_found_diagnostic(&error, schema).unwrap_or_else(|| Error::InvalidInput {
message: format!("{context}: {error}"),
})
}
/// Plain scan: filter / projection / limit over base SSTables in-memory.
/// The plain scan applies limit and offset inside the planner.
async fn plain_plan(
+1 -3
View File
@@ -55,9 +55,7 @@ pub struct DropColumnsResult {
pub struct FieldMetadataUpdate {
/// Dot-separated path to the field (e.g. `"embedding"` or `"address.zip"`).
pub path: String,
/// Keys to set (`Some`) or delete (`None`). See
/// [`Table::update_field_metadata`](crate::Table::update_field_metadata) for
/// the conventional `lancedb:*` keys.
/// Keys to set (`Some`) or delete (`None`).
pub metadata: HashMap<String, Option<String>>,
/// If `true`, replace the field's entire metadata map instead of merging.
pub replace: bool,
+2 -13
View File
@@ -62,33 +62,22 @@ impl UpdateBuilder {
}
/// Executes the update operation.
pub async fn execute(mut self) -> Result<UpdateResult> {
pub async fn execute(self) -> Result<UpdateResult> {
if self.columns.is_empty() {
Err(Error::InvalidInput {
message: "at least one column must be specified in an update operation".to_string(),
})
} else {
self.canonicalize_filter()?;
self.parent.clone().update(self).await
}
}
pub(crate) fn canonicalize_filter(&mut self) -> Result<()> {
self.filter = self
.filter
.take()
.map(|predicate| crate::expr::canonicalize_sql_predicate(&predicate))
.transpose()?;
Ok(())
}
}
/// Internal implementation of the update logic
pub(crate) async fn execute_update(
table: &NativeTable,
mut update: UpdateBuilder,
update: UpdateBuilder,
) -> Result<UpdateResult> {
update.canonicalize_filter()?;
table.dataset.ensure_mutable()?;
// 1. Snapshot the current dataset
-183
View File
@@ -225,159 +225,6 @@ pub(crate) fn resolve_arrow_field_path(schema: &Schema, column: &str) -> Result<
Ok((canonical_path, Field::from(*field)))
}
pub(crate) struct ResolvedFtsField {
pub canonical_path: String,
pub field: Field,
pub list_depth: usize,
}
/// Canonicalize a public FTS field path while keeping Arrow list item names hidden.
pub(crate) fn resolve_lance_fts_field_path(
schema: &lance_core::datatypes::Schema,
column: &str,
) -> Result<ResolvedFtsField> {
let names =
lance_core::datatypes::parse_field_path(column).map_err(|e| Error::InvalidInput {
message: format!("Invalid field path `{}`: {}", column, e),
})?;
let (root_name, remaining_names) = names.split_first().ok_or_else(|| Error::InvalidInput {
message: "FTS field path cannot be empty".to_string(),
})?;
let mut field = schema
.fields
.iter()
.find(|field| field.name == *root_name)
.or_else(|| {
schema
.fields
.iter()
.find(|field| field.name.eq_ignore_ascii_case(root_name))
})
.ok_or_else(|| fts_field_not_found(schema, column))?;
let mut canonical_names = vec![field.name.clone()];
let mut list_depth = 0;
for name in remaining_names {
while matches!(
field.data_type(),
DataType::List(_) | DataType::LargeList(_)
) {
list_depth += 1;
field = field.children.first().ok_or_else(|| Error::Schema {
message: format!(
"FTS field path `{}` has a list without an item field",
column
),
})?;
}
if !matches!(field.data_type(), DataType::Struct(_)) {
return Err(fts_field_not_found(schema, column));
}
field = field
.children
.iter()
.find(|field| field.name == *name)
.or_else(|| {
field
.children
.iter()
.find(|field| field.name.eq_ignore_ascii_case(name))
})
.ok_or_else(|| fts_field_not_found(schema, column))?;
canonical_names.push(field.name.clone());
}
let mut terminal = field;
while matches!(
terminal.data_type(),
DataType::List(_) | DataType::LargeList(_)
) {
list_depth += 1;
terminal = terminal.children.first().ok_or_else(|| Error::Schema {
message: format!(
"FTS field path `{}` has a list without an item field",
column
),
})?;
}
let canonical_path = lance_core::datatypes::format_field_path(
&canonical_names
.iter()
.map(String::as_str)
.collect::<Vec<_>>(),
);
Ok(ResolvedFtsField {
canonical_path,
field: Field::from(field),
list_depth,
})
}
fn fts_field_not_found(schema: &lance_core::datatypes::Schema, column: &str) -> Error {
Error::Schema {
message: format!(
"Field path `{}` not found in schema. Available field paths: {}",
column,
schema.field_paths().join(", ")
),
}
}
fn find_public_fts_field_path_by_id(
field: &lance_core::datatypes::Field,
field_id: i32,
path: &mut Vec<String>,
) -> bool {
if field.id == field_id {
return true;
}
match field.data_type() {
DataType::List(_) | DataType::LargeList(_) => field
.children
.first()
.is_some_and(|child| find_public_fts_field_path_by_id(child, field_id, path)),
DataType::Struct(_) => field.children.iter().any(|child| {
path.push(child.name.clone());
let found = find_public_fts_field_path_by_id(child, field_id, path);
if !found {
path.pop();
}
found
}),
_ => false,
}
}
pub(crate) fn public_fts_field_path_by_id(
schema: &lance_core::datatypes::Schema,
field_id: i32,
) -> Result<String> {
for root in &schema.fields {
let mut path = vec![root.name.clone()];
if find_public_fts_field_path_by_id(root, field_id, &mut path) {
return Ok(lance_core::datatypes::format_field_path(
&path.iter().map(String::as_str).collect::<Vec<_>>(),
));
}
}
Err(Error::Schema {
message: format!("Field id `{}` not found in schema", field_id),
})
}
pub(crate) fn resolve_arrow_fts_field_path(
schema: &Schema,
column: &str,
) -> Result<(String, Field)> {
let lance_schema =
lance_core::datatypes::Schema::try_from(schema).map_err(|e| Error::Schema {
message: format!("Invalid schema: {}", e),
})?;
let resolved = resolve_lance_fts_field_path(&lance_schema, column)?;
Ok((resolved.canonical_path, resolved.field))
}
pub fn supported_btree_data_type(dtype: &DataType) -> bool {
dtype.is_integer()
|| dtype.is_floating()
@@ -633,36 +480,6 @@ mod tests {
use super::*;
#[test]
fn test_public_fts_field_path_prefers_exact_case() {
let text_list = || {
DataType::List(Arc::new(Field::new(
"item",
DataType::Struct(vec![Field::new("content", DataType::Utf8, true)].into()),
true,
)))
};
let schema = Schema::new(vec![
Field::new("Docs", text_list(), true),
Field::new("docs", text_list(), true),
]);
let (path, _) = resolve_arrow_fts_field_path(&schema, "docs.content").unwrap();
assert_eq!(path, "docs.content");
let lance_schema = lance_core::datatypes::Schema::try_from(&schema).unwrap();
let field_id = lance_schema
.resolve_case_insensitive("docs.item.content")
.unwrap()
.last()
.unwrap()
.id;
assert_eq!(
public_fts_field_path_by_id(&lance_schema, field_id).unwrap(),
"docs.content"
);
}
#[test]
fn test_guess_default_column() {
let schema_no_vector = Schema::new(vec![
+4 -226
View File
@@ -5,14 +5,12 @@ use std::sync::Arc;
use arrow_array::{
Array, ArrayRef, BinaryArray, Int64Array, LargeBinaryArray, RecordBatch, StringArray,
StructArray, UInt64Array, new_null_array,
StructArray, UInt64Array,
};
use arrow_schema::{DataType, Field, Fields, Schema};
use futures::TryStreamExt;
use lance::Dataset;
use lance::dataset::WriteParams;
use lance_file::version::{ConcreteFileVersion, LanceFileVersion};
use lance_table::format::BasePath;
use lancedb::{
Connection, Error, Result, Table,
blob::{BlobRangeRequest, blob},
@@ -21,7 +19,7 @@ use lancedb::{
ListingDatabaseOptions, NewTableConfig, OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS,
},
query::{ExecutableQuery, QueryBase},
table::{AddDataMode, CompactionOptions, OptimizeAction, OptimizeStats, WriteOptions},
table::{AddDataMode, CompactionOptions, OptimizeAction, OptimizeStats},
};
use tempfile::tempdir;
@@ -263,11 +261,11 @@ async fn add_rejects_uncoercible_blob_input() -> Result<()> {
let batch = RecordBatch::try_new(
Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("image", DataType::Int64, true),
Field::new("image", DataType::Utf8, true),
])),
vec![
Arc::new(Int64Array::from(vec![1])),
Arc::new(Int64Array::from(vec![42])),
Arc::new(StringArray::from(vec!["not bytes"])),
],
)
.unwrap();
@@ -1334,223 +1332,3 @@ async fn optimize_preserves_blob_v2_null_and_empty_distinction() -> Result<()> {
);
Ok(())
}
fn uri_struct_batch(id: i64, uri: &str) -> RecordBatch {
let image_field = blob("image", true);
let DataType::Struct(child_fields) = image_field.data_type().clone() else {
unreachable!("blob field is a struct");
};
let children: Vec<ArrayRef> = child_fields
.iter()
.map(|field| match field.name().as_str() {
"uri" => Arc::new(StringArray::from(vec![Some(uri)])) as ArrayRef,
_ => new_null_array(field.data_type(), 1),
})
.collect();
let image = StructArray::new(child_fields, children, None);
RecordBatch::try_new(
Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
image_field,
])),
vec![Arc::new(Int64Array::from(vec![id])), Arc::new(image)],
)
.unwrap()
}
fn uri_string_batch(id: i64, uri: &str) -> RecordBatch {
RecordBatch::try_new(
Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("image", DataType::Utf8, true),
])),
vec![
Arc::new(Int64Array::from(vec![id])),
Arc::new(StringArray::from(vec![Some(uri)])),
],
)
.unwrap()
}
fn write_payload_file_uri(dir: &std::path::Path, name: &str, payload: &[u8]) -> String {
let path = dir.join(name);
std::fs::write(&path, payload).unwrap();
url::Url::from_file_path(&path).unwrap().to_string()
}
#[tokio::test]
async fn external_uri_struct_round_trips_with_flag() -> Result<()> {
let tmp = tempdir().unwrap();
let db = connect(tmp.path().join("db").to_str().unwrap())
.execute()
.await?;
let payload: &[u8] = b"external-struct-payload";
let uri = write_payload_file_uri(tmp.path(), "payload.bin", payload);
let table = db
.create_empty_table("t", blob_table_schema())
.execute()
.await?;
table
.add(uri_struct_batch(1, &uri))
.allow_external_blob_outside_bases(true)
.execute()
.await?;
let ids = collect_row_ids(&table).await?;
let bytes = table.fetch_blobs("image", &ids).await?;
assert_eq!(bytes.value(0), payload);
Ok(())
}
#[tokio::test]
async fn external_uri_add_requires_opt_in() -> Result<()> {
let tmp = tempdir().unwrap();
let db = connect(tmp.path().join("db").to_str().unwrap())
.execute()
.await?;
let uri = write_payload_file_uri(tmp.path(), "payload.bin", b"unreachable");
let table = db
.create_empty_table("t", blob_table_schema())
.execute()
.await?;
let err = table
.add(uri_struct_batch(1, &uri))
.execute()
.await
.unwrap_err();
assert!(
err.to_string()
.contains("allow_external_blob_outside_bases"),
"got: {err}"
);
assert_eq!(table.count_rows(None).await?, 0);
Ok(())
}
#[tokio::test]
async fn string_uri_input_round_trips_as_external_reference() -> Result<()> {
let tmp = tempdir().unwrap();
let db = connect(tmp.path().join("db").to_str().unwrap())
.execute()
.await?;
let payload: &[u8] = b"external-string-payload";
let uri = write_payload_file_uri(tmp.path(), "payload.bin", payload);
let table = db
.create_empty_table("t", blob_table_schema())
.execute()
.await?;
table
.add(uri_string_batch(1, &uri))
.allow_external_blob_outside_bases(true)
.execute()
.await?;
let ids = collect_row_ids(&table).await?;
let bytes = table.fetch_blobs("image", &ids).await?;
assert_eq!(bytes.value(0), payload);
let files = table.fetch_blob_files("image", &ids).await?;
let file = files[0].as_ref().expect("missing blob file");
assert_eq!(file.uri(), Some(uri.as_str()));
Ok(())
}
#[tokio::test]
async fn string_uri_inside_registered_base_does_not_need_the_flag() -> Result<()> {
let tmp = tempdir().unwrap();
let db_path = tmp.path().join("db");
let external_base = tmp.path().join("external_base");
let object_dir = external_base.join("objects");
std::fs::create_dir_all(&object_dir).unwrap();
let payload: &[u8] = b"mapped-in-base";
let object_path = object_dir.join("mapped.bin");
std::fs::write(&object_path, payload).unwrap();
let object_uri = url::Url::from_file_path(&object_path).unwrap().to_string();
let base_uri = url::Url::from_file_path(&external_base)
.unwrap()
.to_string();
let db = connect(db_path.to_str().unwrap()).execute().await?;
let table = db
.create_empty_table("t", blob_table_schema())
.write_options(WriteOptions {
lance_write_params: Some(WriteParams {
initial_bases: Some(vec![BasePath {
id: 1,
name: Some("external".to_string()),
path: base_uri,
is_dataset_root: false,
}]),
..Default::default()
}),
})
.execute()
.await?;
table
.add(uri_string_batch(1, &object_uri))
.execute()
.await?;
let ids = collect_row_ids(&table).await?;
let bytes = table.fetch_blobs("image", &ids).await?;
assert_eq!(bytes.value(0), payload);
Ok(())
}
#[tokio::test]
async fn external_uri_rows_mix_with_inline_rows() -> Result<()> {
let tmp = tempdir().unwrap();
let db = connect(tmp.path().join("db").to_str().unwrap())
.execute()
.await?;
let external_payload: &[u8] = b"external-bytes";
let uri = write_payload_file_uri(tmp.path(), "payload.bin", external_payload);
let table =
create_inline_blob_table(&db, "t", &[1], &[Some(b"inline-bytes".as_slice())]).await?;
table
.add(uri_string_batch(2, &uri))
.allow_external_blob_outside_bases(true)
.execute()
.await?;
let pairs = collect_id_rowid(&table).await?;
let row_ids: Vec<u64> = pairs.iter().map(|(_, r)| *r).collect();
let bytes = table.fetch_blobs("image", &row_ids).await?;
for (i, (id, _)) in pairs.iter().enumerate() {
match id {
1 => assert_eq!(bytes.value(i), b"inline-bytes"),
2 => assert_eq!(bytes.value(i), external_payload),
_ => unreachable!(),
}
}
Ok(())
}
#[tokio::test]
async fn malformed_string_uri_is_rejected_at_write() -> Result<()> {
let tmp = tempdir().unwrap();
let db = connect(tmp.path().join("db").to_str().unwrap())
.execute()
.await?;
let table = db
.create_empty_table("t", blob_table_schema())
.execute()
.await?;
let err = table
.add(uri_string_batch(1, "not a uri"))
.allow_external_blob_outside_bases(true)
.execute()
.await
.unwrap_err();
assert!(err.to_string().contains("not a uri"), "got: {err}");
assert_eq!(table.count_rows(None).await?, 0);
Ok(())
}