Compare commits

..

1 Commits

Author SHA1 Message Date
dependabot[bot]
80b8022255 chore(deps): bump snafu from 0.8.9 to 0.9.0
Bumps [snafu](https://github.com/shepmaster/snafu) from 0.8.9 to 0.9.0.
- [Changelog](https://github.com/shepmaster/snafu/blob/main/CHANGELOG.md)
- [Commits](https://github.com/shepmaster/snafu/compare/0.8.9...0.9.0)

---
updated-dependencies:
- dependency-name: snafu
  dependency-version: 0.9.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-05-20 16:12:28 +00:00
42 changed files with 277 additions and 1720 deletions

View File

@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.30.0-beta.1"
current_version = "0.29.1-beta.0"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.

View File

@@ -37,13 +37,10 @@ Before committing changes, run formatting for every language you touched. At min
and run targeted tests through `cd python && uv run ...`.
* TypeScript changes: run the relevant `npm`/`pnpm` lint, format, build, and docs commands in `nodejs`.
Before creating a PR, the exact value passed to `gh pr create --title` must follow
Conventional Commits, such as `fix: support nested field paths in native index creation`
or `feat(python): add dataset multiprocessing support`. Do not use a plain natural
language summary like `Support nested field paths in native index creation` as the PR
title. The semantic-release check uses the PR title and body as the merge commit message,
so a non-conventional PR title will fail CI. After creating a PR, read the remote PR title
back and fix it immediately if it is not conventional.
Before creating a PR, make sure the PR title follows Conventional Commits, such as
`fix: support nested field paths in native index creation` or
`feat(python): add dataset multiprocessing support`. The semantic-release check uses the
PR title and body as the merge commit message, so a non-conventional PR title will fail CI.
## Coding tips

54
Cargo.lock generated
View File

@@ -5014,7 +5014,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.30.0-beta.1"
version = "0.29.1-beta.0"
dependencies = [
"ahash",
"anyhow",
@@ -5084,7 +5084,6 @@ dependencies = [
"serde",
"serde_json",
"serde_with",
"serial_test",
"snafu 0.8.9",
"tempfile",
"test-log",
@@ -5097,7 +5096,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.30.0-beta.1"
version = "0.29.1-beta.0"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5120,7 +5119,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.33.0-beta.1"
version = "0.32.1-beta.0"
dependencies = [
"arrow",
"async-trait",
@@ -6997,7 +6996,7 @@ version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
dependencies = [
"heck 0.5.0",
"heck 0.4.1",
"itertools 0.14.0",
"log",
"multimap",
@@ -8129,15 +8128,6 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "scc"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46e6f046b7fef48e2660c57ed794263155d713de679057f2d0c169bfc6e756cc"
dependencies = [
"sdd",
]
[[package]]
name = "schannel"
version = "0.1.29"
@@ -8204,12 +8194,6 @@ dependencies = [
"untrusted 0.9.0",
]
[[package]]
name = "sdd"
version = "3.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca"
[[package]]
name = "sec1"
version = "0.3.0"
@@ -8400,32 +8384,6 @@ dependencies = [
"unsafe-libyaml",
]
[[package]]
name = "serial_test"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "911bd979bf1070a3f3aa7b691a3b3e9968f339ceeec89e08c280a8a22207a32f"
dependencies = [
"futures-executor",
"futures-util",
"log",
"once_cell",
"parking_lot",
"scc",
"serial_test_derive",
]
[[package]]
name = "serial_test_derive"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "sha1"
version = "0.10.6"
@@ -8623,7 +8581,7 @@ version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
dependencies = [
"heck 0.5.0",
"heck 0.4.1",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -8635,7 +8593,7 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40"
dependencies = [
"heck 0.5.0",
"heck 0.4.1",
"proc-macro2",
"quote",
"syn 2.0.117",

View File

@@ -112,25 +112,25 @@ def fetch_remote_tags() -> List[TagInfo]:
"api",
"-X",
"GET",
f"repos/{LANCE_REPO}/releases",
f"repos/{LANCE_REPO}/git/refs/tags",
"--paginate",
"--jq",
".[].tag_name",
"-F",
"per_page=20",
".[].ref",
]
)
tags: List[TagInfo] = []
for line in output.splitlines():
tag = line.strip()
if not tag.startswith("v"):
ref = line.strip()
if not ref.startswith("refs/tags/v"):
continue
tag = ref.split("refs/tags/")[-1]
version = tag.lstrip("v")
try:
tags.append(TagInfo(tag=tag, version=version, semver=parse_semver(version)))
except ValueError:
continue
if not tags:
raise RuntimeError("No Lance releases could be parsed from GitHub API output")
raise RuntimeError("No Lance tags could be parsed from GitHub API output")
return tags

View File

@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.30.0-beta.1</version>
<version>0.29.1-beta.0</version>
</dependency>
```

View File

@@ -70,20 +70,16 @@ client used by manifest-enabled native connections.
optional readConsistencyInterval: number;
```
The interval, in seconds, at which to check for updates to the table
from other processes. If None, then consistency is not checked. For
performance reasons, this is the default. For strong consistency, set
this to zero seconds. Then every read will check for updates from other
processes. As a compromise, you can set this to a non-zero value for
eventual consistency. If more than that interval has passed since the
last check, then the table will be checked for updates. Note: this
consistency only applies to read operations. Write operations are
(For LanceDB OSS only): The interval, in seconds, at which to check for
updates to the table from other processes. If None, then consistency is not
checked. For performance reasons, this is the default. For strong
consistency, set this to zero seconds. Then every read will check for
updates from other processes. As a compromise, you can set this to a
non-zero value for eventual consistency. If more than that interval
has passed since the last check, then the table will be checked for updates.
Note: this consistency only applies to read operations. Write operations are
always consistent.
Stronger consistency is not free. The smaller the interval, the more
often each read pays the cost of checking for updates against object
storage, raising per-read latency and cost.
***
### region?

View File

@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.30.0-beta.1</version>
<version>0.29.1-beta.0</version>
<relativePath>../pom.xml</relativePath>
</parent>

View File

@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.30.0-beta.1</version>
<version>0.29.1-beta.0</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>

View File

@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.30.0-beta.1"
version = "0.29.1-beta.0"
publish = false
license.workspace = true
description.workspace = true

View File

@@ -28,7 +28,6 @@ import {
List,
Schema,
SchemaLike,
Struct,
Type,
Uint8,
Utf8,
@@ -781,113 +780,6 @@ describe("When creating an index", () => {
expect(indices2.length).toBe(0);
});
it("should create and search a nested vector index", async () => {
const db = await connect(tmpDir.name);
const nestedSchema = new Schema([
new Field("id", new Int32(), true),
new Field(
"image",
new Struct([
new Field(
"embedding",
new FixedSizeList(2, new Field("item", new Float32(), true)),
true,
),
]),
true,
),
]);
const nestedTable = await db.createTable(
"nested_vector",
makeArrowTable(
Array.from({ length: 300 }, (_, id) => ({
id,
image: { embedding: [id, id + 1] },
})),
{ schema: nestedSchema },
),
);
await nestedTable.createIndex("image.embedding", {
name: "image_embedding_idx",
});
const indices = await nestedTable.listIndices();
expect(indices).toContainEqual({
name: "image_embedding_idx",
indexType: "IvfPq",
columns: ["image.embedding"],
});
const explicit = await nestedTable
.query()
.nearestTo([0.0, 1.0])
.column("image.embedding")
.limit(1)
.toArray();
const inferred = await nestedTable
.query()
.nearestTo([0.0, 1.0])
.limit(1)
.toArray();
expect(inferred[0].id).toEqual(explicit[0].id);
});
it("should report multiple nested vector candidates", async () => {
const db = await connect(tmpDir.name);
const nestedSchema = new Schema([
new Field(
"image",
new Struct([
new Field(
"embedding",
new FixedSizeList(2, new Field("item", new Float32(), true)),
true,
),
]),
true,
),
new Field(
"text",
new Struct([
new Field(
"embedding",
new FixedSizeList(2, new Field("item", new Float32(), true)),
true,
),
]),
true,
),
]);
const nestedTable = await db.createTable(
"multiple_nested_vectors",
makeArrowTable(
[
{
image: { embedding: [0.0, 1.0] },
text: { embedding: [2.0, 3.0] },
},
],
{ schema: nestedSchema },
),
);
await expect(
nestedTable.query().nearestTo([0.0, 1.0]).limit(1).toArray(),
).rejects.toThrow(/image\.embedding.*text\.embedding/);
});
it("should report when no default vector column exists", async () => {
const db = await connect(tmpDir.name);
const noVectorTable = await db.createTable(
"no_vector",
makeArrowTable([{ id: 0, label: "cat" }]),
);
await expect(
noVectorTable.query().nearestTo([0.0, 1.0]).limit(1).toArray(),
).rejects.toThrow(/No vector column/);
});
it("should wait for index readiness", async () => {
// Create an index and then wait for it to be ready
await tbl.createIndex("vec");

View File

@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.30.0-beta.1",
"version": "0.29.1-beta.0",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",

View File

@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.30.0-beta.1",
"version": "0.29.1-beta.0",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",

View File

@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.30.0-beta.1",
"version": "0.29.1-beta.0",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",

View File

@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.30.0-beta.1",
"version": "0.29.1-beta.0",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",

View File

@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.30.0-beta.1",
"version": "0.29.1-beta.0",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",

View File

@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.30.0-beta.1",
"version": "0.29.1-beta.0",
"os": [
"win32"
],

View File

@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.30.0-beta.1",
"version": "0.29.1-beta.0",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",

View File

@@ -1,12 +1,12 @@
{
"name": "@lancedb/lancedb",
"version": "0.30.0-beta.1",
"version": "0.29.1-beta.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@lancedb/lancedb",
"version": "0.30.0-beta.1",
"version": "0.29.1-beta.0",
"cpu": [
"x64",
"arm64"

View File

@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.30.0-beta.1",
"version": "0.29.1-beta.0",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",

View File

@@ -24,19 +24,15 @@ mod util;
#[napi(object)]
#[derive(Debug)]
pub struct ConnectionOptions {
/// The interval, in seconds, at which to check for updates to the table
/// from other processes. If None, then consistency is not checked. For
/// performance reasons, this is the default. For strong consistency, set
/// this to zero seconds. Then every read will check for updates from other
/// processes. As a compromise, you can set this to a non-zero value for
/// eventual consistency. If more than that interval has passed since the
/// last check, then the table will be checked for updates. Note: this
/// consistency only applies to read operations. Write operations are
/// (For LanceDB OSS only): The interval, in seconds, at which to check for
/// updates to the table from other processes. If None, then consistency is not
/// checked. For performance reasons, this is the default. For strong
/// consistency, set this to zero seconds. Then every read will check for
/// updates from other processes. As a compromise, you can set this to a
/// non-zero value for eventual consistency. If more than that interval
/// has passed since the last check, then the table will be checked for updates.
/// Note: this consistency only applies to read operations. Write operations are
/// always consistent.
///
/// Stronger consistency is not free. The smaller the interval, the more
/// often each read pays the cost of checking for updates against object
/// storage, raising per-read latency and cost.
pub read_consistency_interval: Option<f64>,
/// (For LanceDB OSS only): configuration for object storage.
///

View File

@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.33.0-beta.1"
current_version = "0.32.1-beta.0"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.

View File

@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.33.0-beta.1"
version = "0.32.1-beta.0"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"

View File

@@ -94,6 +94,7 @@ def connect(
host_override: str, optional
The override url for LanceDB Cloud.
read_consistency_interval: timedelta, default None
(For LanceDB OSS only)
The interval at which to check for updates to the table from other
processes. If None, then consistency is not checked. For performance
reasons, this is the default. For strong consistency, set this to
@@ -103,10 +104,6 @@ def connect(
the last check, then the table will be checked for updates. Note: this
consistency only applies to read operations. Write operations are
always consistent.
Stronger consistency is not free. The smaller the interval, the more
often each read pays the cost of checking for updates against object
storage, raising per-read latency and cost.
client_config: ClientConfig or dict, optional
Configuration options for the LanceDB Cloud HTTP client. If a dict, then
the keys are the attributes of the ClientConfig class. If None, then the
@@ -150,13 +147,6 @@ def connect(
>>> db = lancedb.connect("s3://my-bucket/lancedb",
... storage_options={"aws_access_key_id": "***"})
For tests and temporary data, use an in-memory database:
>>> db = lancedb.connect("memory://")
In-memory databases are not persisted. Tables are dropped when the last
connection or table handle referencing them is closed.
Connect to LanceDB cloud:
>>> db = lancedb.connect("db://my_database", api_key="ldb_...",
@@ -220,7 +210,6 @@ def connect(
request_thread_pool=request_thread_pool,
client_config=client_config,
storage_options=storage_options,
read_consistency_interval=read_consistency_interval,
**kwargs,
)
_check_s3_bucket_with_dots(str(uri), storage_options)
@@ -347,6 +336,7 @@ async def connect_async(
host_override: str, optional
The override url for LanceDB Cloud.
read_consistency_interval: timedelta, default None
(For LanceDB OSS only)
The interval at which to check for updates to the table from other
processes. If None, then consistency is not checked. For performance
reasons, this is the default. For strong consistency, set this to
@@ -356,10 +346,6 @@ async def connect_async(
the last check, then the table will be checked for updates. Note: this
consistency only applies to read operations. Write operations are
always consistent.
Stronger consistency is not free. The smaller the interval, the more
often each read pays the cost of checking for updates against object
storage, raising per-read latency and cost.
client_config: ClientConfig or dict, optional
Configuration options for the LanceDB Cloud HTTP client. If a dict, then
the keys are the attributes of the ClientConfig class. If None, then the
@@ -392,8 +378,6 @@ async def connect_async(
... db = await lancedb.connect_async("s3://my-bucket/lancedb",
... storage_options={
... "aws_access_key_id": "***"})
... # For tests and temporary data, use an in-memory database
... db = await lancedb.connect_async("memory://")
... # Connect to LanceDB cloud
... db = await lancedb.connect_async("db://my_database", api_key="ldb_...",
... client_config={

View File

@@ -8,17 +8,7 @@ from abc import abstractmethod
from datetime import timedelta
from pathlib import Path
import sys
from typing import (
TYPE_CHECKING,
Any,
Dict,
Generator,
Iterable,
List,
Literal,
Optional,
Union,
)
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Literal, Optional, Union
if sys.version_info >= (3, 12):
from typing import override
@@ -323,7 +313,7 @@ class DBConnection(EnforceOverrides):
>>> data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
... {"vector": [0.2, 1.8], "lat": 40.1, "long": -74.1}]
>>> db.create_table("my_table", data)
LanceTable(name='my_table', ...)
LanceTable(name='my_table', version=1, ...)
>>> db["my_table"].head()
pyarrow.Table
vector: fixed_size_list<item: float>[2]
@@ -344,7 +334,7 @@ class DBConnection(EnforceOverrides):
... "long": [-122.7, -74.1]
... })
>>> db.create_table("table2", data)
LanceTable(name='table2', ...)
LanceTable(name='table2', version=1, ...)
>>> db["table2"].head()
pyarrow.Table
vector: fixed_size_list<item: float>[2]
@@ -367,7 +357,7 @@ class DBConnection(EnforceOverrides):
... pa.field("long", pa.float32())
... ])
>>> db.create_table("table3", data, schema = custom_schema)
LanceTable(name='table3', ...)
LanceTable(name='table3', version=1, ...)
>>> db["table3"].head()
pyarrow.Table
vector: fixed_size_list<item: float>[2]
@@ -401,7 +391,7 @@ class DBConnection(EnforceOverrides):
... pa.field("price", pa.float32()),
... ])
>>> db.create_table("table4", make_batches(), schema=schema)
LanceTable(name='table4', ...)
LanceTable(name='table4', version=1, ...)
"""
raise NotImplementedError
@@ -578,15 +568,15 @@ class LanceDBConnection(DBConnection):
>>> db = lancedb.connect("./.lancedb")
>>> db.create_table("my_table", data=[{"vector": [1.1, 1.2], "b": 2},
... {"vector": [0.5, 1.3], "b": 4}])
LanceTable(name='my_table', ...)
LanceTable(name='my_table', version=1, ...)
>>> db.create_table("another_table", data=[{"vector": [0.4, 0.4], "b": 6}])
LanceTable(name='another_table', ...)
LanceTable(name='another_table', version=1, ...)
>>> sorted(db.table_names())
['another_table', 'my_table']
>>> len(db)
2
>>> db["my_table"]
LanceTable(name='my_table', ...)
LanceTable(name='my_table', version=1, ...)
>>> "my_table" in db
True
>>> db.drop_table("my_table")
@@ -857,20 +847,11 @@ class LanceDBConnection(DBConnection):
)
)
def _all_table_names(self) -> Generator[str, None, None]:
page_token = None
while True:
response = self.list_tables(page_token=page_token)
yield from response.tables
page_token = response.page_token
if not page_token:
return
def __len__(self) -> int:
return sum(1 for _ in self._all_table_names())
return len(self.table_names())
def __contains__(self, name: str) -> bool:
return name in self._all_table_names()
return name in self.table_names()
@override
def create_table(

View File

@@ -3,14 +3,12 @@
from __future__ import annotations
import asyncio
from abc import ABC, abstractmethod
from concurrent.futures import ThreadPoolExecutor
from datetime import timedelta
from enum import Enum
from datetime import timedelta
from typing import (
TYPE_CHECKING,
Any,
Dict,
List,
Literal,
@@ -19,40 +17,41 @@ from typing import (
Type,
TypeVar,
Union,
Any,
)
import asyncio
import deprecation
import numpy as np
import pyarrow as pa
import pyarrow.compute as pc
import pydantic
from typing_extensions import Annotated
from lancedb._lancedb import fts_query_to_json
from lancedb.background_loop import LOOP
from lancedb.pydantic import PYDANTIC_VERSION
from lancedb.background_loop import LOOP
from . import __version__
from .arrow import AsyncRecordBatchReader
from .dependencies import pandas as pd
from .expr import Expr
from .rerankers.base import Reranker
from .rerankers.rrf import RRFReranker
from .rerankers.util import check_reranker_result
from .util import flatten_columns
from .expr import Expr
from lancedb._lancedb import fts_query_to_json
from typing_extensions import Annotated
if TYPE_CHECKING:
import sys
import PIL
import polars as pl
from ._lancedb import Query as LanceQuery
from ._lancedb import FTSQuery as LanceFTSQuery
from ._lancedb import HybridQuery as LanceHybridQuery
from ._lancedb import PyQueryRequest
from ._lancedb import Query as LanceQuery
from ._lancedb import TakeQuery as LanceTakeQuery
from ._lancedb import VectorQuery as LanceVectorQuery
from ._lancedb import TakeQuery as LanceTakeQuery
from ._lancedb import PyQueryRequest
from .common import VEC
from .pydantic import LanceModel
from .table import Table
@@ -3349,18 +3348,16 @@ class BaseQueryBuilder(object):
If not specified, no timeout is applied. If the query does not
complete within the specified time, an error will be raised.
"""
async_reader = LOOP.run(
self._inner.to_batches(max_batch_length=max_batch_length, timeout=timeout)
)
async_iter = LOOP.run(self._inner.execute(max_batch_length, timeout))
def iter_sync():
try:
while True:
yield LOOP.run(async_reader.__anext__())
yield LOOP.run(async_iter.__anext__())
except StopAsyncIteration:
return
return pa.RecordBatchReader.from_batches(async_reader.schema, iter_sync())
return pa.RecordBatchReader.from_batches(async_iter.schema, iter_sync())
def to_arrow(self, timeout: Optional[timedelta] = None) -> pa.Table:
"""

View File

@@ -50,7 +50,6 @@ class RemoteDBConnection(DBConnection):
connection_timeout: Optional[float] = None,
read_timeout: Optional[float] = None,
storage_options: Optional[Dict[str, str]] = None,
read_consistency_interval: Optional[timedelta] = None,
):
"""Connect to a remote LanceDB database."""
if isinstance(client_config, dict):
@@ -104,7 +103,6 @@ class RemoteDBConnection(DBConnection):
host_override=host_override,
client_config=client_config,
storage_options=storage_options,
read_consistency_interval=read_consistency_interval,
)
)

View File

@@ -2178,7 +2178,7 @@ class LanceTable(Table):
return LOOP.run(self._table.count_rows(filter))
def __repr__(self) -> str:
val = f"{self.__class__.__name__}(name={self.name!r}"
val = f"{self.__class__.__name__}(name={self.name!r}, version={self.version}"
if self._conn.read_consistency_interval is not None:
val += ", read_consistency_interval={!r}".format(
self._conn.read_consistency_interval

View File

@@ -10,7 +10,7 @@ import pathlib
import warnings
from datetime import date, datetime
from functools import singledispatch
from typing import Tuple, Union, Optional, Any, List
from typing import Tuple, Union, Optional, Any
from urllib.parse import urlparse
import numpy as np
@@ -189,33 +189,7 @@ def flatten_columns(tbl: pa.Table, flatten: Optional[Union[int, bool]] = None):
return tbl
def _format_field_path(path: List[str]) -> str:
def format_segment(segment: str) -> str:
if all(char.isalnum() or char == "_" for char in segment):
return segment
return f"`{segment.replace('`', '``')}`"
return ".".join(format_segment(segment) for segment in path)
def _iter_vector_columns(
field: pa.Field, path: List[str], dim: Optional[int] = None
) -> List[str]:
field_path = [*path, field.name]
if is_vector_column(field.type):
vector_dim = infer_vector_column_dim(field.type)
if dim is None or vector_dim == dim:
return [_format_field_path(field_path)]
return []
if pa.types.is_struct(field.type):
columns = []
for idx in range(field.type.num_fields):
columns.extend(_iter_vector_columns(field.type.field(idx), field_path, dim))
return columns
return []
def inf_vector_column_query(schema: pa.Schema, dim: Optional[int] = None) -> str:
def inf_vector_column_query(schema: pa.Schema) -> str:
"""
Get the vector column name
@@ -228,21 +202,26 @@ def inf_vector_column_query(schema: pa.Schema, dim: Optional[int] = None) -> str
-------
str: the vector column name.
"""
vector_col_names = []
for field in schema:
vector_col_names.extend(_iter_vector_columns(field, [], dim))
if len(vector_col_names) > 1:
raise ValueError(
"Schema has more than one vector column. "
"Please specify the vector column name "
f"for vector search. Candidates: {vector_col_names}"
)
if len(vector_col_names) == 0:
vector_col_name = ""
vector_col_count = 0
for field_name in schema.names:
field = schema.field(field_name)
if is_vector_column(field.type):
vector_col_count += 1
if vector_col_count > 1:
raise ValueError(
"Schema has more than one vector column. "
"Please specify the vector column name "
"for vector search"
)
elif vector_col_count == 1:
vector_col_name = field_name
if vector_col_count == 0:
raise ValueError(
"There is no vector column in the data. "
"Please specify the vector column name for vector search"
)
return vector_col_names[0]
return vector_col_name
def is_vector_column(data_type: pa.DataType) -> bool:
@@ -268,29 +247,6 @@ def is_vector_column(data_type: pa.DataType) -> bool:
return False
def infer_vector_column_dim(data_type: pa.DataType) -> Optional[int]:
if pa.types.is_fixed_size_list(data_type):
return data_type.list_size
if pa.types.is_list(data_type):
return infer_vector_column_dim(data_type.value_type)
return None
def _query_vector_dim(query: Optional[Any]) -> Optional[int]:
if query is None:
return None
if isinstance(query, np.ndarray):
if query.ndim == 0:
return None
return query.shape[-1]
if isinstance(query, list) and query:
first = query[0]
if isinstance(first, (list, tuple, np.ndarray)):
return len(first)
return len(query)
return None
def infer_vector_column_name(
schema: pa.Schema,
query_type: str,
@@ -306,9 +262,7 @@ def infer_vector_column_name(
if query is not None or query_type == "hybrid":
try:
vector_column_name = inf_vector_column_query(
schema, dim=_query_vector_dim(query)
)
vector_column_name = inf_vector_column_query(schema)
except Exception as e:
raise e

View File

@@ -6,7 +6,6 @@ import re
import sys
from datetime import timedelta
import os
from types import SimpleNamespace
import lancedb
import numpy as np
@@ -189,43 +188,6 @@ def test_table_names(tmp_db: lancedb.DBConnection):
assert len(result) == 3
def test_db_contains_and_len_include_all_table_name_pages(tmp_db: lancedb.DBConnection):
for idx in range(20):
tmp_db.create_table(f"table_{idx}", data=[{"id": idx}])
assert len(tmp_db) == 20
for idx in range(20):
assert f"table_{idx}" in tmp_db
assert "does_not_exist" not in tmp_db
def test_db_contains_stops_after_matching_table_page(
tmp_db: lancedb.DBConnection, monkeypatch
):
calls = []
pages = {
None: SimpleNamespace(tables=["table_0", "table_1"], page_token="next"),
"next": SimpleNamespace(tables=["table_2"], page_token=None),
}
def list_tables(*, page_token=None, **_kwargs):
calls.append(page_token)
return pages[page_token]
monkeypatch.setattr(tmp_db, "list_tables", list_tables)
assert "table_1" in tmp_db
assert calls == [None]
calls.clear()
assert "table_2" in tmp_db
assert calls == [None, "next"]
calls.clear()
assert len(tmp_db) == 3
assert calls == [None, "next"]
@pytest.mark.asyncio
async def test_table_names_async(tmp_path):
db = lancedb.connect(tmp_path)

View File

@@ -563,7 +563,7 @@ def test_create_index_multiple_columns(tmp_path, table):
def test_nested_schema(tmp_path, table):
table.create_fts_index("nested.text", with_position=True)
table.create_fts_index("nested.text")
indices = table.list_indices()
assert len(indices) == 1
assert indices[0].index_type == "FTS"
@@ -577,98 +577,6 @@ def test_nested_schema(tmp_path, table):
assert len(results) > 0
assert all("puppy" in row["nested"]["text"] for row in results)
results = table.search(MatchQuery("puppy", "nested.text")).limit(5).to_list()
assert len(results) > 0
assert all("puppy" in row["nested"]["text"] for row in results)
phrase_results = (
table.search(PhraseQuery("puppy runs", "nested.text")).limit(5).to_list()
)
assert len(phrase_results) > 0
assert all("puppy runs" in row["nested"]["text"] for row in phrase_results)
hybrid_results = (
table.search(query_type="hybrid", fts_columns="nested.text")
.vector([0 for _ in range(128)])
.text("puppy")
.limit(5)
.to_list()
)
assert len(hybrid_results) > 0
@pytest.mark.asyncio
async def test_nested_schema_async(async_table):
await async_table.create_index("nested.text", config=FTS(with_position=True))
indices = await async_table.list_indices()
assert len(indices) == 1
assert indices[0].index_type == "FTS"
assert indices[0].columns == ["nested.text"]
results = await (
async_table.query()
.nearest_to_text("puppy", columns="nested.text")
.limit(5)
.to_list()
)
assert len(results) > 0
assert all("puppy" in row["nested"]["text"] for row in results)
results = await (
async_table.query()
.nearest_to_text(MatchQuery("puppy", "nested.text"))
.limit(5)
.to_list()
)
assert len(results) > 0
assert all("puppy" in row["nested"]["text"] for row in results)
phrase_results = await (
async_table.query()
.nearest_to_text(PhraseQuery("puppy runs", "nested.text"))
.limit(5)
.to_list()
)
assert len(phrase_results) > 0
assert all("puppy runs" in row["nested"]["text"] for row in phrase_results)
hybrid_results = await (
async_table.query()
.nearest_to([0 for _ in range(128)])
.nearest_to_text("puppy", columns="nested.text")
.limit(5)
.to_list()
)
assert len(hybrid_results) > 0
def test_nested_schema_rejects_invalid_fts_fields(tmp_path):
db = ldb.connect(tmp_path)
data = pa.table(
{
"payload": pa.array(
[
{"text": "puppy runs", "count": 1},
{"text": "car drives", "count": 2},
]
),
"vector": pa.array(
[[0.1, 0.1], [0.2, 0.2]],
type=pa.list_(pa.float32(), list_size=2),
),
}
)
table = db.create_table("test", data=data)
with pytest.raises(ValueError, match="FTS index cannot be created.*payload"):
table.create_fts_index("payload")
with pytest.raises(ValueError, match="FTS index cannot be created.*count"):
table.create_fts_index("payload.count")
with pytest.raises(ValueError, match="Field path `payload.missing` not found"):
table.create_fts_index("payload.missing")
def test_search_index_with_filter(table):
table.create_fts_index("text")

View File

@@ -105,46 +105,6 @@ async def test_create_scalar_index(some_table: AsyncTable):
assert len(indices) == 0
@pytest.mark.asyncio
async def test_create_nested_scalar_index_lists_canonical_paths(db_async):
metadata_type = pa.struct(
[
pa.field("user_id", pa.int32()),
pa.field("user.id", pa.int32()),
]
)
data = pa.Table.from_arrays(
[
pa.array([1, 2, 3], type=pa.int32()),
pa.array(
[
{"user_id": 10, "user.id": 100},
{"user_id": 20, "user.id": 200},
{"user_id": 30, "user.id": 300},
],
type=metadata_type,
),
],
names=["user_id", "metadata"],
)
table = await db_async.create_table("nested_scalar_index", data)
await table.create_index("user_id", config=BTree(), name="top_user_id_idx")
await table.create_index(
"metadata.user_id", config=BTree(), name="nested_user_id_idx"
)
await table.create_index(
"metadata.`user.id`", config=BTree(), name="escaped_user_id_idx"
)
columns_by_name = {
index.name: index.columns for index in await table.list_indices()
}
assert columns_by_name["top_user_id_idx"] == ["user_id"]
assert columns_by_name["nested_user_id_idx"] == ["metadata.user_id"]
assert columns_by_name["escaped_user_id_idx"] == ["metadata.`user.id`"]
@pytest.mark.asyncio
async def test_create_fixed_size_binary_index(some_table: AsyncTable):
await some_table.create_index("fsb", config=BTree())

View File

@@ -1512,37 +1512,6 @@ def test_take_queries(tmp_path):
]
def test_take_queries_to_batches(tmp_path):
# Regression test for the sync take-query path: `to_batches` previously
# raised ``AttributeError: 'AsyncTakeQuery' object has no attribute
# 'execute'`` because the inherited ``BaseQueryBuilder.to_batches`` called
# ``execute`` on the async wrapper instead of the native query.
db = lancedb.connect(tmp_path)
data = pa.table({"idx": list(range(100)), "label": [str(i) for i in range(100)]})
table = db.create_table("test", data)
# Take by offset → to_batches
rs = list(table.take_offsets([5, 2, 17]).to_batches())
assert all(isinstance(b, pa.RecordBatch) for b in rs)
assert sum(b.num_rows for b in rs) == 3
assert sorted(v for b in rs for v in b.column("idx").to_pylist()) == [2, 5, 17]
# Take by row id → to_batches
rs = list(table.take_row_ids([5, 2, 17]).to_batches())
assert all(isinstance(b, pa.RecordBatch) for b in rs)
assert sum(b.num_rows for b in rs) == 3
assert sorted(v for b in rs for v in b.column("idx").to_pylist()) == [2, 5, 17]
# Take with select projection → to_batches preserves the projection
rs = list(table.take_row_ids([5, 2, 17]).select(["label"]).to_batches())
assert all(b.schema.names == ["label"] for b in rs)
assert sorted(v for b in rs for v in b.column("label").to_pylist()) == [
"17",
"2",
"5",
]
def test_getitems(tmp_path):
db = lancedb.connect(tmp_path)
data = pa.table(

View File

@@ -362,22 +362,6 @@ def test_table_create_indices():
schema=dict(
fields=[
dict(name="id", type={"type": "int64"}, nullable=False),
dict(name="text", type={"type": "string"}, nullable=False),
dict(
name="vector",
type={
"type": "fixed_size_list",
"fields": [
dict(
name="item",
type={"type": "float"},
nullable=True,
)
],
"length": 2,
},
nullable=False,
),
]
),
)

View File

@@ -33,7 +33,7 @@ def test_basic(mem_db: DBConnection):
table = mem_db.create_table("test", data=data)
assert table.name == "test"
assert "LanceTable(name='test', _conn=LanceDBConnection(" in repr(table)
assert "LanceTable(name='test', version=1, _conn=LanceDBConnection(" in repr(table)
expected_schema = pa.schema(
{
"vector": pa.list_(pa.float32(), 2),
@@ -1934,10 +1934,6 @@ def test_create_index_nested_field_paths(mem_db: DBConnection):
assert len(vector_results) == 1
assert vector_results[0]["metadata"]["user_id"] == 0
default_vector_results = table.search([0.0, 1.0]).limit(1).to_list()
assert len(default_vector_results) == 1
assert default_vector_results[0]["metadata"]["user_id"] == 0
filtered_results = table.search().where("metadata.user_id = 42").limit(1).to_list()
assert len(filtered_results) == 1
assert filtered_results[0]["metadata"]["user_id"] == 42
@@ -2017,74 +2013,6 @@ def test_search_with_schema_inf_multiple_vector(mem_db: DBConnection):
table.search(q).limit(1).to_arrow()
def test_search_infers_single_nested_vector(mem_db: DBConnection):
schema = pa.schema(
[
pa.field("id", pa.int32()),
pa.field(
"image",
pa.struct([pa.field("embedding", pa.list_(pa.float32(), 2))]),
),
]
)
data = pa.Table.from_pylist(
[
{"id": 0, "image": {"embedding": [0.0, 1.0]}},
{"id": 1, "image": {"embedding": [10.0, 11.0]}},
],
schema=schema,
)
table = mem_db.create_table("nested_vector_default_search", data=data)
result = table.search([0.0, 1.0]).limit(1).to_list()
assert result[0]["id"] == 0
def test_search_nested_vector_multiple_candidates(mem_db: DBConnection):
schema = pa.schema(
[
pa.field(
"image",
pa.struct([pa.field("embedding", pa.list_(pa.float32(), 2))]),
),
pa.field(
"text",
pa.struct([pa.field("embedding", pa.list_(pa.float32(), 2))]),
),
]
)
data = pa.Table.from_pylist(
[
{
"image": {"embedding": [0.0, 1.0]},
"text": {"embedding": [2.0, 3.0]},
}
],
schema=schema,
)
table = mem_db.create_table("nested_vector_multiple_candidates", data=data)
with pytest.raises(ValueError, match="image.embedding.*text.embedding"):
table.search([0.0, 1.0]).limit(1).to_arrow()
def test_search_nested_vector_no_candidates(mem_db: DBConnection):
schema = pa.schema(
[
pa.field("id", pa.int32()),
pa.field("metadata", pa.struct([pa.field("label", pa.string())])),
]
)
data = pa.Table.from_pylist(
[{"id": 0, "metadata": {"label": "cat"}}],
schema=schema,
)
table = mem_db.create_table("nested_vector_no_candidates", data=data)
with pytest.raises(ValueError, match="no vector column"):
table.search([0.0, 1.0]).limit(1).to_arrow()
def test_compact_cleanup(tmp_db: DBConnection):
pytest.importorskip("lance")
table = tmp_db.create_table(

View File

@@ -1,6 +1,6 @@
[package]
name = "lancedb"
version = "0.30.0-beta.1"
version = "0.29.1-beta.0"
edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true
@@ -104,7 +104,6 @@ datafusion.workspace = true
http-body = "1" # Matching reqwest
rstest = "0.23.0"
test-log = "0.2"
serial_test = "3"
[features]

View File

@@ -812,7 +812,8 @@ impl ConnectBuilder {
self
}
/// The interval at which to check for updates from other processes.
/// The interval at which to check for updates from other processes. This
/// only affects LanceDB OSS.
///
/// If left unset, consistency is not checked. For maximum read
/// performance, this is the default. For strong consistency, set this to
@@ -824,11 +825,8 @@ impl ConnectBuilder {
/// This only affects read operations. Write operations are always
/// consistent.
///
/// # Cost
///
/// Stronger consistency is not free. The smaller the interval, the more
/// often each read pays the cost of checking for updates against object
/// storage, raising per-read latency and cost.
/// LanceDB Cloud uses eventual consistency under the hood, and is not
/// currently configurable.
pub fn read_consistency_interval(
mut self,
read_consistency_interval: std::time::Duration,
@@ -888,7 +886,6 @@ impl ConnectBuilder {
options.host_override,
self.request.client_config,
storage_options.into(),
self.request.read_consistency_interval,
)?);
Ok(Connection {
internal,

View File

@@ -23,12 +23,17 @@ impl VectorIndex {
.fields
.iter()
.map(|field_id| {
manifest.schema.field_path(*field_id).unwrap_or_else(|_| {
panic!(
"field {field_id} of index {} must exist in schema",
index.name
)
})
manifest
.schema
.field_by_id(*field_id)
.unwrap_or_else(|| {
panic!(
"field {field_id} of index {} must exist in schema",
index.name
)
})
.name
.clone()
})
.collect();
Self {

View File

@@ -245,9 +245,6 @@ pub struct RestfulLanceDbClient<S: HttpSend = Sender> {
pub(crate) sender: S,
pub(crate) id_delimiter: String,
pub(crate) header_provider: Option<Arc<dyn HeaderProvider>>,
/// Connection-level read consistency interval. Drives the
/// `x-lancedb-min-timestamp` freshness header sent on read requests.
pub(crate) read_consistency_interval: Option<Duration>,
}
impl<S: HttpSend> std::fmt::Debug for RestfulLanceDbClient<S> {
@@ -341,7 +338,6 @@ impl RestfulLanceDbClient<Sender> {
host_override: Option<String>,
default_headers: HeaderMap,
client_config: ClientConfig,
read_consistency_interval: Option<Duration>,
) -> Result<Self> {
// Get the timeouts
let timeout =
@@ -439,7 +435,6 @@ impl RestfulLanceDbClient<Sender> {
.clone()
.unwrap_or("$".to_string()),
header_provider: client_config.header_provider,
read_consistency_interval,
})
}
}
@@ -845,16 +840,6 @@ pub mod test_utils {
pub fn client_with_handler<T>(
handler: impl Fn(reqwest::Request) -> http::response::Response<T> + Send + Sync + 'static,
) -> RestfulLanceDbClient<MockSender>
where
T: Into<reqwest::Body>,
{
client_with_handler_and_interval(handler, None)
}
pub fn client_with_handler_and_interval<T>(
handler: impl Fn(reqwest::Request) -> http::response::Response<T> + Send + Sync + 'static,
read_consistency_interval: Option<Duration>,
) -> RestfulLanceDbClient<MockSender>
where
T: Into<reqwest::Body>,
{
@@ -872,7 +857,6 @@ pub mod test_utils {
},
id_delimiter: "$".to_string(),
header_provider: None,
read_consistency_interval,
}
}
@@ -897,7 +881,6 @@ pub mod test_utils {
},
id_delimiter: config.id_delimiter.unwrap_or_else(|| "$".to_string()),
header_provider: config.header_provider,
read_consistency_interval: None,
}
}
}
@@ -905,7 +888,6 @@ pub mod test_utils {
#[cfg(test)]
mod tests {
use super::*;
use serial_test::serial;
use std::time::Duration;
#[test]
@@ -1064,7 +1046,6 @@ mod tests {
sender: Sender,
id_delimiter: "+".to_string(),
header_provider: Some(Arc::new(provider) as Arc<dyn HeaderProvider>),
read_consistency_interval: None,
};
// Apply dynamic headers
@@ -1100,7 +1081,6 @@ mod tests {
sender: Sender,
id_delimiter: "+".to_string(),
header_provider: Some(Arc::new(provider) as Arc<dyn HeaderProvider>),
read_consistency_interval: None,
};
// Apply dynamic headers
@@ -1138,7 +1118,6 @@ mod tests {
sender: Sender,
id_delimiter: "+".to_string(),
header_provider: Some(Arc::new(provider) as Arc<dyn HeaderProvider>),
read_consistency_interval: None,
};
// Header provider errors should fail the request
@@ -1164,7 +1143,6 @@ mod tests {
}
#[test]
#[serial(user_id_env)]
fn test_resolve_user_id_none() {
let config = ClientConfig::default();
// Clear env vars that might be set from other tests
@@ -1177,7 +1155,6 @@ mod tests {
}
#[test]
#[serial(user_id_env)]
fn test_resolve_user_id_from_env() {
// SAFETY: This is only called in tests
unsafe {
@@ -1192,7 +1169,6 @@ mod tests {
}
#[test]
#[serial(user_id_env)]
fn test_resolve_user_id_from_env_key() {
// SAFETY: This is only called in tests
unsafe {
@@ -1213,7 +1189,6 @@ mod tests {
}
#[test]
#[serial(user_id_env)]
fn test_resolve_user_id_direct_takes_precedence() {
// SAFETY: This is only called in tests
unsafe {
@@ -1231,7 +1206,6 @@ mod tests {
}
#[test]
#[serial(user_id_env)]
fn test_resolve_user_id_empty_env_ignored() {
// SAFETY: This is only called in tests
unsafe {

View File

@@ -206,7 +206,6 @@ impl RemoteDatabase {
host_override: Option<String>,
client_config: ClientConfig,
options: RemoteOptions,
read_consistency_interval: Option<std::time::Duration>,
) -> Result<Self> {
let parsed = super::client::parse_db_url(uri)?;
let header_map = RestfulLanceDbClient::<Sender>::default_headers(
@@ -234,7 +233,6 @@ impl RemoteDatabase {
host_override,
header_map,
client_config.clone(),
read_consistency_interval,
)?;
let table_cache = Cache::builder()

File diff suppressed because it is too large Load Diff

View File

@@ -656,30 +656,6 @@ mod test_utils {
}
}
pub fn new_with_handler_and_interval<T>(
name: impl Into<String>,
handler: impl Fn(reqwest::Request) -> http::Response<T> + Clone + Send + Sync + 'static,
read_consistency_interval: Option<std::time::Duration>,
) -> Self
where
T: Into<reqwest::Body>,
{
let inner = Arc::new(
crate::remote::table::RemoteTable::new_mock_with_consistency_interval(
name.into(),
handler.clone(),
read_consistency_interval,
),
);
let database = Arc::new(crate::remote::db::RemoteDatabase::new_mock(handler));
Self {
inner,
database: Some(database),
// Registry is unused.
embedding_registry: Arc::new(MemoryRegistry::new()),
}
}
pub fn new_with_handler_version<T>(
name: impl Into<String>,
version: semver::Version,
@@ -2712,13 +2688,16 @@ impl BaseTable for NativeTable {
message: "Multi-column (composite) indices are not yet supported".to_string(),
});
}
self.dataset.ensure_mutable()?;
let mut dataset = (*self.dataset.get().await?).clone();
let dataset = self.dataset.get().await?;
let (column, field) = Self::resolve_index_field(dataset.schema(), &opts.columns[0])?;
drop(dataset);
let lance_idx_params = self.make_index_params(&field, opts.index.clone()).await?;
let index_type = self.get_index_type_for_field(&field, &opts.index);
let columns = [column.as_str()];
self.dataset.ensure_mutable()?;
let mut dataset = (*self.dataset.get().await?).clone();
let mut builder = dataset
.create_index_builder(&columns, index_type, lance_idx_params.as_ref())
.train(opts.train)
@@ -2836,88 +2815,63 @@ impl BaseTable for NativeTable {
async fn list_indices(&self) -> Result<Vec<IndexConfig>> {
let dataset = self.dataset.get().await?;
let indices = dataset.load_indices().await?;
let results = futures::stream::iter(indices.as_slice())
.then(|idx| async {
// skip Lance internal indexes
if idx.name == FRAG_REUSE_INDEX_NAME {
let results = futures::stream::iter(indices.as_slice()).then(|idx| async {
// skip Lance internal indexes
if idx.name == FRAG_REUSE_INDEX_NAME {
return None;
}
let stats = match dataset.index_statistics(idx.name.as_str()).await {
Ok(stats) => stats,
Err(e) => {
log::warn!("Failed to get statistics for index {} ({}): {}", idx.name, idx.uuid, e);
return None;
}
};
let stats = match dataset.index_statistics(idx.name.as_str()).await {
Ok(stats) => stats,
Err(e) => {
log::warn!(
"Failed to get statistics for index {} ({}): {}",
idx.name,
idx.uuid,
e
);
return None;
}
};
let stats: serde_json::Value = match serde_json::from_str(&stats) {
Ok(stats) => stats,
Err(e) => {
log::warn!(
"Failed to deserialize index statistics for index {} ({}): {}",
idx.name,
idx.uuid,
e
);
return None;
}
};
let Some(index_type) = stats.get("index_type").and_then(|v| v.as_str()) else {
log::warn!(
"Index statistics was missing 'index_type' field for index {} ({})",
idx.name,
idx.uuid
);
let stats: serde_json::Value = match serde_json::from_str(&stats) {
Ok(stats) => stats,
Err(e) => {
log::warn!("Failed to deserialize index statistics for index {} ({}): {}", idx.name, idx.uuid, e);
return None;
};
}
};
let index_type: crate::index::IndexType = match index_type.parse() {
Ok(index_type) => index_type,
let Some(index_type) = stats.get("index_type").and_then(|v| v.as_str()) else {
log::warn!("Index statistics was missing 'index_type' field for index {} ({})", idx.name, idx.uuid);
return None;
};
let index_type: crate::index::IndexType = match index_type.parse() {
Ok(index_type) => index_type,
Err(e) => {
log::warn!("Failed to parse index type for index {} ({}): {}", idx.name, idx.uuid, e);
return None;
}
};
let mut columns = Vec::with_capacity(idx.fields.len());
for field_id in &idx.fields {
let column = match dataset.schema().field_path(*field_id) {
Ok(column) => column,
Err(e) => {
log::warn!(
"Failed to parse index type for index {} ({}): {}",
"The index {} ({}) referenced a field with id {} which does not exist in the schema: {}",
idx.name,
idx.uuid,
field_id,
e
);
return None;
}
};
columns.push(column);
}
let mut columns = Vec::with_capacity(idx.fields.len());
for field_id in &idx.fields {
let field_path = match dataset.schema().field_path(*field_id) {
Ok(field_path) => field_path,
Err(e) => {
log::warn!(
"Failed to resolve field path for index {} ({}) field id {}: {}",
idx.name,
idx.uuid,
field_id,
e
);
return None;
}
};
columns.push(field_path);
}
let name = idx.name.clone();
Some(IndexConfig {
index_type,
columns,
name,
})
})
.collect::<Vec<_>>()
.await;
let name = idx.name.clone();
Some(IndexConfig { index_type, columns, name })
}).collect::<Vec<_>>().await;
Ok(results.into_iter().flatten().collect())
}
@@ -3120,7 +3074,6 @@ pub struct FragmentSummaryStats {
#[cfg(test)]
#[allow(deprecated)]
mod tests {
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
@@ -3901,25 +3854,6 @@ mod tests {
1
);
let default_vector_results = table
.query()
.nearest_to(&[0.0; 8])
.unwrap()
.limit(1)
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
assert_eq!(
default_vector_results
.iter()
.map(|batch| batch.num_rows())
.sum::<usize>(),
1
);
let fts_results = table
.query()
.full_text_search(FullTextSearchQuery::new("document".to_string()))

View File

@@ -6,7 +6,7 @@ pub(crate) mod background_cache;
use std::sync::Arc;
use arrow_array::RecordBatch;
use arrow_schema::{DataType, Field, Schema, SchemaRef};
use arrow_schema::{DataType, Schema, SchemaRef};
use datafusion_common::{DataFusionError, Result as DataFusionResult};
use datafusion_execution::RecordBatchStream;
use futures::{FutureExt, Stream};
@@ -152,10 +152,14 @@ pub fn validate_namespace(namespace: &[String]) -> Result<()> {
/// Find one default column to create index or perform vector query.
pub(crate) fn default_vector_column(schema: &Schema, dim: Option<i32>) -> Result<String> {
// Try to find a vector column.
let mut candidates = Vec::new();
for field in schema.fields() {
collect_vector_columns(field, &mut Vec::new(), dim, &mut candidates);
}
let candidates = schema
.fields()
.iter()
.filter_map(|field| match infer_vector_dim(field.data_type()) {
Ok(d) if dim.is_none() || dim == Some(d as i32) => Some(field.name()),
_ => None,
})
.collect::<Vec<_>>();
if candidates.is_empty() {
Err(Error::InvalidInput {
message: format!(
@@ -176,57 +180,6 @@ pub(crate) fn default_vector_column(schema: &Schema, dim: Option<i32>) -> Result
}
}
fn collect_vector_columns(
field: &Field,
path: &mut Vec<String>,
dim: Option<i32>,
candidates: &mut Vec<String>,
) {
path.push(field.name().clone());
match infer_vector_dim(field.data_type()) {
Ok(d) if dim.is_none() || dim == Some(d as i32) => {
let path_segments = path.iter().map(String::as_str).collect::<Vec<_>>();
candidates.push(lance_core::datatypes::format_field_path(&path_segments));
}
_ => {
if let DataType::Struct(fields) = field.data_type() {
for child in fields {
collect_vector_columns(child, path, dim, candidates);
}
}
}
}
path.pop();
}
pub(crate) fn resolve_arrow_field_path(schema: &Schema, column: &str) -> Result<(String, Field)> {
lance_core::datatypes::parse_field_path(column).map_err(|e| Error::InvalidInput {
message: format!("Invalid field path `{}`: {}", column, e),
})?;
let lance_schema =
lance_core::datatypes::Schema::try_from(schema).map_err(|e| Error::Schema {
message: format!("Invalid schema: {}", e),
})?;
let field_path = lance_schema
.resolve_case_insensitive(column)
.ok_or_else(|| Error::Schema {
message: format!(
"Field path `{}` not found in schema. Available field paths: {}",
column,
lance_schema.field_paths().join(", ")
),
})?;
let field = field_path.last().expect("field path should be non-empty");
let path_segments = field_path
.iter()
.map(|field| field.name.as_str())
.collect::<Vec<_>>();
let canonical_path = lance_core::datatypes::format_field_path(&path_segments);
Ok((canonical_path, Field::from(*field)))
}
pub fn supported_btree_data_type(dtype: &DataType) -> bool {
dtype.is_integer()
|| dtype.is_floating()
@@ -497,49 +450,6 @@ mod tests {
"vec"
);
let schema_with_nested_vec_col = Schema::new(vec![
Field::new("id", DataType::Int16, true),
Field::new(
"image",
DataType::Struct(
vec![Field::new(
"embedding",
DataType::FixedSizeList(
Arc::new(Field::new("item", DataType::Float32, false)),
10,
),
false,
)]
.into(),
),
false,
),
]);
assert_eq!(
default_vector_column(&schema_with_nested_vec_col, None).unwrap(),
"image.embedding"
);
let schema_with_escaped_nested_vec_col = Schema::new(vec![Field::new(
"image-meta",
DataType::Struct(
vec![Field::new(
"embedding.v1",
DataType::FixedSizeList(
Arc::new(Field::new("item", DataType::Float32, false)),
10,
),
false,
)]
.into(),
),
false,
)]);
assert_eq!(
default_vector_column(&schema_with_escaped_nested_vec_col, None).unwrap(),
"`image-meta`.`embedding.v1`"
);
let multi_vec_col = Schema::new(vec![
Field::new("id", DataType::Int16, true),
Field::new(
@@ -559,48 +469,6 @@ mod tests {
.to_string()
.contains("More than one")
);
let multi_nested_vec_col = Schema::new(vec![
Field::new(
"image",
DataType::Struct(
vec![Field::new(
"embedding",
DataType::FixedSizeList(
Arc::new(Field::new("item", DataType::Float32, false)),
10,
),
false,
)]
.into(),
),
false,
),
Field::new(
"text",
DataType::Struct(
vec![Field::new(
"embedding",
DataType::FixedSizeList(
Arc::new(Field::new("item", DataType::Float32, false)),
50,
),
false,
)]
.into(),
),
false,
),
]);
assert_eq!(
default_vector_column(&multi_nested_vec_col, Some(50)).unwrap(),
"text.embedding"
);
let err = default_vector_column(&multi_nested_vec_col, None)
.unwrap_err()
.to_string();
assert!(err.contains("image.embedding"));
assert!(err.contains("text.embedding"));
}
#[test]