rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-12 23:02:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge main into gatekeeper/fix-4273-1

Browse Source 
Retain the upstream nested-blob missing-field checks and regression tests while preserving non-scan blob pandas conversion and remote snapshot safeguards.
This commit is contained in:
Gatefixer
2026-10-12 20:40:44 +00:00
parent febb9b8731 792ffd1082
commit 10e876db6a
64 changed files with 2857 additions and 892 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.41.0-beta.1"
version = "0.41.0-beta.6"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
python/pyproject.toml
+3 -2
View File
@@ -10,7 +10,7 @@ dependencies = [
"pyarrow>=16",
"pydantic>=2.7.4,<3",
"tqdm>=4.27.0",
"lance-namespace>=0.3.2"
"lance-namespace>=0.13.0"
]
description = "lancedb"
authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }]
@@ -63,7 +63,7 @@ tests = [
"polars>=0.19, <=1.32.3",
"pyarrow<25",
"pyarrow-stubs>=16.0",
"pylance==14.0.0b6",
"pylance==14.0.0b12",
"requests>=2.31.0",
"datafusion>=54,<55",
"opentelemetry-sdk>=1.30.0",
@@ -73,6 +73,7 @@ dev = [
"pre-commit>=3.5.0",
"pyright>=1.1.350",
'typing-extensions>=4.0.0; python_version < "3.11"',
"maturin>=1.10",
]
docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings-python"]
clip = ["torch", "pillow>=12.1.1", "open-clip-torch"]
python/python/lancedb/_blob.py
+1
View File
@@ -328,6 +328,7 @@ def _set_blob_column(
leaf_name, blobs.type, nullable=field.nullable
)
return
raise KeyError(f"Blob field '{leaf_name}' not found in struct '{parent_path}'")
return _transform_struct_column(tbl, parent_path, replace_blob)
python/python/lancedb/_lancedb.pyi
+4
View File
@@ -727,6 +727,7 @@ class VectorQuery:
def use_lsm(self, enable: bool): ...
def refine_factor(self, refine_factor: int): ...
def nprobes(self, nprobes: int): ...
def search_effort(self, search_effort: Optional[float]): ...
def minimum_nprobes(self, minimum_nprobes: int): ...
def maximum_nprobes(self, maximum_nprobes: int): ...
def bypass_vector_index(self): ...
@@ -752,6 +753,7 @@ class HybridQuery:
def distance_type(self, distance_type: str): ...
def refine_factor(self, refine_factor: int): ...
def nprobes(self, nprobes: int): ...
def search_effort(self, search_effort: Optional[float]): ...
def minimum_nprobes(self, minimum_nprobes: int): ...
def maximum_nprobes(self, maximum_nprobes: int): ...
def bypass_vector_index(self): ...
@@ -778,8 +780,10 @@ class PyQueryRequest:
use_lsm: Optional[bool]
column: Optional[str]
query_vector: Optional[List[pa.Array]]
nprobes: Optional[int]
minimum_nprobes: Optional[int]
maximum_nprobes: Optional[int]
search_effort: Optional[float]
lower_bound: Optional[float]
upper_bound: Optional[float]
ef: Optional[int]
python/python/lancedb/functions.py
+1 -1
View File
@@ -372,7 +372,7 @@ class FunctionVersion(_RemoteValue):
image: FunctionImage
signature: FunctionSignature
secret_bindings: tuple[SecretBinding, ...] = ()
created_at: str
created_at_millis: int
metadata: Mapping[str, str]
disabled: bool
python/python/lancedb/index.py
+35 -2
View File
@@ -112,9 +112,42 @@ class Fm:
class FTS:
"""Describe a FTS index configuration.
`FTS` is a full-text search index that can be used on `String` columns
`FTS` supports string columns, lists of strings, and JSON columns created
from PyArrow ``pa.json_()``. JSON columns use Lance's JSON-aware tokenizer
and are stored as ``LargeBinary`` with the ``lance.json`` logical type.
Arbitrary binary columns are not supported.
For example, it works with `title`, `description`, `content`, etc.
JSON queries use ``path,str,value`` triples. Paths are relative to the JSON
column and use dots for nested object keys. For example,
``meta.tag,str,nature`` matches the string value at ``$.meta.tag``;
it does not search other paths or JSON key names. The value is processed
with the configured text tokenizer. Use ``str``, not ``string``.
Create and query a JSON index through the modern API::
import pyarrow as pa
from lancedb.index import FTS
table = db.create_table(
"documents",
pa.table({"doc": pa.array(
['{"meta":{"tag":"nature"}}'], type=pa.json_()
)}),
)
table.create_index("doc", config=FTS())
table.search(
"meta.tag,str,nature", query_type="fts", fts_columns="doc"
).to_list()
Create the index on the JSON column itself, not a JSON subpath such as
``doc.meta.tag``. JSON-path scalar indexes do not provide FTS path scoping.
Plain text such as ``nature`` is not a valid JSON FTS query. The current
Lance tokenizer can panic on malformed triples; see
https://github.com/lance-format/lance/issues/8812.
Remote tables use the same ``FTS()`` configuration and require a server
that supports JSON FTS. Wait for ``table.wait_for_index(["doc_idx"])``
before searching a newly created remote index with its default name.
Examples
--------
python/python/lancedb/namespace.py
+17 -1
View File
@@ -105,6 +105,11 @@ def _query_to_namespace_request(
if not isinstance(query, Query):
raise TypeError(f"Expected Query, got {type(query)}")
if query.search_effort is not None:
raise ValueError(
"namespace QueryTable does not support search_effort; "
"execute the query locally"
)
vector_request = None
if query.vector is not None:
@@ -190,8 +195,12 @@ def _query_to_namespace_request(
kwargs["with_row_id"] = query.with_row_id
if query.ef is not None:
kwargs["ef"] = query.ef
if query.nprobes is not None:
kwargs["nprobes"] = query.nprobes
if query.minimum_nprobes is not None:
kwargs["nprobes"] = query.minimum_nprobes
kwargs["minimum_nprobes"] = query.minimum_nprobes
if query.maximum_nprobes is not None:
kwargs["maximum_nprobes"] = query.maximum_nprobes
if query.refine_factor is not None:
kwargs["refine_factor"] = query.refine_factor
if query.lower_bound is not None:
@@ -205,6 +214,13 @@ def _query_to_namespace_request(
if query.distance_type is not None:
kwargs["distance_type"] = query.distance_type
if kwargs.get("maximum_nprobes") == 0:
# The generated 0.13 model rejects LanceDB's unbounded sentinel;
# model_copy preserves direct wire pass-through to the Lance receiver.
maximum_nprobes = kwargs.pop("maximum_nprobes")
return QueryTableRequest(**kwargs).model_copy(
update={"maximum_nprobes": maximum_nprobes}
)
return QueryTableRequest(**kwargs)
python/python/lancedb/query.py
+118 -22
View File
@@ -551,8 +551,8 @@ class MatchQuery(FullTextQuery):
----------
query : str
The query string to match against.
column : str
The name of the column to match against.
column : str, optional
The column to match against. None leaves column selection to query planning.
boost : float, default 1.0
The boost factor for the query.
The score of each matching document is multiplied by this value.
@@ -581,7 +581,7 @@ class MatchQuery(FullTextQuery):
"""
query: str
column: str
column: Optional[str] = None
boost: float = pydantic.Field(1.0, kw_only=True)
fuzziness: Optional[int] = pydantic.Field(0, kw_only=True)
max_expansions: int = pydantic.Field(50, kw_only=True)
@@ -769,8 +769,9 @@ class Query(pydantic.BaseModel):
if True then apply the filter after vector / FTS search. This is ignored for
plain SQL filtering.
nprobes : Optional[int]
The number of IVF partitions to search. If this is None then a default
number of partitions will be used.
The legacy number of IVF partitions to search. Lance sets both probe
bounds to this value. If this is None then Lance's default probe settings
will be used.
- A higher number makes search more accurate but also slower.
@@ -849,22 +850,28 @@ class Query(pydantic.BaseModel):
# distance type to use for vector search
distance_type: Optional[str] = None
# legacy number of IVF partitions to search
#
# Lance sets both probe bounds to this value. Explicit bounds can override it.
nprobes: Optional[int] = None
# which columns to return in the results (dict values may be str or Expr)
columns: QueryProjection = None
# minimum number of IVF partitions to search
#
# If None then a default value (20) will be used.
# If None then Lance's default will be used.
minimum_nprobes: Optional[int] = None
# maximum number of IVF partitions to search
#
# If None then a default value (20) will be used.
#
# If 0 then no limit will be applied and all partitions could be searched
# if needed to satisfy the limit.
# If None then Lance's default will be used. If 0 then no limit will be applied
# and all partitions could be searched if needed to satisfy the limit.
maximum_nprobes: Optional[int] = None
# Optional initial IVF effort; None preserves Auto and explicit nprobes.
search_effort: Optional[float] = None
# lower bound for distance search
lower_bound: Optional[float] = None
@@ -923,8 +930,10 @@ class Query(pydantic.BaseModel):
query.vector_column = req.column
query.vector = req.query_vector
query.distance_type = req.distance_type
query.nprobes = req.nprobes
query.minimum_nprobes = req.minimum_nprobes
query.maximum_nprobes = req.maximum_nprobes
query.search_effort = req.search_effort
query.lower_bound = req.lower_bound
query.upper_bound = req.upper_bound
query.ef = req.ef
@@ -1715,8 +1724,10 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
super().__init__(table)
self._query = query
self._distance_type = None
self._nprobes = None
self._minimum_nprobes = None
self._maximum_nprobes = None
self._search_effort = None
self._lower_bound = None
self._upper_bound = None
self._refine_factor = None
@@ -1779,9 +1790,9 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
See discussion in [Querying an ANN Index](https://lancedb.com/docs/indexing/)
for tuning advice.
This method sets both the minimum and maximum number of probes to the same
value. See `minimum_nprobes` and `maximum_nprobes` for more fine-grained
control.
The value is retained as `nprobes` through client and server request
construction. Lance sets both probe bounds to this value. Explicit minimum
or maximum settings can override their respective bounds.
Parameters
----------
@@ -1795,8 +1806,28 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
"""
if nprobes <= 0:
raise ValueError("Invalid input, nprobes must be greater than 0")
self._minimum_nprobes = nprobes
self._maximum_nprobes = nprobes
self._nprobes = nprobes
return self
def search_effort(self, search_effort: Optional[float]) -> LanceVectorQueryBuilder:
"""Set the initial IVF search effort to a finite value in [0, 1].
None (the default) uses Auto, internally 0.5. Zero starts at the caller
minimum; one searches all partitions within the caller maximum. Later
probing may continue when too few candidates are found. Any explicit
effort, including 0.5, overrides nprobes in either call order. Explicit
minimum and maximum bounds still apply. This is not a recall guarantee
and does not change refinement or HNSW ef. MemWAL LSM queries do not
yet support this option.
Examples
--------
>>> import lancedb
>>> db = lancedb.connect("memory://")
>>> table = db.create_table("effort_vectors", [{"vector": [0.1, 0.2]}])
>>> query = table.search([0.1, 0.2]).search_effort(0.75)
"""
self._search_effort = search_effort
return self
def minimum_nprobes(self, minimum_nprobes: int) -> LanceVectorQueryBuilder:
@@ -1938,8 +1969,10 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
limit=self._limit,
distance_type=self._distance_type,
columns=self._columns,
nprobes=self._nprobes,
minimum_nprobes=self._minimum_nprobes,
maximum_nprobes=self._maximum_nprobes,
search_effort=self._search_effort,
lower_bound=self._lower_bound,
upper_bound=self._upper_bound,
refine_factor=self._refine_factor,
@@ -2296,8 +2329,10 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
self._fts_columns = fts_columns
self._norm = None
self._reranker = None
self._nprobes = None
self._minimum_nprobes = None
self._maximum_nprobes = None
self._search_effort = None
self._refine_factor = None
self._distance_type = None
self._phrase_query = None
@@ -2587,6 +2622,9 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
Higher values will yield better recall (more likely to find vectors if
they exist) at the expense of latency.
The value is retained as `nprobes` until Lance sets both probe bounds to it.
Explicit minimum or maximum settings can override their respective bounds.
Parameters
----------
nprobes: int
@@ -2599,8 +2637,33 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
"""
if nprobes <= 0:
raise ValueError("Invalid input, nprobes must be greater than 0")
self._minimum_nprobes = nprobes
self._maximum_nprobes = nprobes
self._nprobes = nprobes
return self
def search_effort(self, search_effort: Optional[float]) -> LanceHybridQueryBuilder:
"""Set the initial IVF search effort to a finite value in [0, 1].
None (the default) uses Auto, internally 0.5. Zero starts at the caller
minimum; one searches all partitions within the caller maximum. Later
probing may continue when too few candidates are found. Any explicit
effort, including 0.5, overrides nprobes in either call order. Explicit
minimum and maximum bounds still apply. This is not a recall guarantee
and does not change refinement or HNSW ef. MemWAL LSM queries do not
yet support this option.
Examples
--------
>>> import lancedb
>>> db = lancedb.connect("memory://")
>>> table = db.create_table(
... "effort_hybrid", [{"vector": [0.1, 0.2], "text": "dog"}]
... )
>>> query = (
... table.search(query_type="hybrid").vector([0.1, 0.2])
... .text("dog").search_effort(0.75)
... )
"""
self._search_effort = search_effort
return self
def minimum_nprobes(self, minimum_nprobes: int) -> LanceHybridQueryBuilder:
@@ -2879,6 +2942,10 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
self._fts_query.phrase_query(True)
if self._distance_type:
self._vector_query.metric(self._distance_type)
if self._nprobes is not None:
self._vector_query.nprobes(self._nprobes)
if self._search_effort is not None:
self._vector_query.search_effort(self._search_effort)
if self._minimum_nprobes is not None:
self._vector_query.minimum_nprobes(self._minimum_nprobes)
if self._maximum_nprobes is not None:
@@ -3782,7 +3849,7 @@ class AsyncVectorQueryBase:
def nprobes(self, nprobes: int) -> Self:
"""
Set the number of partitions to search (probe)
Set the legacy IVF probe parameter
The number of probes must be greater than 0.
@@ -3794,20 +3861,49 @@ class AsyncVectorQueryBase:
The partition whose centroids are closest to the query vector will be
exhaustiely searched to find matches. This parameter controls how many
partitions should be searched.
partitions will be searched.
Increasing this value will increase the recall of your query but will
also increase the latency of your query. The default value is 20. This
default is good for many cases but the best value to use will depend on
your data and the recall that you need to achieve.
also increase the latency of your query. If this method is not called,
Lance's adaptive probe defaults are used.
For best results we recommend tuning this parameter with a benchmark against
your actual data to find the smallest possible value that will still give
you the desired recall.
LanceDB retains this as `nprobes` through local and remote request
construction. Lance sets both probe bounds to this value. Explicit minimum
or maximum settings can override their respective bounds.
"""
self._inner.nprobes(nprobes)
return self
def search_effort(self, search_effort: Optional[float]) -> Self:
"""Set the initial IVF search effort to a finite value in [0, 1].
None preserves Auto (internally 0.5) and fixed nprobes. Any explicit
effort, including 0.5, overrides nprobes in either call order; explicit
minimum and maximum bounds remain effective. Zero starts at the caller
minimum and one searches all partitions within the maximum. Later
probing may continue to obtain enough candidates. This is not a recall
guarantee and does not change refinement or HNSW ef. MemWAL LSM queries
do not yet support this option.
Examples
--------
>>> import asyncio
>>> import lancedb
>>> async def build_query():
... db = await lancedb.connect_async("memory://")
... table = await db.create_table(
... "effort_async", [{"vector": [0.1, 0.2]}]
... )
... return table.vector_search([0.1, 0.2]).search_effort(0.75)
>>> query = asyncio.run(build_query())
"""
self._inner.search_effort(search_effort)
return self
def minimum_nprobes(self, minimum_nprobes: int) -> Self:
"""Set the minimum number of probes to use.
python/python/lancedb/table.py
+7 -2
View File
@@ -4440,6 +4440,7 @@ class LanceTable(Table):
and not self._route_pushdown_to_rust
and self.current_branch() is None
and query.take_offsets is None
and query.search_effort is None
):
from lancedb.namespace import _execute_server_side_query
@@ -6439,9 +6440,12 @@ class AsyncTable:
)
if query.distance_type is not None:
async_query = async_query.distance_type(query.distance_type)
if query.nprobes is not None:
async_query = async_query.nprobes(query.nprobes)
if query.search_effort is not None:
async_query = async_query.search_effort(query.search_effort)
if query.minimum_nprobes is not None and query.maximum_nprobes is not None:
# Set both to the minimum first to avoid min > max error.
async_query = async_query.nprobes(
async_query = async_query.minimum_nprobes(
query.minimum_nprobes
).maximum_nprobes(query.maximum_nprobes)
elif query.minimum_nprobes is not None:
@@ -6480,6 +6484,7 @@ class AsyncTable:
)
and not self._route_pushdown_to_rust
and query.take_offsets is None
and query.search_effort is None
):
from lancedb.namespace import _execute_server_side_query
python/python/lancedb/util.py
+9
View File
@@ -10,6 +10,7 @@ import os
import pathlib
import warnings
from datetime import date, datetime
from decimal import Decimal
from functools import singledispatch
from typing import Tuple, Union, Optional, Any, List
from urllib.parse import urlparse
@@ -373,6 +374,14 @@ def _(value: float):
return str(value)
@value_to_sql.register(Decimal)
def _(value: Decimal):
if not value.is_finite():
raise ValueError("Non-finite Decimal values cannot be converted to SQL")
# Quote exact decimal text so the target-column cast avoids float rounding.
return value_to_sql(format(value, "f"))
@value_to_sql.register(bool)
def _(value: bool):
return str(value).upper()
python/python/tests/test_blob.py
+16
View File
@@ -15,6 +15,7 @@ from lance.blob import Blob, BlobType as LanceBlobType
import lancedb
from lancedb._blob import (
_set_blob_column,
blob_v2_projection_sources,
read_row_ids_from_hits,
stash_auto_row_ids,
@@ -312,6 +313,21 @@ def test_blob_v2_projection_sources_include_selected_struct():
}
@pytest.mark.parametrize("parent_path", ["info", "outer.info"])
def test_set_blob_column_missing_nested_field_raises(parent_path):
row = {"name": "first"}
for name in reversed(parent_path.split(".")):
row = {name: row}
table = pa.Table.from_pylist([row])
blobs = pa.array([b"hello"], type=pa.large_binary())
with pytest.raises(KeyError) as exc_info:
_set_blob_column(table, f"{parent_path}.blob", blobs)
assert parent_path in str(exc_info.value)
assert "'blob'" in str(exc_info.value)
def _nested_blob_table(db, name):
schema = pa.schema(
[
python/python/tests/test_e2e_remote_db.py
+65
View File
@@ -2,13 +2,78 @@
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import os
from datetime import timedelta
from uuid import uuid4
import lancedb
import numpy as np
import pyarrow as pa
import pytest
from lancedb import LanceDBConnection
from lancedb.index import FTS
# TODO: setup integ test mark and script
@pytest.mark.skipif(
not os.environ.get("LANCEDB_URI"), reason="Set LANCEDB_URI to a JSON FTS server"
)
def test_json_fts_against_server():
"""Requires a writable database and capacity to run the distributed indexer.
Configure LANCEDB_URI, LANCEDB_API_KEY, and optionally LANCEDB_REGION and
LANCEDB_HOST_OVERRIDE before running this test.
"""
uri = os.environ["LANCEDB_URI"]
assert uri.startswith("db://"), "LANCEDB_URI must identify a remote database"
conn = lancedb.connect(
uri,
api_key=os.environ.get("LANCEDB_API_KEY"),
region=os.environ.get("LANCEDB_REGION", "us-east-1"),
host_override=os.environ.get("LANCEDB_HOST_OVERRIDE"),
)
name = f"json_fts_{uuid4().hex}"
table = conn.create_table(
name,
pa.table(
{
"id": [0, 1, 2],
"doc": pa.array(
[
'{"meta":{"tag":"nature"},"author":"alice"}',
'{"meta":{"tag":"city"},"author":"nature"}',
'{"meta":{"tag":"nature"},"author":"bob"}',
],
type=pa.json_(),
),
"raw": pa.array([b"nature"] * 3, type=pa.large_binary()),
}
),
)
try:
table.create_index("doc", config=FTS(), name="doc_fts")
table.wait_for_index(["doc_fts"], timeout=timedelta(minutes=5))
stats = table.index_stats("doc_fts")
assert stats.num_indexed_rows == 3
assert stats.num_unindexed_rows == 0
assert stats.index_type == "FTS"
for query, expected in [
("meta.tag,str,nature", [0, 2]),
("meta.tag,str,alice", []),
("meta.tag,str,tag", []),
]:
search = table.search(query, query_type="fts", fts_columns="doc")
assert sorted(row["id"] for row in search.to_list()) == expected
plan = search.explain_plan(True)
assert "MatchQuery" in plan, plan
assert "FlatMatchQuery" not in plan, plan
with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"):
table.create_index("raw", config=FTS())
finally:
conn.drop_table(name)
@pytest.mark.skip(reason="Need to set up a local server")
def test_against_local_server():
conn = LanceDBConnection("lancedb+http://localhost:10024")
python/python/tests/test_first_class_function_slice2.py
+1 -1
View File
@@ -1555,7 +1555,7 @@ def _mock_remote_function_catalog():
)["image"],
"signature": body["signature"],
"secret_bindings": body.get("secret_bindings", []),
"created_at": "2026-08-21T00:00:00Z",
"created_at_millis": 1787270400000, # 2026-08-21T00:00:00Z
}
response = {"job_id": "job-register"}
status = 202
python/python/tests/test_fts.py
+69
View File
@@ -45,6 +45,75 @@ from utils import exception_output
TEST_LANGUAGE_MODEL_HOME = Path(__file__).parent / "models"
@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
@pytest.mark.parametrize("block_size", [128, 256])
def test_json_fts(tmp_path, block_size):
db = ldb.connect(tmp_path)
data = pa.table(
{
"id": [0, 1, 2, 3],
"doc": pa.array(
[
'{"title":"brown fox","meta":{"tag":"nature"},"author":"alice"}',
'{"title":"blue car","meta":{"tag":"city"}}',
'{"title":"brown bear","meta":{"tag":"nature"}}',
'{"title":"red fox","author":"alice"}',
],
type=pa.json_(),
),
}
)
table = db.create_table("json_fts", data)
table.create_index("doc", config=FTS(block_size=block_size), name="doc_fts")
table = db.open_table("json_fts")
index = table.list_indices()[0]
assert index.name == "doc_fts"
assert index.index_type == "FTS"
assert index.columns == ["doc"]
stats = table.index_stats("doc_fts")
assert stats.num_indexed_rows == 4
assert stats.num_unindexed_rows == 0
for query, expected in [
("title,str,brown", [0, 2]),
("meta.tag,str,nature", [0, 2]),
("author,str,alice", [0, 3]),
("title,str,alice", []),
("title,str,title", []),
]:
search = table.search(query, query_type="fts", fts_columns="doc")
assert sorted(row["id"] for row in search.to_list()) == expected
plan = search.explain_plan(True)
assert "MatchQuery" in plan, plan
assert "FlatMatchQuery" not in plan, plan
@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
@pytest.mark.asyncio
async def test_json_fts_async(tmp_path):
db = await ldb.connect_async(tmp_path)
table = await db.create_table(
"json_fts",
pa.table({"doc": pa.array(['{"meta":{"tag":"nature"}}'], type=pa.json_())}),
)
await table.create_index("doc", config=FTS())
search = table.query().nearest_to_text(MatchQuery("meta.tag,str,nature", "doc"))
assert len(await search.to_list()) == 1
plan = await search.explain_plan(True)
assert "MatchQuery" in plan, plan
assert "FlatMatchQuery" not in plan, plan
@pytest.mark.parametrize("dtype", [pa.binary(), pa.large_binary()])
def test_fts_rejects_raw_binary(tmp_path, dtype):
table = ldb.connect(tmp_path).create_table(
"binary_fts", pa.table({"doc": pa.array([b'{"title":"brown"}'], type=dtype)})
)
with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"):
table.create_index("doc", config=FTS())
assert table.list_indices() == []
@pytest.fixture
def table(tmp_path) -> ldb.table.LanceTable:
# Use local random state to avoid affecting other tests
python/python/tests/test_hybrid_query.py
+14
View File
@@ -355,6 +355,20 @@ def test_hybrid_query_minimum_nprobes_zero_raises(sync_table: Table):
)
def test_hybrid_query_nprobes_remains_independent(sync_table: Table):
query = sync_table.search(query_type="hybrid").nprobes(20)
assert query._nprobes == 20
assert query._minimum_nprobes is None
assert query._maximum_nprobes is None
query = query.minimum_nprobes(5).nprobes(20)
assert query._nprobes == 20
assert query._minimum_nprobes == 5
assert query._maximum_nprobes is None
def test_hybrid_query_distance_range(sync_table: Table):
reranker = RRFReranker(return_score="all")
result = (
python/python/tests/test_namespace.py
+37 -1
View File
@@ -15,7 +15,8 @@ import pyarrow as pa
import lancedb
from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
from lancedb.listing import AsyncListing, Listing
from lancedb.namespace import _MAX_QUERY_K
from lancedb.namespace import _MAX_QUERY_K, _query_to_namespace_request
from lancedb.query import Query
from lancedb.table import AsyncTable, LanceTable
@@ -1111,6 +1112,29 @@ class TestPushdownOperations:
]
assert all(r.k <= 2**31 - 1 for r in namespace_client.requests)
def test_probe_fields_pass_through_independently(self):
request = _query_to_namespace_request(
["geneva", "hist"],
Query(
vector=[1.0, 2.0],
nprobes=20,
minimum_nprobes=3,
maximum_nprobes=10,
),
)
assert request.nprobes == 20
assert request.minimum_nprobes == 3
assert request.maximum_nprobes == 10
request = _query_to_namespace_request(
["geneva", "hist"],
Query(vector=[1.0, 2.0], nprobes=5, maximum_nprobes=0),
)
assert request.nprobes == 5
assert request.maximum_nprobes == 0
assert request.to_dict()["maximum_nprobes"] == 0
@pytest.mark.asyncio
class TestAsyncPushdownOperations:
@@ -1185,3 +1209,15 @@ def test_local_table_to_arrow_and_to_pandas_are_unchanged(tmp_path):
assert table.to_arrow().column("id").to_pylist() == [1, 2]
assert table.to_pandas()["id"].tolist() == [1, 2]
@pytest.mark.parametrize("effort", [0.0, 0.5, 1.0])
def test_namespace_request_rejects_search_effort(effort):
from lancedb.query import Query
with pytest.raises(
ValueError, match="namespace QueryTable does not support search_effort"
):
_query_to_namespace_request(
["test"], Query(vector=[1.0, 0.0], search_effort=effort)
)
python/python/tests/test_query.py
+212 -28
View File
@@ -1174,12 +1174,22 @@ def test_invalid_nprobes_sync(table):
ValueError,
match="maximum_nprobes must be greater than or equal to minimum_nprobes",
):
LanceVectorQueryBuilder(table, [0, 0], "vector").maximum_nprobes(5).to_list()
(
LanceVectorQueryBuilder(table, [0, 0], "vector")
.minimum_nprobes(10)
.maximum_nprobes(5)
.to_list()
)
with pytest.raises(
ValueError,
match="minimum_nprobes must be less than or equal to maximum_nprobes",
match="maximum_nprobes must be greater than or equal to minimum_nprobes",
):
LanceVectorQueryBuilder(table, [0, 0], "vector").minimum_nprobes(100).to_list()
(
LanceVectorQueryBuilder(table, [0, 0], "vector")
.maximum_nprobes(5)
.minimum_nprobes(100)
.to_list()
)
def test_nprobes_works_sync(table):
@@ -1218,12 +1228,22 @@ async def test_invalid_nprobes_async(table_async: AsyncTable):
ValueError,
match="maximum_nprobes must be greater than or equal to minimum_nprobes",
):
await table_async.vector_search([0, 0]).maximum_nprobes(5).to_list()
await (
table_async.vector_search([0, 0])
.minimum_nprobes(10)
.maximum_nprobes(5)
.to_list()
)
with pytest.raises(
ValueError,
match="minimum_nprobes must be less than or equal to maximum_nprobes",
):
await table_async.vector_search([0, 0]).minimum_nprobes(100).to_list()
await (
table_async.vector_search([0, 0])
.maximum_nprobes(5)
.minimum_nprobes(100)
.to_list()
)
def test_query_builder_with_prefilter(table):
@@ -1814,8 +1834,9 @@ def test_query_serialization_sync(table: lancedb.table.Table):
q,
vector_column="vector",
vector=[5.0, 6.0],
minimum_nprobes=10,
maximum_nprobes=10,
nprobes=10,
minimum_nprobes=None,
maximum_nprobes=None,
refine_factor=5,
)
@@ -1833,8 +1854,19 @@ def test_query_serialization_sync(table: lancedb.table.Table):
q,
vector_column="vector",
vector=[5.0, 6.0],
minimum_nprobes=50,
maximum_nprobes=50,
nprobes=50,
minimum_nprobes=None,
maximum_nprobes=None,
)
q = table.search([5.0, 6.0]).minimum_nprobes(5).nprobes(50).to_query_object()
check_set_props(
q,
vector_column="vector",
vector=[5.0, 6.0],
nprobes=50,
minimum_nprobes=5,
maximum_nprobes=None,
)
q = table.search([5.0, 6.0]).maximum_nprobes(10).to_query_object()
@@ -1846,6 +1878,15 @@ def test_query_serialization_sync(table: lancedb.table.Table):
minimum_nprobes=None,
)
q = table.search([5.0, 6.0]).nprobes(5).maximum_nprobes(0).to_query_object()
check_set_props(
q,
vector_column="vector",
vector=[5.0, 6.0],
nprobes=5,
maximum_nprobes=0,
)
q = table.search([5.0, 6.0]).distance_range(0.0, 1.0).to_query_object()
check_set_props(
q, vector_column="vector", vector=[5.0, 6.0], lower_bound=0.0, upper_bound=1.0
@@ -1895,8 +1936,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
limit=10,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
)
@@ -1906,8 +1947,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
q,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1918,8 +1959,9 @@ async def test_query_serialization_async(table_async: AsyncTable):
q,
vector=sample_vector,
postfilter=False,
minimum_nprobes=50,
maximum_nprobes=50,
nprobes=50,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1938,8 +1980,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
filter="id = 1",
postfilter=True,
vector=sample_vector,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
)
@@ -1953,8 +1995,9 @@ async def test_query_serialization_async(table_async: AsyncTable):
check_set_props(
q,
vector=sample_vector,
minimum_nprobes=10,
maximum_nprobes=10,
nprobes=10,
minimum_nprobes=None,
maximum_nprobes=None,
refine_factor=5,
postfilter=False,
with_row_id=False,
@@ -1967,7 +2010,24 @@ async def test_query_serialization_async(table_async: AsyncTable):
q,
vector=sample_vector,
minimum_nprobes=5,
maximum_nprobes=20,
maximum_nprobes=None,
postfilter=False,
with_row_id=False,
bypass_vector_index=False,
limit=10,
)
q = (
(await table_async.search([5.0, 6.0]))
.nprobes(5)
.maximum_nprobes(0)
.to_query_object()
)
check_set_props(
q,
vector=sample_vector,
nprobes=5,
maximum_nprobes=0,
postfilter=False,
with_row_id=False,
bypass_vector_index=False,
@@ -1985,8 +2045,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
lower_bound=0.0,
upper_bound=1.0,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1998,8 +2058,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
distance_type="cosine",
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -2011,8 +2071,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
ef=7,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -2024,8 +2084,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
bypass_vector_index=True,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
limit=10,
)
@@ -2384,3 +2444,127 @@ def test_blob_v2_to_batches_row_id(tmp_db):
assert "_rowid" in hits.column_names
blobs = table.fetch_blobs("blob", hits)
assert [blobs[i].as_py() for i in range(len(blobs))] == [b"one", b"two"]
@pytest.mark.parametrize("metric", ["l2", "cosine", "dot"])
@pytest.mark.parametrize(
"effort,probes", [(None, 4), (0, 1), (0.25, 2), (0.5, 4), (0.75, 8), (1, 16)]
)
def test_search_effort_probe_budget(tmp_path, monkeypatch, metric, effort, probes):
import re
from lancedb.index import IvfFlat
monkeypatch.setenv("LANCE_AUTO_PROBE_MARGIN", "0")
monkeypatch.setenv("LANCE_AUTO_MIN_INITIAL_NPROBES", "4")
monkeypatch.setenv("LANCE_AUTO_MAX_INITIAL_NPROBES", "4")
rng = np.random.default_rng(42)
vectors = np.repeat(np.eye(16, dtype=np.float32), 32, axis=0)
vectors += rng.normal(0, 0.001, vectors.shape).astype(np.float32)
data = pa.table(
{
"id": np.arange(512),
"text": ["dog"] * 512,
"vector": pa.FixedSizeListArray.from_arrays(pa.array(vectors.ravel()), 16),
}
)
table = lancedb.connect(tmp_path).create_table("effort", data.slice(0, 256))
table.add(data.slice(256))
table.create_index(
"vector", config=IvfFlat(distance_type=metric, num_partitions=16)
)
vector = np.eye(16, dtype=np.float32)[0]
def search(value=effort):
return table.search(vector).distance_type(metric).search_effort(value).limit(8)
def partitions(query):
vector_plan = query.analyze_plan().split("FTS Search Plan:", 1)[0]
values = re.findall(r"partitions_searched=(\d+)", vector_plan)
assert values
return sum(map(int, values))
assert partitions(search()) == probes
exact = set(
table.search(vector)
.distance_type(metric)
.bypass_vector_index()
.limit(8)
.to_arrow()["id"]
.to_pylist()
)
actual = set(search().to_arrow()["id"].to_pylist())
assert len(actual & exact) / 8 >= 0.5
expected = 1 if effort is None else probes
assert partitions(search().nprobes(1)) == expected
assert (
partitions(
table.search(vector)
.distance_type(metric)
.nprobes(1)
.search_effort(effort)
.limit(8)
)
== expected
)
assert partitions(search().nprobes(1).search_effort(None)) == 1
assert partitions(search().maximum_nprobes(2)) <= 2
assert partitions(search().minimum_nprobes(3).maximum_nprobes(3)) == 3
assert partitions(search(0).nprobes(16).maximum_nprobes(0)) == 1
if effort is None:
assert search().to_arrow() == search(0.5).to_arrow()
filtered = search(0).maximum_nprobes(2).where("id % 8 = 0", prefilter=True)
assert partitions(filtered) <= 2
assert all(row["id"] % 8 == 0 for row in filtered.to_list())
if metric == "l2" and effort == 0.5:
table.create_index("text", config=FTS())
hybrid = (
table.search(query_type="hybrid")
.vector(vector)
.text("dog")
.nprobes(1)
.search_effort(0.5)
.limit(8)
)
assert partitions(hybrid) == 4
assert partitions(hybrid.search_effort(None)) == 1
@pytest.mark.parametrize(
"effort", [-0.1, 1.1, float("nan"), float("inf"), -float("inf")]
)
def test_search_effort_invalid_sync(table, effort):
with pytest.raises(ValueError, match="search_effort must be finite and in"):
table.search([0, 0]).search_effort(effort).to_list()
@pytest.mark.asyncio
@pytest.mark.parametrize("hybrid", [False, True])
@pytest.mark.parametrize("effort", [None, 0, 0.25, 0.5, 0.75, 1])
async def test_search_effort_async_roundtrip(table_async, hybrid, effort):
query = table_async.vector_search([0, 0])
if hybrid:
query = query.nearest_to_text("dog")
query = query.nprobes(5).search_effort(effort).minimum_nprobes(2).maximum_nprobes(3)
request = query.to_query_object()
assert request.search_effort == effort
assert request.nprobes == 5
assert request.minimum_nprobes == 2
assert request.maximum_nprobes == 3
assert query.search_effort(None).to_query_object().search_effort is None
query.search_effort(effort)
if not hybrid:
assert len(await query.limit(1).to_list()) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("hybrid", [False, True])
@pytest.mark.parametrize(
"effort", [-0.1, 1.1, float("nan"), float("inf"), -float("inf")]
)
async def test_search_effort_invalid_async(table_async, hybrid, effort):
query = table_async.vector_search([0, 0])
if hybrid:
query = query.nearest_to_text("dog")
with pytest.raises(ValueError, match="search_effort must be finite and in"):
query.search_effort(effort)
python/python/tests/test_remote_db.py
+89 -14
View File
@@ -1410,6 +1410,72 @@ def test_remote_job_wait_raises_on_failure():
job.wait()
@pytest.mark.parametrize("server_version", ["0.2.0", "0.6.0"])
def test_remote_json_fts_config(server_version):
from lancedb.index import FTS
received_requests = []
def handler(request):
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.send_header("phalanx-version", server_version)
request.end_headers()
if request.path == "/v1/table/test/describe/":
fields = [
{
"name": "doc",
"type": {"type": "large_binary"},
"nullable": True,
"metadata": {"ARROW:extension:name": "lance.json"},
},
{
"name": "raw",
"type": {"type": "large_binary"},
"nullable": True,
},
]
request.wfile.write(
json.dumps({"version": 1, "schema": {"fields": fields}}).encode()
)
elif request.path == "/v1/table/test/create_index/":
received_requests.append(read_json_body(request))
request.wfile.write(b"{}")
else:
raise AssertionError(f"Unexpected request: {request.path}")
with mock_lancedb_connection(handler) as db:
table = db.open_table("test")
table.create_index(
"doc",
config=FTS(
with_position=True,
base_tokenizer="whitespace",
stem=False,
remove_stop_words=False,
lower_case=False,
block_size=256,
),
name="doc_fts",
replace=False,
)
with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"):
table.create_index("raw", config=FTS())
assert len(received_requests) == 1
request = received_requests[0]
assert request["column"] == "doc"
assert request["index_type"] == "FTS"
assert request["name"] == "doc_fts"
assert request["replace"] is False
assert request["with_position"] is True
assert request["base_tokenizer"] == "whitespace"
assert request["stem"] is False
assert request["remove_stop_words"] is False
assert request["lower_case"] is False
assert request["block_size"] == 256
def test_remote_create_index_new_api(caplog):
received_requests = []
@@ -2098,9 +2164,6 @@ def test_query_sync_minimal():
"ef": None,
"vector": [1.0, 2.0, 3.0],
"vector_column": "vector",
"nprobes": 20,
"minimum_nprobes": 20,
"maximum_nprobes": 20,
"version": None,
}
@@ -2272,8 +2335,6 @@ def test_query_sync_maximal():
"refine_factor": 10,
"vector": [1.0, 2.0, 3.0],
"nprobes": 5,
"minimum_nprobes": 5,
"maximum_nprobes": 5,
"lower_bound": None,
"upper_bound": None,
"ef": None,
@@ -2369,7 +2430,7 @@ async def test_query_async_nprobes_zero(hybrid):
assert requests == ["/v1/table/test/describe/"]
def test_query_sync_nprobes():
def test_query_sync_probe_bounds():
def handler(body):
assert body == {
"k": 10,
@@ -2381,7 +2442,6 @@ def test_query_sync_nprobes():
"upper_bound": None,
"ef": None,
"vector": [1.0, 2.0, 3.0],
"nprobes": 5,
"minimum_nprobes": 5,
"maximum_nprobes": 15,
"version": None,
@@ -2398,7 +2458,28 @@ def test_query_sync_nprobes():
)
def test_query_sync_no_max_nprobes():
def test_query_sync_nprobes_passthrough():
def handler(body):
assert body["nprobes"] == 20
assert "minimum_nprobes" not in body
assert "maximum_nprobes" not in body
return pa.table({"id": [1]})
with query_test_table(handler) as table:
table.search([1, 2, 3]).nprobes(20).to_list()
def test_query_sync_nprobes_with_unbounded_maximum():
def handler(body):
assert body["nprobes"] == 20
assert body["maximum_nprobes"] == 0
return pa.table({"id": [1]})
with query_test_table(handler) as table:
table.search([1, 2, 3]).nprobes(20).maximum_nprobes(0).to_list()
def test_query_sync_minimum_nprobes_only():
def handler(body):
assert body == {
"k": 10,
@@ -2410,9 +2491,7 @@ def test_query_sync_no_max_nprobes():
"upper_bound": None,
"ef": None,
"vector": [1.0, 2.0, 3.0],
"nprobes": 5,
"minimum_nprobes": 5,
"maximum_nprobes": 0,
"version": None,
}
@@ -2422,7 +2501,6 @@ def test_query_sync_no_max_nprobes():
(
table.search([1, 2, 3], vector_column_name="vector2", fast_search=True)
.minimum_nprobes(5)
.maximum_nprobes(0)
.to_list()
)
@@ -2577,9 +2655,6 @@ def test_query_sync_hybrid():
"refine_factor": None,
"vector": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
"vector_column": "vector",
"nprobes": 20,
"minimum_nprobes": 20,
"maximum_nprobes": 20,
"lower_bound": None,
"upper_bound": None,
"ef": None,
python/python/tests/test_table.py
+46 -1
View File
@@ -12,7 +12,7 @@ import warnings
import weakref
from concurrent.futures import ThreadPoolExecutor
from datetime import date, datetime, timedelta
from decimal import Decimal
from decimal import Decimal, localcontext
from time import sleep
from typing import List
from unittest.mock import patch
@@ -2612,6 +2612,51 @@ def test_update_with_arrow_scalar(mem_db: DBConnection):
assert table.to_arrow()["vector"].to_pylist() == [[1.0, 2.0, 3.0, 4.0]]
@pytest.mark.parametrize("use_arrow_scalar", [False, True])
@pytest.mark.parametrize(
"decimal_type, value",
[
(pa.decimal128(10, 2), "9.99"),
(pa.decimal128(10, 2), "-9.99"),
(pa.decimal128(10, 2), "0.00"),
(pa.decimal128(10, 2), "1.23E+5"),
(pa.decimal128(38, 18), "1E-18"),
(pa.decimal128(38, 18), "12345678901234567.123456789012345678"),
(pa.decimal128(38, 18), "12345678901234567890.123456789012345678"),
(pa.decimal128(38, 18), "-12345678901234567890.123456789012345678"),
(
pa.decimal256(76, 38),
"12345678901234567890123456789012345678."
"12345678901234567890123456789012345678",
),
(
pa.decimal256(76, 38),
"-12345678901234567890123456789012345678."
"12345678901234567890123456789012345678",
),
],
)
def test_update_decimal(mem_db: DBConnection, decimal_type, value, use_arrow_scalar):
expected = Decimal(value)
table = mem_db.create_table(
"my_table",
pa.table(
{
"id": [1, 2],
"price": pa.array([Decimal("1"), expected], type=decimal_type),
}
),
)
update = table.to_arrow()["price"][1] if use_arrow_scalar else expected
with localcontext() as context:
context.prec = 6
result = table.update(where="id = 1", values={"price": update})
assert result.rows_updated == 1
actual = table.to_arrow()
assert actual.schema.field("price").type == decimal_type
assert actual["price"].to_pylist() == [expected, expected]
def test_update_types(mem_db: DBConnection):
table = mem_db.create_table(
"my_table",
python/python/tests/test_util.py
+36
View File
@@ -4,6 +4,7 @@
import os
import pathlib
from decimal import Decimal, localcontext
from typing import Optional
import lance
@@ -157,6 +158,41 @@ def test_value_to_sql_string(tmp_path):
assert table.to_pandas().query("search == @value")["replace"].item() == value
@pytest.mark.parametrize(
"value, expected",
[
("9.99", "'9.99'"),
("-9.99", "'-9.99'"),
("0", "'0'"),
("-0.00", "'-0.00'"),
("1.23E+5", "'123000'"),
("1E-18", "'0.000000000000000001'"),
(
"12345678901234567890.123456789012345678",
"'12345678901234567890.123456789012345678'",
),
(
"12345678901234567890123456789012345678."
"12345678901234567890123456789012345678",
"'12345678901234567890123456789012345678."
"12345678901234567890123456789012345678'",
),
],
)
def test_value_to_sql_decimal(value, expected):
with localcontext() as context:
context.prec = 6
assert value_to_sql(Decimal(value)) == expected
@pytest.mark.parametrize("value", ["NaN", "sNaN", "NaN123456", "Infinity", "-Infinity"])
def test_value_to_sql_decimal_non_finite(value):
with pytest.raises(
ValueError, match="^Non-finite Decimal values cannot be converted to SQL$"
):
value_to_sql(Decimal(value))
def test_value_to_sql_dict():
# Simple flat struct
assert value_to_sql({"a": 1, "b": "hello"}) == "named_struct('a', 1, 'b', 'hello')"
python/src/query.rs
+36 -10
View File
@@ -71,7 +71,7 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB<FtsQuery> {
match class_name(&ob)?.as_str() {
"MatchQuery" => {
let query = ob.getattr("query")?.extract()?;
let column = ob.getattr("column")?.extract()?;
let column = ob.getattr("column")?.extract::<Option<String>>()?;
let boost = ob.getattr("boost")?.extract()?;
let fuzziness = ob.getattr("fuzziness")?.extract()?;
let max_expansions = ob.getattr("max_expansions")?.extract()?;
@@ -88,7 +88,7 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB<FtsQuery> {
let mut query =
MatchQuery::new(query)
.with_column(Some(column))
.with_column(column)
.with_boost(boost)
.with_fuzziness(fuzziness)
.with_max_expansions(max_expansions)
@@ -199,7 +199,7 @@ impl<'py> IntoPyObject<'py> for PyLanceDB<FtsQuery> {
}
namespace
.getattr(intern!(py, "MatchQuery"))?
.call((query.terms, query.column.unwrap()), Some(&kwargs))
.call((query.terms, query.column), Some(&kwargs))
}
FtsQuery::Phrase(query) => {
let kwargs = PyDict::new(py);
@@ -336,10 +336,10 @@ pub struct PyQueryRequest {
pub use_lsm: Option<bool>,
pub column: Option<String>,
pub query_vector: Option<PyQueryVectors>,
pub nprobes: Option<usize>,
pub minimum_nprobes: Option<usize>,
// None means user did not set it and default should be used (currently 20)
// Some(0) means user set it to None and there is no limit
pub maximum_nprobes: Option<usize>,
pub search_effort: Option<f64>,
pub lower_bound: Option<f32>,
pub upper_bound: Option<f32>,
pub ef: Option<usize>,
@@ -369,8 +369,10 @@ impl From<AnyQuery> for PyQueryRequest {
use_lsm: query_request.use_lsm,
column: None,
query_vector: None,
nprobes: None,
minimum_nprobes: None,
maximum_nprobes: None,
search_effort: None,
lower_bound: None,
upper_bound: None,
ef: None,
@@ -396,11 +398,10 @@ impl From<AnyQuery> for PyQueryRequest {
use_lsm: vector_query.base.use_lsm,
column: vector_query.column,
query_vector: Some(PyQueryVectors(vector_query.query_vector)),
minimum_nprobes: Some(vector_query.minimum_nprobes),
maximum_nprobes: match vector_query.maximum_nprobes {
None => Some(0),
Some(value) => Some(value),
},
nprobes: vector_query.nprobes,
minimum_nprobes: vector_query.minimum_nprobes,
maximum_nprobes: vector_query.maximum_nprobes,
search_effort: vector_query.search_effort,
lower_bound: vector_query.lower_bound,
upper_bound: vector_query.upper_bound,
ef: vector_query.ef,
@@ -1000,6 +1001,15 @@ impl VectorQuery {
Ok(())
}
pub fn search_effort(&mut self, search_effort: Option<f64>) -> PyResult<()> {
self.inner = self
.inner
.clone()
.search_effort(search_effort)
.infer_error()?;
Ok(())
}
pub fn minimum_nprobes(&mut self, minimum_nprobes: u32) -> PyResult<()> {
self.inner = self
.inner
@@ -1193,6 +1203,18 @@ impl HybridQuery {
self.inner_vec.nprobes(nprobe)
}
pub fn search_effort(&mut self, search_effort: Option<f64>) -> PyResult<()> {
self.inner_vec.search_effort(search_effort)
}
pub fn minimum_nprobes(&mut self, minimum_nprobes: u32) -> PyResult<()> {
self.inner_vec.minimum_nprobes(minimum_nprobes)
}
pub fn maximum_nprobes(&mut self, maximum_nprobes: u32) -> PyResult<()> {
self.inner_vec.maximum_nprobes(maximum_nprobes)
}
pub fn ef(&mut self, ef: u32) {
self.inner_vec.ef(ef);
}
@@ -1237,6 +1259,10 @@ impl HybridQuery {
req.query_vector = vec_req.query_vector;
req.column = vec_req.column;
req.distance_type = vec_req.distance_type;
req.nprobes = vec_req.nprobes;
req.minimum_nprobes = vec_req.minimum_nprobes;
req.maximum_nprobes = vec_req.maximum_nprobes;
req.search_effort = vec_req.search_effort;
req.ef = vec_req.ef;
req.refine_factor = vec_req.refine_factor;
req.lower_bound = vec_req.lower_bound;
python/uv.lock
Generated
+41 -15
View File
@@ -1849,19 +1849,19 @@ wheels = [
[[package]]
name = "lance-namespace"
version = "0.11.1"
version = "0.13.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "lance-namespace-urllib3-client" },
]
sdist = { url = "https://files.pythonhosted.org/packages/bf/93/da5f7fcac690db9b282a3439ed9e34960c147619a0d6e1f4eb8cd240e7a5/lance_namespace-0.11.1.tar.gz", hash = "sha256:f67cfbbe0647b7cb42f23b673e7edf8a75b7d8a047265a916492f8d247ee1bc2", size = 11631, upload-time = "2026-08-18T17:40:06.294Z" }
sdist = { url = "https://files.pythonhosted.org/packages/c0/e7/d5d46594678ee479c0eda830c47b2f5c46133bd100e84e9aa6e01306eca7/lance_namespace-0.13.0.tar.gz", hash = "sha256:24554a0997bdb39595c6e4cb3ac6722069f6cd3bd1a74e7acbba7bfaf774a40d", size = 11636, upload-time = "2026-09-14T02:47:03.871523Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/fa/bc/601f2b3cc4cfa0070d858a33223bc823fffdd7981a25c45984a5216ca952/lance_namespace-0.11.1-py3-none-any.whl", hash = "sha256:07643fce9a42ad4d58cc8bf91e3f592bc7f4cbd8d0ad5233223506debf67551c", size = 13507, upload-time = "2026-08-18T17:40:03.561Z" },
{ url = "https://files.pythonhosted.org/packages/e7/47/cfa33cca1ba7c749fd2918cfc5c8ded788f378cc6424d23e4fead5a14125/lance_namespace-0.13.0-py3-none-any.whl", hash = "sha256:438c7b17aef421c21c138196e715f2510d62f07e865372047f87c1e75e618c7a", size = 13507, upload-time = "2026-09-14T02:47:01.765655Z" },
]
[[package]]
name = "lance-namespace-urllib3-client"
version = "0.11.1"
version = "0.13.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pydantic" },
@@ -1869,9 +1869,9 @@ dependencies = [
{ name = "typing-extensions" },
{ name = "urllib3" },
]
sdist = { url = "https://files.pythonhosted.org/packages/e4/c5/2bdd0ff98b469894c8a73be809d26ffdad5402517b0e5f9e758026cba29e/lance_namespace_urllib3_client-0.11.1.tar.gz", hash = "sha256:145a9e9424d7597487249b5b95ee274423bf2910e1a9160b6a07b676b61ea46a", size = 237345, upload-time = "2026-08-18T17:40:07.308Z" }
sdist = { url = "https://files.pythonhosted.org/packages/69/25/9aaa4a5e8999693fb0f227c2c0c4b97bd8f0539066408cdff0b89d41b2d5/lance_namespace_urllib3_client-0.13.0.tar.gz", hash = "sha256:1e8a79c6e4e6277033597fd76aa0e2f33d909ca1436c935936e9e569221f43ef", size = 238348, upload-time = "2026-09-14T02:47:04.521878Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/2a/2a/eaaefd55d1190291207049fedc6b3eb22b506e57d6de91bae46bbaaa9c60/lance_namespace_urllib3_client-0.11.1-py3-none-any.whl", hash = "sha256:36537f529294da6d884ba0fe783704483f0a75463497c7705fd083a4d0257990", size = 406311, upload-time = "2026-08-18T17:40:04.842Z" },
{ url = "https://files.pythonhosted.org/packages/60/4c/7b8f0712a7fe1b8655b711342bc4989696635a79e7027ee56ba3cae23e99/lance_namespace_urllib3_client-0.13.0-py3-none-any.whl", hash = "sha256:fb361eb4f6c7f2d1f9e92809609657b73e38241393e9c4516e12d6b6ee643a0a", size = 406848, upload-time = "2026-09-14T02:47:02.715954Z" },
]
[[package]]
@@ -1900,6 +1900,7 @@ clip = [
{ name = "torch" },
]
dev = [
{ name = "maturin" },
{ name = "pre-commit" },
{ name = "pyright" },
{ name = "ruff" },
@@ -1980,7 +1981,8 @@ requires-dist = [
{ name = "huggingface-hub", marker = "extra == 'embeddings'", specifier = ">=0.19.0" },
{ name = "ibm-watsonx-ai", marker = "python_full_version >= '3.10' and extra == 'embeddings'", specifier = ">=1.1.2" },
{ name = "instructorembedding", marker = "extra == 'embeddings'", specifier = ">=1.0.1" },
{ name = "lance-namespace", specifier = ">=0.3.2" },
{ name = "lance-namespace", specifier = ">=0.13.0" },
{ name = "maturin", marker = "extra == 'dev'", specifier = ">=1.10" },
{ name = "mkdocs", marker = "extra == 'docs'" },
{ name = "mkdocs-jupyter", marker = "extra == 'docs'" },
{ name = "mkdocs-material", marker = "extra == 'docs'" },
@@ -2005,7 +2007,7 @@ requires-dist = [
{ name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" },
{ name = "pydantic", specifier = ">=2.7.4,<3" },
{ name = "pylance", marker = "extra == 'pylance'", specifier = ">=5.0.0b5" },
{ name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b6" },
{ name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b12" },
{ name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" },
{ name = "pytest", marker = "extra == 'tests'", specifier = ">=7.0" },
{ name = "pytest-asyncio", marker = "extra == 'tests'", specifier = ">=0.21" },
@@ -2156,6 +2158,30 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" },
]
[[package]]
name = "maturin"
version = "1.15.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "tomli", marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/b9/c8/22e5e21b2679c9bce6415ca578034ca2cc9316be0642ae21e051a2d5198c/maturin-1.15.0.tar.gz", hash = "sha256:94b26cc8e8aba61a5f2099715fe640e18c5f678e9a500408b38761263954228a", size = 385504, upload-time = "2026-08-24T12:11:22.665Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/14/69/5c01b461044eb1f45ddcce006706eb88110c793cdb11c7ae0b5e08492e94/maturin-1.15.0-py3-none-linux_armv6l.whl", hash = "sha256:6bf6dc62e22d4dcfd5a51244ff0d58975fa4979c48209fe84159617648956d82", size = 10206220, upload-time = "2026-08-24T12:10:53.327Z" },
{ url = "https://files.pythonhosted.org/packages/eb/1f/2b431554e11687cdb1077e0cdadcc118c53f611086b3af00c8545a67c6a5/maturin-1.15.0-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:cd35772633f489841132bc8e71d6fc7f842df30b9c05cd5cdf1ee1ddcb744cc7", size = 19416513, upload-time = "2026-08-24T12:10:56.126Z" },
{ url = "https://files.pythonhosted.org/packages/51/36/e23a21cb34a648b711036b9b2fe1d4f3f4ee24f8db54215d73f1a9a3a3ec/maturin-1.15.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c40b4eae7bf5ef1f4b1af8d623fe4105016f93578fb15b764e741d08ec3b92dd", size = 10014962, upload-time = "2026-08-24T12:10:58.486Z" },
{ url = "https://files.pythonhosted.org/packages/8c/80/33b15cb2d8f30f12c807955e8f2fd775027692904e30ec0784744ce8cd83/maturin-1.15.0-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:7eb066372f541f8eb4909c79c5d9bd0b9e8125980bdf1ec9e8aba23c6c8d6c55", size = 10196223, upload-time = "2026-08-24T12:11:00.696Z" },
{ url = "https://files.pythonhosted.org/packages/fe/91/b495e19e2f5c503b540452b2039115e7b2363867e8c5ad4179eb752fa92c/maturin-1.15.0-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:653020a63525bb224e5ab0adf02e17a2e08bc86dbea7fc1399c9a56d7529b99e", size = 10541186, upload-time = "2026-08-24T12:11:02.857Z" },
{ url = "https://files.pythonhosted.org/packages/5d/2b/2abff58037188d852b124871b1f0d720e1c2bfb3d4f1b03d87c52cd66488/maturin-1.15.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:0ebf9767892725083138e671c34482c660317a2f3d6a29fc0e0f34e9d8c99136", size = 10083468, upload-time = "2026-08-24T12:11:05.012Z" },
{ url = "https://files.pythonhosted.org/packages/3f/07/b7e9f8be99a6627849e81ac7b6694876bce8f50a92995fe17e3cf2610f0a/maturin-1.15.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:7ab7eebffd7b8debca2265985de4eaeb332141276d24b9560b5ad484d4b3add1", size = 10047786, upload-time = "2026-08-24T12:11:07.042Z" },
{ url = "https://files.pythonhosted.org/packages/ca/ab/167e3cb7accee11b507dbe53e0e87aeccb376d44ae66284c96ee4df3a9fd/maturin-1.15.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:126e12e618b4db42f68c779a56d41f82a390145ba36ac3f621d057eb34f5ad9d", size = 13315332, upload-time = "2026-08-24T12:11:09.433Z" },
{ url = "https://files.pythonhosted.org/packages/14/4d/801379f646cbc6b00998e5289b0630a886be3a4ee4c75b6bc9b87478a7f1/maturin-1.15.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4f9d33e6c3f9615c8caceecbbbd440f8eb25a3ddeb687077682cd5eca2e9ae15", size = 10807183, upload-time = "2026-08-24T12:11:11.73Z" },
{ url = "https://files.pythonhosted.org/packages/89/27/2e612e1cbd1580e9e94d4722c227b5180dca27b32b955a34b79918aa1292/maturin-1.15.0-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:bf29beddd0c6708f112db51d5275fc28b28b9e9c9c5faae387eaef662918b176", size = 10413274, upload-time = "2026-08-24T12:11:14.04Z" },
{ url = "https://files.pythonhosted.org/packages/70/d8/202a7b4d75a51f20f84ec9ce3b7345b12b822207072164e2b1c6ef665125/maturin-1.15.0-py3-none-win32.whl", hash = "sha256:da649988be98e87e009e51b1bf0d301b6a301bc0cecbdd60d40d8ba60748d1ca", size = 8928744, upload-time = "2026-08-24T12:11:16.269Z" },
{ url = "https://files.pythonhosted.org/packages/40/dc/4e90da594986ba78dd3bc8a5921ecdcdb11085b22b02a412caab3b225601/maturin-1.15.0-py3-none-win_amd64.whl", hash = "sha256:552c2be4afd43fe8d5c9f3ec8d4c4756d973b8dcbe94c14084390301f50243e1", size = 10335085, upload-time = "2026-08-24T12:11:18.326Z" },
{ url = "https://files.pythonhosted.org/packages/8b/10/15d4314edf130955edf2dc237aa393a8a7c10f2b9b57b89fa2f61f915659/maturin-1.15.0-py3-none-win_arm64.whl", hash = "sha256:c7dc0c66c78d3debdd9c5aa807e861fbcbf07f3505d34b125df74c03986b0f48", size = 9713795, upload-time = "2026-08-24T12:11:20.83Z" },
]
[[package]]
name = "mdit-py-plugins"
version = "0.6.1"
@@ -3912,7 +3938,7 @@ crypto = [
[[package]]
name = "pylance"
version = "14.0.0b6"
version = "14.0.0b12"
source = { registry = "https://pypi.fury.io/lance-format/" }
dependencies = [
{ name = "lance-namespace" },
@@ -3922,12 +3948,12 @@ dependencies = [
{ name = "pyarrow" },
]
wheels = [
{ url = "https://pypi.fury.io/lance-format/-/ver_DuQEn/pylance-14.0.0b6-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:3eade86fa30d9ff3248ee7ffb0b27ce29495394d12327da4d22b5c54bc2b59fe" },
{ url = "https://pypi.fury.io/lance-format/-/ver_K5sqj/pylance-14.0.0b6-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:10e291974ee738a1d5d7e9dad17a634394a91cb60080b5ceac209077fbaec43c" },
{ url = "https://pypi.fury.io/lance-format/-/ver_EnbkT/pylance-14.0.0b6-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61bac10f5081c1210edd017f81c8c784d800b89446bd0ed05d4a24e7001965bc" },
{ url = "https://pypi.fury.io/lance-format/-/ver_19ffYg/pylance-14.0.0b6-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3c627ad4a9870611becf84409d8e5ecddc6be64e1dd53bee4bd5fb10ab561e5c" },
{ url = "https://pypi.fury.io/lance-format/-/ver_2eLLY5/pylance-14.0.0b6-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:bc603a7dd690233f1fe000600171e1c092f153bc9060cdf198825c4bbe1418aa" },
{ url = "https://pypi.fury.io/lance-format/-/ver_t8ggc/pylance-14.0.0b6-cp310-abi3-win_amd64.whl", hash = "sha256:1d32df32a3d157611450def07eae8ace3a7401286eac68cad9a24e0d5deed2db" },
{ url = "https://pypi.fury.io/lance-format/-/ver_1Ixasc/pylance-14.0.0b12-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:84350a3c5c1eea3398f4c4fad7cfe85b831d831a5e43b5ca2a783dbdb0e11969" },
{ url = "https://pypi.fury.io/lance-format/-/ver_sCEkG/pylance-14.0.0b12-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76b386c182a6a8e7b86fbef28f84d1dd39e4626ac8681a0cf36f40a45dd7cb87" },
{ url = "https://pypi.fury.io/lance-format/-/ver_1vdT1C/pylance-14.0.0b12-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6aa548c7937fcdfa8f1943a2d59041a42f8b500199880dbb509c7195fa253381" },
{ url = "https://pypi.fury.io/lance-format/-/ver_2fXu1V/pylance-14.0.0b12-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:79f82d6a6c50f37aab338470e3733436e6a27fbb5972d29fcf453ece56d3b3c6" },
{ url = "https://pypi.fury.io/lance-format/-/ver_24UavN/pylance-14.0.0b12-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:de0305999257002785f60169e6926896584d66f1da050bb5afd1d48c6e3d9bb9" },
{ url = "https://pypi.fury.io/lance-format/-/ver_XNynX/pylance-14.0.0b12-cp310-abi3-win_amd64.whl", hash = "sha256:4d0cdb689e898ce71e8e434d00a3b97639cea8fb97da21f1c921543be363268f" },
]
[[package]]