rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-26 13:36:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(rust): accept usize::MAX as a local query limit (#4567)

Browse Source 
Fixes #2211.

Local queries converted `limit`/`offset` with `as i64`, so
`limit(usize::MAX)` wrapped to `-1` and Lance failed with `Limit must be
non-negative`. The remote client already worked around this (#2354);
this fixes the local path.

- Saturate scanner offsets to `i64::MAX` and cap limits at `i64::MAX -
offset`, keeping both conversion and full-text pagination addition safe.
- Use `saturating_add` for vector `top_k = limit + offset`, then cap
candidates at the dataset row count to avoid indexed-search capacity and
refinement overflows. Keep a positive candidate bound for empty tables
because Lance rejects `k = 0`.
- Leave public APIs and remote query handling unchanged.

### Testing
- Regression coverage includes plain scans, indexed IVF search,
refinement, index bypass, empty vector tables, and indexed full-text
pagination with finite and extreme limits/offsets.
- Reproduced the indexed-search and FTS overflows before applying the
follow-up fix.
- `cargo test --quiet --features remote -p lancedb --lib query::` — 81
passed.
- `cargo clippy --quiet --features remote -p lancedb --tests --examples`
— clean.
- `cargo fmt --all` and `git diff --check` — clean.

Vector planning now reads the dataset row count (metadata fast path) to
bound candidate allocation; this was not performance-benchmarked.

---------

Co-authored-by: Romin Parekh <roparekh@adobe.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Co-authored-by: Xuanwo <github@xuanwo.io>
This commit is contained in:
Romin Parekh
2026-10-24 11:08:40 +08:00
committed by GitHub
co-authored by Romin Parekh Copilot Xuanwo
parent 6777a40dc2
commit a8305e0722
3 changed files with 188 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/python/lancedb/query.py
+17 -12
View File
@@ -1561,9 +1561,10 @@ class LanceQueryBuilder(ABC):
ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
GlobalLimitExec: skip=0, fetch=10
FilterExec: _distance@2 IS NOT NULL
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
SortExec: TopK(fetch=1), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
KNNVectorDistance: metric=l2
LanceRead: uri=..., projection=[vector], ...
LanceRead: uri=..., projection=[vector], num_fragments=..., ...
<BLANKLINE>
Parameters
----------
@@ -1619,8 +1620,8 @@ class LanceQueryBuilder(ABC):
GlobalLimitExec: elapsed=..., skip=0, fetch=10,
metrics=[output_rows=..., elapsed_compute=..., output_bytes=...]
FilterExec: elapsed=..., _distance@2 IS NOT NULL, metrics=[...]
SortExec: elapsed=..., TopK(fetch=10), expr=[...],
preserve_partitioning=[...],
SortExec: elapsed=..., TopK(fetch=1), expr=[...],
preserve_partitioning=[...], filter=[...],
metrics=[output_rows=..., elapsed_compute=...,
output_bytes=..., row_replacements=...]
KNNVectorDistance: elapsed=..., metric=l2,
@@ -1633,6 +1634,7 @@ class LanceQueryBuilder(ABC):
metrics=[output_rows=..., elapsed_compute=..., output_bytes=...,
fragments_scanned=..., ranges_scanned=1, rows_scanned=1,
bytes_read=..., iops=..., requests=..., task_wait_time=...]
<BLANKLINE>
Parameters
----------
@@ -2959,9 +2961,10 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
GlobalLimitExec: skip=0, fetch=10
FilterExec: _distance@2 IS NOT NULL
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
SortExec: TopK(fetch=1), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
KNNVectorDistance: metric=l2
LanceRead: uri=..., projection=[vector], ...
LanceRead: uri=..., projection=[vector], num_fragments=..., ...
<BLANKLINE>
Parameters
----------
@@ -3601,9 +3604,10 @@ class AsyncQueryBase(object):
ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
GlobalLimitExec: skip=0, fetch=10
FilterExec: _distance@2 IS NOT NULL
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
SortExec: TopK(fetch=1), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
KNNVectorDistance: metric=l2
LanceRead: uri=..., projection=[vector], ...
LanceRead: uri=..., projection=[vector], num_fragments=..., ...
<BLANKLINE>
<BLANKLINE>
Parameters
@@ -4521,9 +4525,9 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
LanceRead: uri=..., projection=[text], source=stream(_rowid)
GlobalLimitExec: skip=0, fetch=10
FilterExec: _distance@2 IS NOT NULL
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
SortExec: TopK(fetch=1), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
KNNVectorDistance: metric=l2
LanceRead: uri=..., projection=[vector], ...
LanceRead: uri=..., projection=[vector], num_fragments=..., ...
ProjectionExec: expr=[vector@2 as vector, text@3 as text, _score@1 as _score, _rowid@0 as _rowid]
LanceRead: uri=..., projection=[vector, text], source=stream(_rowid)
GlobalLimitExec: skip=0, fetch=10
@@ -4877,9 +4881,10 @@ class BaseQueryBuilder(object):
ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
GlobalLimitExec: skip=0, fetch=10
FilterExec: _distance@2 IS NOT NULL
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
SortExec: TopK(fetch=1), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
KNNVectorDistance: metric=l2
LanceRead: uri=..., projection=[vector], ...
LanceRead: uri=..., projection=[vector], num_fragments=..., ...
<BLANKLINE>
<BLANKLINE>
Parameters