mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-02 11:38:49 +00:00
fix: preserve duplicate take offsets (#4024)
## Summary - preserve repeated table offsets without adding a public ordering guarantee - retain exact requested ordering in identity and persisted permutations - cover local, projected, multi-batch, and mocked-remote query paths ## Root cause Take queries lowered offsets to a set-like IN predicate and discarded repeated occurrences. Persisted permutation loading also compared the distinct base-table result count with the requested occurrence count, rejecting repeated row IDs before its existing reordering step could expand them. ## Fix The shared take-query path now deduplicates the predicate for efficient lookup, requests row-offset metadata internally, and expands each matching row to the requested multiplicity in backend result order. An internal opt-in keeps exact requested order for identity PermutationReader reads, while persisted permutations continue using their existing ordering map. ## Validation - cargo test --quiet --features remote --tests - cargo check --quiet --features remote --tests --examples - cargo clippy --quiet --features remote --tests --examples - targeted Python local and mocked-remote regression tests - exact issue reproduction Fixes #2820 <!-- lance-gatekeeper-fix:v1 agent=75acf840afa6f4be4bff98b567b504bd generation=1 --> --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Co-authored-by: Xuanwo <github@xuanwo.io>
This commit is contained in:
committed by
GitHub
parent
5cbd979455
commit
19232f9c50
@@ -323,6 +323,7 @@ impl<'py> IntoPyObject<'py> for PyQueryVectors {
|
||||
pub struct PyQueryRequest {
|
||||
pub limit: Option<usize>,
|
||||
pub offset: Option<usize>,
|
||||
pub take_offsets: Option<Vec<u64>>,
|
||||
pub filter: Option<PyQueryFilter>,
|
||||
pub full_text_search: Option<PyLanceDB<FtsQuery>>,
|
||||
pub select: PySelect,
|
||||
@@ -353,6 +354,7 @@ impl From<AnyQuery> for PyQueryRequest {
|
||||
AnyQuery::Query(query_request) => Self {
|
||||
limit: query_request.limit,
|
||||
offset: query_request.offset,
|
||||
take_offsets: query_request.take_offsets,
|
||||
filter: query_request.filter.map(PyQueryFilter),
|
||||
full_text_search: query_request
|
||||
.full_text_search
|
||||
@@ -381,6 +383,7 @@ impl From<AnyQuery> for PyQueryRequest {
|
||||
AnyQuery::VectorQuery(vector_query) => Self {
|
||||
limit: vector_query.base.limit,
|
||||
offset: vector_query.base.offset,
|
||||
take_offsets: vector_query.base.take_offsets,
|
||||
filter: vector_query.base.filter.map(PyQueryFilter),
|
||||
full_text_search: None,
|
||||
select_source_columns: PySelect::source_columns(&vector_query.base.select),
|
||||
|
||||
Reference in New Issue
Block a user