rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-28 14:36:13 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
e3dce2cbad45c14422a077dfb34ee68050173827
lancedb/Cargo.toml
T
lancedb-gatefixer[bot]andXuanwo e3dce2cbad fix(rust): prevent oversized SQ remap panics (#3861) 
## Summary

- preflight IVF-SQ and IVF-HNSW-SQ storage segments touched by
compaction
- preserve unknown legacy fragment coverage as conservatively affected
- cover both fixed-size vector and multivector SQ storage dimensions
- reject only affected partitions whose fixed-size-list child exceeds
Arrow's `u32` take-index limit
- return an actionable `InvalidInput` error before Arrow can panic

## Root cause

Arrow 58 builds fixed-size-list child indices with `u32`. For a large SQ
partition, `partition_rows * vector_dimension` can exceed `u32::MAX`;
the child range then wraps inside `take_fixed_size_list`, producing the
reported `ArrayData::slice` assertion during index remapping.

The preflight now reads raw segment metadata, mirrors Lance's
fragment-bitmap selection (including `None` as affected/unknown), opens
only affected vector segments, and checks their actual storage quantizer
and partition sizes. It derives the stored dimension for both
`FixedSizeList` vectors and flattened `List<FixedSizeList>`
multivectors. Stable-row-ID datasets, deferred remapping, custom
remappers, unaffected segments, non-SQ indexes, safe partitions, and
no-op compactions are unchanged.

Because the upstream Arrow kernel still has the limit, an affected index
must be recreated with default IVF partitioning (or enough partitions to
stay below the reported maximum) before compaction. This changes the
failure from a process panic to a targeted recovery error; it does not
silently alter index structure.

## Validation

- `cargo fmt --all -- --check`
- `cargo test --quiet --features aws,remote -p lancedb
table::optimize::tests --lib`
- `cargo test --quiet --features remote -p lancedb
table::create_index::tests::test_create_index_ivf_hnsw_sq`
- `cargo check --quiet --features remote --tests --examples`
- `cargo clippy --quiet --features remote --tests --examples -- -A
clippy::redundant-pub-crate` (the allowance is for pre-existing Rust
1.97 warnings in remote-table code)

Fixes #2866

<!-- lance-gatekeeper-fix:v1 agent=ba2f07b1ee6f9e68a0de1961c164f70e
generation=1 -->

---------

Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
Co-authored-by: Xuanwo <github@xuanwo.io>
2026-10-27 08:27:27 +08:00

110 lines
4.9 KiB
TOML
Raw Blame History

[workspace]
members = [
"rust/lancedb",
"nodejs",
"python",
"docs/web-tests/blob-v2/rs",
]
resolver = "2"
[workspace.package]
edition = "2024"
authors = ["LanceDB Devs <dev@lancedb.com>"]
license = "Apache-2.0"
repository = "https://github.com/lancedb/lancedb"
description = "Serverless, low-latency vector database for AI applications"
keywords = ["lancedb", "lance", "database", "vector", "search"]
categories = ["database-implementations"]
rust-version = "1.94.0"
[workspace.dependencies]
lance = { "version" = "=15.0.0-beta.1", default-features = false, "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=15.0.0-beta.1", default-features = false, "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-jsonb = { version = "=15.0.0-beta.1", tag = "v15.0.0-beta.1", git = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=15.0.0-beta.1", default-features = false, "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { version = "=15.0.0-beta.1", path = "python/vendor/lance-arrow" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
arrow = { version = "58.0.0", optional = false }
arrow-array = "58.0.0"
arrow-buffer = "58.0.0"
arrow-data = "58.0.0"
arrow-ipc = "58.0.0"
arrow-ord = "58.0.0"
arrow-schema = "58.0.0"
arrow-select = "58.0.0"
arrow-cast = "58.0.0"
arrow-flight = { version = "58.0.0", features = ["flight-sql-experimental"] }
async-trait = "0"
# Smithy JSON 0.63 requires the pre-1.7 Document representation; allow the MSRV pin.
aws-smithy-types = ">=1.3.6, <1.7"
bytes = "1"
datafusion = { version = "54.0.0", default-features = false }
datafusion-catalog = "54.0.0"
datafusion-common = { version = "54.0.0", default-features = false }
datafusion-execution = "54.0.0"
datafusion-expr = "54.0.0"
datafusion-functions = "54.0.0"
datafusion-physical-plan = "54.0.0"
datafusion-physical-expr = "54.0.0"
datafusion-sql = "54.0.0"
env_logger = "0.11"
half = { "version" = "2.7.1", default-features = false, features = [
"num-traits",
] }
futures = "0.3"
log = "0.4"
metrics = "0.24"
metrics-util = "0.19"
moka = { version = "0.12", features = ["future"] }
object_store = "0.14.1"
pin-project = "1.0.7"
rand = "0.9"
snafu = "0.8"
url = "2"
num-traits = "0.2"
oauth2 = { version = "5.0", default-features = false }
regex = "1.10"
semver = "1.0.25"
serde = "1"
serde_json = { version = "1", features = ["preserve_order"] }
tempfile = "3.5.0"
tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
tonic = { version = "0.14", features = ["tls-native-roots", "tls-aws-lc"] }
tracing = "0.1"
uuid = { version = "1.7.0", features = ["v4", "v7"] }
chrono = { version = "0.4", default-features = false, features = ["clock"] }
[patch."https://github.com/lance-format/lance.git"]
# Carry the offset-safe validity fix until it is available in a Lance release.
lance-arrow = { path = "python/vendor/lance-arrow" }
[patch.crates-io]
# Lance v14.0.0-beta.18 requires multipart write modes from this upstream revision.
object_store = { git = "https://github.com/apache/arrow-rs-object-store.git", rev = "c10758c207c0103355f940f0c736fb8ecb157b04" }
[profile.ci]
debug = "line-tables-only"
inherits = "dev"
incremental = false
# This rule applies to every package except workspace members (dependencies
# such as `arrow` and `tokio`). It disables debug info and related features on
# dependencies so their binaries stay smaller, improving cache reuse.
[profile.ci.package."*"]
debug = false
debug-assertions = false
strip = "debuginfo"
incremental = false
Reference in New Issue View Git Blame Copy Permalink