rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-11-13 14:16:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(node): expose targetPartitionSize on IVF index options (#4594)

Browse Source 
The Node SDK's vector index options only take `numPartitions`. The Rust index builders (`target_partition_size` in `rust/lancedb/src/index/vector.rs`) and the Python SDK already support a target partition size, so this adds `targetPartitionSize` to `IvfPqOptions`, `IvfRqOptions`, `IvfFlatOptions`, `HnswPqOptions` and `HnswSqOptions` and passes it to their builders. When both are set, `numPartitions` is used, as it is for Rust and Python callers.

The reason to want it: an index built from a partition count records no target partition size, so `table.optimize()` splits and joins it against the index type's default and can merge small partitions down to one or two. Building with a target records it in the index details, and optimize keeps to it. https://github.com/lance-format/lance/pull/10154 fixes this on the Lance side for indexes built from a partition count; this lets Node users set the size directly.

The new tests build IvfPq, IvfFlat, IvfRq, IvfHnswPq and IvfHnswSq indexes with `targetPartitionSize` and check the size recorded in the index details, and check that `numPartitions` wins when both are set. Each option's doc comment states that index type's default target. Docs regenerated with `pnpm run docs`.
This commit is contained in:
geoffgscott
2026-10-29 13:13:00 +08:00
committed by GitHub
parent 03bd909dab
commit 7ea10fa1e0
10 changed files with 271 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nodejs/src/index.rs
+21
View File
@@ -104,6 +104,7 @@ impl Index {
num_bits: Option<u32>,
max_iterations: Option<u32>,
sample_rate: Option<u32>,
target_partition_size: Option<u32>,
) -> napi::Result<Self> {
let mut ivf_pq_builder = IvfPqIndexBuilder::default();
if let Some(distance_type) = distance_type {
@@ -113,6 +114,9 @@ impl Index {
if let Some(num_partitions) = num_partitions {
ivf_pq_builder = ivf_pq_builder.num_partitions(num_partitions);
}
if let Some(target_partition_size) = target_partition_size {
ivf_pq_builder = ivf_pq_builder.target_partition_size(target_partition_size);
}
if let Some(num_sub_vectors) = num_sub_vectors {
ivf_pq_builder = ivf_pq_builder.num_sub_vectors(num_sub_vectors);
}
@@ -137,6 +141,7 @@ impl Index {
num_bits: Option<u32>,
max_iterations: Option<u32>,
sample_rate: Option<u32>,
target_partition_size: Option<u32>,
) -> napi::Result<Self> {
let mut ivf_rq_builder = IvfRqIndexBuilder::default();
if let Some(distance_type) = distance_type {
@@ -146,6 +151,9 @@ impl Index {
if let Some(num_partitions) = num_partitions {
ivf_rq_builder = ivf_rq_builder.num_partitions(num_partitions);
}
if let Some(target_partition_size) = target_partition_size {
ivf_rq_builder = ivf_rq_builder.target_partition_size(target_partition_size);
}
if let Some(num_bits) = num_bits {
ivf_rq_builder = ivf_rq_builder.num_bits(num_bits);
}
@@ -166,6 +174,7 @@ impl Index {
num_partitions: Option<u32>,
max_iterations: Option<u32>,
sample_rate: Option<u32>,
target_partition_size: Option<u32>,
) -> napi::Result<Self> {
let mut ivf_flat_builder = IvfFlatIndexBuilder::default();
if let Some(distance_type) = distance_type {
@@ -175,6 +184,9 @@ impl Index {
if let Some(num_partitions) = num_partitions {
ivf_flat_builder = ivf_flat_builder.num_partitions(num_partitions);
}
if let Some(target_partition_size) = target_partition_size {
ivf_flat_builder = ivf_flat_builder.target_partition_size(target_partition_size);
}
if let Some(max_iterations) = max_iterations {
ivf_flat_builder = ivf_flat_builder.max_iterations(max_iterations);
}
@@ -283,6 +295,7 @@ impl Index {
}
#[napi(factory)]
#[allow(clippy::too_many_arguments)]
pub fn hnsw_pq(
distance_type: Option<String>,
num_partitions: Option<u32>,
@@ -291,6 +304,7 @@ impl Index {
sample_rate: Option<u32>,
m: Option<u32>,
ef_construction: Option<u32>,
target_partition_size: Option<u32>,
) -> napi::Result<Self> {
let mut hnsw_pq_builder = IvfHnswPqIndexBuilder::default();
if let Some(distance_type) = distance_type {
@@ -300,6 +314,9 @@ impl Index {
if let Some(num_partitions) = num_partitions {
hnsw_pq_builder = hnsw_pq_builder.num_partitions(num_partitions);
}
if let Some(target_partition_size) = target_partition_size {
hnsw_pq_builder = hnsw_pq_builder.target_partition_size(target_partition_size);
}
if let Some(num_sub_vectors) = num_sub_vectors {
hnsw_pq_builder = hnsw_pq_builder.num_sub_vectors(num_sub_vectors);
}
@@ -328,6 +345,7 @@ impl Index {
sample_rate: Option<u32>,
m: Option<u32>,
ef_construction: Option<u32>,
target_partition_size: Option<u32>,
) -> napi::Result<Self> {
let mut hnsw_sq_builder = IvfHnswSqIndexBuilder::default();
if let Some(distance_type) = distance_type {
@@ -337,6 +355,9 @@ impl Index {
if let Some(num_partitions) = num_partitions {
hnsw_sq_builder = hnsw_sq_builder.num_partitions(num_partitions);
}
if let Some(target_partition_size) = target_partition_size {
hnsw_sq_builder = hnsw_sq_builder.target_partition_size(target_partition_size);
}
if let Some(max_iterations) = max_iterations {
hnsw_sq_builder = hnsw_sq_builder.max_iterations(max_iterations);
}