rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-11-13 22:26:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(node): expose targetPartitionSize on IVF index options (#4594)

Browse Source 
The Node SDK's vector index options only take `numPartitions`. The Rust index builders (`target_partition_size` in `rust/lancedb/src/index/vector.rs`) and the Python SDK already support a target partition size, so this adds `targetPartitionSize` to `IvfPqOptions`, `IvfRqOptions`, `IvfFlatOptions`, `HnswPqOptions` and `HnswSqOptions` and passes it to their builders. When both are set, `numPartitions` is used, as it is for Rust and Python callers.

The reason to want it: an index built from a partition count records no target partition size, so `table.optimize()` splits and joins it against the index type's default and can merge small partitions down to one or two. Building with a target records it in the index details, and optimize keeps to it. https://github.com/lance-format/lance/pull/10154 fixes this on the Lance side for indexes built from a partition count; this lets Node users set the size directly.

The new tests build IvfPq, IvfFlat, IvfRq, IvfHnswPq and IvfHnswSq indexes with `targetPartitionSize` and check the size recorded in the index details, and check that `numPartitions` wins when both are set. Each option's doc comment states that index type's default target. Docs regenerated with `pnpm run docs`.
This commit is contained in:
geoffgscott
2026-10-29 13:13:00 +08:00
committed by GitHub
parent 03bd909dab
commit 7ea10fa1e0
10 changed files with 271 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nodejs/lancedb/indices.ts
+93 -6
View File
@@ -11,8 +11,9 @@ export interface IvfPqOptions {
* The number of IVF partitions to create.
*
* This value should generally scale with the number of rows in the dataset.
* By default the number of partitions is the square root of the number of
* rows.
* When set, this takes precedence over `targetPartitionSize`. When omitted,
* the partition count is derived from `targetPartitionSize`, which defaults
* to 8192 rows for this index type.
*
* If this value is too large then the first part of the search (picking the
* right partition) will be slow. If this value is too small then the second
@@ -20,6 +21,21 @@ export interface IvfPqOptions {
*/
numPartitions?: number;
/**
* The target number of rows in each IVF partition.
*
* This is an alternative to `numPartitions`. When it is set and
* `numPartitions` is not, the number of partitions is derived by dividing
* the number of rows by this value. When both are set, `numPartitions` is
* used and this value is ignored. When neither is set, the target is
* 8192 rows.
*
* Larger partitions mean fewer of them, which makes the first part of the
* search (picking the right partitions) faster and the second part
* (searching within a partition) slower.
*/
targetPartitionSize?: number;
/**
* Number of sub-vectors of PQ.
*
@@ -117,8 +133,9 @@ export interface IvfRqOptions {
* The number of IVF partitions to create.
*
* This value should generally scale with the number of rows in the dataset.
* By default the number of partitions is the square root of the number of
* rows.
* When set, this takes precedence over `targetPartitionSize`. When omitted,
* the partition count is derived from `targetPartitionSize`, which defaults
* to 4096 rows for this index type.
*
* If this value is too large then the first part of the search (picking the
* right partition) will be slow. If this value is too small then the second
@@ -126,6 +143,21 @@ export interface IvfRqOptions {
*/
numPartitions?: number;
/**
* The target number of rows in each IVF partition.
*
* This is an alternative to `numPartitions`. When it is set and
* `numPartitions` is not, the number of partitions is derived by dividing
* the number of rows by this value. When both are set, `numPartitions` is
* used and this value is ignored. When neither is set, the target is
* 4096 rows.
*
* Larger partitions mean fewer of them, which makes the first part of the
* search (picking the right partitions) faster and the second part
* (searching within a partition) slower.
*/
targetPartitionSize?: number;
/**
* Number of bits per dimension for residual quantization.
*
@@ -215,6 +247,8 @@ export interface HnswPqOptions {
/**
* The number of IVF partitions to create.
*
* When set, this takes precedence over `targetPartitionSize`.
*
* For HNSW, we recommend a small number of partitions. Setting this to 1 works
* well for most tables. For very large tables, training just one HNSW graph
* will require too much memory. Each partition becomes its own HNSW graph, so
@@ -223,6 +257,21 @@ export interface HnswPqOptions {
*/
numPartitions?: number;
/**
* The target number of rows in each IVF partition.
*
* This is an alternative to `numPartitions`. When it is set and
* `numPartitions` is not, the number of partitions is derived by dividing
* the number of rows by this value. When both are set, `numPartitions` is
* used and this value is ignored. When neither is set, the target is
* 1,048,576 rows.
*
* Larger partitions mean fewer of them, which makes the first part of the
* search (picking the right partitions) faster and the second part
* (searching within a partition) slower.
*/
targetPartitionSize?: number;
/**
* Number of sub-vectors of PQ.
*
@@ -331,6 +380,8 @@ export interface HnswSqOptions {
/**
* The number of IVF partitions to create.
*
* When set, this takes precedence over `targetPartitionSize`.
*
* For HNSW, we recommend a small number of partitions. Setting this to 1 works
* well for most tables. For very large tables, training just one HNSW graph
* will require too much memory. Each partition becomes its own HNSW graph, so
@@ -339,6 +390,21 @@ export interface HnswSqOptions {
*/
numPartitions?: number;
/**
* The target number of rows in each IVF partition.
*
* This is an alternative to `numPartitions`. When it is set and
* `numPartitions` is not, the number of partitions is derived by dividing
* the number of rows by this value. When both are set, `numPartitions` is
* used and this value is ignored. When neither is set, the target is
* 1,048,576 rows.
*
* Larger partitions mean fewer of them, which makes the first part of the
* search (picking the right partitions) faster and the second part
* (searching within a partition) slower.
*/
targetPartitionSize?: number;
/**
* Max iterations to train kmeans.
*
@@ -407,8 +473,9 @@ export interface IvfFlatOptions {
* The number of IVF partitions to create.
*
* This value should generally scale with the number of rows in the dataset.
* By default the number of partitions is the square root of the number of
* rows.
* When set, this takes precedence over `targetPartitionSize`. When omitted,
* the partition count is derived from `targetPartitionSize`, which defaults
* to 4096 rows for this index type.
*
* If this value is too large then the first part of the search (picking the
* right partition) will be slow. If this value is too small then the second
@@ -416,6 +483,21 @@ export interface IvfFlatOptions {
*/
numPartitions?: number;
/**
* The target number of rows in each IVF partition.
*
* This is an alternative to `numPartitions`. When it is set and
* `numPartitions` is not, the number of partitions is derived by dividing
* the number of rows by this value. When both are set, `numPartitions` is
* used and this value is ignored. When neither is set, the target is
* 4096 rows.
*
* Larger partitions mean fewer of them, which makes the first part of the
* search (picking the right partitions) faster and the second part
* (searching within a partition) slower.
*/
targetPartitionSize?: number;
/**
* Distance type to use to build the index.
*
@@ -633,6 +715,7 @@ export class Index {
options?.numBits,
options?.maxIterations,
options?.sampleRate,
options?.targetPartitionSize,
),
);
}
@@ -661,6 +744,7 @@ export class Index {
options?.numBits,
options?.maxIterations,
options?.sampleRate,
options?.targetPartitionSize,
),
);
}
@@ -688,6 +772,7 @@ export class Index {
options?.numPartitions,
options?.maxIterations,
options?.sampleRate,
options?.targetPartitionSize,
),
);
}
@@ -798,6 +883,7 @@ export class Index {
options?.sampleRate,
options?.m,
options?.efConstruction,
options?.targetPartitionSize,
),
);
}
@@ -820,6 +906,7 @@ export class Index {
options?.sampleRate,
options?.m,
options?.efConstruction,
options?.targetPartitionSize,
),
);
}
nodejs/lancedb/table.ts
+5
View File
@@ -381,6 +381,11 @@ export abstract class Table {
* }),
* });
* @example
* // Or size the partitions by row count instead of choosing how many.
* await table.createIndex("vector", {
* config: lancedb.Index.ivfPq({ targetPartitionSize: 4096 }),
* });
* @example
* // Or create a Scalar index
* await table.createIndex("my_float_col");
*/