mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-18 12:08:35 +00:00
fix(python): support deterministic IVF-PQ training
This commit is contained in:
@@ -769,6 +769,11 @@ class IvfPq:
|
||||
|
||||
The default value is 256.
|
||||
|
||||
seed: int, optional
|
||||
Seed used for deterministic sampling and training. Given identical data in
|
||||
the same row order and identical index parameters, the same seed produces
|
||||
the same IVF centroids and PQ codebook. If omitted, training remains random.
|
||||
|
||||
target_partition_size: int, default is 8192
|
||||
|
||||
The target size of each partition.
|
||||
@@ -783,6 +788,7 @@ class IvfPq:
|
||||
num_bits: int = 8
|
||||
max_iterations: int = 50
|
||||
sample_rate: int = 256
|
||||
seed: Optional[int] = None
|
||||
target_partition_size: Optional[int] = None
|
||||
# Name of the accelerator (e.g. "cuda") to use for IVF training. When set,
|
||||
# create_index() dispatches to pylance to build the index on the accelerator.
|
||||
|
||||
@@ -375,7 +375,7 @@ async def test_create_vector_index(some_table: AsyncTable):
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_4bit_ivfpq_index(some_table: AsyncTable):
|
||||
# Can create
|
||||
await some_table.create_index("vector", config=IvfPq(num_bits=4))
|
||||
await some_table.create_index("vector", config=IvfPq(num_bits=4, seed=42))
|
||||
# Can recreate if replace=True
|
||||
await some_table.create_index("vector", config=IvfPq(num_bits=4), replace=True)
|
||||
# Can't recreate if replace=False
|
||||
|
||||
@@ -90,6 +90,9 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
|
||||
.max_iterations(params.max_iterations)
|
||||
.sample_rate(params.sample_rate)
|
||||
.num_bits(params.num_bits);
|
||||
if let Some(seed) = params.seed {
|
||||
ivf_pq_builder = ivf_pq_builder.seed(seed);
|
||||
}
|
||||
if let Some(num_partitions) = params.num_partitions {
|
||||
ivf_pq_builder = ivf_pq_builder.num_partitions(num_partitions);
|
||||
}
|
||||
@@ -232,6 +235,7 @@ struct IvfPqParams {
|
||||
num_bits: u32,
|
||||
max_iterations: u32,
|
||||
sample_rate: u32,
|
||||
seed: Option<u64>,
|
||||
target_partition_size: Option<u32>,
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user