rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-01-05 19:32:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add 'target_partition_size' param (#2642)

Browse Source 
this exposes the param `target_partition_size` from lance

---------

Signed-off-by: BubbleCal <bubble-cal@outlook.com>
This commit is contained in:
BubbleCal
2025-09-11 22:56:16 +08:00
committed by GitHub
parent 6ea6884260
commit f7d78c3420
5 changed files with 110 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
python/python/lancedb/index.py
View File

@@ -251,6 +251,13 @@ class HnswPq:
results. In most cases, there is no benefit to setting this higher than 500.
This value should be set to a value that is not less than `ef` in the
search phase.
target_partition_size, default is 1,048,576
The target size of each partition.
This value controls the tradeoff between search performance and accuracy.
faster search but less accurate results as higher value.
"""
distance_type: Literal["l2", "cosine", "dot"] = "l2"
@@ -261,6 +268,7 @@ class HnswPq:
sample_rate: int = 256
m: int = 20
ef_construction: int = 300
target_partition_size: Optional[int] = None
@dataclass
@@ -351,6 +359,12 @@ class HnswSq:
This value should be set to a value that is not less than `ef` in the search
phase.
target_partition_size, default is 1,048,576
The target size of each partition.
This value controls the tradeoff between search performance and accuracy.
faster search but less accurate results as higher value.
"""
distance_type: Literal["l2", "cosine", "dot"] = "l2"
@@ -359,6 +373,7 @@ class HnswSq:
sample_rate: int = 256
m: int = 20
ef_construction: int = 300
target_partition_size: Optional[int] = None
@dataclass
@@ -444,12 +459,20 @@ class IvfFlat:
cases the default should be sufficient.
The default value is 256.
target_partition_size, default is 8192
The target size of each partition.
This value controls the tradeoff between search performance and accuracy.
faster search but less accurate results as higher value.
"""
distance_type: Literal["l2", "cosine", "dot", "hamming"] = "l2"
num_partitions: Optional[int] = None
max_iterations: int = 50
sample_rate: int = 256
target_partition_size: Optional[int] = None
@dataclass
@@ -564,6 +587,13 @@ class IvfPq:
cases the default should be sufficient.
The default value is 256.
target_partition_size, default is 8192
The target size of each partition.
This value controls the tradeoff between search performance and accuracy.
faster search but less accurate results as higher value.
"""
distance_type: Literal["l2", "cosine", "dot"] = "l2"
@@ -572,6 +602,7 @@ class IvfPq:
num_bits: int = 8
max_iterations: int = 50
sample_rate: int = 256
target_partition_size: Optional[int] = None
__all__ = [

7
python/python/lancedb/table.py
View File

@@ -691,6 +691,7 @@ class Table(ABC):
ef_construction: int = 300,
name: Optional[str] = None,
train: bool = True,
target_partition_size: Optional[int] = None,
):
"""Create an index on the table.
@@ -2002,6 +2003,7 @@ class LanceTable(Table):
*,
name: Optional[str] = None,
train: bool = True,
target_partition_size: Optional[int] = None,
):
"""Create an index on the table."""
if accelerator is not None:
@@ -2018,6 +2020,7 @@ class LanceTable(Table):
num_bits=num_bits,
m=m,
ef_construction=ef_construction,
target_partition_size=target_partition_size,
)
self.checkout_latest()
return
@@ -2027,6 +2030,7 @@ class LanceTable(Table):
num_partitions=num_partitions,
max_iterations=max_iterations,
sample_rate=sample_rate,
target_partition_size=target_partition_size,
)
elif index_type == "IVF_PQ":
config = IvfPq(
@@ -2036,6 +2040,7 @@ class LanceTable(Table):
num_bits=num_bits,
max_iterations=max_iterations,
sample_rate=sample_rate,
target_partition_size=target_partition_size,
)
elif index_type == "IVF_HNSW_PQ":
config = HnswPq(
@@ -2047,6 +2052,7 @@ class LanceTable(Table):
sample_rate=sample_rate,
m=m,
ef_construction=ef_construction,
target_partition_size=target_partition_size,
)
elif index_type == "IVF_HNSW_SQ":
config = HnswSq(
@@ -2056,6 +2062,7 @@ class LanceTable(Table):
sample_rate=sample_rate,
m=m,
ef_construction=ef_construction,
target_partition_size=target_partition_size,
)
else:
raise ValueError(f"Unknown index type {index_type}")

39
python/python/tests/test_table.py
View File

@@ -674,6 +674,45 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
"vector", replace=True, config=expected_config, name=None, train=True
)
# Test with target_partition_size
table.create_index(
metric="l2",
num_sub_vectors=96,
vector_column_name="vector",
replace=True,
index_cache_size=256,
num_bits=4,
target_partition_size=8192,
)
expected_config = IvfPq(
distance_type="l2",
num_sub_vectors=96,
num_bits=4,
target_partition_size=8192,
)
mock_create_index.assert_called_with(
"vector", replace=True, config=expected_config, name=None, train=True
)
# target_partition_size has a default value,
# so `num_partitions` and `target_partition_size` are not required
table.create_index(
metric="l2",
num_sub_vectors=96,
vector_column_name="vector",
replace=True,
index_cache_size=256,
num_bits=4,
)
expected_config = IvfPq(
distance_type="l2",
num_sub_vectors=96,
num_bits=4,
)
mock_create_index.assert_called_with(
"vector", replace=True, config=expected_config, name=None, train=True
)
table.create_index(
vector_column_name="my_vector",
metric="dot",

16
python/src/index.rs
View File

@@ -63,6 +63,9 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
if let Some(num_partitions) = params.num_partitions {
ivf_flat_builder = ivf_flat_builder.num_partitions(num_partitions);
}
if let Some(target_partition_size) = params.target_partition_size {
ivf_flat_builder = ivf_flat_builder.target_partition_size(target_partition_size);
}
Ok(LanceDbIndex::IvfFlat(ivf_flat_builder))
},
"IvfPq" => {
@@ -76,6 +79,9 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
if let Some(num_partitions) = params.num_partitions {
ivf_pq_builder = ivf_pq_builder.num_partitions(num_partitions);
}
if let Some(target_partition_size) = params.target_partition_size {
ivf_pq_builder = ivf_pq_builder.target_partition_size(target_partition_size);
}
if let Some(num_sub_vectors) = params.num_sub_vectors {
ivf_pq_builder = ivf_pq_builder.num_sub_vectors(num_sub_vectors);
}
@@ -94,6 +100,9 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
if let Some(num_partitions) = params.num_partitions {
hnsw_pq_builder = hnsw_pq_builder.num_partitions(num_partitions);
}
if let Some(target_partition_size) = params.target_partition_size {
hnsw_pq_builder = hnsw_pq_builder.target_partition_size(target_partition_size);
}
if let Some(num_sub_vectors) = params.num_sub_vectors {
hnsw_pq_builder = hnsw_pq_builder.num_sub_vectors(num_sub_vectors);
}
@@ -111,6 +120,9 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
if let Some(num_partitions) = params.num_partitions {
hnsw_sq_builder = hnsw_sq_builder.num_partitions(num_partitions);
}
if let Some(target_partition_size) = params.target_partition_size {
hnsw_sq_builder = hnsw_sq_builder.target_partition_size(target_partition_size);
}
Ok(LanceDbIndex::IvfHnswSq(hnsw_sq_builder))
},
not_supported => Err(PyValueError::new_err(format!(
@@ -144,6 +156,7 @@ struct IvfFlatParams {
num_partitions: Option<u32>,
max_iterations: u32,
sample_rate: u32,
target_partition_size: Option<u32>,
}
#[derive(FromPyObject)]
@@ -154,6 +167,7 @@ struct IvfPqParams {
num_bits: u32,
max_iterations: u32,
sample_rate: u32,
target_partition_size: Option<u32>,
}
#[derive(FromPyObject)]
@@ -166,6 +180,7 @@ struct IvfHnswPqParams {
sample_rate: u32,
m: u32,
ef_construction: u32,
target_partition_size: Option<u32>,
}
#[derive(FromPyObject)]
@@ -176,6 +191,7 @@ struct IvfHnswSqParams {
sample_rate: u32,
m: u32,
ef_construction: u32,
target_partition_size: Option<u32>,
}
#[pyclass(get_all)]

17
rust/lancedb/src/index/vector.rs
View File

@@ -112,6 +112,15 @@ macro_rules! impl_ivf_params_setter {
self.max_iterations = max_iterations;
self
}
/// The target size of each partition.
///
/// This value controls the tradeoff between search performance and accuracy.
/// The higher the value the faster the search but the less accurate the results will be.
pub fn target_partition_size(mut self, target_partition_size: u32) -> Self {
self.target_partition_size = Some(target_partition_size);
self
}
};
}
@@ -182,6 +191,7 @@ pub struct IvfFlatIndexBuilder {
pub(crate) num_partitions: Option<u32>,
pub(crate) sample_rate: u32,
pub(crate) max_iterations: u32,
pub(crate) target_partition_size: Option<u32>,
}
impl Default for IvfFlatIndexBuilder {
@@ -191,6 +201,7 @@ impl Default for IvfFlatIndexBuilder {
num_partitions: None,
sample_rate: 256,
max_iterations: 50,
target_partition_size: None,
}
}
}
@@ -228,6 +239,7 @@ pub struct IvfPqIndexBuilder {
pub(crate) num_partitions: Option<u32>,
pub(crate) sample_rate: u32,
pub(crate) max_iterations: u32,
pub(crate) target_partition_size: Option<u32>,
// PQ
pub(crate) num_sub_vectors: Option<u32>,
@@ -243,6 +255,7 @@ impl Default for IvfPqIndexBuilder {
num_bits: None,
sample_rate: 256,
max_iterations: 50,
target_partition_size: None,
}
}
}
@@ -293,6 +306,7 @@ pub struct IvfHnswPqIndexBuilder {
pub(crate) num_partitions: Option<u32>,
pub(crate) sample_rate: u32,
pub(crate) max_iterations: u32,
pub(crate) target_partition_size: Option<u32>,
// HNSW
pub(crate) m: u32,
@@ -314,6 +328,7 @@ impl Default for IvfHnswPqIndexBuilder {
max_iterations: 50,
m: 20,
ef_construction: 300,
target_partition_size: None,
}
}
}
@@ -341,6 +356,7 @@ pub struct IvfHnswSqIndexBuilder {
pub(crate) num_partitions: Option<u32>,
pub(crate) sample_rate: u32,
pub(crate) max_iterations: u32,
pub(crate) target_partition_size: Option<u32>,
// HNSW
pub(crate) m: u32,
@@ -358,6 +374,7 @@ impl Default for IvfHnswSqIndexBuilder {
max_iterations: 50,
m: 20,
ef_construction: 300,
target_partition_size: None,
}
}
}