rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-29 23:06:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(sql): use Lance beta 14 native pair tiles

Browse Source
This commit is contained in:
Yang Cen
2026-09-25 00:53:41 +08:00
parent 1069ba8745
commit 78a6976bbf
4 changed files with 83 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+44 -44
View File
@@ -3588,8 +3588,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -5143,8 +5143,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arc-swap",
"arrow",
@@ -5216,8 +5216,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5239,7 +5239,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5253,7 +5253,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5262,8 +5262,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arrayref",
"crunchy",
@@ -5273,8 +5273,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5311,8 +5311,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arrow",
"arrow-array",
@@ -5343,8 +5343,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arrow",
"arrow-array",
@@ -5361,8 +5361,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"proc-macro2",
"quote",
@@ -5371,8 +5371,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5405,8 +5405,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5438,8 +5438,8 @@ dependencies = [
[[package]]
name = "lance-geo"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"datafusion",
"geo-traits",
@@ -5453,8 +5453,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arc-swap",
"arrow",
@@ -5524,8 +5524,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5547,8 +5547,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arrow",
"arrow-array",
@@ -5590,8 +5590,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5605,8 +5605,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arrow",
"async-trait",
@@ -5620,8 +5620,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5674,8 +5674,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5689,8 +5689,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arrow",
"arrow-array",
@@ -5730,8 +5730,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5744,8 +5744,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.0.0-beta.14"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
dependencies = [
"frostem",
"icu_segmenter",
Cargo.toml
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=13.0.0-beta.12", default-features = false, "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=13.0.0-beta.12", default-features = false, "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=13.0.0-beta.12", default-features = false, "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=13.0.0-beta.14", default-features = false, "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=13.0.0-beta.14", default-features = false, "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=13.0.0-beta.14", default-features = false, "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
rust/lancedb/src/table/datafusion/udtf/duplicate_pairs.rs
+24 -9
View File
@@ -35,12 +35,17 @@ use crate::table::NativeTableExt;
// Admission estimates for Lance 13's bounded pairwise kernel. Keep these with
// the minimum dependency when the kernel's buffering contract changes.
/// Version of the native task descriptor and its execution semantics.
pub const DUPLICATE_PAIRS_OPERATOR_VERSION: u32 = 1;
pub const DUPLICATE_PAIRS_OPERATOR_VERSION: u32 = 2;
const CODE_STAGING_BYTES: usize = 16 * 1024 * 1024;
const INDEX_READ_ROWS: usize = 8192;
const VECTOR_BATCH_ROWS: usize = 1024;
const VECTOR_BATCH_ROWS: usize = 8192;
const VECTOR_BATCH_BYTES: usize = 16 * 1024 * 1024;
const MIN_VECTOR_BATCH_ROWS: usize = 32;
const ANCHOR_BLOCK_ROWS: usize = 32;
const SCORING_CONCURRENCY: usize = 4;
const MAX_COORDINATE_BYTES: usize = 8;
const STAGED_ROW_OVERHEAD_BYTES: usize = 64;
const PAIR_ROW_BYTES: usize = 20;
/// Configuration belongs to one immutable source snapshot. Threshold uses
@@ -431,9 +436,9 @@ impl ExecutionPlan for DuplicatePairsExec {
let output_rows = MetricBuilder::new(&self.metrics).counter("output_rows", partition);
let completed =
MetricBuilder::new(&self.metrics).counter("completed_partitions", partition);
// Admission estimate for the released Lance kernel: compact-code staging,
// one 8192-row read and two decoded 1024-row batches. This reservation
// limits concurrent kernels; Lance still owns its bounded buffers/spill.
// Beta.14 stages native codes and scores bounded tiles concurrently.
// Keep the host staging budget explicit rather than inheriting Lance's
// larger default or multiplying CPU-pool concurrency across PE tasks.
let dimension = match dataset
.schema()
.field(&config.column)
@@ -442,9 +447,16 @@ impl ExecutionPlan for DuplicatePairsExec {
Some(DataType::FixedSizeList(_, dimension)) if dimension > 0 => dimension as usize,
_ => return plan_err!("duplicate pairs requires a fixed-size vector column"),
};
let budget = CODE_STAGING_BYTES
+ (INDEX_READ_ROWS + 2 * VECTOR_BATCH_ROWS) * dimension * MAX_COORDINATE_BYTES
+ VECTOR_BATCH_ROWS * PAIR_ROW_BYTES;
let batch_bytes = VECTOR_BATCH_BYTES.max(
MIN_VECTOR_BATCH_ROWS * (dimension * MAX_COORDINATE_BYTES + STAGED_ROW_OVERHEAD_BYTES),
);
// Allow staging/preparation overlap, at least three live spill batches,
// source input, and in-flight output plus the batch being encoded.
// Quantizer models and caches remain outside this admission estimate.
let budget = 2 * CODE_STAGING_BYTES
+ 3 * batch_bytes
+ INDEX_READ_ROWS * (dimension * MAX_COORDINATE_BYTES + STAGED_ROW_OVERHEAD_BYTES)
+ (SCORING_CONCURRENCY + 1) * ANCHOR_BLOCK_ROWS * VECTOR_BATCH_ROWS * PAIR_ROW_BYTES;
let reservation =
datafusion_execution::memory_pool::MemoryConsumer::new("DuplicatePairsExec")
.register(context.memory_pool());
@@ -461,12 +473,15 @@ impl ExecutionPlan for DuplicatePairsExec {
let output_rows = output_rows.clone();
let completed = completed.clone();
async move {
let reader = lance::index::vector::dedup::find_duplicate_pairs_in_partition(
let reader = lance::index::vector::dedup::find_duplicate_pairs_in_partition_with_options(
dataset,
&config.column,
task.segment_id,
task.partition_id,
config.distance_threshold,
lance::index::vector::dedup::DuplicatePairsOptions::default()
.with_memory_limit(CODE_STAGING_BYTES)
.with_max_concurrency(SCORING_CONCURRENCY),
)
.await?;
let measured = stream::try_unfold(
rust/lancedb/tests/duplicate_pairs.rs
+1 -1
View File
@@ -162,7 +162,7 @@ async fn sql_pairs_match_native_snapshot_and_partition_tasks() -> anyhow::Result
// execution must consume scoped readers sequentially.
let runtime = Arc::new(
datafusion_execution::runtime_env::RuntimeEnvBuilder::new()
.with_memory_limit(32 * 1024 * 1024, 1.0)
.with_memory_limit(128 * 1024 * 1024, 1.0)
.build()?,
);
let ctx =