mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-14 07:42:28 +00:00
feat!: use source statistics to plan merge_insert joins
`MergeInsertBuilder::execute` now takes any `Scannable`, and the source is routed so Lance's merge join can pick its build side from statistics: * Rescannable sources become a provider reporting their exact row count. * One-shot sources are collected up to `source_collect_threshold_bytes` (default 64MiB). If they end first they are merged from memory; otherwise the prefix is re-chained, spilled for retries, and reported as an inexact lower bound. The threshold can be set per merge or as a connection default. BREAKING CHANGE: `MergeInsertBuilder::execute` and `BaseTable::merge_insert` take a `Scannable` instead of `Box<dyn RecordBatchReader + Send>`. Pass a `Box<dyn RecordBatchReader + Send>` (e.g. `Box::new(reader) as Box<dyn RecordBatchReader + Send>`), a `RecordBatch`, or a `Vec<RecordBatch>`. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c2c8269fc8
commit
8bdc515f1e
+4
-1
@@ -1601,7 +1601,10 @@ impl Table {
|
||||
}
|
||||
|
||||
future_into_py(self_.py(), async move {
|
||||
let res = builder.execute(Box::new(batches)).await.infer_error()?;
|
||||
let res = builder
|
||||
.execute(Box::new(batches) as Box<dyn arrow::array::RecordBatchReader + Send>)
|
||||
.await
|
||||
.infer_error()?;
|
||||
Ok(MergeResult::from(res))
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user