feat(python): pass merge_insert sources as scannables

Python's `execute_merge_insert` now takes a `PyScannable` (like `add()`),
and the source collection threshold is exposed as a
`source_collect_threshold_bytes` kwarg on `execute()` and a
`merge_insert_source_collect_threshold_bytes` connection option.

Node passes the decoded IPC batches as a `Vec<RecordBatch>`, so the
merge join sees their exact row count.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Will Jones
2026-10-10 17:34:51 +00:00
co-authored by Claude Opus 5.5
parent 300d494d6e
commit 2c2bb9a974
10 changed files with 157 additions and 11 deletions
+6 -7
View File
@@ -20,7 +20,6 @@ use crate::{
use arrow::{
array::{Array, LargeBinaryArray},
datatypes::{DataType, Schema},
ffi_stream::ArrowArrayStreamReader,
pyarrow::{FromPyArrow, PyArrowType, ToPyArrow},
};
use lancedb::blob::{BlobFile, BlobRangeRequest};
@@ -1566,10 +1565,9 @@ impl Table {
pub fn execute_merge_insert<'a>(
self_: PyRef<'a, Self>,
data: Bound<'a, PyAny>,
data: PyScannable,
parameters: MergeInsertParams,
) -> PyResult<Bound<'a, PyAny>> {
let batches: ArrowArrayStreamReader = ArrowArrayStreamReader::from_pyarrow_bound(&data)?;
let on = parameters.on.iter().map(|s| s.as_str()).collect::<Vec<_>>();
let mut builder = self_.inner_ref()?.merge_insert(&on);
if parameters.when_matched_update_all {
@@ -1599,12 +1597,12 @@ impl Table {
if let Some(validate_single_shard) = parameters.validate_single_shard {
builder.validate_single_shard(validate_single_shard);
}
if let Some(bytes) = parameters.source_collect_threshold_bytes {
builder.source_collect_threshold_bytes(bytes);
}
future_into_py(self_.py(), async move {
let res = builder
.execute(Box::new(batches) as Box<dyn arrow::array::RecordBatchReader + Send>)
.await
.infer_error()?;
let res = builder.execute(data).await.infer_error()?;
Ok(MergeResult::from(res))
})
}
@@ -2045,6 +2043,7 @@ pub struct MergeInsertParams {
use_index: Option<bool>,
use_lsm: Option<bool>,
validate_single_shard: Option<bool>,
source_collect_threshold_bytes: Option<usize>,
}
#[pyclass]