feat(python): pass merge_insert sources as scannables

Python's `execute_merge_insert` now takes a `PyScannable` (like `add()`),
and the source collection threshold is exposed as a
`source_collect_threshold_bytes` kwarg on `execute()` and a
`merge_insert_source_collect_threshold_bytes` connection option.

Node passes the decoded IPC batches as a `Vec<RecordBatch>`, so the
merge join sees their exact row count.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Will Jones
2026-10-10 17:34:51 +00:00
co-authored by Claude Opus 5.5
parent 300d494d6e
commit 2c2bb9a974
10 changed files with 157 additions and 11 deletions
+10
View File
@@ -1081,6 +1081,16 @@ describe("merge insert", () => {
expect(res).toEqual(expected);
});
test("empty source with delete by source", async () => {
const empty = arrow.makeEmptyTable(await table.schema());
const res = await table
.mergeInsert("a")
.whenNotMatchedBySourceDelete()
.execute(empty);
expect(res.numDeletedRows).toBe(3);
expect(await table.countRows()).toBe(0);
});
test("timeout", async () => {
const newData = [
{ a: 2, b: "x" },
+19 -1
View File
@@ -3,6 +3,7 @@
use std::time::Duration;
use arrow_array::RecordBatch;
use lancedb::{ipc::ipc_file_to_batches, table::merge::MergeInsertBuilder};
use napi::bindgen_prelude::*;
use napi_derive::napi;
@@ -66,9 +67,26 @@ impl NativeMergeInsertBuilder {
#[napi(catch_unwind)]
pub async fn execute(&self, buf: Buffer) -> napi::Result<MergeResult> {
let data = ipc_file_to_batches(buf.to_vec()).map_err(|e| {
let reader = ipc_file_to_batches(buf.to_vec()).map_err(|e| {
napi::Error::from_reason(format!("Failed to read IPC file: {}", convert_error(&e)))
})?;
let schema = reader.schema();
// Decoded batches are rescannable, so the merge join sees their exact
// row count.
let mut data = reader
.map(|batch| {
batch.map_err(|e| {
napi::Error::from_reason(format!(
"Failed to read record batch from IPC file: {}",
e
))
})
})
.collect::<Result<Vec<_>>>()?;
// An empty Vec has no schema, so keep the source schema in an empty batch.
if data.is_empty() {
data.push(RecordBatch::new_empty(schema));
}
let this = self.clone();