mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-13 07:12:27 +00:00
feat(python): pass merge_insert sources as scannables
Python's `execute_merge_insert` now takes a `PyScannable` (like `add()`), and the source collection threshold is exposed as a `source_collect_threshold_bytes` kwarg on `execute()` and a `merge_insert_source_collect_threshold_bytes` connection option. Node passes the decoded IPC batches as a `Vec<RecordBatch>`, so the merge join sees their exact row count. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
300d494d6e
commit
2c2bb9a974
@@ -1081,6 +1081,16 @@ describe("merge insert", () => {
|
||||
expect(res).toEqual(expected);
|
||||
});
|
||||
|
||||
test("empty source with delete by source", async () => {
|
||||
const empty = arrow.makeEmptyTable(await table.schema());
|
||||
const res = await table
|
||||
.mergeInsert("a")
|
||||
.whenNotMatchedBySourceDelete()
|
||||
.execute(empty);
|
||||
expect(res.numDeletedRows).toBe(3);
|
||||
expect(await table.countRows()).toBe(0);
|
||||
});
|
||||
|
||||
test("timeout", async () => {
|
||||
const newData = [
|
||||
{ a: 2, b: "x" },
|
||||
|
||||
+19
-1
@@ -3,6 +3,7 @@
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use arrow_array::RecordBatch;
|
||||
use lancedb::{ipc::ipc_file_to_batches, table::merge::MergeInsertBuilder};
|
||||
use napi::bindgen_prelude::*;
|
||||
use napi_derive::napi;
|
||||
@@ -66,9 +67,26 @@ impl NativeMergeInsertBuilder {
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn execute(&self, buf: Buffer) -> napi::Result<MergeResult> {
|
||||
let data = ipc_file_to_batches(buf.to_vec()).map_err(|e| {
|
||||
let reader = ipc_file_to_batches(buf.to_vec()).map_err(|e| {
|
||||
napi::Error::from_reason(format!("Failed to read IPC file: {}", convert_error(&e)))
|
||||
})?;
|
||||
let schema = reader.schema();
|
||||
// Decoded batches are rescannable, so the merge join sees their exact
|
||||
// row count.
|
||||
let mut data = reader
|
||||
.map(|batch| {
|
||||
batch.map_err(|e| {
|
||||
napi::Error::from_reason(format!(
|
||||
"Failed to read record batch from IPC file: {}",
|
||||
e
|
||||
))
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
// An empty Vec has no schema, so keep the source schema in an empty batch.
|
||||
if data.is_empty() {
|
||||
data.push(RecordBatch::new_empty(schema));
|
||||
}
|
||||
|
||||
let this = self.clone();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user