mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-19 02:02:27 +00:00
feat(python): pass merge_insert sources as scannables
Python's `execute_merge_insert` now takes a `PyScannable` (like `add()`), and the source collection threshold is exposed as a `source_collect_threshold_bytes` kwarg on `execute()` and a `merge_insert_source_collect_threshold_bytes` connection option. Node passes the decoded IPC batches as a `Vec<RecordBatch>`, so the merge join sees their exact row count. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
300d494d6e
commit
2c2bb9a974
+6
-7
@@ -20,7 +20,6 @@ use crate::{
|
||||
use arrow::{
|
||||
array::{Array, LargeBinaryArray},
|
||||
datatypes::{DataType, Schema},
|
||||
ffi_stream::ArrowArrayStreamReader,
|
||||
pyarrow::{FromPyArrow, PyArrowType, ToPyArrow},
|
||||
};
|
||||
use lancedb::blob::{BlobFile, BlobRangeRequest};
|
||||
@@ -1566,10 +1565,9 @@ impl Table {
|
||||
|
||||
pub fn execute_merge_insert<'a>(
|
||||
self_: PyRef<'a, Self>,
|
||||
data: Bound<'a, PyAny>,
|
||||
data: PyScannable,
|
||||
parameters: MergeInsertParams,
|
||||
) -> PyResult<Bound<'a, PyAny>> {
|
||||
let batches: ArrowArrayStreamReader = ArrowArrayStreamReader::from_pyarrow_bound(&data)?;
|
||||
let on = parameters.on.iter().map(|s| s.as_str()).collect::<Vec<_>>();
|
||||
let mut builder = self_.inner_ref()?.merge_insert(&on);
|
||||
if parameters.when_matched_update_all {
|
||||
@@ -1599,12 +1597,12 @@ impl Table {
|
||||
if let Some(validate_single_shard) = parameters.validate_single_shard {
|
||||
builder.validate_single_shard(validate_single_shard);
|
||||
}
|
||||
if let Some(bytes) = parameters.source_collect_threshold_bytes {
|
||||
builder.source_collect_threshold_bytes(bytes);
|
||||
}
|
||||
|
||||
future_into_py(self_.py(), async move {
|
||||
let res = builder
|
||||
.execute(Box::new(batches) as Box<dyn arrow::array::RecordBatchReader + Send>)
|
||||
.await
|
||||
.infer_error()?;
|
||||
let res = builder.execute(data).await.infer_error()?;
|
||||
Ok(MergeResult::from(res))
|
||||
})
|
||||
}
|
||||
@@ -2045,6 +2043,7 @@ pub struct MergeInsertParams {
|
||||
use_index: Option<bool>,
|
||||
use_lsm: Option<bool>,
|
||||
validate_single_shard: Option<bool>,
|
||||
source_collect_threshold_bytes: Option<usize>,
|
||||
}
|
||||
|
||||
#[pyclass]
|
||||
|
||||
Reference in New Issue
Block a user