From 104cebb1152ed7bf5948ed90603c63565ab6bf60 Mon Sep 17 00:00:00 2001 From: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Wed, 23 Sep 2026 15:41:02 +0000 Subject: [PATCH] fix: preserve nullable struct child offsets --- Cargo.lock | 2 +- Cargo.toml | 5 + python/Cargo.toml | 3 + python/vendor/lance-arrow/Cargo.toml | 33 + python/vendor/lance-arrow/README.lancedb.md | 7 + python/vendor/lance-arrow/README.md | 6 + python/vendor/lance-arrow/src/bfloat16.rs | 405 +++ python/vendor/lance-arrow/src/deepcopy.rs | 355 +++ python/vendor/lance-arrow/src/floats.rs | 291 ++ python/vendor/lance-arrow/src/ipc.rs | 620 ++++ python/vendor/lance-arrow/src/json.rs | 1445 +++++++++ python/vendor/lance-arrow/src/lib.rs | 3048 +++++++++++++++++++ python/vendor/lance-arrow/src/list.rs | 176 ++ python/vendor/lance-arrow/src/memory.rs | 91 + python/vendor/lance-arrow/src/scalar.rs | 301 ++ python/vendor/lance-arrow/src/schema.rs | 207 ++ python/vendor/lance-arrow/src/stream.rs | 553 ++++ python/vendor/lance-arrow/src/struct.rs | 191 ++ rust/lancedb/tests/nullable_struct_merge.rs | 64 + 19 files changed, 7802 insertions(+), 1 deletion(-) create mode 100644 python/vendor/lance-arrow/Cargo.toml create mode 100644 python/vendor/lance-arrow/README.lancedb.md create mode 100644 python/vendor/lance-arrow/README.md create mode 100644 python/vendor/lance-arrow/src/bfloat16.rs create mode 100644 python/vendor/lance-arrow/src/deepcopy.rs create mode 100644 python/vendor/lance-arrow/src/floats.rs create mode 100644 python/vendor/lance-arrow/src/ipc.rs create mode 100644 python/vendor/lance-arrow/src/json.rs create mode 100644 python/vendor/lance-arrow/src/lib.rs create mode 100644 python/vendor/lance-arrow/src/list.rs create mode 100644 python/vendor/lance-arrow/src/memory.rs create mode 100644 python/vendor/lance-arrow/src/scalar.rs create mode 100644 python/vendor/lance-arrow/src/schema.rs create mode 100644 python/vendor/lance-arrow/src/stream.rs create mode 100644 python/vendor/lance-arrow/src/struct.rs create mode 100644 rust/lancedb/tests/nullable_struct_merge.rs diff --git a/Cargo.lock b/Cargo.lock index 873dd159f..f6775b0e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5217,7 +5217,6 @@ dependencies = [ [[package]] name = "lance-arrow" version = "13.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5888,6 +5887,7 @@ dependencies = [ "datafusion-common", "env_logger", "futures", + "lance-arrow", "lance-core", "lance-io", "lance-namespace", diff --git a/Cargo.toml b/Cargo.toml index eb691ac48..a097ec8f3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = ["rust/lancedb", "nodejs", "python"] +exclude = ["python/vendor/lance-arrow"] resolver = "2" [workspace.package] @@ -79,6 +80,10 @@ tonic = { version = "0.14", features = ["tls-native-roots", "tls-ring"] } uuid = { version = "1.7.0", features = ["v4", "v7"] } chrono = { version = "0.4", default-features = false, features = ["clock"] } +[patch."https://github.com/lance-format/lance.git"] +# Carry the offset-safe validity fix until it is available in a Lance release. +lance-arrow = { path = "python/vendor/lance-arrow" } + [profile.ci] debug = "line-tables-only" inherits = "dev" diff --git a/python/Cargo.toml b/python/Cargo.toml index 4cb1a60c5..ec730f182 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -22,6 +22,9 @@ bytes.workspace = true lancedb.workspace = true datafusion-common.workspace = true lance-core.workspace = true +# Make the patched Lance Arrow crate an explicit path dependency so Maturin +# includes it at the path used by the workspace patch in source distributions. +lance-arrow = { version = "=13.0.0-beta.8", path = "vendor/lance-arrow" } lance-namespace.workspace = true lance-namespace-impls.workspace = true lance-io.workspace = true diff --git a/python/vendor/lance-arrow/Cargo.toml b/python/vendor/lance-arrow/Cargo.toml new file mode 100644 index 000000000..8b7c96c7a --- /dev/null +++ b/python/vendor/lance-arrow/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "lance-arrow" +version = "13.0.0-beta.8" +edition = "2024" +authors = ["Lance Devs "] +license = "Apache-2.0" +repository = "https://github.com/lance-format/lance" +readme = "README.md" +description = "Arrow Extension for Lance" +keywords = ["data-format", "data-science", "machine-learning", "apache-arrow", "data-analytics"] +categories = ["database-implementations"] + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +arrow-array = "58.0.0" +arrow-buffer = "58.0.0" +arrow-data = "58.0.0" +arrow-ipc = { version = "58.0.0", features = ["zstd"] } +arrow-ord = "58.0.0" +arrow-schema = "58.0.0" +arrow-select = "58.0.0" +bytemuck = { version = "1", default-features = false, features = ["extern_crate_alloc"] } +bytes = "1.11.1" +futures = "0.3" +half = { version = "2.1", default-features = false, features = ["num-traits", "std", "bytemuck"] } +jsonb = { version = "0.5.3", default-features = false, features = ["databend"] } +num-traits = "0.2" +rand = { version = "0.9.1", features = ["small_rng"] } + +[target.'cfg(target_arch = "wasm32")'.dependencies] +getrandom = { version = "0.2", features = ["js"] } diff --git a/python/vendor/lance-arrow/README.lancedb.md b/python/vendor/lance-arrow/README.lancedb.md new file mode 100644 index 000000000..e0f84c4a2 --- /dev/null +++ b/python/vendor/lance-arrow/README.lancedb.md @@ -0,0 +1,7 @@ +This directory contains `rust/lance-arrow` from Lance tag `v13.0.0-beta.8` +(commit `3c20b7875f621c929fdcecabe74dbca507ec0f0d`), licensed under Apache-2.0. + +LanceDB carries a local fix in `src/lib.rs` for sliced child validity bitmaps +until a Lance release includes the fix. The patch is selected in the root +`Cargo.toml`; the rest of Lance stays on the pinned release. An explicit unsafe +block in `src/bfloat16.rs` lets this copy pass LanceDB's Rust 2024 warning gate. diff --git a/python/vendor/lance-arrow/README.md b/python/vendor/lance-arrow/README.md new file mode 100644 index 000000000..e0f129a3a --- /dev/null +++ b/python/vendor/lance-arrow/README.md @@ -0,0 +1,6 @@ +# lance-arrow + +`lance-arrow` is an internal sub-crate, containing [Apache-Arrow](https://github.com/apache/arrow-rs) +extensions used by [Lance](https://github.com/lance-format/lance). + +**Important Note**: This crate is **not intended for external usage**. diff --git a/python/vendor/lance-arrow/src/bfloat16.rs b/python/vendor/lance-arrow/src/bfloat16.rs new file mode 100644 index 000000000..b29b9aa88 --- /dev/null +++ b/python/vendor/lance-arrow/src/bfloat16.rs @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! bfloat16 support for Apache Arrow. + +use std::fmt::Formatter; +use std::slice; + +use arrow_array::{Array, FixedSizeBinaryArray, builder::BooleanBufferBuilder}; +use arrow_buffer::{Buffer, MutableBuffer}; +use arrow_data::ArrayData; +use arrow_schema::{ArrowError, DataType, Field as ArrowField}; +use half::bf16; + +use crate::{ARROW_EXT_NAME_KEY, FloatArray}; + +/// The name of the bfloat16 extension in Arrow metadata +pub const BFLOAT16_EXT_NAME: &str = "lance.bfloat16"; + +/// Check whether the given field is a bfloat16 field +/// +/// A field is a bfloat16 field if it has a data type of `FixedSizeBinary(2)` and the metadata +/// contains the bfloat16 extension name. +pub fn is_bfloat16_field(field: &ArrowField) -> bool { + field.data_type() == &DataType::FixedSizeBinary(2) + && field + .metadata() + .get(ARROW_EXT_NAME_KEY) + .map(|name| name == BFLOAT16_EXT_NAME) + .unwrap_or_default() +} + +/// The bfloat16 data type +/// +/// This implements the [`ArrowFloatType`](crate::floats::ArrowFloatType) trait for bfloat16 values. +#[derive(Debug)] +pub struct BFloat16Type {} + +/// An array of bfloat16 values +/// +/// Note that bfloat16 is not the same thing as fp16 which is supported natively by arrow-rs. +#[derive(Clone)] +pub struct BFloat16Array { + inner: FixedSizeBinaryArray, +} + +impl std::fmt::Debug for BFloat16Array { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "BFloat16Array\n[\n")?; + from_arrow::print_long_array(&self.inner, f, |array, i, f| { + if array.is_null(i) { + write!(f, "null") + } else { + let binary_values = array.value(i); + let value = + bf16::from_bits(u16::from_le_bytes([binary_values[0], binary_values[1]])); + write!(f, "{:?}", value) + } + })?; + write!(f, "]") + } +} + +impl BFloat16Array { + pub fn from_iter_values(iter: impl IntoIterator) -> Self { + let values: Vec = iter.into_iter().collect(); + values.into() + } + + pub fn len(&self) -> usize { + self.inner.len() + } + + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + pub fn is_null(&self, i: usize) -> bool { + self.inner.is_null(i) + } + + pub fn null_count(&self) -> usize { + self.inner.null_count() + } + + pub fn iter(&self) -> BFloat16Iter<'_> { + BFloat16Iter { + array: self, + index: 0, + } + } + + pub fn value(&self, i: usize) -> bf16 { + assert!( + i < self.len(), + "Trying to access an element at index {} from a BFloat16Array of length {}", + i, + self.len() + ); + // Safety: + // `i < self.len() + unsafe { self.value_unchecked(i) } + } + + /// # Safety + /// Caller must ensure that `i < self.len()` + pub unsafe fn value_unchecked(&self, i: usize) -> bf16 { + // SAFETY: the caller guarantees that `i` is within this array's bounds. + let binary_value = unsafe { self.inner.value_unchecked(i) }; + bf16::from_bits(u16::from_le_bytes([binary_value[0], binary_value[1]])) + } + + pub fn into_inner(self) -> FixedSizeBinaryArray { + self.inner + } +} + +impl FromIterator> for BFloat16Array { + fn from_iter>>(iter: I) -> Self { + let mut buffer = MutableBuffer::new(10); + // No null buffer builder :( + let mut nulls = BooleanBufferBuilder::new(10); + let mut len = 0; + + for maybe_value in iter { + if let Some(value) = maybe_value { + let bytes = value.to_le_bytes(); + buffer.extend(bytes); + } else { + buffer.extend([0u8, 0u8]); + } + nulls.append(maybe_value.is_some()); + len += 1; + } + + let null_buffer = nulls.finish(); + let num_valid = null_buffer.count_set_bits(); + let null_buffer = if num_valid == len { + None + } else { + Some(null_buffer.into_inner()) + }; + + let array_data = ArrayData::builder(DataType::FixedSizeBinary(2)) + .len(len) + .add_buffer(buffer.into()) + .null_bit_buffer(null_buffer); + // SAFETY: the value buffer contains exactly `2 * len` bytes (two bytes + // pushed per iteration of the loop above, including the zero-fill for + // null slots), which matches the `FixedSizeBinary(2)` storage layout. + // The null bit buffer, when present, has `len` bits appended above, so + // its length covers the array's logical range. + let array_data = unsafe { array_data.build_unchecked() }; + Self { + inner: FixedSizeBinaryArray::from(array_data), + } + } +} + +impl FromIterator for BFloat16Array { + fn from_iter>(iter: I) -> Self { + Self::from_iter_values(iter) + } +} + +impl From> for BFloat16Array { + fn from(data: Vec) -> Self { + let len = data.len(); + // Zero-copy: `bf16` is `#[repr(transparent)]` over `u16` and derives + // `bytemuck::Pod`, so `cast_vec` reinterprets the allocation in place — + // no per-element copy or heap alloc. The crate-root `compile_error!` + // pins `target_endian = "little"`, so the resulting bytes match the + // `FixedSizeBinary(2)` on-disk order Lance writes elsewhere. + let raw: Vec = bytemuck::cast_vec(data); + let array_data = ArrayData::builder(DataType::FixedSizeBinary(2)) + .len(len) + .add_buffer(Buffer::from_vec(raw)); + // SAFETY: the value buffer contains exactly `2 * len` bytes — one + // `u16` per element after the layout-compatible cast — matching the + // `FixedSizeBinary(2)` storage layout. No null buffer is attached, so + // every element is logically valid. + let array_data = unsafe { array_data.build_unchecked() }; + Self { + inner: FixedSizeBinaryArray::from(array_data), + } + } +} + +impl TryFrom for BFloat16Array { + type Error = ArrowError; + + fn try_from(value: FixedSizeBinaryArray) -> Result { + if value.value_length() == 2 { + Ok(Self { inner: value }) + } else { + Err(ArrowError::InvalidArgumentError( + "FixedSizeBinaryArray must have a value length of 2".to_string(), + )) + } + } +} + +impl PartialEq for BFloat16Array { + fn eq(&self, other: &Self) -> bool { + self.inner.eq(&other.inner) + } +} + +pub struct BFloat16Iter<'a> { + array: &'a BFloat16Array, + index: usize, +} + +impl<'a> Iterator for BFloat16Iter<'a> { + type Item = Option; + + fn next(&mut self) -> Option { + if self.index >= self.array.len() { + return None; + } + let i = self.index; + self.index += 1; + if self.array.is_null(i) { + Some(None) + } else { + Some(Some(self.array.value(i))) + } + } +} + +/// Methods that are lifted from arrow-rs temporarily until they are made public. +mod from_arrow { + use arrow_array::Array; + + /// Helper function for printing potentially long arrays. + pub(super) fn print_long_array( + array: &A, + f: &mut std::fmt::Formatter, + print_item: F, + ) -> std::fmt::Result + where + A: Array, + F: Fn(&A, usize, &mut std::fmt::Formatter) -> std::fmt::Result, + { + let head = std::cmp::min(10, array.len()); + + for i in 0..head { + if array.is_null(i) { + writeln!(f, " null,")?; + } else { + write!(f, " ")?; + print_item(array, i, f)?; + writeln!(f, ",")?; + } + } + if array.len() > 10 { + if array.len() > 20 { + writeln!(f, " ...{} elements...,", array.len() - 20)?; + } + + let tail = std::cmp::max(head, array.len() - 10); + + for i in tail..array.len() { + if array.is_null(i) { + writeln!(f, " null,")?; + } else { + write!(f, " ")?; + print_item(array, i, f)?; + writeln!(f, ",")?; + } + } + } + Ok(()) + } +} + +impl FloatArray for FixedSizeBinaryArray { + type FloatType = BFloat16Type; + + /// Returns the underlying `bf16` values as a borrowed slice. + /// + /// # Preconditions + /// + /// - `value_length()` must be 2 (the `FixedSizeBinary(2)` storage shape + /// used by [`BFloat16Array`]). Asserted at entry. + /// - The value buffer must be at least 2-byte aligned. Lance's in-tree + /// constructors always satisfy this: value buffers are built either via + /// `MutableBuffer` (aligned to arrow-buffer's `ALIGNMENT` constant, ≥32 + /// bytes) or via `Buffer::from_vec::` (aligned to `align_of::()` + /// == 2); both meet `bf16`'s 2-byte requirement. Externally-built + /// `FixedSizeBinaryArray`s arriving via FFI, IPC, or + /// `Buffer::from_custom_allocation` are not required by arrow-rs to be + /// aligned beyond a single byte; passing one to this method violates the + /// precondition. A `debug_assert` below catches such inputs in debug and + /// test builds. + /// + /// # Endianness + /// + /// `lance-arrow` is gated on `target_endian = "little"` at the crate root, + /// so this method always returns values in the same byte order Lance writes + /// (see [`BFloat16Array::value`] and the [`FromIterator`] impls). + fn as_slice(&self) -> &[bf16] { + assert_eq!( + self.value_length(), + 2, + "BFloat16 arrays must use FixedSizeBinary(2) storage" + ); + debug_assert_eq!( + (self.value_data().as_ptr() as usize) % std::mem::align_of::(), + 0, + "BFloat16 value buffer must be at least 2-byte aligned" + ); + // SAFETY: + // - The assert above pins `value_size == 2`, so `value_data().len() / 2` + // equals the array's logical element count. + // `FixedSizeBinaryArray::From` constructs its value buffer + // as `buffers[0].slice_with_length(offset * 2, len * 2)` (arrow-array + // `fixed_size_binary_array.rs`), so `value_data()` already returns + // the offset-adjusted slice. Do not replace `value_data()` with an + // accessor that returns the un-sliced backing buffer. + // - `bf16` is `#[repr(transparent)]` over `u16` (size 2, alignment 2); + // every `u16` bit pattern is a valid `bf16`, so any byte content + // yields a defined value — never UB. + // - Alignment is the caller's responsibility per the precondition + // documented above. The `debug_assert_eq!` immediately preceding this + // block catches violations in debug and test builds only — release + // builds rely on callers honoring the precondition. arrow-rs + // declares `FixedSizeBinary(n)`'s + // `BufferSpec::FixedWidth { alignment: align_of::() == 1 }` + // (arrow-data `data.rs`), so arrow-rs alone does not guarantee + // 2-byte alignment. Lance's in-tree construction paths build value + // buffers via `MutableBuffer` (arrow-buffer `ALIGNMENT` constant, + // ≥32 bytes) or `Buffer::from_vec::` (2-byte aligned), both of + // which satisfy `bf16`'s 2-byte requirement. + // - The returned slice borrows from `self`; the underlying ref-counted, + // immutable Arrow buffer cannot be mutated or freed for the slice's + // lifetime. + unsafe { + slice::from_raw_parts( + self.value_data().as_ptr() as *const bf16, + self.value_data().len() / 2, + ) + } + } + + fn from_values(values: Vec) -> Self { + BFloat16Array::from(values).into_inner() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_basics() { + let values: Vec = vec![1.0, 2.0, 3.0]; + let values: Vec = values.iter().map(|v| bf16::from_f32(*v)).collect(); + + let array = BFloat16Array::from_iter_values(values.clone()); + let array2 = BFloat16Array::from(values.clone()); + assert_eq!(array, array2); + assert_eq!(array.len(), 3); + + // Pin the raw little-endian bytes emitted by `From>` (rewritten to + // reinterpret the Vec via `bytemuck::cast_vec`), so a layout/byte-order + // regression is caught directly rather than only through Debug formatting. + // bf16 is the high 16 bits of the f32: 1.0->0x3F80, 2.0->0x4000, 3.0->0x4040. + let inner = array2.clone().into_inner(); + let raw_bytes: Vec = (0..inner.len()) + .flat_map(|i| inner.value(i).to_vec()) + .collect(); + assert_eq!(raw_bytes, vec![0x80, 0x3F, 0x00, 0x40, 0x40, 0x40]); + + let expected_fmt = "BFloat16Array\n[\n 1.0,\n 2.0,\n 3.0,\n]"; + assert_eq!(expected_fmt, format!("{:?}", array)); + + for (expected, value) in values.iter().zip(array.iter()) { + assert_eq!(Some(*expected), value); + } + + for (expected, value) in values.as_slice().iter().zip(array2.iter()) { + assert_eq!(Some(*expected), value); + } + + let arrow_array = array.into_inner(); + assert_eq!(arrow_array.as_slice(), values.as_slice()); + } + + #[test] + fn test_nulls() { + let values: Vec> = + vec![Some(bf16::from_f32(1.0)), None, Some(bf16::from_f32(3.0))]; + let array = BFloat16Array::from_iter(values.clone()); + assert_eq!(array.len(), 3); + assert_eq!(array.null_count(), 1); + + let expected_fmt = "BFloat16Array\n[\n 1.0,\n null,\n 3.0,\n]"; + assert_eq!(expected_fmt, format!("{:?}", array)); + + for (expected, value) in values.iter().zip(array.iter()) { + assert_eq!(*expected, value); + } + } +} diff --git a/python/vendor/lance-arrow/src/deepcopy.rs b/python/vendor/lance-arrow/src/deepcopy.rs new file mode 100644 index 000000000..0d859ef1b --- /dev/null +++ b/python/vendor/lance-arrow/src/deepcopy.rs @@ -0,0 +1,355 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use arrow_array::{Array, RecordBatch, make_array}; +use arrow_buffer::{BooleanBuffer, Buffer, NullBuffer}; +use arrow_data::{ArrayData, ArrayDataBuilder, transform::MutableArrayData}; +use arrow_schema::DataType; + +pub fn deep_copy_buffer(buffer: &Buffer) -> Buffer { + Buffer::from(buffer.as_slice()) +} + +pub fn deep_copy_nulls(nulls: Option<&NullBuffer>) -> Option { + let nulls = nulls?; + let bit_buffer = deep_copy_buffer(nulls.inner().inner()); + // SAFETY: `null_count` is taken from the source `NullBuffer`, which already + // upheld `NullBuffer::new_unchecked`'s invariant — the unset-bit count over + // the logical bit slice `[bit_offset, bit_offset + bit_len)`. `NullBuffer::slice` + // adjusts only `BooleanBuffer::bit_offset` / `bit_len` and never byte-advances + // the inner `Buffer`, so `deep_copy_buffer` (which copies the source `Buffer`'s + // `as_slice()` view from byte 0) reproduces the exact bit pattern at the same + // bit offsets; the unset-bit count is therefore preserved. `BooleanBuffer::new` + // panics (does not UB) if `bit_offset + bit_len > 8 * buffer.len()`, and the + // copy has the same length, so that check still passes. + Some(unsafe { + NullBuffer::new_unchecked( + BooleanBuffer::new(bit_buffer, nulls.offset(), nulls.len()), + nulls.null_count(), + ) + }) +} + +pub fn deep_copy_array_data(data: &ArrayData) -> ArrayData { + let data_type = data.data_type().clone(); + let len = data.len(); + let nulls = deep_copy_nulls(data.nulls()); + let offset = data.offset(); + let buffers = data + .buffers() + .iter() + .map(deep_copy_buffer) + .collect::>(); + let child_data = data + .child_data() + .iter() + .map(deep_copy_array_data) + .collect::>(); + // SAFETY: `build_unchecked` inherits `ArrayData::new_unchecked`'s contract — + // `(data_type, len, offset, nulls, buffers, child_data)` must form a valid + // Arrow array. This call reproduces `data` structurally: `data_type`, `len`, + // and `offset` are forwarded unchanged; each buffer is replaced by a byte- + // identical copy of its offset-applied `as_slice()` view (the output buffer + // is `MutableBuffer`-allocated, at least as aligned as the source); `nulls` + // is deep-copied with the same bit offset/length and unset-bit count (see + // `deep_copy_nulls`); `child_data` is recursively cloned with the same + // guarantee. Every value-level invariant the source upheld — UTF-8 validity, + // monotonic offsets, in-bounds dictionary indices, run-end monotonicity, + // struct child-length matching — therefore transfers to the copy. If the + // source `ArrayData` was itself constructed via `new_unchecked` with an + // invalid payload, this function faithfully reproduces that invalidity. + unsafe { + ArrayDataBuilder::new(data_type) + .len(len) + .nulls(nulls) + .offset(offset) + .buffers(buffers) + .child_data(child_data) + .build_unchecked() + } +} + +pub fn deep_copy_array(array: &dyn Array) -> Arc { + let data = array.to_data(); + let data = deep_copy_array_data(&data); + make_array(data) +} + +pub fn deep_copy_batch(batch: &RecordBatch) -> crate::Result { + let arrays = batch + .columns() + .iter() + .map(|array| deep_copy_array(array)) + .collect::>(); + RecordBatch::try_new(batch.schema(), arrays) +} + +/// Deep copy array data, extracting only the sliced portion using MutableArrayData +/// This is the most efficient and correct way to copy just the sliced data +pub fn deep_copy_array_data_sliced(data: &ArrayData) -> ArrayData { + // Use MutableArrayData to efficiently copy just the slice + let mut mutable = MutableArrayData::new(vec![data], false, data.len()); + + // Which index this takes depends on the layout, because arrow's extenders + // disagree about who applies `ArrayData::offset()`. The bit-packed one adds + // it to the raw values buffer itself, so an offset-applied index there + // reads from twice the offset and runs off the end of the buffer. Every + // other layout either reads through an already-offset-applied view or + // forwards the index to children carrying their own offsets, and takes the + // absolute index it has always been given. + let start = match data.data_type() { + DataType::Boolean => 0, + _ => data.offset(), + }; + mutable.extend(0, start, start + data.len()); + + // Freeze into immutable ArrayData + mutable.freeze() +} + +/// Deep copy an array, extracting only the sliced portion using MutableArrayData +pub fn deep_copy_array_sliced(array: &dyn Array) -> Arc { + let data = array.to_data(); + let data = deep_copy_array_data_sliced(&data); + make_array(data) +} + +/// Deep copy a RecordBatch, extracting only the sliced portion using MutableArrayData +pub fn deep_copy_batch_sliced(batch: &RecordBatch) -> crate::Result { + let arrays = batch + .columns() + .iter() + .map(|array| deep_copy_array_sliced(array)) + .collect::>(); + RecordBatch::try_new(batch.schema(), arrays) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow_array::{Array, BooleanArray, Int32Array, RecordBatch, StringArray}; + use arrow_data::ArrayDataBuilder; + use arrow_schema::{DataType, Field, Schema}; + + #[test] + fn raw_sliced_fixed_size_list_data_keeps_its_child_offset() { + // `ArrayData::slice` records the offset on the parent and leaves the + // child whole -- a shape `FixedSizeListArray::slice` never produces, + // but a legal one this public helper can be handed directly. Arrow's + // extender forwards `start * size` to that unsliced child and adds + // nothing, so the index it gets has to be the absolute one; this is + // here to stop the boolean fix from being generalised over it. + let child = Int32Array::from(vec![10, 11, 20, 21]).to_data(); + let field = Arc::new(Field::new_list_field(DataType::Int32, false)); + let data = ArrayDataBuilder::new(DataType::FixedSizeList(field, 2)) + .len(2) + .add_child_data(child) + .build() + .unwrap(); + let sliced = data.slice(1, 1); + + let copied = super::deep_copy_array_data_sliced(&sliced); + let copied_child = Int32Array::from(copied.child_data()[0].clone()); + assert_eq!(copied_child.values().as_ref(), &[20, 21]); + } + + #[test] + fn raw_sliced_struct_of_fixed_size_list_reaches_the_right_values() { + // Slicing raw struct data pushes the window into the struct's children, + // and a fixed-size-list child takes it as its own parent offset with + // its values left whole. Two levels of offset, and the absolute index + // has to remain right through both of them. + let values = Int32Array::from(vec![10, 11, 20, 21, 30, 31]).to_data(); + let item = Arc::new(Field::new_list_field(DataType::Int32, false)); + let list = ArrayDataBuilder::new(DataType::FixedSizeList(item, 2)) + .len(3) + .add_child_data(values) + .build() + .unwrap(); + let field = Arc::new(Field::new("l", list.data_type().clone(), false)); + let data = ArrayDataBuilder::new(DataType::Struct(vec![field].into())) + .len(3) + .add_child_data(list) + .build() + .unwrap(); + let sliced = data.slice(2, 1); + + let copied = super::deep_copy_array_data_sliced(&sliced); + let copied_values = Int32Array::from(copied.child_data()[0].child_data()[0].clone()); + assert_eq!(copied_values.values().as_ref(), &[30, 31]); + } + + #[test] + fn raw_parent_offset_struct_keeps_the_selected_child_row() { + // A struct whose children are whole and whose window is the parent + // offset: arrow's struct extender forwards the index to those children + // untouched, so it has to be the absolute one. + let field = Arc::new(Field::new("a", DataType::Int32, false)); + let data = ArrayDataBuilder::new(DataType::Struct(vec![field].into())) + .len(1) + .offset(1) + .add_child_data(Int32Array::from(vec![10, 20]).to_data()) + .build() + .unwrap(); + + let copied = super::deep_copy_array_data_sliced(&data); + let copied_child = Int32Array::from(copied.child_data()[0].clone()); + assert_eq!(copied_child.values().as_ref(), &[20]); + } + + #[test] + fn sliced_boolean_deep_copy_reads_from_the_slice() { + // A boolean slice keeps its `ArrayData::offset()` -- the buffer cannot + // advance by a fraction of a byte -- so a copy that adds the offset on + // top of what `MutableArrayData` already applies reads past the end of + // the values buffer and panics. + let array = BooleanArray::from(vec![true, false, true, false, true, false, true, false]); + for (offset, len) in [(0usize, 8usize), (1, 7), (3, 5), (7, 1)] { + let sliced = array.slice(offset, len); + let copied = super::deep_copy_array_sliced(&sliced); + let copied = copied.as_any().downcast_ref::().unwrap(); + let expected: Vec = (0..len).map(|i| sliced.value(i)).collect(); + let actual: Vec = (0..len).map(|i| copied.value(i)).collect(); + assert_eq!(actual, expected, "offset={offset} len={len}"); + } + } + + #[test] + fn sliced_boolean_deep_copy_keeps_its_nulls() { + let array = BooleanArray::from(vec![ + Some(true), + None, + Some(false), + Some(true), + None, + Some(false), + ]); + let sliced = array.slice(1, 4); + let copied = super::deep_copy_array_sliced(&sliced); + let copied = copied.as_any().downcast_ref::().unwrap(); + let expected: Vec> = (0..sliced.len()) + .map(|i| (!sliced.is_null(i)).then(|| sliced.value(i))) + .collect(); + let actual: Vec> = (0..copied.len()) + .map(|i| (!copied.is_null(i)).then(|| copied.value(i))) + .collect(); + assert_eq!(actual, expected); + } + + #[test] + fn test_deep_copy_sliced_array_with_nulls() { + let array = Arc::new(Int32Array::from(vec![ + Some(1), + None, + Some(3), + None, + Some(5), + ])); + let sliced_array = array.slice(1, 3); + let copied_array = super::deep_copy_array(&sliced_array); + assert_eq!(sliced_array.len(), copied_array.len()); + assert_eq!(sliced_array.nulls(), copied_array.nulls()); + } + + #[test] + fn test_deep_copy_array_data_sliced() { + let array = Int32Array::from((0..1000).collect::>()); + let sliced = array.slice(100, 10); + + let sliced_data = sliced.to_data(); + let copied_data = super::deep_copy_array_data_sliced(&sliced_data); + + assert_eq!(copied_data.len(), 10); + assert_eq!(copied_data.offset(), 0); + + // Verify data correctness + let copied_array = Int32Array::from(copied_data); + for i in 0..10 { + assert_eq!(copied_array.value(i), 100 + i as i32); + } + } + + #[test] + fn test_deep_copy_array_sliced() { + let array = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])); + let sliced = array.slice(1, 3); + + let copied = super::deep_copy_array_sliced(&sliced); + + assert_eq!(copied.len(), 3); + let copied_int = copied.as_any().downcast_ref::().unwrap(); + assert_eq!(copied_int.value(0), 2); + assert_eq!(copied_int.value(1), 3); + assert_eq!(copied_int.value(2), 4); + } + + #[test] + fn test_deep_copy_batch_sliced() { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + ])); + + let id_array = Arc::new(Int32Array::from((0..100).collect::>())); + let name_array = Arc::new(StringArray::from( + (0..100) + .map(|i| format!("name_{}", i)) + .collect::>(), + )); + + let batch = RecordBatch::try_new( + schema, + vec![id_array as Arc, name_array as Arc], + ) + .unwrap(); + + let sliced = batch.slice(10, 5); + let copied = super::deep_copy_batch_sliced(&sliced).unwrap(); + + assert_eq!(copied.num_rows(), 5); + assert_eq!(copied.num_columns(), 2); + + // Verify data correctness + let id_col = copied + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let name_col = copied + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + + for i in 0..5 { + assert_eq!(id_col.value(i), 10 + i as i32); + assert_eq!(name_col.value(i), format!("name_{}", 10 + i)); + } + } + + #[test] + fn test_deep_copy_array_sliced_with_nulls() { + let array = Arc::new(Int32Array::from(vec![ + Some(1), + None, + Some(3), + None, + Some(5), + ])); + let sliced = array.slice(1, 3); // [None, Some(3), None] + + let copied = super::deep_copy_array_sliced(&sliced); + + assert_eq!(copied.len(), 3); + assert_eq!(copied.null_count(), 2); // Two nulls in the slice + + let copied_int = copied.as_any().downcast_ref::().unwrap(); + assert!(!copied_int.is_valid(0)); // None + assert!(copied_int.is_valid(1)); // Some(3) + assert!(!copied_int.is_valid(2)); // None + assert_eq!(copied_int.value(1), 3); + } +} diff --git a/python/vendor/lance-arrow/src/floats.rs b/python/vendor/lance-arrow/src/floats.rs new file mode 100644 index 000000000..ad5f37689 --- /dev/null +++ b/python/vendor/lance-arrow/src/floats.rs @@ -0,0 +1,291 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Floats Array + +use std::fmt::{Debug, Display}; +use std::iter::Sum; +use std::sync::Arc; +use std::{ + fmt::Formatter, + ops::{AddAssign, DivAssign}, +}; + +use arrow_array::{ + Array, FixedSizeBinaryArray, Float16Array, Float32Array, Float64Array, + types::{Float16Type, Float32Type, Float64Type}, +}; +use arrow_schema::{DataType, Field}; +use half::{bf16, f16}; +use num_traits::{AsPrimitive, Bounded, Float, FromPrimitive}; + +use super::bfloat16::{BFloat16Array, BFloat16Type}; +use crate::Result; +use crate::bfloat16::is_bfloat16_field; + +/// Float data type. +/// +/// This helps differentiate between the different float types, +/// because bf16 is not officially supported [DataType] in arrow-rs. +#[derive(Debug)] +pub enum FloatType { + BFloat16, + Float16, + Float32, + Float64, +} + +impl std::fmt::Display for FloatType { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::BFloat16 => write!(f, "bfloat16"), + Self::Float16 => write!(f, "float16"), + Self::Float32 => write!(f, "float32"), + Self::Float64 => write!(f, "float64"), + } + } +} + +/// Try to convert a [DataType] to a [FloatType]. To support bfloat16, always +/// prefer using the `TryFrom<&Field>` implementation. +impl TryFrom<&DataType> for FloatType { + type Error = crate::ArrowError; + + fn try_from(value: &DataType) -> Result { + match *value { + DataType::Float16 => Ok(Self::Float16), + DataType::Float32 => Ok(Self::Float32), + DataType::Float64 => Ok(Self::Float64), + _ => Err(crate::ArrowError::InvalidArgumentError(format!( + "{:?} is not a floating type", + value + ))), + } + } +} + +impl TryFrom<&Field> for FloatType { + type Error = crate::ArrowError; + + fn try_from(field: &Field) -> Result { + match field.data_type() { + DataType::FixedSizeBinary(2) if is_bfloat16_field(field) => Ok(Self::BFloat16), + _ => Self::try_from(field.data_type()), + } + } +} + +/// Trait for float types used in Lance indexes +/// +/// This mimics the utilities provided by [`arrow_array::ArrowPrimitiveType`] +/// but applies to all float types (including bfloat16) +pub trait ArrowFloatType: Debug { + type Native: FromPrimitive + + FloatToArrayType + + AsPrimitive + + Debug + + Display; + + const FLOAT_TYPE: FloatType; + const MIN: Self::Native; + const MAX: Self::Native; + + /// Arrow Float Array Type. + type ArrayType: FloatArray; + + /// Returns empty array of this type. + fn empty_array() -> Self::ArrayType { + >::from_values(Vec::new()) + } +} + +/// Trait to be implemented by native types that have a corresponding [`ArrowFloatType`] +/// implementation. +/// +/// This helps define what operations are supported by native floats and also helps convert +/// from a native type back to the corresponding Arrow float type. +pub trait FloatToArrayType: + Float + + Bounded + + Sum + + AddAssign + + AsPrimitive + + AsPrimitive + + DivAssign + + Send + + Sync + + Copy +{ + /// The corresponding [`ArrowFloatType`] implementation for this native type + type ArrowType: ArrowFloatType; +} + +impl FloatToArrayType for bf16 { + type ArrowType = BFloat16Type; +} + +impl FloatToArrayType for f16 { + type ArrowType = Float16Type; +} + +impl FloatToArrayType for f32 { + type ArrowType = Float32Type; +} + +impl FloatToArrayType for f64 { + type ArrowType = Float64Type; +} + +impl ArrowFloatType for BFloat16Type { + type Native = bf16; + + const FLOAT_TYPE: FloatType = FloatType::BFloat16; + const MIN: Self::Native = bf16::MIN; + const MAX: Self::Native = bf16::MAX; + + type ArrayType = FixedSizeBinaryArray; +} + +impl ArrowFloatType for Float16Type { + type Native = f16; + + const FLOAT_TYPE: FloatType = FloatType::Float16; + const MIN: Self::Native = f16::MIN; + const MAX: Self::Native = f16::MAX; + + type ArrayType = Float16Array; +} + +impl ArrowFloatType for Float32Type { + type Native = f32; + + const FLOAT_TYPE: FloatType = FloatType::Float32; + const MIN: Self::Native = f32::MIN; + const MAX: Self::Native = f32::MAX; + + type ArrayType = Float32Array; +} + +impl ArrowFloatType for Float64Type { + type Native = f64; + + const FLOAT_TYPE: FloatType = FloatType::Float64; + const MIN: Self::Native = f64::MIN; + const MAX: Self::Native = f64::MAX; + + type ArrayType = Float64Array; +} + +/// [FloatArray] is a trait that is implemented by all float type arrays +/// +/// This is similar to [`arrow_array::PrimitiveArray`] but applies to all float types (including bfloat16) +/// and is implemented as a trait and not a struct +pub trait FloatArray: Array + Clone + 'static { + type FloatType: ArrowFloatType; + + /// Returns a reference to the underlying data as a slice. + /// + /// # Panics + /// + /// Implementations may panic if the array's storage shape does not match + /// the expected element layout. In particular, the `bf16` impl panics if + /// `value_length() != 2` (the `FixedSizeBinary(2)` shape required by + /// `BFloat16Array`). + /// + /// # Preconditions + /// + /// Implementations may impose additional invariants on the underlying + /// buffer. The `bf16` impl requires the value buffer to be at least + /// 2-byte aligned — satisfied automatically by every in-tree Lance + /// constructor, but external callers passing externally-built arrays + /// (FFI, IPC, `Buffer::from_custom_allocation`) must ensure alignment. + /// See the impl's docstring for details. + fn as_slice(&self) -> &[T::Native]; + + /// Construct an array from a vector of values. + fn from_values(values: Vec) -> Self; + + /// Construct an array from an iterator of values. + fn from_iter_values(values: impl IntoIterator) -> Self + where + Self: Sized, + { + Self::from_values(values.into_iter().collect()) + } +} + +impl FloatArray for Float16Array { + type FloatType = Float16Type; + + fn as_slice(&self) -> &[::Native] { + self.values() + } + + fn from_values(values: Vec<::Native>) -> Self { + Self::from(values) + } +} + +impl FloatArray for Float32Array { + type FloatType = Float32Type; + + fn as_slice(&self) -> &[::Native] { + self.values() + } + + fn from_values(values: Vec<::Native>) -> Self { + Self::from(values) + } +} + +impl FloatArray for Float64Array { + type FloatType = Float64Type; + + fn as_slice(&self) -> &[::Native] { + self.values() + } + + fn from_values(values: Vec<::Native>) -> Self { + Self::from(values) + } +} + +/// Convert a float32 array to another float array +/// +/// This is used during queries as query vectors are always provided as float32 arrays +/// and need to be converted to the appropriate float type for the index. +pub fn coerce_float_vector(input: &Float32Array, float_type: FloatType) -> Result> { + match float_type { + FloatType::BFloat16 => Ok(Arc::new( + BFloat16Array::from_iter_values(input.values().iter().map(|v| bf16::from_f32(*v))) + .into_inner(), + )), + FloatType::Float16 => Ok(Arc::new(Float16Array::from_iter_values( + input.values().iter().map(|v| f16::from_f32(*v)), + ))), + FloatType::Float32 => Ok(Arc::new(input.clone())), + FloatType::Float64 => Ok(Arc::new(Float64Array::from_iter_values( + input.values().iter().map(|v| *v as f64), + ))), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_coerce_float_vector_bfloat16() { + let input = Float32Array::from(vec![1.0f32, 2.0, 3.0]); + let array = coerce_float_vector(&input, FloatType::BFloat16).unwrap(); + + assert_eq!(array.data_type(), &DataType::FixedSizeBinary(2)); + + let fixed = array + .as_any() + .downcast_ref::() + .unwrap(); + let expected: Vec = input.values().iter().map(|v| bf16::from_f32(*v)).collect(); + assert_eq!(fixed.as_slice(), expected.as_slice()); + } +} diff --git a/python/vendor/lance-arrow/src/ipc.rs b/python/vendor/lance-arrow/src/ipc.rs new file mode 100644 index 000000000..8b6e5cf41 --- /dev/null +++ b/python/vendor/lance-arrow/src/ipc.rs @@ -0,0 +1,620 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Zero-copy Arrow IPC stream read/write utilities. +//! +//! Provides helpers for serializing and deserializing [`RecordBatch`]es as +//! self-delimiting Arrow IPC streams using synchronous [`Read`]/[`Write`] I/O. +//! +//! These are designed for embedding IPC streams inside larger binary formats +//! (e.g. a cache entry that contains multiple IPC sections). Each stream is +//! self-delimiting (schema + batches + EOS marker) and can be read back +//! independently. +//! +//! # Zero-copy reads +//! +//! [`read_ipc_stream`] and [`read_ipc_stream_single`] take `&Bytes` and use +//! [`Bytes::slice`] to produce each message buffer. Because `Bytes::slice` +//! increments a reference count rather than copying, the resulting +//! [`Buffer`]s — and the array data decoded from them by [`FileDecoder`] — +//! are all backed by the same allocation as the input. + +use std::io::{Read, Write}; +use std::sync::Arc; + +use arrow_array::RecordBatch; +use arrow_buffer::Buffer; +use arrow_ipc::convert::fb_to_schema; +use arrow_ipc::reader::FileDecoder; +use arrow_ipc::root_as_message; +use arrow_ipc::writer::StreamWriter; +use arrow_schema::ArrowError; +use bytes::Bytes; + +// --------------------------------------------------------------------------- +// Length-prefixed byte utilities +// --------------------------------------------------------------------------- + +/// Write `data` prefixed by its length as a little-endian `u64`. +/// +/// Paired with [`read_len_prefixed_bytes`]. +pub fn write_len_prefixed_bytes(writer: &mut dyn Write, data: &[u8]) -> Result<(), ArrowError> { + writer + .write_all(&(data.len() as u64).to_le_bytes()) + .map_err(|e| ArrowError::IoError(e.to_string(), e))?; + writer + .write_all(data) + .map_err(|e| ArrowError::IoError(e.to_string(), e)) +} + +/// Read a byte slice written by [`write_len_prefixed_bytes`]. +/// +/// Reads an 8-byte little-endian length then exactly that many bytes. +pub fn read_len_prefixed_bytes(reader: &mut dyn Read) -> Result, ArrowError> { + let mut len_buf = [0u8; 8]; + reader + .read_exact(&mut len_buf) + .map_err(|e| ArrowError::IoError(e.to_string(), e))?; + let len = u64::from_le_bytes(len_buf) as usize; + let mut buf = vec![0u8; len]; + reader + .read_exact(&mut buf) + .map_err(|e| ArrowError::IoError(e.to_string(), e))?; + Ok(buf) +} + +// --------------------------------------------------------------------------- +// IPC stream utilities +// --------------------------------------------------------------------------- + +// 4-byte continuation marker used by modern Arrow IPC streams. +const IPC_CONTINUATION: [u8; 4] = [0xff; 4]; + +/// Write `batch` as a single-batch Arrow IPC stream to `writer`. +pub fn write_ipc_stream(batch: &RecordBatch, writer: &mut dyn Write) -> Result<(), ArrowError> { + let mut sw = StreamWriter::try_new(&mut *writer, batch.schema_ref())?; + sw.write(batch)?; + sw.finish() +} + +/// Write all batches from `iter` as a single Arrow IPC stream to `writer`. +/// +/// `iter` must yield at least one batch; the schema is inferred from the first +/// batch. Returns `ArrowError::InvalidArgumentError` if the iterator is empty. +/// If you need to write an empty stream (schema only, no rows), construct a +/// `StreamWriter` directly. +pub fn write_ipc_stream_batches(iter: I, writer: &mut dyn Write) -> Result<(), ArrowError> +where + I: IntoIterator, +{ + let mut iter = iter.into_iter(); + let first = iter + .next() + .ok_or_else(|| ArrowError::InvalidArgumentError("no batches to serialize".into()))?; + let mut sw = StreamWriter::try_new(&mut *writer, first.schema_ref())?; + sw.write(&first)?; + for batch in iter { + sw.write(&batch)?; + } + sw.finish() +} + +/// Read one complete Arrow IPC stream message from `data` as a zero-copy [`Buffer`]. +/// +/// Parses the first message starting at byte 0 of `data`. Returns `None` on +/// EOS (size field == 0) or empty input. The returned [`Buffer`] is backed by +/// `data`'s allocation — no bytes are copied. +/// +/// The caller should advance its position by `buf.len()` after each call. +fn read_one_ipc_message(data: &Bytes) -> Result, ArrowError> { + let bytes = data.as_ref(); + + if bytes.is_empty() { + return Ok(None); + } + if bytes.len() < 4 { + return Err(ArrowError::IoError( + "IPC: truncated header".into(), + std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated IPC header"), + )); + } + + let has_continuation = bytes[..4] == IPC_CONTINUATION; + let (size_bytes, prefix_len): ([u8; 4], usize) = if has_continuation { + if bytes.len() < 8 { + return Err(ArrowError::IoError( + "IPC: truncated header after continuation".into(), + std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "truncated after continuation", + ), + )); + } + (bytes[4..8].try_into().unwrap(), 8) + } else { + (bytes[..4].try_into().unwrap(), 4) + }; + + let meta_size = u32::from_le_bytes(size_bytes) as usize; + if meta_size == 0 { + return Ok(None); // EOS + } + + let meta_end = prefix_len + meta_size; + if bytes.len() < meta_end { + return Err(ArrowError::IoError( + "IPC: truncated metadata".into(), + std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated IPC metadata"), + )); + } + + let msg = root_as_message(&bytes[prefix_len..meta_end]) + .map_err(|e| ArrowError::ParseError(format!("IPC message parse error: {e}")))?; + let body_len = msg.bodyLength() as usize; + + let total = meta_end + body_len; + if bytes.len() < total { + return Err(ArrowError::IoError( + "IPC: truncated body".into(), + std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated IPC body"), + )); + } + + // Zero-copy: Bytes::slice shares the backing allocation; Buffer::from + // wraps it without copying. + Ok(Some(Buffer::from(data.slice(0..total)))) +} + +/// Read a length-prefixed byte slice at `offset` in `data`, advancing `offset`. +/// +/// Reads an 8-byte little-endian length, then slices exactly that many bytes +/// from `data`. The returned [`Bytes`] is zero-copy (shares `data`'s allocation). +pub fn read_len_prefixed_bytes_at(data: &Bytes, offset: &mut usize) -> Result { + let bytes = data.as_ref(); + let len_end = offset + .checked_add(8) + .filter(|&e| e <= bytes.len()) + .ok_or_else(|| { + ArrowError::IoError( + "length-prefixed bytes: truncated length field".into(), + std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated length"), + ) + })?; + let len = u64::from_le_bytes(bytes[*offset..len_end].try_into().unwrap()) as usize; + *offset = len_end; + let data_end = offset + .checked_add(len) + .filter(|&e| e <= bytes.len()) + .ok_or_else(|| { + ArrowError::IoError( + "length-prefixed bytes: truncated data".into(), + std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated data"), + ) + })?; + let result = data.slice(*offset..data_end); + *offset = data_end; + Ok(result) +} + +/// Read all [`RecordBatch`]es from one Arrow IPC stream starting at `offset`, +/// advancing `offset` past the stream (including the EOS marker). +/// +/// Zero-copy: array buffers borrow from `data`'s allocation. +pub fn read_ipc_stream_at( + data: &Bytes, + offset: &mut usize, +) -> Result, ArrowError> { + let batches = read_ipc_stream(&data.slice(*offset..))?; + + // Recompute how many bytes were consumed by re-parsing message sizes. + // We can't get this from read_ipc_stream directly, so we re-walk the + // message headers (metadata only, no body re-read) to sum up lengths. + let slice = &data.as_ref()[*offset..]; + let mut consumed = 0usize; + loop { + let rem = &slice[consumed..]; + if rem.is_empty() { + break; + } + let has_cont = rem.len() >= 4 && rem[..4] == IPC_CONTINUATION; + let (size_bytes, prefix_len): ([u8; 4], usize) = if has_cont { + if rem.len() < 8 { + break; + } + (rem[4..8].try_into().unwrap(), 8) + } else { + if rem.len() < 4 { + break; + } + (rem[..4].try_into().unwrap(), 4) + }; + let meta_size = u32::from_le_bytes(size_bytes) as usize; + if meta_size == 0 { + // EOS — consume it and stop. + consumed += prefix_len; + break; + } + let meta_end = prefix_len + meta_size; + if rem.len() < meta_end { + break; + } + let msg = root_as_message(&rem[prefix_len..meta_end]) + .map_err(|e| ArrowError::ParseError(format!("IPC message parse error: {e}")))?; + let body_len = msg.bodyLength() as usize; + consumed += meta_end + body_len; + } + *offset += consumed; + + Ok(batches) +} + +/// Read exactly one [`RecordBatch`] from one Arrow IPC stream starting at `offset`, +/// advancing `offset` past the stream (including the EOS marker). +/// +/// Zero-copy: array buffers borrow from `data`'s allocation. +pub fn read_ipc_stream_single_at( + data: &Bytes, + offset: &mut usize, +) -> Result { + let mut batches = read_ipc_stream_at(data, offset)?; + match batches.len() { + 1 => Ok(batches.remove(0)), + n => Err(ArrowError::ParseError(format!( + "expected exactly 1 IPC record batch, got {n}" + ))), + } +} + +/// Extract the prefix length and metadata size from a raw IPC message buffer. +/// +/// Modern IPC streams have an 8-byte prefix `[continuation: 4][size: 4]`. +/// Legacy streams have a 4-byte prefix `[size: 4]`. Returns `(prefix_len, meta_size)`. +fn parse_ipc_message_prefix(buf: &Buffer) -> Result<(usize, usize), ArrowError> { + let has_continuation = buf.len() >= 4 && buf[..4] == IPC_CONTINUATION; + if has_continuation { + if buf.len() < 8 { + return Err(ArrowError::ParseError( + "IPC message buffer too short".into(), + )); + } + let meta_size = u32::from_le_bytes(buf[4..8].try_into().unwrap()) as usize; + Ok((8, meta_size)) + } else { + if buf.len() < 4 { + return Err(ArrowError::ParseError( + "IPC message buffer too short".into(), + )); + } + let meta_size = u32::from_le_bytes(buf[..4].try_into().unwrap()) as usize; + Ok((4, meta_size)) + } +} + +/// Read all [`RecordBatch`]es from one Arrow IPC stream. +/// +/// Zero-copy: each batch's array data buffers are borrowed from the input +/// message buffer(s) and not copied during decoding. +/// +/// Uses [`FileDecoder`] directly (rather than `StreamDecoder`) to avoid a +/// known edge case where `StreamDecoder` does not produce a batch for messages +/// with a zero-length body when the message exactly fills the decode buffer. +pub fn read_ipc_stream(data: &Bytes) -> Result, ArrowError> { + let mut offset = 0usize; + + let schema_buf = read_one_ipc_message(&data.slice(offset..))?.ok_or_else(|| { + ArrowError::ParseError("IPC stream: expected schema message, got EOS".into()) + })?; + offset += schema_buf.len(); + + let (prefix_len, meta_size) = parse_ipc_message_prefix(&schema_buf)?; + let schema_msg = root_as_message(&schema_buf[prefix_len..prefix_len + meta_size]) + .map_err(|e| ArrowError::ParseError(format!("IPC schema parse error: {e}")))?; + let schema = Arc::new(fb_to_schema(schema_msg.header_as_schema().ok_or_else( + || ArrowError::ParseError("IPC stream: first message is not a schema".into()), + )?)); + let mut decoder = FileDecoder::new(schema, schema_msg.version()); + + let mut batches = Vec::new(); + + loop { + let Some(buf) = read_one_ipc_message(&data.slice(offset..))? else { + break; + }; + offset += buf.len(); + + let (prefix_len, meta_size) = parse_ipc_message_prefix(&buf)?; + let msg = root_as_message(&buf[prefix_len..prefix_len + meta_size]) + .map_err(|e| ArrowError::ParseError(format!("IPC message parse error: {e}")))?; + let body_len = msg.bodyLength() as usize; + + // Block offset = 0 since the buffer starts at the message boundary. + // metaDataLength = prefix_len + meta_size (prefix + flatbuf + padding). + let block = arrow_ipc::Block::new(0, (prefix_len + meta_size) as i32, body_len as i64); + + match msg.header_type() { + arrow_ipc::MessageHeader::RecordBatch => { + if let Some(batch) = decoder.read_record_batch(&block, &buf)? { + batches.push(batch); + } + } + arrow_ipc::MessageHeader::DictionaryBatch => { + decoder.read_dictionary(&block, &buf)?; + } + _ => break, + } + } + + Ok(batches) +} + +/// Read exactly one [`RecordBatch`] from one Arrow IPC stream. +pub fn read_ipc_stream_single(data: &Bytes) -> Result { + let mut batches = read_ipc_stream(data)?; + match batches.len() { + 1 => Ok(batches.remove(0)), + n => Err(ArrowError::ParseError(format!( + "expected exactly 1 IPC record batch, got {n}" + ))), + } +} + +// --------------------------------------------------------------------------- +// Aligned IPC sections +// --------------------------------------------------------------------------- + +/// Byte alignment that each IPC section's stream start is padded to. +/// +/// When several IPC streams are concatenated into one larger blob (e.g. a +/// cache entry), a section that starts at an arbitrary offset would leave its +/// array data misaligned. [`FileDecoder`] with `require_alignment = false` +/// then silently copies each buffer into a freshly aligned allocation on +/// every read, defeating zero-copy. Padding each section start to a 64-byte +/// boundary keeps the decoded buffers borrowed directly from the input. +pub const IPC_SECTION_ALIGNMENT: usize = 64; + +/// Number of zero-padding bytes needed to advance `pos` to the next +/// [`IPC_SECTION_ALIGNMENT`] boundary. +fn section_padding(pos: usize) -> usize { + (IPC_SECTION_ALIGNMENT - (pos % IPC_SECTION_ALIGNMENT)) % IPC_SECTION_ALIGNMENT +} + +/// A [`Write`] adapter that counts the bytes written through it. +struct CountingWriter<'a> { + inner: &'a mut dyn Write, + count: usize, +} + +impl Write for CountingWriter<'_> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let n = self.inner.write(buf)?; + self.count += n; + Ok(n) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} + +/// Write zero padding so the next byte lands on an [`IPC_SECTION_ALIGNMENT`] +/// boundary, advancing `pos` past it. +fn write_section_padding(writer: &mut dyn Write, pos: &mut usize) -> Result<(), ArrowError> { + let pad = section_padding(*pos); + if pad > 0 { + const ZEROS: [u8; IPC_SECTION_ALIGNMENT] = [0u8; IPC_SECTION_ALIGNMENT]; + writer + .write_all(&ZEROS[..pad]) + .map_err(|e| ArrowError::IoError(e.to_string(), e))?; + *pos += pad; + } + Ok(()) +} + +/// Write `batch` as a 64-byte-aligned single-batch Arrow IPC section. +/// +/// `pos` is the absolute byte offset of `writer` within the enclosing blob. +/// Zero padding is written first so the IPC stream begins on an +/// [`IPC_SECTION_ALIGNMENT`] boundary, then the stream itself. `pos` is +/// advanced past both the padding and the stream so the caller can write +/// further aligned sections. +/// +/// Paired with [`read_ipc_section_at`]. For the decoded buffers to be borrowed +/// zero-copy, the blob must ultimately be read back from a buffer whose base +/// address is at least 64-byte aligned. +pub fn write_ipc_section( + writer: &mut dyn Write, + pos: &mut usize, + batch: &RecordBatch, +) -> Result<(), ArrowError> { + write_section_padding(writer, pos)?; + + let mut counting = CountingWriter { + inner: writer, + count: 0, + }; + write_ipc_stream(batch, &mut counting)?; + *pos += counting.count; + Ok(()) +} + +/// Read a single [`RecordBatch`] from an aligned IPC section at `offset`. +/// +/// Skips the alignment padding written by [`write_ipc_section`], then reads +/// the stream, advancing `offset` past the section (padding + stream + EOS). +/// +/// Zero-copy: array buffers borrow from `data`'s allocation when `data`'s base +/// address is at least 64-byte aligned (see [`write_ipc_section`]). +pub fn read_ipc_section_at(data: &Bytes, offset: &mut usize) -> Result { + *offset += section_padding(*offset); + read_ipc_stream_single_at(data, offset) +} + +/// Write `batches` as a single 64-byte-aligned multi-batch Arrow IPC section. +/// +/// Like [`write_ipc_section`] but emits every batch from `iter` into one IPC +/// stream (schema + N batches + EOS). `iter` must yield at least one batch. +/// Paired with [`read_ipc_section_batches_at`]. +pub fn write_ipc_section_batches( + writer: &mut dyn Write, + pos: &mut usize, + iter: I, +) -> Result<(), ArrowError> +where + I: IntoIterator, +{ + write_section_padding(writer, pos)?; + + let mut counting = CountingWriter { + inner: writer, + count: 0, + }; + write_ipc_stream_batches(iter, &mut counting)?; + *pos += counting.count; + Ok(()) +} + +/// Read all [`RecordBatch`]es from an aligned multi-batch IPC section at +/// `offset`, advancing `offset` past the section (padding + stream + EOS). +/// +/// Zero-copy: array buffers borrow from `data`'s allocation when `data`'s base +/// address is at least 64-byte aligned (see [`write_ipc_section_batches`]). +pub fn read_ipc_section_batches_at( + data: &Bytes, + offset: &mut usize, +) -> Result, ArrowError> { + *offset += section_padding(*offset); + read_ipc_stream_at(data, offset) +} + +#[cfg(test)] +mod tests { + use arrow_array::{ArrayRef, record_batch}; + + use super::*; + + #[test] + fn test_ipc_roundtrip() { + let batch1 = record_batch!( + ("int", Int32, [1, 2, 3]), + ("str", Utf8, ["foo", "bar", "baz"]) + ) + .unwrap(); + let batch2 = record_batch!(("int", Int32, [4, 5]), ("str", Utf8, ["qux", "quux"])).unwrap(); + let batches = vec![batch1.clone(), batch2.clone()]; + + let mut buf = Vec::new(); + write_ipc_stream_batches(batches, &mut buf).unwrap(); + + let data = Bytes::from(buf); + + let batches = read_ipc_stream(&data).unwrap(); + assert_eq!(batches.len(), 2); + assert_eq!(batches[0], batch1); + assert_eq!(batches[1], batch2); + + let data_base = data.as_ptr() as usize; + let data_end = data_base + data.len(); + let assert_col_zero_copy = |array: &ArrayRef| { + for buffer in array.to_data().buffers() { + let ptr = buffer.as_ptr() as usize; + assert!( + ptr >= data_base && ptr < data_end, + "buffer at {ptr:#x} is not backed by the input Bytes allocation \ + [{data_base:#x}..{data_end:#x})" + ); + } + }; + + for batch in &batches { + assert_eq!(batch.schema(), batch1.schema()); + assert_col_zero_copy(batch.column(0)); + assert_col_zero_copy(batch.column(1)); + } + } + + /// Allocate a [`Bytes`] whose base address is 64-byte aligned, modelling a + /// backend that reads cache entries into an aligned buffer. A plain + /// `Bytes::from(vec)` only guarantees the allocator's alignment for `u8`. + fn aligned_bytes(payload: &[u8]) -> Bytes { + let mut v = vec![0u8; payload.len() + IPC_SECTION_ALIGNMENT]; + let pad = section_padding(v.as_ptr() as usize); + v[pad..pad + payload.len()].copy_from_slice(payload); + Bytes::from(v).slice(pad..pad + payload.len()) + } + + #[test] + fn test_aligned_ipc_sections_are_zero_copy() { + // A LargeBinary column exercises the i64-offset buffer whose 8-byte + // alignment requirement triggers a realigning memcpy when misaligned. + let blocks = arrow_array::LargeBinaryArray::from_vec(vec![&b"hello"[..], b"world"]); + let section_a = RecordBatch::try_from_iter([("a", Arc::new(blocks) as ArrayRef)]).unwrap(); + let section_b = record_batch!(("b", Int64, [10i64, 20, 30, 40, 50])).unwrap(); + + let mut buf = Vec::new(); + // Arbitrary, deliberately non-64-aligned preamble so the first section + // must be padded rather than landing at offset 0 by luck. + buf.extend_from_slice(&[0xABu8; 7]); + let mut pos = buf.len(); + // The first section's stream begins after padding the 7-byte preamble + // up to the next 64-byte boundary. + assert_eq!(7 + section_padding(7), IPC_SECTION_ALIGNMENT); + write_ipc_section(&mut buf, &mut pos, §ion_a).unwrap(); + write_ipc_section(&mut buf, &mut pos, §ion_b).unwrap(); + + let data = aligned_bytes(&buf); + assert_eq!( + section_padding(data.as_ptr() as usize), + 0, + "base not aligned" + ); + + let mut offset = 7; + let read_a = read_ipc_section_at(&data, &mut offset).unwrap(); + let read_b = read_ipc_section_at(&data, &mut offset).unwrap(); + assert_eq!(read_a, section_a); + assert_eq!(read_b, section_b); + + let data_base = data.as_ptr() as usize; + let data_end = data_base + data.len(); + for batch in [&read_a, &read_b] { + for buffer in batch.column(0).to_data().buffers() { + let ptr = buffer.as_ptr() as usize; + assert!( + ptr >= data_base && ptr < data_end, + "section buffer at {ptr:#x} was realigned out of the input \ + [{data_base:#x}..{data_end:#x}) — misaligned section", + ); + } + } + } + + #[test] + fn test_aligned_multi_batch_section_roundtrip_zero_copy() { + // A multi-batch section (e.g. IVF SQ storage chunks) must round-trip + // every batch and decode the first batch's buffers zero-copy. + let b1 = record_batch!(("v", Int64, [1i64, 2, 3])).unwrap(); + let b2 = record_batch!(("v", Int64, [4i64, 5])).unwrap(); + let b3 = record_batch!(("v", Int64, [6i64])).unwrap(); + + let mut buf = vec![0xCDu8; 5]; + let mut pos = buf.len(); + write_ipc_section_batches(&mut buf, &mut pos, [b1.clone(), b2.clone(), b3.clone()]) + .unwrap(); + + let data = aligned_bytes(&buf); + let mut offset = 5; + let read = read_ipc_section_batches_at(&data, &mut offset).unwrap(); + assert_eq!(read, vec![b1, b2, b3]); + assert_eq!(offset, buf.len(), "offset should land at section end"); + + let data_base = data.as_ptr() as usize; + let data_end = data_base + data.len(); + for buffer in read[0].column(0).to_data().buffers() { + let ptr = buffer.as_ptr() as usize; + assert!( + ptr >= data_base && ptr < data_end, + "first batch buffer at {ptr:#x} was realigned out of the input", + ); + } + } +} diff --git a/python/vendor/lance-arrow/src/json.rs b/python/vendor/lance-arrow/src/json.rs new file mode 100644 index 000000000..96ede058a --- /dev/null +++ b/python/vendor/lance-arrow/src/json.rs @@ -0,0 +1,1445 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! JSON support for Apache Arrow. + +use std::convert::TryFrom; +use std::sync::Arc; + +use arrow_array::builder::{LargeBinaryBuilder, StringBuilder}; +use arrow_array::cast::AsArray; +use arrow_array::{ + Array, ArrayRef, FixedSizeListArray, LargeBinaryArray, LargeListArray, LargeStringArray, + ListArray, MapArray, RecordBatch, StringArray, StructArray, +}; +use arrow_schema::{ArrowError, DataType, Field as ArrowField, Fields, Schema}; + +use crate::ARROW_EXT_NAME_KEY; + +/// Arrow extension type name for JSON data (Lance internal) +pub const JSON_EXT_NAME: &str = "lance.json"; + +/// Arrow extension type name for JSON data (Arrow official) +pub const ARROW_JSON_EXT_NAME: &str = "arrow.json"; + +/// Check if a field is a JSON extension field (Lance internal JSONB storage) +pub fn is_json_field(field: &ArrowField) -> bool { + field.data_type() == &DataType::LargeBinary + && field + .metadata() + .get(ARROW_EXT_NAME_KEY) + .map(|name| name == JSON_EXT_NAME) + .unwrap_or_default() +} + +/// Check if a field is an Arrow JSON extension field (PyArrow pa.json() type) +pub fn is_arrow_json_field(field: &ArrowField) -> bool { + // Arrow JSON extension type uses Utf8 or LargeUtf8 as storage type + (field.data_type() == &DataType::Utf8 || field.data_type() == &DataType::LargeUtf8) + && field + .metadata() + .get(ARROW_EXT_NAME_KEY) + .map(|name| name == ARROW_JSON_EXT_NAME) + .unwrap_or_default() +} + +/// Check if a field or any of its descendants is a JSON field +pub fn has_json_fields(field: &ArrowField) -> bool { + if is_json_field(field) { + return true; + } + + match field.data_type() { + DataType::Struct(fields) => fields.iter().any(|f| has_json_fields(f)), + DataType::List(f) | DataType::LargeList(f) | DataType::FixedSizeList(f, _) => { + has_json_fields(f) + } + DataType::Map(f, _) => has_json_fields(f), + _ => false, + } +} + +/// Check if a field or any of its descendants is an Arrow JSON field +pub fn has_arrow_json_fields(field: &ArrowField) -> bool { + if is_arrow_json_field(field) { + return true; + } + + match field.data_type() { + DataType::Struct(fields) => fields.iter().any(|f| has_arrow_json_fields(f)), + DataType::List(f) | DataType::LargeList(f) | DataType::FixedSizeList(f, _) => { + has_arrow_json_fields(f) + } + DataType::Map(f, _) => has_arrow_json_fields(f), + _ => false, + } +} + +/// Create a JSON field with the appropriate extension metadata +pub fn json_field(name: &str, nullable: bool) -> ArrowField { + let mut field = ArrowField::new(name, DataType::LargeBinary, nullable); + let mut metadata = std::collections::HashMap::new(); + metadata.insert(ARROW_EXT_NAME_KEY.to_string(), JSON_EXT_NAME.to_string()); + field.set_metadata(metadata); + field +} + +/// A specialized array for JSON data stored as JSONB binary format +#[derive(Debug, Clone)] +pub struct JsonArray { + inner: LargeBinaryArray, +} + +impl JsonArray { + /// Create a new JsonArray from an iterator of JSON strings + pub fn try_from_iter(iter: I) -> Result + where + I: IntoIterator>, + S: AsRef, + { + let mut builder = LargeBinaryBuilder::new(); + + for json_str in iter { + match json_str { + Some(s) => { + let encoded = encode_json(s.as_ref()).map_err(|e| { + ArrowError::InvalidArgumentError(format!("Failed to encode JSON: {}", e)) + })?; + builder.append_value(&encoded); + } + None => builder.append_null(), + } + } + + Ok(Self { + inner: builder.finish(), + }) + } + + /// Get the underlying LargeBinaryArray + pub fn into_inner(self) -> LargeBinaryArray { + self.inner + } + + /// Get a reference to the underlying LargeBinaryArray + pub fn inner(&self) -> &LargeBinaryArray { + &self.inner + } + + /// Get the value at index i as decoded JSON string + pub fn value(&self, i: usize) -> Result { + if self.inner.is_null(i) { + return Err(ArrowError::InvalidArgumentError( + "Value is null".to_string(), + )); + } + + let jsonb_bytes = self.inner.value(i); + Ok(decode_json(jsonb_bytes)) + } + + /// Get the value at index i as raw JSONB bytes + pub fn value_bytes(&self, i: usize) -> &[u8] { + self.inner.value(i) + } + + /// Get JSONPath value from the JSON at index i + pub fn json_path(&self, i: usize, path: &str) -> Result, ArrowError> { + if self.inner.is_null(i) { + return Ok(None); + } + + let jsonb_bytes = self.inner.value(i); + get_json_path(jsonb_bytes, path).map_err(|e| { + ArrowError::InvalidArgumentError(format!("Failed to extract JSONPath: {}", e)) + }) + } + + /// Convert to Arrow string array (JSON as UTF-8) + pub fn to_arrow_json(&self) -> ArrayRef { + let mut builder = arrow_array::builder::StringBuilder::new(); + + for i in 0..self.inner.len() { + if self.inner.is_null(i) { + builder.append_null(); + } else { + let jsonb_bytes = self.inner.value(i); + let json_str = decode_json(jsonb_bytes); + builder.append_value(&json_str); + } + } + + // Return as UTF-8 string array (Arrow represents JSON as strings) + Arc::new(builder.finish()) + } + + pub fn len(&self) -> usize { + self.inner.len() + } + + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + pub fn is_null(&self, i: usize) -> bool { + self.inner.is_null(i) + } +} + +// TryFrom implementations for string arrays +impl TryFrom for JsonArray { + type Error = ArrowError; + + fn try_from(array: StringArray) -> Result { + Self::try_from(&array) + } +} + +impl TryFrom<&StringArray> for JsonArray { + type Error = ArrowError; + + fn try_from(array: &StringArray) -> Result { + let mut builder = LargeBinaryBuilder::with_capacity(array.len(), array.value_data().len()); + + for i in 0..array.len() { + if array.is_null(i) { + builder.append_null(); + } else { + let json_str = array.value(i); + let encoded = encode_json(json_str).map_err(|e| { + ArrowError::InvalidArgumentError(format!("Failed to encode JSON: {}", e)) + })?; + builder.append_value(&encoded); + } + } + + Ok(Self { + inner: builder.finish(), + }) + } +} + +impl TryFrom for JsonArray { + type Error = ArrowError; + + fn try_from(array: LargeStringArray) -> Result { + Self::try_from(&array) + } +} + +impl TryFrom<&LargeStringArray> for JsonArray { + type Error = ArrowError; + + fn try_from(array: &LargeStringArray) -> Result { + let mut builder = LargeBinaryBuilder::with_capacity(array.len(), array.value_data().len()); + + for i in 0..array.len() { + if array.is_null(i) { + builder.append_null(); + } else { + let json_str = array.value(i); + let encoded = encode_json(json_str).map_err(|e| { + ArrowError::InvalidArgumentError(format!("Failed to encode JSON: {}", e)) + })?; + builder.append_value(&encoded); + } + } + + Ok(Self { + inner: builder.finish(), + }) + } +} + +impl TryFrom for JsonArray { + type Error = ArrowError; + + fn try_from(array_ref: ArrayRef) -> Result { + match array_ref.data_type() { + DataType::Utf8 => { + // Downcast is guaranteed to succeed after matching on DataType::Utf8 + let string_array = array_ref + .as_any() + .downcast_ref::() + .expect("DataType::Utf8 array must be StringArray"); + Self::try_from(string_array) + } + DataType::LargeUtf8 => { + // Downcast is guaranteed to succeed after matching on DataType::LargeUtf8 + let large_string_array = array_ref + .as_any() + .downcast_ref::() + .expect("DataType::LargeUtf8 array must be LargeStringArray"); + Self::try_from(large_string_array) + } + dt => Err(ArrowError::InvalidArgumentError(format!( + "Unsupported array type for JSON: {:?}. Expected Utf8 or LargeUtf8", + dt + ))), + } + } +} + +/// Encode JSON string to JSONB format +pub fn encode_json(json_str: &str) -> Result, Box> { + let value = jsonb::parse_value(json_str.as_bytes())?; + Ok(value.to_vec()) +} + +/// Decode JSONB bytes to JSON string +pub fn decode_json(jsonb_bytes: &[u8]) -> String { + let raw_jsonb = jsonb::RawJsonb::new(jsonb_bytes); + raw_jsonb.to_string() +} + +/// Extract JSONPath value from JSONB +fn get_json_path( + jsonb_bytes: &[u8], + path: &str, +) -> Result, Box> { + let json_path = jsonb::jsonpath::parse_json_path(path.as_bytes())?; + let raw_jsonb = jsonb::RawJsonb::new(jsonb_bytes); + let mut selector = jsonb::jsonpath::Selector::new(raw_jsonb); + + let values = selector.select_values(&json_path)?; + if values.is_empty() { + Ok(None) + } else { + Ok(Some(values[0].to_string())) + } +} + +/// Convert an Arrow JSON field to Lance JSON field (with JSONB storage) +pub fn arrow_json_to_lance_json(field: &ArrowField) -> ArrowField { + if is_arrow_json_field(field) { + return field_with_extension(field, DataType::LargeBinary, JSON_EXT_NAME); + } + + let data_type = match field.data_type() { + DataType::Struct(fields) => { + let fields = fields + .iter() + .map(|field| Arc::new(arrow_json_to_lance_json(field))) + .collect::>(); + DataType::Struct(Fields::from(fields)) + } + DataType::List(item) => DataType::List(Arc::new(arrow_json_to_lance_json(item))), + DataType::LargeList(item) => DataType::LargeList(Arc::new(arrow_json_to_lance_json(item))), + DataType::FixedSizeList(item, size) => { + DataType::FixedSizeList(Arc::new(arrow_json_to_lance_json(item)), *size) + } + DataType::Map(entries, keys_sorted) => { + DataType::Map(Arc::new(arrow_json_to_lance_json(entries)), *keys_sorted) + } + _ => return field.clone(), + }; + + field_with_data_type(field, data_type) +} + +/// Convert a Lance JSON field to Arrow JSON field. +pub fn lance_json_to_arrow_json(field: &ArrowField) -> ArrowField { + if is_json_field(field) { + return field_with_extension(field, DataType::Utf8, ARROW_JSON_EXT_NAME); + } + + let data_type = match field.data_type() { + DataType::Struct(fields) => { + let fields = fields + .iter() + .map(|field| Arc::new(lance_json_to_arrow_json(field))) + .collect::>(); + DataType::Struct(Fields::from(fields)) + } + DataType::List(item) => DataType::List(Arc::new(lance_json_to_arrow_json(item))), + DataType::LargeList(item) => DataType::LargeList(Arc::new(lance_json_to_arrow_json(item))), + DataType::FixedSizeList(item, size) => { + DataType::FixedSizeList(Arc::new(lance_json_to_arrow_json(item)), *size) + } + DataType::Map(entries, keys_sorted) => { + DataType::Map(Arc::new(lance_json_to_arrow_json(entries)), *keys_sorted) + } + _ => return field.clone(), + }; + + field_with_data_type(field, data_type) +} + +fn field_with_data_type(field: &ArrowField, data_type: DataType) -> ArrowField { + ArrowField::new(field.name(), data_type, field.is_nullable()) + .with_metadata(field.metadata().clone()) +} + +fn field_with_extension( + field: &ArrowField, + data_type: DataType, + extension_name: &str, +) -> ArrowField { + let mut metadata = field.metadata().clone(); + metadata.insert(ARROW_EXT_NAME_KEY.to_string(), extension_name.to_string()); + ArrowField::new(field.name(), data_type, field.is_nullable()).with_metadata(metadata) +} + +fn convert_json_array( + field: &ArrowField, + array: &ArrayRef, + convert_leaf: &F, +) -> Result<(ArrowField, ArrayRef, bool), ArrowError> +where + F: Fn(&ArrowField, &ArrayRef) -> Result, ArrowError>, +{ + if let Some((field, array)) = convert_leaf(field, array)? { + return Ok((field, array, true)); + } + + match field.data_type() { + DataType::Struct(fields) => { + let struct_array = array.as_struct(); + let mut new_fields = Vec::with_capacity(fields.len()); + let mut new_columns = Vec::with_capacity(fields.len()); + let mut changed = false; + + for (field, column) in fields.iter().zip(struct_array.columns()) { + let (new_field, new_column, field_changed) = + convert_json_array(field, column, convert_leaf)?; + changed |= field_changed; + new_fields.push(Arc::new(new_field)); + new_columns.push(new_column); + } + + if changed { + let fields = Fields::from(new_fields); + let new_field = field_with_data_type(field, DataType::Struct(fields.clone())); + let new_array = + StructArray::new(fields, new_columns, struct_array.nulls().cloned()); + Ok((new_field, Arc::new(new_array) as ArrayRef, true)) + } else { + Ok((field.clone(), array.clone(), false)) + } + } + DataType::List(item) => { + let list_array: &ListArray = array.as_list(); + let (new_item, new_values, changed) = + convert_json_array(item, list_array.values(), convert_leaf)?; + if changed { + let new_field = + field_with_data_type(field, DataType::List(Arc::new(new_item.clone()))); + let new_array = ListArray::new( + Arc::new(new_item), + list_array.offsets().clone(), + new_values, + list_array.nulls().cloned(), + ); + Ok((new_field, Arc::new(new_array) as ArrayRef, true)) + } else { + Ok((field.clone(), array.clone(), false)) + } + } + DataType::LargeList(item) => { + let list_array: &LargeListArray = array.as_list(); + let (new_item, new_values, changed) = + convert_json_array(item, list_array.values(), convert_leaf)?; + if changed { + let new_field = + field_with_data_type(field, DataType::LargeList(Arc::new(new_item.clone()))); + let new_array = LargeListArray::new( + Arc::new(new_item), + list_array.offsets().clone(), + new_values, + list_array.nulls().cloned(), + ); + Ok((new_field, Arc::new(new_array) as ArrayRef, true)) + } else { + Ok((field.clone(), array.clone(), false)) + } + } + DataType::FixedSizeList(item, size) => { + let list_array: &FixedSizeListArray = array.as_fixed_size_list(); + let (new_item, new_values, changed) = + convert_json_array(item, list_array.values(), convert_leaf)?; + if changed { + let new_field = field_with_data_type( + field, + DataType::FixedSizeList(Arc::new(new_item.clone()), *size), + ); + let new_array = FixedSizeListArray::try_new_with_length( + Arc::new(new_item), + *size, + new_values, + list_array.nulls().cloned(), + list_array.len(), + )?; + Ok((new_field, Arc::new(new_array) as ArrayRef, true)) + } else { + Ok((field.clone(), array.clone(), false)) + } + } + DataType::Map(entries, keys_sorted) => { + let map_array = array + .as_any() + .downcast_ref::() + .expect("DataType::Map array must be MapArray"); + let entries_array = Arc::new(map_array.entries().clone()) as ArrayRef; + let (new_entries, new_entries_array, changed) = + convert_json_array(entries, &entries_array, convert_leaf)?; + if changed { + let entries_struct = new_entries_array + .as_any() + .downcast_ref::() + .expect("Map entries must be StructArray") + .clone(); + let new_field = field_with_data_type( + field, + DataType::Map(Arc::new(new_entries.clone()), *keys_sorted), + ); + let new_array = MapArray::new( + Arc::new(new_entries), + map_array.offsets().clone(), + entries_struct, + map_array.nulls().cloned(), + *keys_sorted, + ); + Ok((new_field, Arc::new(new_array) as ArrayRef, true)) + } else { + Ok((field.clone(), array.clone(), false)) + } + } + _ => Ok((field.clone(), array.clone(), false)), + } +} + +fn convert_arrow_json_array( + field: &ArrowField, + array: &ArrayRef, +) -> Result<(ArrowField, ArrayRef, bool), ArrowError> { + convert_json_array(field, array, &|field, array| { + if is_arrow_json_field(field) { + let json_array = JsonArray::try_from(array.clone())?; + Ok(Some(( + arrow_json_to_lance_json(field), + Arc::new(json_array.into_inner()) as ArrayRef, + ))) + } else { + Ok(None) + } + }) +} + +fn convert_lance_json_array( + field: &ArrowField, + array: &ArrayRef, +) -> Result<(ArrowField, ArrayRef, bool), ArrowError> { + convert_json_array(field, array, &|field, array| { + if is_json_field(field) { + let binary_array = array + .as_any() + .downcast_ref::() + .expect("Lance JSON field must be LargeBinaryArray"); + let mut builder = StringBuilder::new(); + + for i in 0..binary_array.len() { + if binary_array.is_null(i) { + builder.append_null(); + } else { + let jsonb_bytes = binary_array.value(i); + let json_str = decode_json(jsonb_bytes); + builder.append_value(&json_str); + } + } + + Ok(Some(( + lance_json_to_arrow_json(field), + Arc::new(builder.finish()) as ArrayRef, + ))) + } else { + Ok(None) + } + }) +} + +/// Convert a RecordBatch with Lance JSON columns (JSONB) back to Arrow JSON format (strings) +pub fn convert_lance_json_to_arrow( + batch: &arrow_array::RecordBatch, +) -> Result { + let schema = batch.schema(); + let mut needs_conversion = false; + let mut new_fields = Vec::with_capacity(schema.fields().len()); + let mut new_columns = Vec::with_capacity(batch.num_columns()); + + for (i, field) in schema.fields().iter().enumerate() { + let column = batch.column(i); + let (new_field, new_column, changed) = convert_lance_json_array(field, column)?; + + needs_conversion |= changed; + new_fields.push(new_field); + new_columns.push(new_column); + } + + if needs_conversion { + let new_schema = Arc::new(Schema::new_with_metadata( + new_fields, + schema.metadata().clone(), + )); + RecordBatch::try_new(new_schema, new_columns) + } else { + // No conversion needed, return original batch + Ok(batch.clone()) + } +} + +/// Convert a RecordBatch with Arrow JSON columns to Lance JSON format (JSONB) +pub fn convert_json_columns( + batch: &arrow_array::RecordBatch, +) -> Result { + let schema = batch.schema(); + let mut needs_conversion = false; + let mut new_fields = Vec::with_capacity(schema.fields().len()); + let mut new_columns = Vec::with_capacity(batch.num_columns()); + + for (i, field) in schema.fields().iter().enumerate() { + let column = batch.column(i); + let (new_field, new_column, changed) = convert_arrow_json_array(field, column)?; + + needs_conversion |= changed; + new_fields.push(new_field); + new_columns.push(new_column); + } + + if needs_conversion { + let new_schema = Arc::new(Schema::new_with_metadata( + new_fields, + schema.metadata().clone(), + )); + RecordBatch::try_new(new_schema, new_columns) + } else { + // No conversion needed, return original batch + Ok(batch.clone()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_json_field_creation() { + let field = json_field("data", true); + assert_eq!(field.name(), "data"); + assert_eq!(field.data_type(), &DataType::LargeBinary); + assert!(field.is_nullable()); + assert!(is_json_field(&field)); + } + + #[test] + fn test_json_array_from_strings() { + let json_strings = vec![ + Some(r#"{"name": "Alice", "age": 30}"#), + None, + Some(r#"{"name": "Bob", "age": 25}"#), + ]; + + let array = JsonArray::try_from_iter(json_strings).unwrap(); + assert_eq!(array.len(), 3); + assert!(!array.is_null(0)); + assert!(array.is_null(1)); + assert!(!array.is_null(2)); + + let decoded = array.value(0).unwrap(); + assert!(decoded.contains("Alice")); + } + + #[test] + fn test_json_array_from_string_array() { + let string_array = StringArray::from(vec![ + Some(r#"{"name": "Alice"}"#), + Some(r#"{"name": "Bob"}"#), + None, + ]); + + let json_array = JsonArray::try_from(string_array).unwrap(); + assert_eq!(json_array.len(), 3); + assert!(!json_array.is_null(0)); + assert!(!json_array.is_null(1)); + assert!(json_array.is_null(2)); + } + + #[test] + fn test_json_path_extraction() { + let json_array = JsonArray::try_from_iter(vec![ + Some(r#"{"user": {"name": "Alice", "age": 30}}"#), + Some(r#"{"user": {"name": "Bob"}}"#), + ]) + .unwrap(); + + let name = json_array.json_path(0, "$.user.name").unwrap(); + assert_eq!(name, Some("\"Alice\"".to_string())); + + let age = json_array.json_path(1, "$.user.age").unwrap(); + assert_eq!(age, None); + } + + #[test] + fn test_convert_json_columns() { + // Create a batch with Arrow JSON column + let json_strings = vec![Some(r#"{"name": "Alice"}"#), Some(r#"{"name": "Bob"}"#)]; + let json_arr = StringArray::from(json_strings); + + // Create field with arrow.json extension + let mut field = ArrowField::new("data", DataType::Utf8, false); + let mut metadata = std::collections::HashMap::new(); + metadata.insert( + ARROW_EXT_NAME_KEY.to_string(), + ARROW_JSON_EXT_NAME.to_string(), + ); + field.set_metadata(metadata); + + let schema = Arc::new(Schema::new(vec![field])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(json_arr) as ArrayRef]).unwrap(); + + // Convert the batch + let converted = convert_json_columns(&batch).unwrap(); + + // Check the converted schema + assert_eq!(converted.num_columns(), 1); + let converted_schema = converted.schema(); + let converted_field = converted_schema.field(0); + assert_eq!(converted_field.data_type(), &DataType::LargeBinary); + assert_eq!( + converted_field.metadata().get(ARROW_EXT_NAME_KEY), + Some(&JSON_EXT_NAME.to_string()) + ); + + // Check the data was converted + let converted_column = converted.column(0); + assert_eq!(converted_column.data_type(), &DataType::LargeBinary); + assert_eq!(converted_column.len(), 2); + + // Verify the data is valid JSONB + let binary_array = converted_column + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..binary_array.len() { + let jsonb_bytes = binary_array.value(i); + let decoded = decode_json(jsonb_bytes); + assert!(decoded.contains("name")); + } + } + + #[test] + fn test_convert_nested_json_columns() { + use arrow_buffer::{OffsetBuffer, ScalarBuffer}; + + let uri_field = Arc::new(ArrowField::new("uri", DataType::Utf8, false)); + let mut metadata = std::collections::HashMap::new(); + metadata.insert( + ARROW_EXT_NAME_KEY.to_string(), + ARROW_JSON_EXT_NAME.to_string(), + ); + let extra_field = + Arc::new(ArrowField::new("extra", DataType::Utf8, true).with_metadata(metadata)); + let item_fields = Fields::from(vec![uri_field, extra_field]); + + let values = StructArray::new( + item_fields.clone(), + vec![ + Arc::new(StringArray::from(vec![Some("a.jpg"), Some("b.jpg")])) as ArrayRef, + Arc::new(StringArray::from(vec![ + Some(r#"{"codec":"h264"}"#), + None::<&str>, + ])) as ArrayRef, + ], + None, + ); + let item = Arc::new(ArrowField::new("item", DataType::Struct(item_fields), true)); + let media = ListArray::new( + item, + OffsetBuffer::new(ScalarBuffer::from(vec![0, 1, 2])), + Arc::new(values), + None, + ); + let schema = Arc::new(Schema::new(vec![ArrowField::new( + "media", + media.data_type().clone(), + true, + )])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(media) as ArrayRef]).unwrap(); + + assert!(has_arrow_json_fields(batch.schema().field(0))); + + let converted = convert_json_columns(&batch).unwrap(); + let converted_schema = converted.schema(); + let DataType::List(item) = converted_schema.field(0).data_type() else { + panic!("expected list field"); + }; + let DataType::Struct(fields) = item.data_type() else { + panic!("expected struct item"); + }; + assert!(is_json_field(&fields[1])); + + let list_array: &ListArray = converted.column(0).as_list(); + let values = list_array.values().as_struct(); + let extra = values + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(decode_json(extra.value(0)).contains("h264")); + assert!(extra.is_null(1)); + + let logical = convert_lance_json_to_arrow(&converted).unwrap(); + let logical_schema = logical.schema(); + let DataType::List(item) = logical_schema.field(0).data_type() else { + panic!("expected list field"); + }; + let DataType::Struct(fields) = item.data_type() else { + panic!("expected struct item"); + }; + assert!(is_arrow_json_field(&fields[1])); + + let list_array: &ListArray = logical.column(0).as_list(); + let values = list_array.values().as_struct(); + let extra = values + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(extra.value(0).contains("h264")); + assert!(extra.is_null(1)); + } + + #[test] + fn test_convert_fixed_size_list_zero_json_preserves_length() { + let mut metadata = std::collections::HashMap::new(); + metadata.insert( + ARROW_EXT_NAME_KEY.to_string(), + ARROW_JSON_EXT_NAME.to_string(), + ); + let item = Arc::new(ArrowField::new("item", DataType::Utf8, true).with_metadata(metadata)); + let values = Arc::new(StringArray::from(Vec::>::new())) as ArrayRef; + let lists = FixedSizeListArray::try_new_with_length(item, 0, values, None, 3).unwrap(); + let schema = Arc::new(Schema::new(vec![ArrowField::new( + "lists", + lists.data_type().clone(), + true, + )])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(lists) as ArrayRef]).unwrap(); + + let converted = convert_json_columns(&batch).unwrap(); + assert_eq!(converted.num_rows(), 3); + assert_eq!(converted.column(0).len(), 3); + + let converted_schema = converted.schema(); + let DataType::FixedSizeList(item, size) = converted_schema.field(0).data_type() else { + panic!("expected fixed size list field"); + }; + assert_eq!(*size, 0); + assert!(is_json_field(item)); + + let logical = convert_lance_json_to_arrow(&converted).unwrap(); + assert_eq!(logical.num_rows(), 3); + assert_eq!(logical.column(0).len(), 3); + + let logical_schema = logical.schema(); + let DataType::FixedSizeList(item, size) = logical_schema.field(0).data_type() else { + panic!("expected fixed size list field"); + }; + assert_eq!(*size, 0); + assert!(is_arrow_json_field(item)); + } + + #[test] + fn test_has_json_fields() { + // Test direct JSON field + let json_f = json_field("data", true); + assert!(has_json_fields(&json_f)); + + // Test non-JSON field + let non_json = ArrowField::new("data", DataType::Utf8, true); + assert!(!has_json_fields(&non_json)); + + // Test struct containing JSON field + let struct_field = ArrowField::new( + "struct", + DataType::Struct(vec![json_field("nested_json", true)].into()), + true, + ); + assert!(has_json_fields(&struct_field)); + + // Test struct without JSON field + let struct_no_json = ArrowField::new( + "struct", + DataType::Struct(vec![ArrowField::new("text", DataType::Utf8, true)].into()), + true, + ); + assert!(!has_json_fields(&struct_no_json)); + + // Test List containing JSON field + let list_field = ArrowField::new( + "list", + DataType::List(Arc::new(json_field("item", true))), + true, + ); + assert!(has_json_fields(&list_field)); + + // Test LargeList containing JSON field + let large_list_field = ArrowField::new( + "large_list", + DataType::LargeList(Arc::new(json_field("item", true))), + true, + ); + assert!(has_json_fields(&large_list_field)); + + // Test FixedSizeList containing JSON field + let fixed_list_field = ArrowField::new( + "fixed_list", + DataType::FixedSizeList(Arc::new(json_field("item", true)), 3), + true, + ); + assert!(has_json_fields(&fixed_list_field)); + + // Test Map containing JSON field + let map_field = ArrowField::new( + "map", + DataType::Map( + Arc::new(ArrowField::new( + "entries", + DataType::Struct( + vec![ + ArrowField::new("key", DataType::Utf8, false), + json_field("value", true), + ] + .into(), + ), + false, + )), + false, + ), + true, + ); + assert!(has_json_fields(&map_field)); + } + + #[test] + fn test_json_array_inner() { + let json_array = JsonArray::try_from_iter(vec![Some(r#"{"a": 1}"#)]).unwrap(); + let inner = json_array.inner(); + assert_eq!(inner.len(), 1); + } + + #[test] + fn test_json_array_value_null_error() { + let json_array = JsonArray::try_from_iter(vec![None::<&str>]).unwrap(); + let result = json_array.value(0); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("null")); + } + + #[test] + fn test_json_array_value_bytes() { + let json_array = JsonArray::try_from_iter(vec![Some(r#"{"a": 1}"#)]).unwrap(); + let bytes = json_array.value_bytes(0); + assert!(!bytes.is_empty()); + } + + #[test] + fn test_json_path_with_null() { + let json_array = + JsonArray::try_from_iter(vec![Some(r#"{"user": {"name": "Alice"}}"#), None::<&str>]) + .unwrap(); + + let result = json_array.json_path(1, "$.user.name").unwrap(); + assert_eq!(result, None); + } + + #[test] + fn test_to_arrow_json() { + let json_array = JsonArray::try_from_iter(vec![ + Some(r#"{"name": "Alice"}"#), + None::<&str>, + Some(r#"{"name": "Bob"}"#), + ]) + .unwrap(); + + let arrow_json = json_array.to_arrow_json(); + assert_eq!(arrow_json.len(), 3); + assert!(!arrow_json.is_null(0)); + assert!(arrow_json.is_null(1)); + assert!(!arrow_json.is_null(2)); + + let string_array = arrow_json.as_any().downcast_ref::().unwrap(); + assert!(string_array.value(0).contains("Alice")); + assert!(string_array.value(2).contains("Bob")); + } + + #[test] + fn test_json_array_trait_methods() { + let json_array = + JsonArray::try_from_iter(vec![Some(r#"{"a": 1}"#), Some(r#"{"b": 2}"#)]).unwrap(); + + // Wrapper methods + assert_eq!(json_array.len(), 2); + assert!(!json_array.is_empty()); + assert!(!json_array.is_null(0)); + + // Underlying Arrow array + assert_eq!(json_array.inner().data_type(), &DataType::LargeBinary); + assert_eq!(json_array.inner().len(), 2); + } + + #[test] + fn test_json_array_empty() { + let json_array = JsonArray::try_from_iter(Vec::>::new()).unwrap(); + assert!(json_array.is_empty()); + assert_eq!(json_array.len(), 0); + } + + #[test] + fn test_try_from_large_string_array() { + let large_string_array = LargeStringArray::from(vec![ + Some(r#"{"name": "Alice"}"#), + Some(r#"{"name": "Bob"}"#), + None, + ]); + + // Test TryFrom<&LargeStringArray> + let json_array = JsonArray::try_from(&large_string_array).unwrap(); + assert_eq!(json_array.len(), 3); + assert!(!json_array.is_null(0)); + assert!(!json_array.is_null(1)); + assert!(json_array.is_null(2)); + + // Test TryFrom (owned) + let large_string_array2 = LargeStringArray::from(vec![Some(r#"{"x": 1}"#)]); + let json_array2 = JsonArray::try_from(large_string_array2).unwrap(); + assert_eq!(json_array2.len(), 1); + } + + #[test] + fn test_try_from_array_ref() { + // Test with Utf8 + let string_array: ArrayRef = Arc::new(StringArray::from(vec![ + Some(r#"{"a": 1}"#), + Some(r#"{"b": 2}"#), + ])); + let json_array = JsonArray::try_from(string_array).unwrap(); + assert_eq!(json_array.len(), 2); + + // Test with LargeUtf8 + let large_string_array: ArrayRef = Arc::new(LargeStringArray::from(vec![ + Some(r#"{"c": 3}"#), + Some(r#"{"d": 4}"#), + ])); + let json_array2 = JsonArray::try_from(large_string_array).unwrap(); + assert_eq!(json_array2.len(), 2); + + // Test with unsupported type + let int_array: ArrayRef = Arc::new(arrow_array::Int32Array::from(vec![1, 2, 3])); + let result = JsonArray::try_from(int_array); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("Unsupported")); + } + + #[test] + fn test_arrow_json_to_lance_json_non_json_field() { + // Test that non-JSON fields are returned unchanged + let field = ArrowField::new("text", DataType::Utf8, true); + let converted = arrow_json_to_lance_json(&field); + assert_eq!(converted.data_type(), &DataType::Utf8); + assert_eq!(converted.name(), "text"); + } + + #[test] + fn test_convert_lance_json_to_arrow() { + // Create a batch with Lance JSON column (JSONB) + let json_array = JsonArray::try_from_iter(vec![ + Some(r#"{"name": "Alice"}"#), + None::<&str>, + Some(r#"{"name": "Bob"}"#), + ]) + .unwrap(); + + let lance_json_field = json_field("data", true); + let schema = Arc::new(Schema::new(vec![lance_json_field])); + let batch = + RecordBatch::try_new(schema, vec![Arc::new(json_array.into_inner()) as ArrayRef]) + .unwrap(); + + // Convert back to Arrow JSON + let converted = convert_lance_json_to_arrow(&batch).unwrap(); + + // Check schema + let converted_schema = converted.schema(); + let converted_field = converted_schema.field(0); + assert_eq!(converted_field.data_type(), &DataType::Utf8); + assert_eq!( + converted_field.metadata().get(ARROW_EXT_NAME_KEY), + Some(&ARROW_JSON_EXT_NAME.to_string()) + ); + + // Check data + let string_array = converted + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(!string_array.is_null(0)); + assert!(string_array.is_null(1)); + assert!(!string_array.is_null(2)); + assert!(string_array.value(0).contains("Alice")); + assert!(string_array.value(2).contains("Bob")); + } + + #[test] + fn test_convert_lance_json_to_arrow_empty_batch() { + // Create an empty batch with Lance JSON column + let lance_json_field = json_field("data", true); + let schema = Arc::new(Schema::new(vec![lance_json_field])); + let empty_binary = LargeBinaryBuilder::new().finish(); + let batch = RecordBatch::try_new(schema, vec![Arc::new(empty_binary) as ArrayRef]).unwrap(); + + // Convert back to Arrow JSON + let converted = convert_lance_json_to_arrow(&batch).unwrap(); + assert_eq!(converted.num_rows(), 0); + assert_eq!(converted.schema().field(0).data_type(), &DataType::Utf8); + } + + #[test] + fn test_convert_lance_json_to_arrow_no_json_columns() { + // Create a batch without JSON columns + let field = ArrowField::new("text", DataType::Utf8, true); + let schema = Arc::new(Schema::new(vec![field])); + let string_array = StringArray::from(vec![Some("hello"), Some("world")]); + let batch = RecordBatch::try_new(schema, vec![Arc::new(string_array) as ArrayRef]).unwrap(); + + // Convert - should return the same batch + let converted = convert_lance_json_to_arrow(&batch).unwrap(); + assert_eq!(converted.num_columns(), 1); + assert_eq!(converted.schema().field(0).data_type(), &DataType::Utf8); + } + + #[test] + fn test_convert_json_columns_empty_batch() { + // Create an empty batch with Arrow JSON column + let mut field = ArrowField::new("data", DataType::Utf8, false); + let mut metadata = std::collections::HashMap::new(); + metadata.insert( + ARROW_EXT_NAME_KEY.to_string(), + ARROW_JSON_EXT_NAME.to_string(), + ); + field.set_metadata(metadata); + + let schema = Arc::new(Schema::new(vec![field])); + let empty_strings = arrow_array::builder::StringBuilder::new().finish(); + let batch = + RecordBatch::try_new(schema, vec![Arc::new(empty_strings) as ArrayRef]).unwrap(); + + let converted = convert_json_columns(&batch).unwrap(); + assert_eq!(converted.num_rows(), 0); + assert_eq!( + converted.schema().field(0).data_type(), + &DataType::LargeBinary + ); + } + + #[test] + fn test_convert_json_columns_large_string() { + // Create a batch with Arrow JSON column using LargeUtf8 + let json_strings = LargeStringArray::from(vec![ + Some(r#"{"name": "Alice"}"#), + Some(r#"{"name": "Bob"}"#), + ]); + + let mut field = ArrowField::new("data", DataType::LargeUtf8, false); + let mut metadata = std::collections::HashMap::new(); + metadata.insert( + ARROW_EXT_NAME_KEY.to_string(), + ARROW_JSON_EXT_NAME.to_string(), + ); + field.set_metadata(metadata); + + let schema = Arc::new(Schema::new(vec![field])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(json_strings) as ArrayRef]).unwrap(); + + let converted = convert_json_columns(&batch).unwrap(); + assert_eq!(converted.num_columns(), 1); + assert_eq!( + converted.schema().field(0).data_type(), + &DataType::LargeBinary + ); + assert_eq!(converted.num_rows(), 2); + } + + #[test] + fn test_convert_json_columns_no_json_columns() { + // Create a batch without JSON columns + let field = ArrowField::new("text", DataType::Utf8, true); + let schema = Arc::new(Schema::new(vec![field])); + let string_array = StringArray::from(vec![Some("hello"), Some("world")]); + let batch = RecordBatch::try_new(schema, vec![Arc::new(string_array) as ArrayRef]).unwrap(); + + // Convert - should return the same batch + let converted = convert_json_columns(&batch).unwrap(); + assert_eq!(converted.num_columns(), 1); + assert_eq!(converted.schema().field(0).data_type(), &DataType::Utf8); + } + + #[test] + fn test_convert_json_columns_mixed_columns() { + // Create a batch with both JSON and non-JSON columns + let json_strings = StringArray::from(vec![ + Some(r#"{"name": "Alice"}"#), + Some(r#"{"name": "Bob"}"#), + ]); + let text_strings = StringArray::from(vec![Some("hello"), Some("world")]); + + let mut json_field = ArrowField::new("json_data", DataType::Utf8, false); + let mut metadata = std::collections::HashMap::new(); + metadata.insert( + ARROW_EXT_NAME_KEY.to_string(), + ARROW_JSON_EXT_NAME.to_string(), + ); + json_field.set_metadata(metadata); + + let text_field = ArrowField::new("text_data", DataType::Utf8, true); + + let schema = Arc::new(Schema::new(vec![json_field, text_field])); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(json_strings) as ArrayRef, + Arc::new(text_strings) as ArrayRef, + ], + ) + .unwrap(); + + let converted = convert_json_columns(&batch).unwrap(); + assert_eq!(converted.num_columns(), 2); + assert_eq!( + converted.schema().field(0).data_type(), + &DataType::LargeBinary + ); + assert_eq!(converted.schema().field(1).data_type(), &DataType::Utf8); + } + + #[test] + fn test_is_arrow_json_field_large_utf8() { + // Test with LargeUtf8 storage type + let mut field = ArrowField::new("data", DataType::LargeUtf8, true); + let mut metadata = std::collections::HashMap::new(); + metadata.insert( + ARROW_EXT_NAME_KEY.to_string(), + ARROW_JSON_EXT_NAME.to_string(), + ); + field.set_metadata(metadata); + + assert!(is_arrow_json_field(&field)); + } + + #[test] + fn test_encode_json_invalid() { + // Test encoding invalid JSON + let result = encode_json("not valid json {"); + assert!(result.is_err()); + } + + #[test] + fn test_json_array_from_invalid_json() { + // Test creating JsonArray from invalid JSON strings + let result = JsonArray::try_from_iter(vec![Some("invalid json {")]); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("Failed to encode")); + } + + #[test] + fn test_try_from_string_array_invalid_json() { + let string_array = StringArray::from(vec![Some("invalid json {")]); + let result = JsonArray::try_from(string_array); + assert!(result.is_err()); + } + + #[test] + fn test_try_from_large_string_array_invalid_json() { + let large_string_array = LargeStringArray::from(vec![Some("invalid json {")]); + let result = JsonArray::try_from(large_string_array); + assert!(result.is_err()); + } + + #[test] + fn test_convert_lance_json_to_arrow_mixed_columns() { + // Create a batch with both JSON and non-JSON columns + let json_array = JsonArray::try_from_iter(vec![ + Some(r#"{"name": "Alice"}"#), + Some(r#"{"name": "Bob"}"#), + ]) + .unwrap(); + let text_strings = StringArray::from(vec![Some("hello"), Some("world")]); + + let json_f = json_field("json_data", true); + let text_field = ArrowField::new("text_data", DataType::Utf8, true); + + let schema = Arc::new(Schema::new(vec![json_f, text_field])); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(json_array.into_inner()) as ArrayRef, + Arc::new(text_strings) as ArrayRef, + ], + ) + .unwrap(); + + let converted = convert_lance_json_to_arrow(&batch).unwrap(); + assert_eq!(converted.num_columns(), 2); + assert_eq!(converted.schema().field(0).data_type(), &DataType::Utf8); + assert_eq!(converted.schema().field(1).data_type(), &DataType::Utf8); + } + + #[test] + fn test_json_path_invalid_path() { + let json_array = JsonArray::try_from_iter(vec![Some(r#"{"a": 1}"#)]).unwrap(); + // Invalid JSONPath syntax should return error + let result = json_array.json_path(0, "invalid path without $"); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("Failed to extract JSONPath") + ); + } + + #[test] + fn test_convert_json_columns_invalid_storage_type() { + // Create a batch with Arrow JSON field but wrong storage type (Int32 instead of Utf8) + let int_array = arrow_array::Int32Array::from(vec![1, 2, 3]); + + let mut field = ArrowField::new("data", DataType::Int32, false); + let mut metadata = std::collections::HashMap::new(); + metadata.insert( + ARROW_EXT_NAME_KEY.to_string(), + ARROW_JSON_EXT_NAME.to_string(), + ); + field.set_metadata(metadata); + + let schema = Arc::new(Schema::new(vec![field])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(int_array) as ArrayRef]).unwrap(); + + // This should succeed since Int32 doesn't match is_arrow_json_field check + // (is_arrow_json_field requires Utf8 or LargeUtf8) + let result = convert_json_columns(&batch); + assert!(result.is_ok()); + } + + #[test] + fn test_is_json_field_wrong_extension() { + // LargeBinary field without the correct extension metadata + let field = ArrowField::new("data", DataType::LargeBinary, true); + assert!(!is_json_field(&field)); + + // LargeBinary field with wrong extension name + let mut field2 = ArrowField::new("data", DataType::LargeBinary, true); + let mut metadata = std::collections::HashMap::new(); + metadata.insert( + ARROW_EXT_NAME_KEY.to_string(), + "other.extension".to_string(), + ); + field2.set_metadata(metadata); + assert!(!is_json_field(&field2)); + } + + #[test] + fn test_is_arrow_json_field_wrong_extension() { + // Utf8 field without extension metadata + let field = ArrowField::new("data", DataType::Utf8, true); + assert!(!is_arrow_json_field(&field)); + + // Utf8 field with wrong extension name + let mut field2 = ArrowField::new("data", DataType::Utf8, true); + let mut metadata = std::collections::HashMap::new(); + metadata.insert( + ARROW_EXT_NAME_KEY.to_string(), + "other.extension".to_string(), + ); + field2.set_metadata(metadata); + assert!(!is_arrow_json_field(&field2)); + + // Wrong type entirely + let field3 = ArrowField::new("data", DataType::Int32, true); + assert!(!is_arrow_json_field(&field3)); + } + + #[test] + fn test_convert_json_columns_invalid_json_utf8() { + // Test error propagation when converting invalid JSON (Utf8) + let invalid_json = StringArray::from(vec![Some("invalid json {")]); + + let mut field = ArrowField::new("data", DataType::Utf8, false); + let mut metadata = std::collections::HashMap::new(); + metadata.insert( + ARROW_EXT_NAME_KEY.to_string(), + ARROW_JSON_EXT_NAME.to_string(), + ); + field.set_metadata(metadata); + + let schema = Arc::new(Schema::new(vec![field])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(invalid_json) as ArrayRef]).unwrap(); + + let result = convert_json_columns(&batch); + assert!(result.is_err()); + } + + #[test] + fn test_convert_json_columns_invalid_json_large_utf8() { + // Test error propagation when converting invalid JSON (LargeUtf8) + let invalid_json = LargeStringArray::from(vec![Some("invalid json {")]); + + let mut field = ArrowField::new("data", DataType::LargeUtf8, false); + let mut metadata = std::collections::HashMap::new(); + metadata.insert( + ARROW_EXT_NAME_KEY.to_string(), + ARROW_JSON_EXT_NAME.to_string(), + ); + field.set_metadata(metadata); + + let schema = Arc::new(Schema::new(vec![field])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(invalid_json) as ArrayRef]).unwrap(); + + let result = convert_json_columns(&batch); + assert!(result.is_err()); + } + + #[test] + fn test_json_path_on_corrupted_jsonb() { + // Create corrupted JSONB bytes directly + let corrupted_bytes: &[u8] = &[0xFF, 0xFE, 0x00, 0x01, 0x02]; + let corrupted_binary = LargeBinaryArray::from(vec![Some(corrupted_bytes)]); + + // Wrap in JsonArray + let corrupted_json = JsonArray { + inner: corrupted_binary, + }; + + // Try to use json_path on corrupted data - the selector might fail or return unexpected results + // This exercises the code path but may not produce an error depending on jsonb library behavior + let _result = corrupted_json.json_path(0, "$.a"); + // We don't assert on the result as the behavior depends on the jsonb library + } + + #[test] + fn test_decode_json_on_various_inputs() { + // Test decode_json with various inputs + let valid_jsonb = encode_json(r#"{"key": "value"}"#).unwrap(); + let decoded = decode_json(&valid_jsonb); + assert!(decoded.contains("key")); + + // Empty bytes - jsonb library handles this gracefully + let decoded_empty = decode_json(&[]); + // Just verify it doesn't panic + let _ = decoded_empty; + + // Random bytes - jsonb library handles this gracefully + let decoded_random = decode_json(&[0xFF, 0xFE, 0x00]); + // Just verify it doesn't panic + let _ = decoded_random; + } +} diff --git a/python/vendor/lance-arrow/src/lib.rs b/python/vendor/lance-arrow/src/lib.rs new file mode 100644 index 000000000..777b2ad01 --- /dev/null +++ b/python/vendor/lance-arrow/src/lib.rs @@ -0,0 +1,3048 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Extend Arrow Functionality +//! +//! To improve Arrow-RS ergonomic + +#![warn(clippy::undocumented_unsafe_blocks)] + +// lance-arrow reinterprets value bytes as native numeric types in +// `FloatArray::as_slice` for `bf16` (rust/lance-arrow/src/bfloat16.rs), which +// requires the host byte order to match the on-disk byte order Lance writes. +// Lance writes little-endian; building on a big-endian target would silently +// produce wrong numeric values. +#[cfg(not(target_endian = "little"))] +compile_error!("lance-arrow only supports little-endian targets"); + +use std::sync::Arc; +use std::{collections::HashMap, ptr::NonNull}; + +use arrow_array::{ + Array, ArrayRef, ArrowNumericType, FixedSizeBinaryArray, FixedSizeListArray, GenericListArray, + LargeListArray, ListArray, MapArray, OffsetSizeTrait, PrimitiveArray, RecordBatch, StructArray, + UInt8Array, UInt32Array, cast::AsArray, +}; +use arrow_array::{ + Float32Array, Float64Array, Int8Array, Int16Array, Int32Array, Int64Array, new_null_array, +}; +use arrow_buffer::MutableBuffer; +use arrow_data::ArrayDataBuilder; +use arrow_schema::{ArrowError, DataType, Field, Fields, IntervalUnit, Schema, SortOptions}; +use arrow_select::{interleave::interleave, take::take}; +use rand::prelude::*; + +pub mod deepcopy; +pub mod schema; +pub use schema::*; +pub mod bfloat16; +pub mod floats; +use crate::list::ListArrayExt; +pub use floats::*; + +pub mod ipc; +pub mod json; +pub mod list; +pub mod memory; +pub mod scalar; +pub mod stream; +pub mod r#struct; + +/// Arrow extension metadata key for extension name +pub const ARROW_EXT_NAME_KEY: &str = "ARROW:extension:name"; + +/// Arrow extension metadata key for extension metadata +pub const ARROW_EXT_META_KEY: &str = "ARROW:extension:metadata"; + +/// Key used by lance to mark a field as a blob +/// TODO: Use Arrow extension mechanism instead? +pub const BLOB_META_KEY: &str = "lance-encoding:blob"; +/// Arrow extension type name for Lance blob v2 columns +pub const BLOB_V2_EXT_NAME: &str = "lance.blob.v2"; +/// Metadata key for overriding the dedicated blob size threshold (in bytes) +pub const BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY: &str = + "lance-encoding:blob-dedicated-size-threshold"; +/// Metadata key for overriding the inline blob size threshold (in bytes) +pub const BLOB_INLINE_SIZE_THRESHOLD_META_KEY: &str = "lance-encoding:blob-inline-size-threshold"; +/// Metadata key for overriding the maximum size (in bytes) of a packed blob sidecar file +pub const BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY: &str = + "lance-encoding:blob-pack-file-size-threshold"; + +type Result = std::result::Result; + +pub trait DataTypeExt { + /// Returns true if the data type is binary-like, such as Utf8, Binary, or the large and/or view variants. + fn is_binary_like(&self) -> bool; + + /// Returns true if the data type is a struct. + fn is_struct(&self) -> bool; + + /// Check whether the given Arrow DataType is fixed stride. + /// + /// A fixed stride type has the same byte width for all array elements + /// This includes all PrimitiveType's Boolean, FixedSizeList, FixedSizeBinary, and Decimals + fn is_fixed_stride(&self) -> bool; + + /// Returns true if the [DataType] is a dictionary type. + fn is_dictionary(&self) -> bool; + + /// Returns the byte width of the data type + /// Panics if the data type is not fixed stride. + fn byte_width(&self) -> usize; + + /// Returns the byte width of the data type, if it is fixed stride. + /// Returns None if the data type is not fixed stride. + fn byte_width_opt(&self) -> Option; +} + +impl DataTypeExt for DataType { + fn is_binary_like(&self) -> bool { + use DataType::*; + matches!( + self, + Utf8 | Binary | LargeUtf8 | LargeBinary | Utf8View | BinaryView + ) + } + + fn is_struct(&self) -> bool { + matches!(self, Self::Struct(_)) + } + + fn is_fixed_stride(&self) -> bool { + use DataType::*; + matches!( + self, + Boolean + | UInt8 + | UInt16 + | UInt32 + | UInt64 + | Int8 + | Int16 + | Int32 + | Int64 + | Float16 + | Float32 + | Float64 + | Decimal128(_, _) + | Decimal256(_, _) + | FixedSizeList(_, _) + | FixedSizeBinary(_) + | Duration(_) + | Timestamp(_, _) + | Date32 + | Date64 + | Time32(_) + | Time64(_) + ) + } + + fn is_dictionary(&self) -> bool { + matches!(self, Self::Dictionary(_, _)) + } + + fn byte_width_opt(&self) -> Option { + match self { + Self::Int8 => Some(1), + Self::Int16 => Some(2), + Self::Int32 => Some(4), + Self::Int64 => Some(8), + Self::UInt8 => Some(1), + Self::UInt16 => Some(2), + Self::UInt32 => Some(4), + Self::UInt64 => Some(8), + Self::Float16 => Some(2), + Self::Float32 => Some(4), + Self::Float64 => Some(8), + Self::Date32 => Some(4), + Self::Date64 => Some(8), + Self::Time32(_) => Some(4), + Self::Time64(_) => Some(8), + Self::Timestamp(_, _) => Some(8), + Self::Duration(_) => Some(8), + Self::Decimal128(_, _) => Some(16), + Self::Decimal256(_, _) => Some(32), + Self::Interval(unit) => match unit { + IntervalUnit::YearMonth => Some(4), + IntervalUnit::DayTime => Some(8), + IntervalUnit::MonthDayNano => Some(16), + }, + Self::FixedSizeBinary(s) => Some(*s as usize), + Self::FixedSizeList(dt, s) => dt + .data_type() + .byte_width_opt() + .map(|width| width * *s as usize), + _ => None, + } + } + + fn byte_width(&self) -> usize { + self.byte_width_opt() + .unwrap_or_else(|| panic!("Expecting fixed stride data type, found {:?}", self)) + } +} + +/// Create an [`GenericListArray`] from values and offsets. +/// +/// ``` +/// use arrow_array::{Int32Array, Int64Array, ListArray}; +/// use arrow_array::types::Int64Type; +/// use lance_arrow::try_new_generic_list_array; +/// +/// let offsets = Int32Array::from_iter([0, 2, 7, 10]); +/// let int_values = Int64Array::from_iter(0..10); +/// let list_arr = try_new_generic_list_array(int_values, &offsets).unwrap(); +/// assert_eq!(list_arr, +/// ListArray::from_iter_primitive::(vec![ +/// Some(vec![Some(0), Some(1)]), +/// Some(vec![Some(2), Some(3), Some(4), Some(5), Some(6)]), +/// Some(vec![Some(7), Some(8), Some(9)]), +/// ])) +/// ``` +pub fn try_new_generic_list_array( + values: T, + offsets: &PrimitiveArray, +) -> Result> +where + Offset::Native: OffsetSizeTrait, +{ + let data_type = if Offset::Native::IS_LARGE { + DataType::LargeList(Arc::new(Field::new( + "item", + values.data_type().clone(), + true, + ))) + } else { + DataType::List(Arc::new(Field::new( + "item", + values.data_type().clone(), + true, + ))) + }; + let data = ArrayDataBuilder::new(data_type) + .len(offsets.len() - 1) + .add_buffer(offsets.into_data().buffers()[0].clone()) + .add_child_data(values.into_data()) + .build()?; + + Ok(GenericListArray::from(data)) +} + +pub fn fixed_size_list_type(list_width: i32, inner_type: DataType) -> DataType { + DataType::FixedSizeList(Arc::new(Field::new("item", inner_type, true)), list_width) +} + +pub trait FixedSizeListArrayExt { + /// Create an [`FixedSizeListArray`] from values and list size. + /// + /// ``` + /// use arrow_array::{Int64Array, FixedSizeListArray}; + /// use arrow_array::types::Int64Type; + /// use lance_arrow::FixedSizeListArrayExt; + /// + /// let int_values = Int64Array::from_iter(0..10); + /// let fixed_size_list_arr = FixedSizeListArray::try_new_from_values(int_values, 2).unwrap(); + /// assert_eq!(fixed_size_list_arr, + /// FixedSizeListArray::from_iter_primitive::(vec![ + /// Some(vec![Some(0), Some(1)]), + /// Some(vec![Some(2), Some(3)]), + /// Some(vec![Some(4), Some(5)]), + /// Some(vec![Some(6), Some(7)]), + /// Some(vec![Some(8), Some(9)]) + /// ], 2)) + /// ``` + fn try_new_from_values( + values: T, + list_size: i32, + ) -> Result; + + /// Sample `n` rows from the [FixedSizeListArray] + /// + /// ``` + /// use arrow_array::{Int64Array, FixedSizeListArray, Array}; + /// use lance_arrow::FixedSizeListArrayExt; + /// + /// let int_values = Int64Array::from_iter(0..256); + /// let fixed_size_list_arr = FixedSizeListArray::try_new_from_values(int_values, 16).unwrap(); + /// let sampled = fixed_size_list_arr.sample(10).unwrap(); + /// assert_eq!(sampled.len(), 10); + /// assert_eq!(sampled.value_length(), 16); + /// assert_eq!(sampled.values().len(), 160); + /// ``` + fn sample(&self, n: usize) -> Result; + + /// Ensure the [FixedSizeListArray] of Float16, Float32, Float64, + /// Int8, Int16, Int32, Int64, UInt8, UInt32 type to its closest floating point type. + fn convert_to_floating_point(&self) -> Result; +} + +impl FixedSizeListArrayExt for FixedSizeListArray { + fn try_new_from_values(values: T, list_size: i32) -> Result { + let field = Arc::new(Field::new("item", values.data_type().clone(), true)); + let values = Arc::new(values); + + Self::try_new(field, list_size, values, None) + } + + fn sample(&self, n: usize) -> Result { + if n >= self.len() { + return Ok(self.clone()); + } + let mut rng = SmallRng::from_os_rng(); + let chosen = (0..self.len() as u32).choose_multiple(&mut rng, n); + take(self, &UInt32Array::from(chosen), None).map(|arr| arr.as_fixed_size_list().clone()) + } + + fn convert_to_floating_point(&self) -> Result { + match self.data_type() { + DataType::FixedSizeList(field, size) => match field.data_type() { + DataType::Float16 | DataType::Float32 | DataType::Float64 => Ok(self.clone()), + DataType::Int8 => Ok(Self::new( + Arc::new(arrow_schema::Field::new( + field.name(), + DataType::Float32, + field.is_nullable(), + )), + *size, + Arc::new(Float32Array::from_iter( + self.values() + .as_any() + .downcast_ref::() + .ok_or(ArrowError::ParseError( + "Fail to cast primitive array to Int8Type".to_string(), + ))? + .into_iter() + .map(|x| x.map(|y| y as f32)), + )), + self.nulls().cloned(), + )), + DataType::Int16 => Ok(Self::new( + Arc::new(arrow_schema::Field::new( + field.name(), + DataType::Float32, + field.is_nullable(), + )), + *size, + Arc::new(Float32Array::from_iter( + self.values() + .as_any() + .downcast_ref::() + .ok_or(ArrowError::ParseError( + "Fail to cast primitive array to Int16Type".to_string(), + ))? + .into_iter() + .map(|x| x.map(|y| y as f32)), + )), + self.nulls().cloned(), + )), + DataType::Int32 => Ok(Self::new( + Arc::new(arrow_schema::Field::new( + field.name(), + DataType::Float32, + field.is_nullable(), + )), + *size, + Arc::new(Float32Array::from_iter( + self.values() + .as_any() + .downcast_ref::() + .ok_or(ArrowError::ParseError( + "Fail to cast primitive array to Int32Type".to_string(), + ))? + .into_iter() + .map(|x| x.map(|y| y as f32)), + )), + self.nulls().cloned(), + )), + DataType::Int64 => Ok(Self::new( + Arc::new(arrow_schema::Field::new( + field.name(), + DataType::Float64, + field.is_nullable(), + )), + *size, + Arc::new(Float64Array::from_iter( + self.values() + .as_any() + .downcast_ref::() + .ok_or(ArrowError::ParseError( + "Fail to cast primitive array to Int64Type".to_string(), + ))? + .into_iter() + .map(|x| x.map(|y| y as f64)), + )), + self.nulls().cloned(), + )), + DataType::UInt8 => Ok(Self::new( + Arc::new(arrow_schema::Field::new( + field.name(), + DataType::Float64, + field.is_nullable(), + )), + *size, + Arc::new(Float64Array::from_iter( + self.values() + .as_any() + .downcast_ref::() + .ok_or(ArrowError::ParseError( + "Fail to cast primitive array to UInt8Type".to_string(), + ))? + .into_iter() + .map(|x| x.map(|y| y as f64)), + )), + self.nulls().cloned(), + )), + DataType::UInt32 => Ok(Self::new( + Arc::new(arrow_schema::Field::new( + field.name(), + DataType::Float64, + field.is_nullable(), + )), + *size, + Arc::new(Float64Array::from_iter( + self.values() + .as_any() + .downcast_ref::() + .ok_or(ArrowError::ParseError( + "Fail to cast primitive array to UInt32Type".to_string(), + ))? + .into_iter() + .map(|x| x.map(|y| y as f64)), + )), + self.nulls().cloned(), + )), + data_type => Err(ArrowError::ParseError(format!( + "Expect either floating type or integer got {:?}", + data_type + ))), + }, + data_type => Err(ArrowError::ParseError(format!( + "Expect either FixedSizeList got {:?}", + data_type + ))), + } + } +} + +/// Force downcast of an [`Array`], such as an [`ArrayRef`], to +/// [`FixedSizeListArray`], panic'ing on failure. +pub fn as_fixed_size_list_array(arr: &dyn Array) -> &FixedSizeListArray { + arr.as_any().downcast_ref::().unwrap() +} + +pub trait FixedSizeBinaryArrayExt { + /// Create an [`FixedSizeBinaryArray`] from values and stride. + /// + /// ``` + /// use arrow_array::{UInt8Array, FixedSizeBinaryArray}; + /// use arrow_array::types::UInt8Type; + /// use lance_arrow::FixedSizeBinaryArrayExt; + /// + /// let int_values = UInt8Array::from_iter(0..10); + /// let fixed_size_list_arr = FixedSizeBinaryArray::try_new_from_values(&int_values, 2).unwrap(); + /// assert_eq!(fixed_size_list_arr, + /// FixedSizeBinaryArray::from(vec![ + /// Some(vec![0, 1].as_slice()), + /// Some(vec![2, 3].as_slice()), + /// Some(vec![4, 5].as_slice()), + /// Some(vec![6, 7].as_slice()), + /// Some(vec![8, 9].as_slice()) + /// ])) + /// ``` + fn try_new_from_values(values: &UInt8Array, stride: i32) -> Result; +} + +impl FixedSizeBinaryArrayExt for FixedSizeBinaryArray { + fn try_new_from_values(values: &UInt8Array, stride: i32) -> Result { + let data_type = DataType::FixedSizeBinary(stride); + let data = ArrayDataBuilder::new(data_type) + .len(values.len() / stride as usize) + .add_buffer(values.into_data().buffers()[0].clone()) + .build()?; + Ok(Self::from(data)) + } +} + +pub fn as_fixed_size_binary_array(arr: &dyn Array) -> &FixedSizeBinaryArray { + arr.as_any().downcast_ref::().unwrap() +} + +pub fn iter_str_array(arr: &dyn Array) -> Box> + Send + '_> { + match arr.data_type() { + DataType::Utf8 => Box::new(arr.as_string::().iter()), + DataType::LargeUtf8 => Box::new(arr.as_string::().iter()), + _ => panic!("Expecting Utf8 or LargeUtf8, found {:?}", arr.data_type()), + } +} + +pub fn iter_binary_array( + arr: &dyn Array, +) -> Result> + Send + '_>> { + match arr.data_type() { + DataType::Binary => Ok(Box::new(arr.as_binary::().iter())), + DataType::LargeBinary => Ok(Box::new(arr.as_binary::().iter())), + DataType::BinaryView => Ok(Box::new(arr.as_binary_view().iter())), + DataType::FixedSizeBinary(_) => Ok(Box::new(arr.as_fixed_size_binary().iter())), + data_type => Err(ArrowError::InvalidArgumentError(format!( + "Expecting a binary type, found {data_type}" + ))), + } +} + +/// Extends Arrow's [RecordBatch]. +pub trait RecordBatchExt { + /// Append a new column to this [`RecordBatch`] and returns a new RecordBatch. + /// + /// ``` + /// use std::sync::Arc; + /// use arrow_array::{RecordBatch, Int32Array, StringArray}; + /// use arrow_schema::{Schema, Field, DataType}; + /// use lance_arrow::*; + /// + /// let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])); + /// let int_arr = Arc::new(Int32Array::from(vec![1, 2, 3, 4])); + /// let record_batch = RecordBatch::try_new(schema, vec![int_arr.clone()]).unwrap(); + /// + /// let new_field = Field::new("s", DataType::Utf8, true); + /// let str_arr = Arc::new(StringArray::from(vec!["a", "b", "c", "d"])); + /// let new_record_batch = record_batch.try_with_column(new_field, str_arr.clone()).unwrap(); + /// + /// assert_eq!( + /// new_record_batch, + /// RecordBatch::try_new( + /// Arc::new(Schema::new( + /// vec![ + /// Field::new("a", DataType::Int32, true), + /// Field::new("s", DataType::Utf8, true) + /// ]) + /// ), + /// vec![int_arr, str_arr], + /// ).unwrap() + /// ) + /// ``` + fn try_with_column(&self, field: Field, arr: ArrayRef) -> Result; + + /// Created a new RecordBatch with column at index. + fn try_with_column_at(&self, index: usize, field: Field, arr: ArrayRef) -> Result; + + /// Creates a new [`RecordBatch`] from the provided [`StructArray`]. + /// + /// The fields on the [`StructArray`] need to match this [`RecordBatch`] schema + fn try_new_from_struct_array(&self, arr: StructArray) -> Result; + + /// Merge with another [`RecordBatch`] and returns a new one. + /// + /// Fields are merged based on name. First we iterate the left columns. If a matching + /// name is found in the right then we merge the two columns. If there is no match then + /// we add the left column to the output. + /// + /// To merge two columns we consider the type. If both arrays are struct arrays we recurse. + /// Otherwise we use the left array. + /// + /// Afterwards we add all non-matching right columns to the output. + /// + /// Note: This method likely does not handle nested fields correctly and you may want to consider + /// using [`Self::merge_with_schema`] instead. + /// ``` + /// use std::sync::Arc; + /// use arrow_array::*; + /// use arrow_schema::{Schema, Field, DataType}; + /// use lance_arrow::*; + /// + /// let left_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])); + /// let int_arr = Arc::new(Int32Array::from(vec![1, 2, 3, 4])); + /// let left = RecordBatch::try_new(left_schema, vec![int_arr.clone()]).unwrap(); + /// + /// let right_schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, true)])); + /// let str_arr = Arc::new(StringArray::from(vec!["a", "b", "c", "d"])); + /// let right = RecordBatch::try_new(right_schema, vec![str_arr.clone()]).unwrap(); + /// + /// let new_record_batch = left.merge(&right).unwrap(); + /// + /// assert_eq!( + /// new_record_batch, + /// RecordBatch::try_new( + /// Arc::new(Schema::new( + /// vec![ + /// Field::new("a", DataType::Int32, true), + /// Field::new("s", DataType::Utf8, true) + /// ]) + /// ), + /// vec![int_arr, str_arr], + /// ).unwrap() + /// ) + /// ``` + /// + /// TODO: add merge nested fields support. + fn merge(&self, other: &RecordBatch) -> Result; + + /// Create a batch by merging columns between two batches with a given schema. + /// + /// A reference schema is used to determine the proper ordering of nested fields. + /// + /// For each field in the reference schema we look for corresponding fields in + /// the left and right batches. If a field is found in both batches we recursively merge + /// it. + /// + /// If a field is only in the left or right batch we take it as it is. + fn merge_with_schema(&self, other: &RecordBatch, schema: &Schema) -> Result; + + /// Drop one column specified with the name and return the new [`RecordBatch`]. + /// + /// If the named column does not exist, it returns a copy of this [`RecordBatch`]. + fn drop_column(&self, name: &str) -> Result; + + /// Replace a column (specified by name) and return the new [`RecordBatch`]. + fn replace_column_by_name(&self, name: &str, column: Arc) -> Result; + + /// Replace a column schema (specified by name) and return the new [`RecordBatch`]. + fn replace_column_schema_by_name( + &self, + name: &str, + new_data_type: DataType, + column: Arc, + ) -> Result; + + /// Rename a column at a given index. + fn rename_column(&self, index: usize, new_name: &str) -> Result; + + /// Get (potentially nested) column by qualified name. + fn column_by_qualified_name(&self, name: &str) -> Option<&ArrayRef>; + + /// Project the schema over the [RecordBatch]. + fn project_by_schema(&self, schema: &Schema) -> Result; + + /// metadata of the schema. + fn metadata(&self) -> &HashMap; + + /// Add metadata to the schema. + fn add_metadata(&self, key: String, value: String) -> Result { + let mut metadata = self.metadata().clone(); + metadata.insert(key, value); + self.with_metadata(metadata) + } + + /// Replace the schema metadata with the provided one. + fn with_metadata(&self, metadata: HashMap) -> Result; + + /// Take selected rows from the [RecordBatch]. + fn take(&self, indices: &UInt32Array) -> Result; + + /// Create a new RecordBatch with compacted memory after slicing. + fn shrink_to_fit(&self) -> Result; + + /// Helper method to sort the RecordBatch by a column + fn sort_by_column(&self, column: usize, options: Option) -> Result; +} + +impl RecordBatchExt for RecordBatch { + fn try_with_column(&self, field: Field, arr: ArrayRef) -> Result { + let new_schema = Arc::new(self.schema().as_ref().try_with_column(field)?); + let mut new_columns = self.columns().to_vec(); + new_columns.push(arr); + Self::try_new(new_schema, new_columns) + } + + fn try_with_column_at(&self, index: usize, field: Field, arr: ArrayRef) -> Result { + let new_schema = Arc::new(self.schema().as_ref().try_with_column_at(index, field)?); + let mut new_columns = self.columns().to_vec(); + new_columns.insert(index, arr); + Self::try_new(new_schema, new_columns) + } + + fn try_new_from_struct_array(&self, arr: StructArray) -> Result { + let schema = Arc::new(Schema::new_with_metadata( + arr.fields().to_vec(), + self.schema().metadata.clone(), + )); + let batch = Self::from(arr); + batch.with_schema(schema) + } + + fn merge(&self, other: &Self) -> Result { + if self.num_rows() != other.num_rows() { + return Err(ArrowError::InvalidArgumentError(format!( + "Attempt to merge two RecordBatch with different sizes: {} != {}", + self.num_rows(), + other.num_rows() + ))); + } + let left_struct_array: StructArray = self.clone().into(); + let right_struct_array: StructArray = other.clone().into(); + self.try_new_from_struct_array(merge(&left_struct_array, &right_struct_array)) + } + + fn merge_with_schema(&self, other: &RecordBatch, schema: &Schema) -> Result { + if self.num_rows() != other.num_rows() { + return Err(ArrowError::InvalidArgumentError(format!( + "Attempt to merge two RecordBatch with different sizes: {} != {}", + self.num_rows(), + other.num_rows() + ))); + } + let left_struct_array: StructArray = self.clone().into(); + let right_struct_array: StructArray = other.clone().into(); + self.try_new_from_struct_array(merge_with_schema( + &left_struct_array, + &right_struct_array, + schema.fields(), + )) + } + + fn drop_column(&self, name: &str) -> Result { + let mut fields = vec![]; + let mut columns = vec![]; + for i in 0..self.schema().fields.len() { + if self.schema().field(i).name() != name { + fields.push(self.schema().field(i).clone()); + columns.push(self.column(i).clone()); + } + } + Self::try_new( + Arc::new(Schema::new_with_metadata( + fields, + self.schema().metadata().clone(), + )), + columns, + ) + } + + fn rename_column(&self, index: usize, new_name: &str) -> Result { + let mut fields = self.schema().fields().to_vec(); + if index >= fields.len() { + return Err(ArrowError::InvalidArgumentError(format!( + "Index out of bounds: {}", + index + ))); + } + fields[index] = Arc::new(Field::new( + new_name, + fields[index].data_type().clone(), + fields[index].is_nullable(), + )); + Self::try_new( + Arc::new(Schema::new_with_metadata( + fields, + self.schema().metadata().clone(), + )), + self.columns().to_vec(), + ) + } + + fn replace_column_by_name(&self, name: &str, column: Arc) -> Result { + let mut columns = self.columns().to_vec(); + let field_i = self + .schema() + .fields() + .iter() + .position(|f| f.name() == name) + .ok_or_else(|| ArrowError::SchemaError(format!("Field {} does not exist", name)))?; + columns[field_i] = column; + Self::try_new(self.schema(), columns) + } + + fn replace_column_schema_by_name( + &self, + name: &str, + new_data_type: DataType, + column: Arc, + ) -> Result { + let fields = self + .schema() + .fields() + .iter() + .map(|x| { + if x.name() != name { + x.clone() + } else { + let new_field = Field::new(name, new_data_type.clone(), x.is_nullable()); + Arc::new(new_field) + } + }) + .collect::>(); + let schema = Schema::new_with_metadata(fields, self.schema().metadata.clone()); + let mut columns = self.columns().to_vec(); + let field_i = self + .schema() + .fields() + .iter() + .position(|f| f.name() == name) + .ok_or_else(|| ArrowError::SchemaError(format!("Field {} does not exist", name)))?; + columns[field_i] = column; + Self::try_new(Arc::new(schema), columns) + } + + fn column_by_qualified_name(&self, name: &str) -> Option<&ArrayRef> { + let split = name.split('.').collect::>(); + if split.is_empty() { + return None; + } + + self.column_by_name(split[0]) + .and_then(|arr| get_sub_array(arr, &split[1..])) + } + + fn project_by_schema(&self, schema: &Schema) -> Result { + let struct_array: StructArray = self.clone().into(); + self.try_new_from_struct_array(project(&struct_array, schema.fields())?) + } + + fn metadata(&self) -> &HashMap { + self.schema_ref().metadata() + } + + fn with_metadata(&self, metadata: HashMap) -> Result { + let mut schema = self.schema_ref().as_ref().clone(); + schema.metadata = metadata; + Self::try_new(schema.into(), self.columns().into()) + } + + fn take(&self, indices: &UInt32Array) -> Result { + let struct_array: StructArray = self.clone().into(); + let taken = take(&struct_array, indices, None)?; + self.try_new_from_struct_array(taken.as_struct().clone()) + } + + fn shrink_to_fit(&self) -> Result { + // Deep copy the sliced record batch, instead of whole batch + crate::deepcopy::deep_copy_batch_sliced(self) + } + + fn sort_by_column(&self, column: usize, options: Option) -> Result { + if column >= self.num_columns() { + return Err(ArrowError::InvalidArgumentError(format!( + "Column index out of bounds: {}", + column + ))); + } + let column = self.column(column); + let sorted = arrow_ord::sort::sort_to_indices(column, options, None)?; + self.take(&sorted) + } +} + +/// Recursively projects an array to match the target field's structure. +/// This handles reordering fields inside nested List types. +fn project_array(array: &ArrayRef, target_field: &Field) -> Result { + match target_field.data_type() { + DataType::Struct(subfields) => { + let struct_arr = array.as_struct(); + let projected = project(struct_arr, subfields)?; + Ok(Arc::new(projected)) + } + DataType::List(inner_field) => { + let list_arr: &ListArray = array.as_list(); + let projected_values = project_array(list_arr.values(), inner_field.as_ref())?; + Ok(Arc::new(ListArray::new( + inner_field.clone(), + list_arr.offsets().clone(), + projected_values, + list_arr.nulls().cloned(), + ))) + } + DataType::LargeList(inner_field) => { + let list_arr: &LargeListArray = array.as_list(); + let projected_values = project_array(list_arr.values(), inner_field.as_ref())?; + Ok(Arc::new(LargeListArray::new( + inner_field.clone(), + list_arr.offsets().clone(), + projected_values, + list_arr.nulls().cloned(), + ))) + } + DataType::FixedSizeList(inner_field, size) => { + let list_arr = array.as_fixed_size_list(); + let projected_values = project_array(list_arr.values(), inner_field.as_ref())?; + Ok(Arc::new(FixedSizeListArray::new( + inner_field.clone(), + *size, + projected_values, + list_arr.nulls().cloned(), + ))) + } + // A nullable entries field fails MapArray::try_new unconditionally, + // so a (schema-invalid) map declared that way keeps the clone + // fallthrough it always had rather than gaining a new error. + DataType::Map(entries_field, sorted) if !entries_field.is_nullable() => { + let map_arr = array.as_map(); + let DataType::Struct(entry_fields) = entries_field.data_type() else { + return Err(ArrowError::SchemaError(format!( + "Map entries field must be a struct, got {}", + entries_field.data_type() + ))); + }; + let projected_entries = project(map_arr.entries(), entry_fields)?; + // try_new re-checks the entries invariants (a non-null entries + // struct, two entry columns, offset bounds); null keys are ruled + // out one level down, by the struct rebuild against the + // non-nullable key field. + Ok(Arc::new(MapArray::try_new( + entries_field.clone(), + map_arr.offsets().clone(), + projected_entries, + map_arr.nulls().cloned(), + *sorted, + )?)) + } + _ => Ok(array.clone()), + } +} + +fn project(struct_array: &StructArray, fields: &Fields) -> Result { + if fields.is_empty() { + return Ok(StructArray::new_empty_fields( + struct_array.len(), + struct_array.nulls().cloned(), + )); + } + let mut columns: Vec = vec![]; + for field in fields.iter() { + if let Some(col) = struct_array.column_by_name(field.name()) { + let projected = project_array(col, field.as_ref())?; + columns.push(projected); + } else { + return Err(ArrowError::SchemaError(format!( + "field {} does not exist in the RecordBatch", + field.name() + ))); + } + } + // Preserve the struct's validity when projecting + StructArray::try_new(fields.clone(), columns, struct_array.nulls().cloned()) +} + +fn lists_have_same_offsets_helper(left: &dyn Array, right: &dyn Array) -> bool { + let left_list: &GenericListArray = left.as_list(); + let right_list: &GenericListArray = right.as_list(); + left_list.offsets().inner() == right_list.offsets().inner() +} + +fn merge_list_structs_helper( + left: &dyn Array, + right: &dyn Array, + items_field_name: impl Into, + items_nullable: bool, +) -> Arc { + let left_list: &GenericListArray = left.as_list(); + let right_list: &GenericListArray = right.as_list(); + let left_struct = left_list.values(); + let right_struct = right_list.values(); + let left_struct_arr = left_struct.as_struct(); + let right_struct_arr = right_struct.as_struct(); + let merged_items = Arc::new(merge(left_struct_arr, right_struct_arr)); + let items_field = Arc::new(Field::new( + items_field_name, + merged_items.data_type().clone(), + items_nullable, + )); + Arc::new(GenericListArray::::new( + items_field, + left_list.offsets().clone(), + merged_items, + left_list.nulls().cloned(), + )) +} + +fn merge_list_struct_null_helper( + left: &dyn Array, + right: &dyn Array, + not_null: &dyn Array, + items_field_name: impl Into, +) -> Arc { + let left_list: &GenericListArray = left.as_list::(); + let not_null_list = not_null.as_list::(); + let right_list = right.as_list::(); + + let left_struct = left_list.values().as_struct(); + let not_null_struct: &StructArray = not_null_list.values().as_struct(); + let right_struct = right_list.values().as_struct(); + + let values_len = not_null_list.values().len(); + let mut merged_fields = + Vec::with_capacity(not_null_struct.num_columns() + right_struct.num_columns()); + let mut merged_columns = + Vec::with_capacity(not_null_struct.num_columns() + right_struct.num_columns()); + + for (_, field) in left_struct.columns().iter().zip(left_struct.fields()) { + merged_fields.push(field.clone()); + if let Some(val) = not_null_struct.column_by_name(field.name()) { + merged_columns.push(val.clone()); + } else { + merged_columns.push(new_null_array(field.data_type(), values_len)) + } + } + for (_, field) in right_struct + .columns() + .iter() + .zip(right_struct.fields()) + .filter(|(_, field)| left_struct.column_by_name(field.name()).is_none()) + { + merged_fields.push(field.clone()); + if let Some(val) = not_null_struct.column_by_name(field.name()) { + merged_columns.push(val.clone()); + } else { + merged_columns.push(new_null_array(field.data_type(), values_len)); + } + } + + let merged_struct = Arc::new(StructArray::new( + Fields::from(merged_fields), + merged_columns, + not_null_struct.nulls().cloned(), + )); + let items_field = Arc::new(Field::new( + items_field_name, + merged_struct.data_type().clone(), + true, + )); + Arc::new(GenericListArray::::new( + items_field, + not_null_list.offsets().clone(), + merged_struct, + not_null_list.nulls().cloned(), + )) +} + +fn merge_list_struct_null( + left: &dyn Array, + right: &dyn Array, + not_null: &dyn Array, +) -> Arc { + match left.data_type() { + DataType::List(left_field) => { + merge_list_struct_null_helper::(left, right, not_null, left_field.name()) + } + DataType::LargeList(left_field) => { + merge_list_struct_null_helper::(left, right, not_null, left_field.name()) + } + _ => unreachable!(), + } +} + +fn merge_list_struct(left: &dyn Array, right: &dyn Array) -> Arc { + // Merging fields into a list> is tricky and can only succeed + // in two ways. First, if both lists have the same offsets. Second, if + // one of the lists is all-null + if left.null_count() == left.len() { + return merge_list_struct_null(left, right, right); + } else if right.null_count() == right.len() { + return merge_list_struct_null(left, right, left); + } + match (left.data_type(), right.data_type()) { + (DataType::List(left_field), DataType::List(_)) => { + if !lists_have_same_offsets_helper::(left, right) { + panic!("Attempt to merge list struct arrays which do not have same offsets"); + } + merge_list_structs_helper::( + left, + right, + left_field.name(), + left_field.is_nullable(), + ) + } + (DataType::LargeList(left_field), DataType::LargeList(_)) => { + if !lists_have_same_offsets_helper::(left, right) { + panic!("Attempt to merge list struct arrays which do not have same offsets"); + } + merge_list_structs_helper::( + left, + right, + left_field.name(), + left_field.is_nullable(), + ) + } + _ => unreachable!(), + } +} + +/// Helper function to merge validity buffers from two struct arrays. +/// +/// A row is valid if it is valid in either input. +/// An absent validity buffer means all rows are valid, an all-null buffer acts as the identity for this merge. +fn merge_struct_validity( + left_validity: Option<&arrow_buffer::NullBuffer>, + right_validity: Option<&arrow_buffer::NullBuffer>, +) -> Option { + match (left_validity, right_validity) { + // Fast paths: no computation needed + (None, _) | (_, None) => None, + (Some(left), Some(right)) => { + if left.null_count() == 0 || right.null_count() == 0 { + return None; + } + if left.null_count() == left.len() { + return Some(right.clone()); + } + if right.null_count() == right.len() { + return Some(left.clone()); + } + + let left_buffer = left.inner(); + let right_buffer = right.inner(); + + // Perform bitwise OR directly on BooleanBuffers + // This preserves the correct semantics: 1 = valid, 0 = null + let merged_buffer = left_buffer | right_buffer; + + Some(arrow_buffer::NullBuffer::from(merged_buffer)) + } + } +} + +fn merge_list_child_values( + child_field: &Field, + left_values: ArrayRef, + right_values: ArrayRef, +) -> ArrayRef { + match child_field.data_type() { + DataType::Struct(child_fields) => Arc::new(merge_with_schema( + left_values.as_struct(), + right_values.as_struct(), + child_fields, + )) as ArrayRef, + DataType::List(grandchild) => { + let left_list = left_values + .as_any() + .downcast_ref::() + .expect("left list values should be ListArray"); + let right_list = right_values + .as_any() + .downcast_ref::() + .expect("right list values should be ListArray"); + let merged_values = merge_list_child_values( + grandchild.as_ref(), + left_list.values().clone(), + right_list.values().clone(), + ); + let merged_validity = merge_struct_validity(left_list.nulls(), right_list.nulls()); + Arc::new(ListArray::new( + grandchild.clone(), + left_list.offsets().clone(), + merged_values, + merged_validity, + )) as ArrayRef + } + DataType::LargeList(grandchild) => { + let left_list = left_values + .as_any() + .downcast_ref::() + .expect("left list values should be LargeListArray"); + let right_list = right_values + .as_any() + .downcast_ref::() + .expect("right list values should be LargeListArray"); + let merged_values = merge_list_child_values( + grandchild.as_ref(), + left_list.values().clone(), + right_list.values().clone(), + ); + let merged_validity = merge_struct_validity(left_list.nulls(), right_list.nulls()); + Arc::new(LargeListArray::new( + grandchild.clone(), + left_list.offsets().clone(), + merged_values, + merged_validity, + )) as ArrayRef + } + DataType::FixedSizeList(grandchild, list_size) => { + let left_list = left_values + .as_any() + .downcast_ref::() + .expect("left list values should be FixedSizeListArray"); + let right_list = right_values + .as_any() + .downcast_ref::() + .expect("right list values should be FixedSizeListArray"); + let merged_values = merge_list_child_values( + grandchild.as_ref(), + left_list.values().clone(), + right_list.values().clone(), + ); + let merged_validity = merge_struct_validity(left_list.nulls(), right_list.nulls()); + Arc::new(FixedSizeListArray::new( + grandchild.clone(), + *list_size, + merged_values, + merged_validity, + )) as ArrayRef + } + _ => left_values.clone(), + } +} + +// Helper function to adjust child array validity based on parent struct validity +// When parent struct is null, propagates null to child array +// Optimized with fast paths and SIMD operations +fn adjust_child_validity( + child: &ArrayRef, + parent_validity: Option<&arrow_buffer::NullBuffer>, +) -> ArrayRef { + // Fast path: no parent validity means no adjustment needed + let parent_validity = match parent_validity { + None => return child.clone(), + Some(p) if p.null_count() == 0 => return child.clone(), // No nulls to propagate + Some(p) => p, + }; + + // Fast path: DataType::Null arrays are always entirely null by definition and cannot + // carry an explicit null bitmap (Arrow rejects it). No adjustment is needed. + if child.data_type() == &DataType::Null { + return child.clone(); + } + + let child_validity = child.nulls(); + + // Compute the new validity: child_validity AND parent_validity + let new_validity = match child_validity { + None => { + // Fast path: child has no existing validity, just use parent's + parent_validity.clone() + } + Some(child_nulls) => { + let child_buffer = child_nulls.inner(); + let parent_buffer = parent_validity.inner(); + + // Perform bitwise AND directly on BooleanBuffers + // This preserves the correct semantics: 1 = valid, 0 = null + let merged_buffer = child_buffer & parent_buffer; + + arrow_buffer::NullBuffer::from(merged_buffer) + } + }; + + // ArrayData keeps the child's offset for its value buffers. Its null bitmap + // must cover that offset too, even though the merged validity describes only + // the visible rows. Copy logical bits into an offset-aligned bitmap when needed. + let null_bit_buffer = if child.offset() == 0 && new_validity.inner().offset() == 0 { + new_validity.into_inner().into_inner() + } else { + let mut builder = + arrow_buffer::builder::BooleanBufferBuilder::new(child.offset() + child.len()); + builder.append_n(child.offset(), true); + builder.append_buffer(new_validity.inner()); + builder.build().into_inner() + }; + + // Create new array with adjusted validity + arrow_array::make_array( + arrow_data::ArrayData::try_new( + child.data_type().clone(), + child.len(), + Some(null_bit_buffer), + child.offset(), + child.to_data().buffers().to_vec(), + child.to_data().child_data().to_vec(), + ) + .unwrap(), + ) +} + +fn merge(left_struct_array: &StructArray, right_struct_array: &StructArray) -> StructArray { + let mut fields: Vec = vec![]; + let mut columns: Vec = vec![]; + let right_fields = right_struct_array.fields(); + let right_columns = right_struct_array.columns(); + + // Get the validity buffers from both structs + let left_validity = left_struct_array.nulls(); + let right_validity = right_struct_array.nulls(); + + // Compute merged validity + let merged_validity = merge_struct_validity(left_validity, right_validity); + + // iterate through the fields on the left hand side + for (left_field, left_column) in left_struct_array + .fields() + .iter() + .zip(left_struct_array.columns().iter()) + { + match right_fields + .iter() + .position(|f| f.name() == left_field.name()) + { + // if the field exists on the right hand side, merge them recursively if appropriate + Some(right_index) => { + let right_field = right_fields.get(right_index).unwrap(); + let right_column = right_columns.get(right_index).unwrap(); + // if both fields are struct, merge them recursively + match (left_field.data_type(), right_field.data_type()) { + (DataType::Struct(_), DataType::Struct(_)) => { + let left_sub_array = left_column.as_struct(); + let right_sub_array = right_column.as_struct(); + let merged_sub_array = merge(left_sub_array, right_sub_array); + fields.push(Field::new( + left_field.name(), + merged_sub_array.data_type().clone(), + left_field.is_nullable(), + )); + columns.push(Arc::new(merged_sub_array) as ArrayRef); + } + (DataType::List(left_list), DataType::List(right_list)) + if left_list.data_type().is_struct() + && right_list.data_type().is_struct() => + { + // Identical inner types: nothing to merge, keep the left column. + if left_list.data_type() == right_list.data_type() { + fields.push(left_field.as_ref().clone()); + columns.push(left_column.clone()); + } else { + // The struct fields differ, so merge them structurally. merge_list_struct + // only succeeds when both lists share offsets or one side is all-null; + // it panics otherwise. + let merged_sub_array = merge_list_struct(&left_column, &right_column); + + fields.push(Field::new( + left_field.name(), + merged_sub_array.data_type().clone(), + left_field.is_nullable(), + )); + columns.push(merged_sub_array); + } + } + // otherwise, just use the field on the left hand side + _ => { + // TODO handle list-of-struct and other types + fields.push(left_field.as_ref().clone()); + // Adjust the column validity: if left struct was null, propagate to child + let adjusted_column = adjust_child_validity(left_column, left_validity); + columns.push(adjusted_column); + } + } + } + None => { + fields.push(left_field.as_ref().clone()); + // Adjust the column validity: if left struct was null, propagate to child + let adjusted_column = adjust_child_validity(left_column, left_validity); + columns.push(adjusted_column); + } + } + } + + // now iterate through the fields on the right hand side + right_fields + .iter() + .zip(right_columns.iter()) + .for_each(|(field, column)| { + // add new columns on the right + if !left_struct_array + .fields() + .iter() + .any(|f| f.name() == field.name()) + { + fields.push(field.as_ref().clone()); + // This field doesn't exist on the left + // We use the right's column but need to adjust for struct validity + let adjusted_column = adjust_child_validity(column, right_validity); + columns.push(adjusted_column); + } + }); + + StructArray::try_new(Fields::from(fields), columns, merged_validity).unwrap() +} + +fn merge_with_schema( + left_struct_array: &StructArray, + right_struct_array: &StructArray, + fields: &Fields, +) -> StructArray { + // Helper function that returns true if both types are struct or both are non-struct + fn same_type_kind(left: &DataType, right: &DataType) -> bool { + match (left, right) { + (DataType::Struct(_), DataType::Struct(_)) => true, + (DataType::Struct(_), _) => false, + (_, DataType::Struct(_)) => false, + _ => true, + } + } + + let mut output_fields: Vec = Vec::with_capacity(fields.len()); + let mut columns: Vec = Vec::with_capacity(fields.len()); + + let left_fields = left_struct_array.fields(); + let left_columns = left_struct_array.columns(); + let right_fields = right_struct_array.fields(); + let right_columns = right_struct_array.columns(); + + // Get the validity buffers from both structs + let left_validity = left_struct_array.nulls(); + let right_validity = right_struct_array.nulls(); + + // Compute merged validity + let merged_validity = merge_struct_validity(left_validity, right_validity); + + for field in fields { + let left_match_idx = left_fields.iter().position(|f| { + f.name() == field.name() && same_type_kind(f.data_type(), field.data_type()) + }); + let right_match_idx = right_fields.iter().position(|f| { + f.name() == field.name() && same_type_kind(f.data_type(), field.data_type()) + }); + + match (left_match_idx, right_match_idx) { + (None, Some(right_idx)) => { + output_fields.push(right_fields[right_idx].as_ref().clone()); + // Adjust validity if the right struct was null + let adjusted_column = + adjust_child_validity(&right_columns[right_idx], right_validity); + columns.push(adjusted_column); + } + (Some(left_idx), None) => { + output_fields.push(left_fields[left_idx].as_ref().clone()); + // Adjust validity if the left struct was null + let adjusted_column = adjust_child_validity(&left_columns[left_idx], left_validity); + columns.push(adjusted_column); + } + (Some(left_idx), Some(right_idx)) => { + match field.data_type() { + DataType::Struct(child_fields) => { + let left_sub_array = left_columns[left_idx].as_struct(); + let right_sub_array = right_columns[right_idx].as_struct(); + let merged_sub_array = + merge_with_schema(left_sub_array, right_sub_array, child_fields); + output_fields.push(Field::new( + field.name(), + merged_sub_array.data_type().clone(), + field.is_nullable(), + )); + columns.push(Arc::new(merged_sub_array) as ArrayRef); + } + DataType::List(child_field) => { + let left_list = left_columns[left_idx] + .as_any() + .downcast_ref::() + .unwrap(); + let right_list = right_columns[right_idx] + .as_any() + .downcast_ref::() + .unwrap(); + let merged_values = merge_list_child_values( + child_field.as_ref(), + left_list.trimmed_values(), + right_list.trimmed_values(), + ); + let merged_validity = + merge_struct_validity(left_list.nulls(), right_list.nulls()); + // `trimmed_values` starts at the first used value, so offsets + // must be shifted to match or `ListArray::new` panics when the + // input list was sliced (e.g. from a filtered batch). + let merged_list = ListArray::new( + child_field.clone(), + left_list.trimmed_offsets(), + merged_values, + merged_validity, + ); + output_fields.push(field.as_ref().clone()); + columns.push(Arc::new(merged_list) as ArrayRef); + } + DataType::LargeList(child_field) => { + let left_list = left_columns[left_idx] + .as_any() + .downcast_ref::() + .unwrap(); + let right_list = right_columns[right_idx] + .as_any() + .downcast_ref::() + .unwrap(); + let merged_values = merge_list_child_values( + child_field.as_ref(), + left_list.trimmed_values(), + right_list.trimmed_values(), + ); + let merged_validity = + merge_struct_validity(left_list.nulls(), right_list.nulls()); + let merged_list = LargeListArray::new( + child_field.clone(), + left_list.trimmed_offsets(), + merged_values, + merged_validity, + ); + output_fields.push(field.as_ref().clone()); + columns.push(Arc::new(merged_list) as ArrayRef); + } + DataType::FixedSizeList(child_field, list_size) => { + let left_list = left_columns[left_idx] + .as_any() + .downcast_ref::() + .unwrap(); + let right_list = right_columns[right_idx] + .as_any() + .downcast_ref::() + .unwrap(); + let merged_values = merge_list_child_values( + child_field.as_ref(), + left_list.values().clone(), + right_list.values().clone(), + ); + let merged_validity = + merge_struct_validity(left_list.nulls(), right_list.nulls()); + let merged_list = FixedSizeListArray::new( + child_field.clone(), + *list_size, + merged_values, + merged_validity, + ); + output_fields.push(field.as_ref().clone()); + columns.push(Arc::new(merged_list) as ArrayRef); + } + _ => { + output_fields.push(left_fields[left_idx].as_ref().clone()); + // For fields that exist in both, use left but adjust validity + let adjusted_column = + adjust_child_validity(&left_columns[left_idx], left_validity); + columns.push(adjusted_column); + } + } + } + (None, None) => { + // The field will not be included in the output + } + } + } + + StructArray::try_new(Fields::from(output_fields), columns, merged_validity).unwrap() +} + +fn get_sub_array<'a>(array: &'a ArrayRef, components: &[&str]) -> Option<&'a ArrayRef> { + if components.is_empty() { + return Some(array); + } + if !matches!(array.data_type(), DataType::Struct(_)) { + return None; + } + let struct_arr = array.as_struct(); + struct_arr + .column_by_name(components[0]) + .and_then(|arr| get_sub_array(arr, &components[1..])) +} + +/// Interleave multiple RecordBatches into a single RecordBatch. +/// +/// Behaves like [`arrow_select::interleave::interleave`], but for RecordBatches. +pub fn interleave_batches( + batches: &[RecordBatch], + indices: &[(usize, usize)], +) -> Result { + let first_batch = batches.first().ok_or_else(|| { + ArrowError::InvalidArgumentError("Cannot interleave zero RecordBatches".to_string()) + })?; + let schema = first_batch.schema(); + let num_columns = first_batch.num_columns(); + let mut columns = Vec::with_capacity(num_columns); + let mut chunks = Vec::with_capacity(batches.len()); + + for i in 0..num_columns { + for batch in batches { + chunks.push(batch.column(i).as_ref()); + } + let new_column = interleave(&chunks, indices)?; + columns.push(new_column); + chunks.clear(); + } + + RecordBatch::try_new(schema, columns) +} + +pub trait BufferExt { + /// Create an `arrow_buffer::Buffer`` from a `bytes::Bytes` object + /// + /// The alignment must be specified (as `bytes_per_value`) since we want to make + /// sure we can safely reinterpret the buffer. + /// + /// If the buffer is properly aligned this will be zero-copy. If not, a copy + /// will be made and an owned buffer returned. + /// + /// If `bytes_per_value` is not a power of two, then we assume the buffer is + /// never going to be reinterpreted into another type and we can safely + /// ignore the alignment. + /// + /// Yes, the method name is odd. It's because there is already a `from_bytes` + /// which converts from `arrow_buffer::bytes::Bytes` (not `bytes::Bytes`) + fn from_bytes_bytes(bytes: bytes::Bytes, bytes_per_value: u64) -> Self; + + /// Allocates a new properly aligned arrow buffer and copies `bytes` into it + /// + /// `size_bytes` can be larger than `bytes` and, if so, the trailing bytes will + /// be zeroed out. + /// + /// # Panics + /// + /// Panics if `size_bytes` is less than `bytes.len()` + fn copy_bytes_bytes(bytes: bytes::Bytes, size_bytes: usize) -> Self; +} + +fn is_pwr_two(n: u64) -> bool { + n & (n - 1) == 0 +} + +impl BufferExt for arrow_buffer::Buffer { + fn from_bytes_bytes(bytes: bytes::Bytes, bytes_per_value: u64) -> Self { + if is_pwr_two(bytes_per_value) && bytes.as_ptr().align_offset(bytes_per_value as usize) != 0 + { + // The original buffer is not aligned, cannot zero-copy + let size_bytes = bytes.len(); + Self::copy_bytes_bytes(bytes, size_bytes) + } else { + // The original buffer is aligned, can zero-copy + // SAFETY: the alignment is correct we can make this conversion + unsafe { + Self::from_custom_allocation( + NonNull::new(bytes.as_ptr() as _).expect("should be a valid pointer"), + bytes.len(), + Arc::new(bytes), + ) + } + } + } + + fn copy_bytes_bytes(bytes: bytes::Bytes, size_bytes: usize) -> Self { + assert!(size_bytes >= bytes.len()); + let mut buf = MutableBuffer::with_capacity(size_bytes); + let to_fill = size_bytes - bytes.len(); + buf.extend(bytes); + buf.extend(std::iter::repeat_n(0_u8, to_fill)); + + // FIX for issue #4512: Shrink buffer to actual size before converting to immutable + // This reduces memory overhead from capacity over-allocation + buf.shrink_to_fit(); + + Self::from(buf) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_array::{ + BinaryArray, BinaryViewArray, BooleanArray, FixedSizeBinaryArray, Float32Array, Int32Array, + LargeBinaryArray, ListArray, NullArray, StringArray, StructArray, new_empty_array, + new_null_array, + }; + use arrow_buffer::OffsetBuffer; + + #[test] + fn test_adjust_child_validity_with_sliced_child() { + let child: ArrayRef = Arc::new(BooleanArray::from_iter((0..1030).map(|index| { + if index == 1027 { + None + } else { + Some(index % 2 == 1) + } + }))); + let child = child.slice(1025, 4); + assert_eq!(child.offset(), 1025); + + let parent_validity = arrow_buffer::NullBuffer::from(vec![true, false, true, true]); + let adjusted = adjust_child_validity(&child, Some(&parent_validity)); + assert_eq!(adjusted.offset(), child.offset()); + let adjusted = adjusted.as_any().downcast_ref::().unwrap(); + assert_eq!( + adjusted.iter().collect::>(), + vec![Some(true), None, None, Some(false)] + ); + } + + #[test] + fn test_adjust_child_validity_with_sliced_parent_bitmap() { + let child: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4])); + let parent_validity: arrow_buffer::NullBuffer = + (0..1030).map(|index| index != 1026).collect(); + let parent_validity = parent_validity.slice(1025, 4); + assert_eq!(parent_validity.inner().offset(), 1025); + + let adjusted = adjust_child_validity(&child, Some(&parent_validity)); + let adjusted = adjusted.as_any().downcast_ref::().unwrap(); + assert_eq!( + adjusted.iter().collect::>(), + vec![Some(1), None, Some(3), Some(4)] + ); + } + + #[test] + fn test_convert_to_floating_point_preserves_inner_nulls() { + // A FixedSizeList with a null inner element must convert to a + // FixedSizeList with the null kept in place. Dropping it would + // shorten the values array and shift every later element (and, when the + // remaining count is not a multiple of the list size, panic). + let values = Int8Array::from(vec![Some(1), None, Some(3), Some(4)]); + let fsl = FixedSizeListArray::new( + Arc::new(Field::new("item", DataType::Int8, true)), + 2, + Arc::new(values), + None, + ); + + let converted = fsl.convert_to_floating_point().unwrap(); + + assert_eq!(converted.len(), 2); + let conv_values = converted + .values() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(conv_values.len(), 4); + assert_eq!(conv_values.value(0), 1.0); + assert!(conv_values.is_null(1)); + assert_eq!(conv_values.value(2), 3.0); + assert_eq!(conv_values.value(3), 4.0); + } + + #[test] + fn test_convert_to_floating_point_preserves_inner_nulls_f64_arm() { + // The Float64-producing arms (Int64/UInt8/UInt32) share the same fix as the + // Float32 arms; cover one representative (UInt8 -> Float64) so both branch + // families are exercised. + let values = UInt8Array::from(vec![Some(10u8), None, Some(30), Some(40)]); + let fsl = FixedSizeListArray::new( + Arc::new(Field::new("item", DataType::UInt8, true)), + 2, + Arc::new(values), + None, + ); + + let converted = fsl.convert_to_floating_point().unwrap(); + + assert_eq!(converted.len(), 2); + let conv_values = converted + .values() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(conv_values.len(), 4); + assert_eq!(conv_values.value(0), 10.0); + assert!(conv_values.is_null(1)); + assert_eq!(conv_values.value(2), 30.0); + assert_eq!(conv_values.value(3), 40.0); + } + + #[test] + fn test_merge_recursive() { + let a_array = Int32Array::from(vec![Some(1), Some(2), Some(3)]); + let e_array = Int32Array::from(vec![Some(4), Some(5), Some(6)]); + let c_array = Int32Array::from(vec![Some(7), Some(8), Some(9)]); + let d_array = StringArray::from(vec![Some("a"), Some("b"), Some("c")]); + + let left_schema = Schema::new(vec![ + Field::new("a", DataType::Int32, true), + Field::new( + "b", + DataType::Struct(vec![Field::new("c", DataType::Int32, true)].into()), + true, + ), + ]); + let left_batch = RecordBatch::try_new( + Arc::new(left_schema), + vec![ + Arc::new(a_array.clone()), + Arc::new(StructArray::from(vec![( + Arc::new(Field::new("c", DataType::Int32, true)), + Arc::new(c_array.clone()) as ArrayRef, + )])), + ], + ) + .unwrap(); + + let right_schema = Schema::new(vec![ + Field::new("e", DataType::Int32, true), + Field::new( + "b", + DataType::Struct(vec![Field::new("d", DataType::Utf8, true)].into()), + true, + ), + ]); + let right_batch = RecordBatch::try_new( + Arc::new(right_schema), + vec![ + Arc::new(e_array.clone()), + Arc::new(StructArray::from(vec![( + Arc::new(Field::new("d", DataType::Utf8, true)), + Arc::new(d_array.clone()) as ArrayRef, + )])) as ArrayRef, + ], + ) + .unwrap(); + + let merged_schema = Schema::new(vec![ + Field::new("a", DataType::Int32, true), + Field::new( + "b", + DataType::Struct( + vec![ + Field::new("c", DataType::Int32, true), + Field::new("d", DataType::Utf8, true), + ] + .into(), + ), + true, + ), + Field::new("e", DataType::Int32, true), + ]); + let merged_batch = RecordBatch::try_new( + Arc::new(merged_schema), + vec![ + Arc::new(a_array) as ArrayRef, + Arc::new(StructArray::from(vec![ + ( + Arc::new(Field::new("c", DataType::Int32, true)), + Arc::new(c_array) as ArrayRef, + ), + ( + Arc::new(Field::new("d", DataType::Utf8, true)), + Arc::new(d_array) as ArrayRef, + ), + ])) as ArrayRef, + Arc::new(e_array) as ArrayRef, + ], + ) + .unwrap(); + + let result = left_batch.merge(&right_batch).unwrap(); + assert_eq!(result, merged_batch); + } + + #[test] + fn test_merge_with_schema() { + fn test_batch(names: &[&str], types: &[DataType]) -> (Schema, RecordBatch) { + let fields: Fields = names + .iter() + .zip(types) + .map(|(name, ty)| Field::new(name.to_string(), ty.clone(), false)) + .collect(); + let schema = Schema::new(vec![Field::new( + "struct", + DataType::Struct(fields.clone()), + false, + )]); + let children = types.iter().map(new_empty_array).collect::>(); + let batch = RecordBatch::try_new( + Arc::new(schema.clone()), + vec![Arc::new(StructArray::new(fields, children, None)) as ArrayRef], + ); + (schema, batch.unwrap()) + } + + let (_, left_batch) = test_batch(&["a", "b"], &[DataType::Int32, DataType::Int64]); + let (_, right_batch) = test_batch(&["c", "b"], &[DataType::Int32, DataType::Int64]); + let (output_schema, _) = test_batch( + &["b", "a", "c"], + &[DataType::Int64, DataType::Int32, DataType::Int32], + ); + + // If we use merge_with_schema the schema is respected + let merged = left_batch + .merge_with_schema(&right_batch, &output_schema) + .unwrap(); + assert_eq!(merged.schema().as_ref(), &output_schema); + + // If we use merge we get first-come first-serve based on the left batch + let (naive_schema, _) = test_batch( + &["a", "b", "c"], + &[DataType::Int32, DataType::Int64, DataType::Int32], + ); + let merged = left_batch.merge(&right_batch).unwrap(); + assert_eq!(merged.schema().as_ref(), &naive_schema); + } + + #[test] + fn test_merge_list_struct() { + let x_field = Arc::new(Field::new("x", DataType::Int32, true)); + let y_field = Arc::new(Field::new("y", DataType::Int32, true)); + let x_struct_field = Arc::new(Field::new( + "item", + DataType::Struct(Fields::from(vec![x_field.clone()])), + true, + )); + let y_struct_field = Arc::new(Field::new( + "item", + DataType::Struct(Fields::from(vec![y_field.clone()])), + true, + )); + let both_struct_field = Arc::new(Field::new( + "item", + DataType::Struct(Fields::from(vec![x_field.clone(), y_field.clone()])), + true, + )); + let left_schema = Schema::new(vec![Field::new( + "list_struct", + DataType::List(x_struct_field.clone()), + true, + )]); + let right_schema = Schema::new(vec![Field::new( + "list_struct", + DataType::List(y_struct_field.clone()), + true, + )]); + let both_schema = Schema::new(vec![Field::new( + "list_struct", + DataType::List(both_struct_field.clone()), + true, + )]); + + let x = Arc::new(Int32Array::from(vec![1])); + let y = Arc::new(Int32Array::from(vec![2])); + let x_struct = Arc::new(StructArray::new( + Fields::from(vec![x_field.clone()]), + vec![x.clone()], + None, + )); + let y_struct = Arc::new(StructArray::new( + Fields::from(vec![y_field.clone()]), + vec![y.clone()], + None, + )); + let both_struct = Arc::new(StructArray::new( + Fields::from(vec![x_field.clone(), y_field.clone()]), + vec![x.clone(), y], + None, + )); + let both_null_struct = Arc::new(StructArray::new( + Fields::from(vec![x_field, y_field]), + vec![x, Arc::new(new_null_array(&DataType::Int32, 1))], + None, + )); + let offsets = OffsetBuffer::from_lengths([1]); + let x_s_list = ListArray::new(x_struct_field, offsets.clone(), x_struct, None); + let y_s_list = ListArray::new(y_struct_field, offsets.clone(), y_struct, None); + let both_list = ListArray::new( + both_struct_field.clone(), + offsets.clone(), + both_struct, + None, + ); + let both_null_list = ListArray::new(both_struct_field, offsets, both_null_struct, None); + let x_batch = + RecordBatch::try_new(Arc::new(left_schema), vec![Arc::new(x_s_list)]).unwrap(); + let y_batch = RecordBatch::try_new( + Arc::new(right_schema.clone()), + vec![Arc::new(y_s_list.clone())], + ) + .unwrap(); + let merged = x_batch.merge(&y_batch).unwrap(); + let expected = + RecordBatch::try_new(Arc::new(both_schema.clone()), vec![Arc::new(both_list)]).unwrap(); + assert_eq!(merged, expected); + + let y_null_list = new_null_array(y_s_list.data_type(), 1); + let y_null_batch = + RecordBatch::try_new(Arc::new(right_schema), vec![Arc::new(y_null_list.clone())]) + .unwrap(); + let expected = + RecordBatch::try_new(Arc::new(both_schema), vec![Arc::new(both_null_list)]).unwrap(); + let merged = x_batch.merge(&y_null_batch).unwrap(); + assert_eq!(merged, expected); + } + + #[test] + fn test_merge_list_struct_identical_schema() { + // Merging two batches whose `List` columns have identical types + // should yield a single column equal to the input (there is nothing to + // merge), not a struct with the field pushed twice. + let x_field = Arc::new(Field::new("x", DataType::Int32, true)); + let item_field = Arc::new(Field::new( + "item", + DataType::Struct(Fields::from(vec![x_field.clone()])), + true, + )); + let schema = Arc::new(Schema::new(vec![Field::new( + "list_struct", + DataType::List(item_field.clone()), + true, + )])); + + let build_list = |values: Vec| { + let len = values.len(); + let item_struct = Arc::new(StructArray::new( + Fields::from(vec![x_field.clone()]), + vec![Arc::new(Int32Array::from(values))], + None, + )); + ListArray::new( + item_field.clone(), + OffsetBuffer::from_lengths([len]), + item_struct, + None, + ) + }; + + // Distinct values so the equality assertion proves the left column is kept, + // not that some column with the same values happens to be present. + let left = + RecordBatch::try_new(schema.clone(), vec![Arc::new(build_list(vec![1, 2]))]).unwrap(); + let right = RecordBatch::try_new(schema, vec![Arc::new(build_list(vec![3, 4]))]).unwrap(); + + let merged = left.merge(&right).unwrap(); + + // Exactly one column, equal to the left input: the field must not be duplicated. + assert_eq!(merged.num_columns(), 1); + assert_eq!(merged, left); + } + + #[test] + fn test_merge_nested_list_struct_identical_schema() { + // A `List` nested inside a struct reaches the identical-type + // short-circuit through the recursive `merge` call. The recursion must + // keep the field exactly once, equal to the left input. + let x_field = Arc::new(Field::new("x", DataType::Int32, true)); + let item_field = Arc::new(Field::new( + "item", + DataType::Struct(Fields::from(vec![x_field.clone()])), + true, + )); + let companies_field = Arc::new(Field::new( + "companies", + DataType::List(item_field.clone()), + true, + )); + let schema = Arc::new(Schema::new(vec![Field::new( + "outer", + DataType::Struct(Fields::from(vec![companies_field.clone()])), + true, + )])); + + let build_outer = |x: i32| { + // One list row: [{x}]. + let item_struct = Arc::new(StructArray::new( + Fields::from(vec![x_field.clone()]), + vec![Arc::new(Int32Array::from(vec![x]))], + None, + )); + let companies = Arc::new(ListArray::new( + item_field.clone(), + OffsetBuffer::from_lengths([1]), + item_struct, + None, + )); + StructArray::new( + Fields::from(vec![companies_field.clone()]), + vec![companies], + None, + ) + }; + + // Distinct values so equality proves the left column is kept. + let left = RecordBatch::try_new(schema.clone(), vec![Arc::new(build_outer(10))]).unwrap(); + let right = RecordBatch::try_new(schema, vec![Arc::new(build_outer(20))]).unwrap(); + + let merged = left.merge(&right).unwrap(); + + // The recursive identical-type merge keeps exactly one `companies` field, + // with no double-push inside the nested struct. + assert_eq!(merged.column(0).as_struct().num_columns(), 1); + assert_eq!(merged, left); + } + + #[test] + fn test_byte_width_opt() { + assert_eq!(DataType::Int32.byte_width_opt(), Some(4)); + assert_eq!(DataType::Int64.byte_width_opt(), Some(8)); + assert_eq!(DataType::Float32.byte_width_opt(), Some(4)); + assert_eq!(DataType::Float64.byte_width_opt(), Some(8)); + assert_eq!(DataType::Utf8.byte_width_opt(), None); + assert_eq!(DataType::Binary.byte_width_opt(), None); + assert_eq!( + DataType::List(Arc::new(Field::new("item", DataType::Int32, true))).byte_width_opt(), + None + ); + assert_eq!( + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Int32, true)), 3) + .byte_width_opt(), + Some(12) + ); + assert_eq!( + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Int32, true)), 4) + .byte_width_opt(), + Some(16) + ); + assert_eq!( + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Utf8, true)), 5) + .byte_width_opt(), + None + ); + } + + #[test] + fn test_take_record_batch() { + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Utf8, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..20)), + Arc::new(StringArray::from_iter_values( + (0..20).map(|i| format!("str-{}", i)), + )), + ], + ) + .unwrap(); + let taken = batch.take(&(vec![1_u32, 5_u32, 10_u32].into())).unwrap(); + assert_eq!( + taken, + RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(vec![1, 5, 10])), + Arc::new(StringArray::from(vec!["str-1", "str-5", "str-10"])), + ], + ) + .unwrap() + ) + } + + #[test] + fn test_schema_project_by_schema() { + let metadata = [("key".to_string(), "value".to_string())]; + let schema = Arc::new( + Schema::new(vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Utf8, true), + ]) + .with_metadata(metadata.clone().into()), + ); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from_iter_values(0..20)), + Arc::new(StringArray::from_iter_values( + (0..20).map(|i| format!("str-{}", i)), + )), + ], + ) + .unwrap(); + + // Empty schema + let empty_schema = Schema::empty(); + let empty_projected = batch.project_by_schema(&empty_schema).unwrap(); + let expected_schema = empty_schema.with_metadata(metadata.clone().into()); + assert_eq!( + empty_projected, + RecordBatch::from(StructArray::new_empty_fields(batch.num_rows(), None)) + .with_schema(Arc::new(expected_schema)) + .unwrap() + ); + + // Re-ordered schema + let reordered_schema = Schema::new(vec![ + Field::new("b", DataType::Utf8, true), + Field::new("a", DataType::Int32, true), + ]); + let reordered_projected = batch.project_by_schema(&reordered_schema).unwrap(); + let expected_schema = Arc::new(reordered_schema.with_metadata(metadata.clone().into())); + assert_eq!( + reordered_projected, + RecordBatch::try_new( + expected_schema, + vec![ + Arc::new(StringArray::from_iter_values( + (0..20).map(|i| format!("str-{}", i)), + )), + Arc::new(Int32Array::from_iter_values(0..20)), + ], + ) + .unwrap() + ); + + // Sub schema + let sub_schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]); + let sub_projected = batch.project_by_schema(&sub_schema).unwrap(); + let expected_schema = Arc::new(sub_schema.with_metadata(metadata.into())); + assert_eq!( + sub_projected, + RecordBatch::try_new( + expected_schema, + vec![Arc::new(Int32Array::from_iter_values(0..20))], + ) + .unwrap() + ); + } + + #[test] + fn test_project_rebuilds_sliced_map() { + // A sliced MapArray keeps its full entries array behind sliced + // offsets and validity; the Map projection arm must rebuild it + // without renormalizing either. + let entry_fields = Fields::from(vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Int32, true), + ]); + let entries_field = Arc::new(Field::new( + "entries", + DataType::Struct(entry_fields.clone()), + false, + )); + let entries = StructArray::new( + entry_fields, + vec![ + Arc::new(StringArray::from(vec!["k0", "k1", "k2"])) as ArrayRef, + Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef, + ], + None, + ); + let map = MapArray::new( + entries_field.clone(), + OffsetBuffer::new(vec![0, 1, 1, 3].into()), + entries, + Some(arrow_buffer::NullBuffer::from(vec![true, false, true])), + false, + ); + let schema = Arc::new(Schema::new(vec![Field::new( + "m", + DataType::Map(entries_field, false), + true, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(map) as ArrayRef]).unwrap(); + + // Rows 1..3: a null slot and a two-entry slot, offsets not zero-based. + let projected = batch + .slice(1, 2) + .project_by_schema(schema.as_ref()) + .unwrap(); + let map = projected.column(0).as_map(); + assert!(map.is_null(0)); + assert!(map.is_valid(1)); + assert_eq!(map.value_length(1), 2); + assert_eq!( + map.value(1) + .column(1) + .as_primitive::() + .values(), + &[2, 3] + ); + } + + #[test] + fn test_project_preserves_struct_validity() { + // Test that projecting a struct array preserves its validity (fix for issue #4385) + let fields = Fields::from(vec![ + Field::new("id", DataType::Int32, false), + Field::new("value", DataType::Float32, true), + ]); + + // Create a struct array with validity + let id_array = Int32Array::from(vec![1, 2, 3]); + let value_array = Float32Array::from(vec![Some(1.0), Some(2.0), Some(3.0)]); + let struct_array = StructArray::new( + fields.clone(), + vec![ + Arc::new(id_array) as ArrayRef, + Arc::new(value_array) as ArrayRef, + ], + Some(vec![true, false, true].into()), // Second struct is null + ); + + // Project the struct array + let projected = project(&struct_array, &fields).unwrap(); + + // Verify the validity is preserved + assert_eq!(projected.null_count(), 1); + assert!(!projected.is_null(0)); + assert!(projected.is_null(1)); + assert!(!projected.is_null(2)); + } + + #[test] + fn test_merge_struct_with_different_validity() { + // Test case from Weston's review comment + // File 1 has height field with some nulls + let height_array = Int32Array::from(vec![Some(500), None, Some(600), None]); + let left_fields = Fields::from(vec![Field::new("height", DataType::Int32, true)]); + let left_struct = StructArray::new( + left_fields, + vec![Arc::new(height_array) as ArrayRef], + Some(vec![true, false, true, false].into()), // Rows 2 and 4 are null structs + ); + + // File 2 has width field with some nulls + let width_array = Int32Array::from(vec![Some(300), Some(200), None, None]); + let right_fields = Fields::from(vec![Field::new("width", DataType::Int32, true)]); + let right_struct = StructArray::new( + right_fields, + vec![Arc::new(width_array) as ArrayRef], + Some(vec![true, true, false, false].into()), // Rows 3 and 4 are null structs + ); + + // Merge the two structs + let merged = merge(&left_struct, &right_struct); + + // Expected: + // Row 1: both non-null -> {width: 300, height: 500} + // Row 2: left null, right non-null -> {width: 200, height: null} + // Row 3: left non-null, right null -> {width: null, height: 600} + // Row 4: both null -> null struct + + assert_eq!(merged.null_count(), 1); // Only row 4 is null + assert!(!merged.is_null(0)); + assert!(!merged.is_null(1)); + assert!(!merged.is_null(2)); + assert!(merged.is_null(3)); + + // Check field values + let height_col = merged.column_by_name("height").unwrap(); + let height_values = height_col.as_any().downcast_ref::().unwrap(); + assert_eq!(height_values.value(0), 500); + assert!(height_values.is_null(1)); // height is null when left struct was null + assert_eq!(height_values.value(2), 600); + + let width_col = merged.column_by_name("width").unwrap(); + let width_values = width_col.as_any().downcast_ref::().unwrap(); + assert_eq!(width_values.value(0), 300); + assert_eq!(width_values.value(1), 200); + assert!(width_values.is_null(2)); // width is null when right struct was null + + // An all-null validity buffer is data, not a placeholder meaning "this side has no + // validity": merging two of them keeps the rows null. + let all_null_left = StructArray::new( + Fields::from(vec![Field::new("height", DataType::Int32, true)]), + vec![Arc::new(Int32Array::from(vec![None, None])) as ArrayRef], + Some(vec![false, false].into()), + ); + let all_null_right = StructArray::new( + Fields::from(vec![Field::new("width", DataType::Int32, true)]), + vec![Arc::new(Int32Array::from(vec![None, None])) as ArrayRef], + Some(vec![false, false].into()), + ); + + let merged = merge(&all_null_left, &all_null_right); + assert_eq!(merged.null_count(), 2); + + // An all-null side is the identity of the merge, so the other side decides each row. + let partial_left = StructArray::new( + Fields::from(vec![Field::new("height", DataType::Int32, true)]), + vec![Arc::new(Int32Array::from(vec![Some(1), None])) as ArrayRef], + Some(vec![true, false].into()), + ); + let merged = merge(&partial_left, &all_null_right); + assert!(!merged.is_null(0)); + assert!(merged.is_null(1)); + + // A missing validity buffer means all rows are valid, which absorbs an all-null side. + let all_valid_left = StructArray::new( + Fields::from(vec![Field::new("height", DataType::Int32, true)]), + vec![Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef], + None, + ); + let merged = merge(&all_valid_left, &all_null_right); + assert_eq!(merged.null_count(), 0); + + // An explicit all-valid buffer has the same semantics as a missing buffer. + let all_valid: arrow_buffer::NullBuffer = vec![true, true].into(); + let partial: arrow_buffer::NullBuffer = vec![true, false].into(); + assert!(merge_struct_validity(Some(&all_valid), Some(&partial)).is_none()); + assert!(merge_struct_validity(Some(&partial), Some(&all_valid)).is_none()); + } + + #[test] + fn test_merge_null_typed_column_with_parent_validity() { + // Reproduces ENT-990: panic in adjust_child_validity when a Null-typed column + // exists on one side and the parent struct has null rows. + // Arrow's Null type has no null bitmap, so passing one to ArrayData::try_new panics. + let left_struct = StructArray::new( + Fields::from(vec![Field::new("a", DataType::Int32, true)]), + vec![Arc::new(Int32Array::from(vec![Some(1), None])) as ArrayRef], + Some(vec![true, false].into()), + ); + let right_struct = StructArray::new( + Fields::from(vec![Field::new("b", DataType::Null, true)]), + vec![Arc::new(NullArray::new(2)) as ArrayRef], + Some(vec![true, false].into()), + ); + + // Previously panicked: "Arrays of type Null cannot contain a null bitmask" + let merged = merge(&left_struct, &right_struct); + assert_eq!(merged.len(), 2); + let b_col = merged.column_by_name("b").unwrap(); + // DataType::Null implies all-null by definition; no null bitmap is stored. + assert_eq!(b_col.data_type(), &DataType::Null); + assert_eq!(b_col.len(), 2); + } + + #[test] + fn test_merge_with_schema_with_nullable_struct_list_schema_mismatch() { + // left_list setup + let left_company_id = Arc::new(Int32Array::from(vec![None, None])); + let left_count = Arc::new(Int32Array::from(vec![None, None])); + let left_struct = Arc::new(StructArray::new( + Fields::from(vec![ + Field::new("company_id", DataType::Int32, true), + Field::new("count", DataType::Int32, true), + ]), + vec![left_company_id, left_count], + None, + )); + let left_list = Arc::new(ListArray::new( + Arc::new(Field::new( + "item", + DataType::Struct(left_struct.fields().clone()), + true, + )), + OffsetBuffer::from_lengths([2]), + left_struct, + None, + )); + + // Right List Setup + let right_company_name = Arc::new(StringArray::from(vec!["Google", "Microsoft"])); + let right_struct = Arc::new(StructArray::new( + Fields::from(vec![Field::new("company_name", DataType::Utf8, true)]), + vec![right_company_name], + None, + )); + let right_list = Arc::new(ListArray::new( + Arc::new(Field::new( + "item", + DataType::Struct(right_struct.fields().clone()), + true, + )), + OffsetBuffer::from_lengths([2]), + right_struct, + None, + )); + + let target_fields = Fields::from(vec![Field::new( + "companies", + DataType::List(Arc::new(Field::new( + "item", + DataType::Struct(Fields::from(vec![ + Field::new("company_id", DataType::Int32, true), + Field::new("company_name", DataType::Utf8, true), + Field::new("count", DataType::Int32, true), + ])), + true, + ))), + true, + )]); + + let left_batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "companies", + left_list.data_type().clone(), + true, + )])), + vec![left_list as ArrayRef], + ) + .unwrap(); + + let right_batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "companies", + right_list.data_type().clone(), + true, + )])), + vec![right_list as ArrayRef], + ) + .unwrap(); + + let merged = left_batch + .merge_with_schema(&right_batch, &Schema::new(target_fields.to_vec())) + .unwrap(); + + // Verify the merged structure + let merged_list = merged + .column_by_name("companies") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let merged_struct = merged_list.values().as_struct(); + + // Should have all 3 fields + assert_eq!(merged_struct.num_columns(), 3); + assert!(merged_struct.column_by_name("company_id").is_some()); + assert!(merged_struct.column_by_name("company_name").is_some()); + assert!(merged_struct.column_by_name("count").is_some()); + + // Verify values + let company_id = merged_struct + .column_by_name("company_id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert!(company_id.is_null(0)); + assert!(company_id.is_null(1)); + + let company_name = merged_struct + .column_by_name("company_name") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(company_name.value(0), "Google"); + assert_eq!(company_name.value(1), "Microsoft"); + + let count = merged_struct + .column_by_name("count") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert!(count.is_null(0)); + assert!(count.is_null(1)); + } + + #[test] + fn test_merge_struct_lists() { + test_merge_struct_lists_generic::(); + } + + #[test] + fn test_merge_struct_large_lists() { + test_merge_struct_lists_generic::(); + } + + fn test_merge_struct_lists_generic() { + // left_list setup + let left_company_id = Arc::new(Int32Array::from(vec![ + Some(1), + Some(2), + Some(3), + Some(4), + Some(5), + Some(6), + Some(7), + Some(8), + Some(9), + Some(10), + Some(11), + Some(12), + Some(13), + Some(14), + Some(15), + Some(16), + Some(17), + Some(18), + Some(19), + Some(20), + ])); + let left_count = Arc::new(Int32Array::from(vec![ + Some(10), + Some(20), + Some(30), + Some(40), + Some(50), + Some(60), + Some(70), + Some(80), + Some(90), + Some(100), + Some(110), + Some(120), + Some(130), + Some(140), + Some(150), + Some(160), + Some(170), + Some(180), + Some(190), + Some(200), + ])); + let left_struct = Arc::new(StructArray::new( + Fields::from(vec![ + Field::new("company_id", DataType::Int32, true), + Field::new("count", DataType::Int32, true), + ]), + vec![left_company_id, left_count], + None, + )); + + let left_list = Arc::new(GenericListArray::::new( + Arc::new(Field::new( + "item", + DataType::Struct(left_struct.fields().clone()), + true, + )), + OffsetBuffer::from_lengths([3, 1]), + left_struct.clone(), + None, + )); + + let left_list_struct = Arc::new(StructArray::new( + Fields::from(vec![Field::new( + "companies", + if O::IS_LARGE { + DataType::LargeList(Arc::new(Field::new( + "item", + DataType::Struct(left_struct.fields().clone()), + true, + ))) + } else { + DataType::List(Arc::new(Field::new( + "item", + DataType::Struct(left_struct.fields().clone()), + true, + ))) + }, + true, + )]), + vec![left_list as ArrayRef], + None, + )); + + // right_list setup + let right_company_name = Arc::new(StringArray::from(vec![ + "Google", + "Microsoft", + "Apple", + "Facebook", + ])); + let right_struct = Arc::new(StructArray::new( + Fields::from(vec![Field::new("company_name", DataType::Utf8, true)]), + vec![right_company_name], + None, + )); + let right_list = Arc::new(GenericListArray::::new( + Arc::new(Field::new( + "item", + DataType::Struct(right_struct.fields().clone()), + true, + )), + OffsetBuffer::from_lengths([3, 1]), + right_struct.clone(), + None, + )); + + let right_list_struct = Arc::new(StructArray::new( + Fields::from(vec![Field::new( + "companies", + if O::IS_LARGE { + DataType::LargeList(Arc::new(Field::new( + "item", + DataType::Struct(right_struct.fields().clone()), + true, + ))) + } else { + DataType::List(Arc::new(Field::new( + "item", + DataType::Struct(right_struct.fields().clone()), + true, + ))) + }, + true, + )]), + vec![right_list as ArrayRef], + None, + )); + + // prepare schema + let target_fields = Fields::from(vec![Field::new( + "companies", + if O::IS_LARGE { + DataType::LargeList(Arc::new(Field::new( + "item", + DataType::Struct(Fields::from(vec![ + Field::new("company_id", DataType::Int32, true), + Field::new("company_name", DataType::Utf8, true), + Field::new("count", DataType::Int32, true), + ])), + true, + ))) + } else { + DataType::List(Arc::new(Field::new( + "item", + DataType::Struct(Fields::from(vec![ + Field::new("company_id", DataType::Int32, true), + Field::new("company_name", DataType::Utf8, true), + Field::new("count", DataType::Int32, true), + ])), + true, + ))) + }, + true, + )]); + + // merge left_list and right_list + let merged_array = merge_with_schema(&left_list_struct, &right_list_struct, &target_fields); + assert_eq!(merged_array.len(), 2); + } + + #[test] + fn test_merge_with_schema_sliced_list_struct() { + test_merge_with_schema_sliced_list_struct_generic::(); + } + + #[test] + fn test_merge_with_schema_sliced_large_list_struct() { + test_merge_with_schema_sliced_list_struct_generic::(); + } + + // Regression for #6580: merge_with_schema panicked when the left list was a + // sliced view whose offsets did not start at zero (common after a filtered + // scan). Cloning those offsets alongside `trimmed_values` produced offsets + // larger than the trimmed child, panicking in `(Large)ListArray::new`. + fn test_merge_with_schema_sliced_list_struct_generic() { + let make_list_dtype = |item_field: Arc| { + if O::IS_LARGE { + DataType::LargeList(item_field) + } else { + DataType::List(item_field) + } + }; + + // Build a List with two rows of 5 items each, then slice away + // the first row so the remaining list's offsets start at 5, not 0. + let struct_fields_a = Fields::from(vec![Field::new("a", DataType::Int32, true)]); + let left_values = Arc::new(StructArray::new( + struct_fields_a.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..10)) as ArrayRef], + None, + )); + let full_list = GenericListArray::::new( + Arc::new(Field::new("item", DataType::Struct(struct_fields_a), true)), + OffsetBuffer::::from_lengths([5, 5]), + left_values, + None, + ); + let sliced_left = full_list.slice(1, 1); + assert_eq!(sliced_left.offsets()[0].as_usize(), 5); + assert_eq!(sliced_left.offsets()[1].as_usize(), 10); + + let struct_fields_b = Fields::from(vec![Field::new("b", DataType::Int32, true)]); + let right_values = Arc::new(StructArray::new( + struct_fields_b.clone(), + vec![Arc::new(Int32Array::from_iter_values(100..105)) as ArrayRef], + None, + )); + let right_list = GenericListArray::::new( + Arc::new(Field::new("item", DataType::Struct(struct_fields_b), true)), + OffsetBuffer::::from_lengths([5]), + right_values, + None, + ); + + let target_item_field = Arc::new(Field::new( + "item", + DataType::Struct(Fields::from(vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Int32, true), + ])), + true, + )); + let target_fields = Fields::from(vec![Field::new( + "items", + make_list_dtype(target_item_field), + true, + )]); + + let left_batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "items", + sliced_left.data_type().clone(), + true, + )])), + vec![Arc::new(sliced_left) as ArrayRef], + ) + .unwrap(); + let right_batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "items", + right_list.data_type().clone(), + true, + )])), + vec![Arc::new(right_list) as ArrayRef], + ) + .unwrap(); + + let merged = left_batch + .merge_with_schema(&right_batch, &Schema::new(target_fields.to_vec())) + .unwrap(); + + let merged_list = merged + .column_by_name("items") + .unwrap() + .as_any() + .downcast_ref::>() + .unwrap(); + assert_eq!(merged_list.len(), 1); + assert_eq!(merged_list.value_length(0).as_usize(), 5); + let merged_struct = merged_list.values().as_struct(); + assert_eq!(merged_struct.num_columns(), 2); + let a = merged_struct + .column_by_name("a") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + // After shifting offsets to zero, values 5..10 should be first. + let a_vals: Vec = a.iter().map(|v| v.unwrap()).collect(); + assert_eq!(a_vals, vec![5, 6, 7, 8, 9]); + } + + #[test] + fn test_project_by_schema_list_struct_reorder() { + // Test that project_by_schema correctly reorders fields inside List + // This is a regression test for issue #5702 + + // Source schema with inner struct fields in order: c, b, a + let source_inner_struct = DataType::Struct(Fields::from(vec![ + Field::new("c", DataType::Utf8, true), + Field::new("b", DataType::Utf8, true), + Field::new("a", DataType::Utf8, true), + ])); + let source_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new( + "data", + DataType::List(Arc::new(Field::new( + "item", + source_inner_struct.clone(), + true, + ))), + true, + ), + ])); + + // Create source data with c, b, a order + let c_array = StringArray::from(vec!["c1", "c2"]); + let b_array = StringArray::from(vec!["b1", "b2"]); + let a_array = StringArray::from(vec!["a1", "a2"]); + let inner_struct = StructArray::from(vec![ + ( + Arc::new(Field::new("c", DataType::Utf8, true)), + Arc::new(c_array) as ArrayRef, + ), + ( + Arc::new(Field::new("b", DataType::Utf8, true)), + Arc::new(b_array) as ArrayRef, + ), + ( + Arc::new(Field::new("a", DataType::Utf8, true)), + Arc::new(a_array) as ArrayRef, + ), + ]); + + let list_array = ListArray::new( + Arc::new(Field::new("item", source_inner_struct, true)), + OffsetBuffer::from_lengths([1, 1]), + Arc::new(inner_struct), + None, + ); + + let batch = RecordBatch::try_new( + source_schema, + vec![Arc::new(Int32Array::from(vec![1, 2])), Arc::new(list_array)], + ) + .unwrap(); + + // Target schema with inner struct fields in order: a, b, c + let target_inner_struct = DataType::Struct(Fields::from(vec![ + Field::new("a", DataType::Utf8, true), + Field::new("b", DataType::Utf8, true), + Field::new("c", DataType::Utf8, true), + ])); + let target_schema = Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new( + "data", + DataType::List(Arc::new(Field::new("item", target_inner_struct, true))), + true, + ), + ]); + + // Project should reorder the inner struct fields + let projected = batch.project_by_schema(&target_schema).unwrap(); + + // Verify the schema is correct + assert_eq!(projected.schema().as_ref(), &target_schema); + + // Verify the data is correct by checking inner struct field order + let projected_list = projected.column(1).as_list::(); + let projected_struct = projected_list.values().as_struct(); + + // Fields should now be in order: a, b, c + assert_eq!( + projected_struct.column_by_name("a").unwrap().as_ref(), + &StringArray::from(vec!["a1", "a2"]) as &dyn Array + ); + assert_eq!( + projected_struct.column_by_name("b").unwrap().as_ref(), + &StringArray::from(vec!["b1", "b2"]) as &dyn Array + ); + assert_eq!( + projected_struct.column_by_name("c").unwrap().as_ref(), + &StringArray::from(vec!["c1", "c2"]) as &dyn Array + ); + + // Also verify positional access matches expected order (a=0, b=1, c=2) + assert_eq!( + projected_struct.column(0).as_ref(), + &StringArray::from(vec!["a1", "a2"]) as &dyn Array + ); + assert_eq!( + projected_struct.column(1).as_ref(), + &StringArray::from(vec!["b1", "b2"]) as &dyn Array + ); + assert_eq!( + projected_struct.column(2).as_ref(), + &StringArray::from(vec!["c1", "c2"]) as &dyn Array + ); + } + + #[test] + fn test_project_by_schema_nested_list_struct() { + // Test deeply nested List>> projection + let inner_struct = DataType::Struct(Fields::from(vec![ + Field::new("y", DataType::Int32, true), + Field::new("x", DataType::Int32, true), + ])); + let source_schema = Arc::new(Schema::new(vec![Field::new( + "outer", + DataType::List(Arc::new(Field::new( + "item", + DataType::Struct(Fields::from(vec![ + Field::new("b", DataType::Utf8, true), + Field::new( + "inner_list", + DataType::List(Arc::new(Field::new("item", inner_struct.clone(), true))), + true, + ), + Field::new("a", DataType::Utf8, true), + ])), + true, + ))), + true, + )])); + + // Create deeply nested data + let y_array = Int32Array::from(vec![1, 2]); + let x_array = Int32Array::from(vec![3, 4]); + let innermost_struct = StructArray::from(vec![ + ( + Arc::new(Field::new("y", DataType::Int32, true)), + Arc::new(y_array) as ArrayRef, + ), + ( + Arc::new(Field::new("x", DataType::Int32, true)), + Arc::new(x_array) as ArrayRef, + ), + ]); + let inner_list = ListArray::new( + Arc::new(Field::new("item", inner_struct.clone(), true)), + OffsetBuffer::from_lengths([2]), + Arc::new(innermost_struct), + None, + ); + + let b_array = StringArray::from(vec!["b1"]); + let a_array = StringArray::from(vec!["a1"]); + let middle_struct = StructArray::from(vec![ + ( + Arc::new(Field::new("b", DataType::Utf8, true)), + Arc::new(b_array) as ArrayRef, + ), + ( + Arc::new(Field::new( + "inner_list", + DataType::List(Arc::new(Field::new("item", inner_struct, true))), + true, + )), + Arc::new(inner_list) as ArrayRef, + ), + ( + Arc::new(Field::new("a", DataType::Utf8, true)), + Arc::new(a_array) as ArrayRef, + ), + ]); + + let outer_list = ListArray::new( + Arc::new(Field::new("item", middle_struct.data_type().clone(), true)), + OffsetBuffer::from_lengths([1]), + Arc::new(middle_struct), + None, + ); + + let batch = + RecordBatch::try_new(source_schema, vec![Arc::new(outer_list) as ArrayRef]).unwrap(); + + // Target schema with reordered fields at all levels + let target_inner_struct = DataType::Struct(Fields::from(vec![ + Field::new("x", DataType::Int32, true), // x before y now + Field::new("y", DataType::Int32, true), + ])); + let target_schema = Schema::new(vec![Field::new( + "outer", + DataType::List(Arc::new(Field::new( + "item", + DataType::Struct(Fields::from(vec![ + Field::new("a", DataType::Utf8, true), // a before b now + Field::new( + "inner_list", + DataType::List(Arc::new(Field::new("item", target_inner_struct, true))), + true, + ), + Field::new("b", DataType::Utf8, true), + ])), + true, + ))), + true, + )]); + + let projected = batch.project_by_schema(&target_schema).unwrap(); + + // Verify schema + assert_eq!(projected.schema().as_ref(), &target_schema); + + // Verify deeply nested data is reordered correctly + let outer_list = projected.column(0).as_list::(); + let middle_struct = outer_list.values().as_struct(); + + // Middle struct should have a first, then inner_list, then b + assert_eq!( + middle_struct.column(0).as_ref(), + &StringArray::from(vec!["a1"]) as &dyn Array + ); + assert_eq!( + middle_struct.column(2).as_ref(), + &StringArray::from(vec!["b1"]) as &dyn Array + ); + + // Inner list's struct should have x first, then y + let inner_list = middle_struct.column(1).as_list::(); + let innermost_struct = inner_list.values().as_struct(); + assert_eq!( + innermost_struct.column(0).as_ref(), + &Int32Array::from(vec![3, 4]) as &dyn Array + ); + assert_eq!( + innermost_struct.column(1).as_ref(), + &Int32Array::from(vec![1, 2]) as &dyn Array + ); + } + + #[test] + fn test_iter_binary_array_accepts_binary_variants() { + let binary = BinaryArray::from(vec![b"a".as_slice(), b"bc"]); + assert_eq!( + iter_binary_array(&binary).unwrap().collect::>(), + vec![Some(b"a".as_slice()), Some(b"bc".as_slice())] + ); + + let large_binary = LargeBinaryArray::from(vec![b"x".as_slice(), b"yz"]); + assert_eq!( + iter_binary_array(&large_binary) + .unwrap() + .collect::>(), + vec![Some(b"x".as_slice()), Some(b"yz".as_slice())] + ); + + let binary_view = BinaryViewArray::from(vec![b"1".as_slice(), b"23"]); + assert_eq!( + iter_binary_array(&binary_view).unwrap().collect::>(), + vec![Some(b"1".as_slice()), Some(b"23".as_slice())] + ); + + let fixed_size = FixedSizeBinaryArray::from(vec![b"abcd", b"efgh"]); + assert_eq!( + iter_binary_array(&fixed_size).unwrap().collect::>(), + vec![Some(b"abcd".as_slice()), Some(b"efgh".as_slice())] + ); + } + + #[test] + fn test_iter_binary_array_rejects_non_binary() { + let int_array = Int32Array::from(vec![1, 2, 3]); + let Err(error) = iter_binary_array(&int_array) else { + panic!("expected an error for non-binary array"); + }; + assert!(error.to_string().contains("Expecting a binary type")); + } +} diff --git a/python/vendor/lance-arrow/src/list.rs b/python/vendor/lance-arrow/src/list.rs new file mode 100644 index 000000000..06b0fc592 --- /dev/null +++ b/python/vendor/lance-arrow/src/list.rs @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use arrow_array::{Array, BooleanArray, GenericListArray, OffsetSizeTrait}; +use arrow_buffer::{BooleanBufferBuilder, OffsetBuffer, ScalarBuffer}; +use arrow_schema::Field; + +pub trait ListArrayExt { + /// Filters out masked null items from the list array + /// + /// It is legal for a list array to have a null entry with a non-zero length. The + /// values inside the entry are "garbage" and should be ignored. This function + /// filters the values array to remove the garbage values. + /// + /// The output list will always have zero-length nulls. + fn filter_garbage_nulls(&self) -> Self; + /// Returns a copy of the list's values array that has been sliced to size + /// + /// It is legal for a list array's offsets to not start with zero. It's also legal + /// for a list array's offsets to not extend to the entire values array. This function + /// behaves similarly to `values()` except it slices the array so that it starts at + /// the first list offset and ends at the last list offset. + fn trimmed_values(&self) -> Arc; + /// The offset type of the underlying list array. + type Offset: OffsetSizeTrait; + /// Returns offsets shifted so the first offset is zero, matching + /// [`Self::trimmed_values`]. + /// + /// Sliced list arrays (e.g. a filtered batch) keep offsets that reference the + /// original values buffer, so combining them with trimmed values produces + /// offsets that exceed the values length. Use this together with + /// `trimmed_values` when constructing a new list array. + fn trimmed_offsets(&self) -> OffsetBuffer; +} + +impl ListArrayExt for GenericListArray { + fn filter_garbage_nulls(&self) -> Self { + if self.is_empty() { + return self.clone(); + } + let Some(validity) = self.nulls().cloned() else { + return self.clone(); + }; + + let mut should_keep = BooleanBufferBuilder::new(self.values().len()); + + // Handle case where offsets do not start at 0 + let preamble_len = self.offsets().first().unwrap().to_usize().unwrap(); + should_keep.append_n(preamble_len, false); + + let mut new_offsets: Vec = Vec::with_capacity(self.len() + 1); + new_offsets.push(OffsetSize::zero()); + let mut cur_len = OffsetSize::zero(); + for (offset, is_valid) in self.offsets().windows(2).zip(validity.iter()) { + let len = offset[1] - offset[0]; + if is_valid { + cur_len += len; + should_keep.append_n(len.to_usize().unwrap(), true); + new_offsets.push(cur_len); + } else { + should_keep.append_n(len.to_usize().unwrap(), false); + new_offsets.push(cur_len); + } + } + + // Offsets may not reference entire values buffer + let trailer = self.values().len() - should_keep.len(); + should_keep.append_n(trailer, false); + + let should_keep = should_keep.finish(); + let should_keep = BooleanArray::new(should_keep, None); + let new_values = arrow_select::filter::filter(self.values(), &should_keep).unwrap(); + let new_offsets = ScalarBuffer::from(new_offsets); + let new_offsets = OffsetBuffer::new(new_offsets); + + Self::new( + Arc::new(Field::new( + "item", + self.value_type(), + self.values().is_nullable(), + )), + new_offsets, + new_values, + Some(validity), + ) + } + + fn trimmed_values(&self) -> Arc { + let first_value = self + .offsets() + .first() + .map(|v| v.to_usize().unwrap()) + .unwrap_or(0); + let last_value = self + .offsets() + .last() + .map(|v| v.to_usize().unwrap()) + .unwrap_or(0); + self.values().slice(first_value, last_value - first_value) + } + + type Offset = OffsetSize; + + fn trimmed_offsets(&self) -> OffsetBuffer { + let offsets = self.offsets(); + let Some(&first) = offsets.first() else { + return offsets.clone(); + }; + if first == OffsetSize::zero() { + return offsets.clone(); + } + let shifted: Vec = offsets.iter().map(|&o| o - first).collect(); + OffsetBuffer::new(ScalarBuffer::from(shifted)) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow_array::{ListArray, UInt64Array}; + use arrow_buffer::{BooleanBuffer, NullBuffer, OffsetBuffer, ScalarBuffer}; + use arrow_schema::{DataType, Field}; + + use super::ListArrayExt; + + #[test] + fn test_filter_garbage_nulls() { + let items = UInt64Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + let offsets = ScalarBuffer::::from(vec![2, 5, 8, 9]); + let offsets = OffsetBuffer::new(offsets); + let list_validity = NullBuffer::new(BooleanBuffer::from(vec![true, false, true])); + let list_arr = ListArray::new( + Arc::new(Field::new("item", DataType::UInt64, true)), + offsets, + Arc::new(items), + Some(list_validity.clone()), + ); + + let filtered = list_arr.filter_garbage_nulls(); + + let expected_items = UInt64Array::from(vec![2, 3, 4, 8]); + let offsets = ScalarBuffer::::from(vec![0, 3, 3, 4]); + let expected = ListArray::new( + Arc::new(Field::new("item", DataType::UInt64, false)), + OffsetBuffer::new(offsets), + Arc::new(expected_items), + Some(list_validity), + ); + + assert_eq!(filtered, expected); + } + + #[test] + fn test_trim_values() { + let items = UInt64Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + let offsets = ScalarBuffer::::from(vec![2, 5, 6, 8, 9]); + let offsets = OffsetBuffer::new(offsets); + let list_arr = ListArray::new( + Arc::new(Field::new("item", DataType::UInt64, true)), + offsets, + Arc::new(items), + None, + ); + let list_arr = list_arr.slice(1, 2); + + let trimmed = list_arr.trimmed_values(); + + let expected_items = UInt64Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + let expected_items = expected_items.slice(5, 3); + + assert_eq!(trimmed.as_ref(), &expected_items); + } +} diff --git a/python/vendor/lance-arrow/src/memory.rs b/python/vendor/lance-arrow/src/memory.rs new file mode 100644 index 000000000..6b8db9da7 --- /dev/null +++ b/python/vendor/lance-arrow/src/memory.rs @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::collections::HashSet; + +use arrow_array::{Array, RecordBatch}; +use arrow_data::ArrayData; + +/// Counts memory used by buffers of Arrow arrays and RecordBatches. +/// +/// This is meant to capture how much memory is being used by the Arrow data +/// structures as they are. It does not represent the memory used if the data +/// were to be serialized and then deserialized. In particular: +/// +/// * This does not double count memory used by buffers shared by multiple +/// arrays or batches. Round-tripped data may use more memory because of this. +/// * This counts the **total** size of the buffers, even if the array is a slice. +/// Round-tripped data may use less memory because of this. +#[derive(Default)] +pub struct MemoryAccumulator { + seen: HashSet, + total: usize, +} + +impl MemoryAccumulator { + pub fn record_array(&mut self, array: &dyn Array) { + let data = array.to_data(); + self.record_array_data(&data); + } + + fn record_array_data(&mut self, data: &ArrayData) { + for buffer in data.buffers() { + let ptr = buffer.as_ptr(); + if self.seen.insert(ptr as usize) { + self.total += buffer.capacity(); + } + } + + if let Some(nulls) = data.nulls() { + let null_buf = nulls.inner().inner(); + let ptr = null_buf.as_ptr(); + if self.seen.insert(ptr as usize) { + self.total += null_buf.capacity(); + } + } + + for child in data.child_data() { + self.record_array_data(child); + } + } + + pub fn record_batch(&mut self, batch: &RecordBatch) { + for array in batch.columns() { + self.record_array(array); + } + } + + pub fn total(&self) -> usize { + self.total + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow_array::Int32Array; + use arrow_schema::{DataType, Field, Schema}; + + use super::*; + + #[test] + fn test_memory_accumulator() { + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])), + vec![Arc::new(Int32Array::from(vec![1, 2, 3]))], + ) + .unwrap(); + let slice = batch.slice(1, 2); + + let mut acc = MemoryAccumulator::default(); + + // Should record whole buffer, not just slice + acc.record_batch(&slice); + assert_eq!(acc.total(), 3 * std::mem::size_of::()); + + // Should not double count + acc.record_batch(&slice); + assert_eq!(acc.total(), 3 * std::mem::size_of::()); + } +} diff --git a/python/vendor/lance-arrow/src/scalar.rs b/python/vendor/lance-arrow/src/scalar.rs new file mode 100644 index 000000000..6003eb134 --- /dev/null +++ b/python/vendor/lance-arrow/src/scalar.rs @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use arrow_array::{ArrayRef, UInt64Array, make_array}; +use arrow_buffer::Buffer; +use arrow_data::ArrayDataBuilder; +use arrow_schema::{ArrowError, DataType}; +use arrow_select::take::take; + +use crate::DataTypeExt; + +type Result = std::result::Result; + +pub const INLINE_VALUE_MAX_BYTES: usize = 32; + +pub fn extract_scalar_value(array: &ArrayRef, idx: usize) -> Result { + if idx >= array.len() { + return Err(ArrowError::InvalidArgumentError( + "Scalar index out of bounds".to_string(), + )); + } + + take(array.as_ref(), &UInt64Array::from(vec![idx as u64]), None) +} + +fn read_u32(buf: &[u8], offset: &mut usize) -> Result { + if *offset + 4 > buf.len() { + return Err(ArrowError::InvalidArgumentError( + "Invalid scalar value buffer: unexpected EOF".to_string(), + )); + } + let bytes = [ + buf[*offset], + buf[*offset + 1], + buf[*offset + 2], + buf[*offset + 3], + ]; + *offset += 4; + Ok(u32::from_le_bytes(bytes)) +} + +fn read_bytes<'a>(buf: &'a [u8], offset: &mut usize, len: usize) -> Result<&'a [u8]> { + if *offset + len > buf.len() { + return Err(ArrowError::InvalidArgumentError( + "Invalid scalar value buffer: unexpected EOF".to_string(), + )); + } + let slice = &buf[*offset..*offset + len]; + *offset += len; + Ok(slice) +} + +fn write_u32(out: &mut Vec, v: u32) { + out.extend_from_slice(&v.to_le_bytes()); +} + +fn write_bytes(out: &mut Vec, bytes: &[u8]) { + out.extend_from_slice(bytes); +} + +pub fn encode_scalar_value_buffer(scalar: &ArrayRef) -> Result> { + if scalar.len() != 1 || scalar.null_count() != 0 { + return Err(ArrowError::InvalidArgumentError( + "Scalar value buffer must be a single non-null value".to_string(), + )); + } + let data = scalar.to_data(); + if data.offset() != 0 { + return Err(ArrowError::InvalidArgumentError( + "Scalar value buffer must have offset=0".to_string(), + )); + } + if !data.child_data().is_empty() { + return Err(ArrowError::InvalidArgumentError( + "Scalar value buffer does not support nested types".to_string(), + )); + } + + // Minimal format (RFC): store the Arrow value buffers for a length-1 array. + // Null bitmap and child data are intentionally not supported here. + // + // | u32 num_buffers | + // | u32 buffer_0_len | ... | u32 buffer_{n-1}_len | + // | buffer_0 bytes | ... | buffer_{n-1} bytes | + let mut out = Vec::with_capacity(128); + let buffers = data.buffers(); + write_u32(&mut out, buffers.len() as u32); + for b in buffers { + write_u32(&mut out, b.len() as u32); + } + for b in buffers { + write_bytes(&mut out, b.as_slice()); + } + Ok(out) +} + +pub fn decode_scalar_from_value_buffer( + data_type: &DataType, + value_buffer: &[u8], +) -> Result { + if matches!( + data_type, + DataType::Struct(_) | DataType::FixedSizeList(_, _) + ) { + return Err(ArrowError::InvalidArgumentError(format!( + "Scalar value buffer does not support nested data type {:?}", + data_type + ))); + } + + let mut offset = 0; + let num_buffers = read_u32(value_buffer, &mut offset)? as usize; + let buffer_lens = (0..num_buffers) + .map(|_| read_u32(value_buffer, &mut offset).map(|l| l as usize)) + .collect::>>()?; + + let mut buffers = Vec::with_capacity(num_buffers); + for len in buffer_lens { + let bytes = read_bytes(value_buffer, &mut offset, len)?; + buffers.push(Buffer::from_vec(bytes.to_vec())); + } + + if offset != value_buffer.len() { + return Err(ArrowError::InvalidArgumentError( + "Invalid scalar value buffer: trailing bytes".to_string(), + )); + } + + let mut builder = ArrayDataBuilder::new(data_type.clone()) + .len(1) + .null_count(0); + for b in buffers { + builder = builder.add_buffer(b); + } + Ok(make_array(builder.build()?)) +} + +pub fn decode_scalar_from_inline_value( + data_type: &DataType, + inline_value: &[u8], +) -> Result { + // I expect our input to be safe here, but I added some debug_assert_eq statements just in case. + // If they are triggered, we may need to change them to return actual errors. + // + // Boolean values are bit-packed in Arrow and therefore are not "fixed-stride" in bytes. + // As a result, `byte_width_opt()` returns `None` for `DataType::Boolean`, even though a + // length-1 scalar can be represented inline using a single byte (matching `try_inline_value`). + if matches!(data_type, DataType::Boolean) { + debug_assert_eq!( + inline_value.len(), + 1, + "Invalid boolean inline scalar length (expected 1 byte, got {})", + inline_value.len() + ); + } else if let Some(byte_width) = data_type.byte_width_opt() { + debug_assert_eq!( + inline_value.len(), + byte_width, + "Inline constant length mismatch for {:?}: expected {} bytes but got {}", + data_type, + byte_width, + inline_value.len() + ); + } + + let data = ArrayDataBuilder::new(data_type.clone()) + .len(1) + .null_count(0) + .add_buffer(Buffer::from_vec(inline_value.to_vec())) + .build()?; + Ok(make_array(data)) +} + +pub fn try_inline_value(scalar: &ArrayRef) -> Option> { + if scalar.null_count() != 0 || scalar.len() != 1 { + return None; + } + let data = scalar.to_data(); + if !data.child_data().is_empty() { + return None; + } + if data.buffers().len() != 1 { + return None; + } + let bytes = data.buffers()[0].as_slice(); + if bytes.len() > INLINE_VALUE_MAX_BYTES { + return None; + } + Some(bytes.to_vec()) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow_array::{ + BooleanArray, DictionaryArray, FixedSizeBinaryArray, Int8Array, Int32Array, StringArray, + cast::AsArray, types::Int8Type, + }; + + use super::*; + + #[test] + fn test_extract_scalar_value() { + let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None, Some(3)])); + let scalar = extract_scalar_value(&array, 2).unwrap(); + assert_eq!(scalar.len(), 1); + assert_eq!( + scalar + .as_primitive::() + .value(0), + 3 + ); + } + + #[test] + fn test_extract_scalar_value_from_full_dictionary() { + let values = Arc::new(StringArray::from( + (0..=i8::MAX) + .map(|value| format!("value-{value}")) + .collect::>(), + )); + let keys = Int8Array::from((0..=i8::MAX).collect::>()); + let array: ArrayRef = Arc::new(DictionaryArray::::new(keys, values)); + + let scalar = extract_scalar_value(&array, i8::MAX as usize).unwrap(); + + let scalar = scalar.as_dictionary::(); + assert_eq!(scalar.len(), 1); + assert_eq!(scalar.key(0), Some(i8::MAX as usize)); + assert_eq!(scalar.values().len(), i8::MAX as usize + 1); + } + + #[test] + fn test_scalar_value_buffer_utf8_round_trip() { + let scalar: ArrayRef = Arc::new(StringArray::from(vec!["hello"])); + let buf = encode_scalar_value_buffer(&scalar).unwrap(); + let decoded = decode_scalar_from_value_buffer(&DataType::Utf8, &buf).unwrap(); + assert_eq!(decoded.len(), 1); + assert_eq!(decoded.null_count(), 0); + assert_eq!(decoded.as_string::().value(0), "hello"); + } + + #[test] + fn test_scalar_value_buffer_fixed_size_binary_round_trip() { + let val = vec![0xABu8; 33]; + let scalar: ArrayRef = Arc::new( + FixedSizeBinaryArray::try_from_sparse_iter_with_size( + std::iter::once(Some(val.as_slice())), + 33, + ) + .unwrap(), + ); + let buf = encode_scalar_value_buffer(&scalar).unwrap(); + let decoded = + decode_scalar_from_value_buffer(&DataType::FixedSizeBinary(33), &buf).unwrap(); + assert_eq!(decoded.len(), 1); + assert_eq!(decoded.as_fixed_size_binary().value(0), val.as_slice()); + } + + #[test] + fn test_inline_value_boolean_round_trip() { + let scalar: ArrayRef = Arc::new(BooleanArray::from_iter([Some(true)])); + let inline = try_inline_value(&scalar).unwrap(); + let decoded = decode_scalar_from_inline_value(&DataType::Boolean, &inline).unwrap(); + assert_eq!(decoded.len(), 1); + assert_eq!(decoded.null_count(), 0); + assert!(decoded.as_boolean().value(0)); + } + + #[test] + fn test_scalar_value_buffer_rejects_nested_type() { + let field = Arc::new(arrow_schema::Field::new("item", DataType::Int32, false)); + let list: ArrayRef = Arc::new(arrow_array::FixedSizeListArray::new( + field, + 2, + Arc::new(Int32Array::from(vec![1, 2])), + None, + )); + let scalar = list.slice(0, 1); + assert!(encode_scalar_value_buffer(&scalar).is_err()); + } + + #[test] + fn test_decode_scalar_from_value_buffer_rejects_nested_type() { + let buf = Vec::::new(); + let res = + decode_scalar_from_value_buffer(&DataType::Struct(arrow_schema::Fields::empty()), &buf); + assert!(res.is_err()); + } + + #[test] + fn test_decode_scalar_from_value_buffer_trailing_bytes() { + // num_buffers = 0, plus an extra byte + let mut bytes = Vec::new(); + bytes.extend_from_slice(&0u32.to_le_bytes()); + bytes.push(1); + let res = decode_scalar_from_value_buffer(&DataType::Int32, &bytes); + assert!(res.is_err()); + } +} diff --git a/python/vendor/lance-arrow/src/schema.rs b/python/vendor/lance-arrow/src/schema.rs new file mode 100644 index 000000000..8ce9442b4 --- /dev/null +++ b/python/vendor/lance-arrow/src/schema.rs @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Extension to arrow schema + +use arrow_schema::{ArrowError, DataType, Field, FieldRef, Schema}; + +use crate::{ARROW_EXT_NAME_KEY, BLOB_META_KEY, BLOB_V2_EXT_NAME}; + +pub enum Indentation { + OneLine, + MultiLine(u8), +} + +impl Indentation { + fn value(&self) -> String { + match self { + Self::OneLine => "".to_string(), + Self::MultiLine(spaces) => " ".repeat(*spaces as usize), + } + } + + fn deepen(&self) -> Self { + match self { + Self::OneLine => Self::OneLine, + Self::MultiLine(spaces) => Self::MultiLine(spaces + 2), + } + } +} + +/// Extends the functionality of [arrow_schema::Field]. +pub trait FieldExt { + /// Create a compact string representation of the field + /// + /// This is intended for display purposes and not for serialization + fn to_compact_string(&self, indent: Indentation) -> String; + + /// Check if the field is marked as a packed struct + fn is_packed_struct(&self) -> bool; + + /// Check if the field is marked as a blob + fn is_blob(&self) -> bool; + + /// Check if the field is marked as a blob + fn is_blob_v2(&self) -> bool; +} + +impl FieldExt for Field { + fn to_compact_string(&self, indent: Indentation) -> String { + let mut result = format!("{}: ", self.name().clone()); + match self.data_type() { + DataType::Struct(fields) => { + result += "{"; + result += &indent.value(); + for (field_idx, field) in fields.iter().enumerate() { + result += field.to_compact_string(indent.deepen()).as_str(); + if field_idx < fields.len() - 1 { + result += ","; + } + result += indent.value().as_str(); + } + result += "}"; + } + DataType::List(field) + | DataType::LargeList(field) + | DataType::ListView(field) + | DataType::LargeListView(field) => { + result += "["; + result += field.to_compact_string(indent.deepen()).as_str(); + result += "]"; + } + DataType::FixedSizeList(child, dimension) => { + result += &format!( + "[{}; {}]", + child.to_compact_string(indent.deepen()), + dimension + ); + } + DataType::Dictionary(key_type, value_type) => { + result += &value_type.to_string(); + result += "@"; + result += &key_type.to_string(); + } + _ => { + result += &self.data_type().to_string(); + } + } + if self.is_nullable() { + result += "?"; + } + result + } + + // Check if field has metadata `packed` set to true, this check is case insensitive. + fn is_packed_struct(&self) -> bool { + let field_metadata = self.metadata(); + const PACKED_KEYS: [&str; 2] = ["packed", "lance-encoding:packed"]; + PACKED_KEYS.iter().any(|key| { + field_metadata + .get(*key) + .map(|value| value.eq_ignore_ascii_case("true")) + .unwrap_or(false) + }) + } + + fn is_blob(&self) -> bool { + let field_metadata = self.metadata(); + field_metadata.get(BLOB_META_KEY).is_some() + || field_metadata + .get(ARROW_EXT_NAME_KEY) + .map(|value| value == BLOB_V2_EXT_NAME) + .unwrap_or(false) + } + + fn is_blob_v2(&self) -> bool { + let field_metadata = self.metadata(); + field_metadata + .get(ARROW_EXT_NAME_KEY) + .map(|value| value == BLOB_V2_EXT_NAME) + .unwrap_or(false) + } +} + +/// Extends the functionality of [arrow_schema::Schema]. +pub trait SchemaExt { + /// Create a new [`Schema`] with one extra field. + fn try_with_column(&self, field: Field) -> std::result::Result; + + fn try_with_column_at( + &self, + index: usize, + field: Field, + ) -> std::result::Result; + + fn field_names(&self) -> Vec<&String>; + + fn without_column(&self, column_name: &str) -> Schema; + + /// Create a compact string representation of the schema + /// + /// This is intended for display purposes and not for serialization + fn to_compact_string(&self, indent: Indentation) -> String; +} + +impl SchemaExt for Schema { + fn try_with_column(&self, field: Field) -> std::result::Result { + if self.column_with_name(field.name()).is_some() { + return Err(ArrowError::SchemaError(format!( + "Can not append column {} on schema: {:?}", + field.name(), + self + ))); + }; + let mut fields: Vec = self.fields().iter().cloned().collect(); + fields.push(FieldRef::new(field)); + Ok(Self::new_with_metadata(fields, self.metadata.clone())) + } + + fn try_with_column_at( + &self, + index: usize, + field: Field, + ) -> std::result::Result { + if self.column_with_name(field.name()).is_some() { + return Err(ArrowError::SchemaError(format!( + "Failed to modify schema: Inserting column {} would create a duplicate column in schema: {:?}", + field.name(), + self + ))); + }; + let mut fields: Vec = self.fields().iter().cloned().collect(); + fields.insert(index, FieldRef::new(field)); + Ok(Self::new_with_metadata(fields, self.metadata.clone())) + } + + /// Project the schema to remove the given column. + /// + /// This only works on top-level fields right now. If a field does not exist, + /// the schema will be returned as is. + fn without_column(&self, column_name: &str) -> Schema { + let fields: Vec = self + .fields() + .iter() + .filter(|f| f.name() != column_name) + .cloned() + .collect(); + Self::new_with_metadata(fields, self.metadata.clone()) + } + + fn field_names(&self) -> Vec<&String> { + self.fields().iter().map(|f| f.name()).collect() + } + + fn to_compact_string(&self, indent: Indentation) -> String { + let mut result = "{".to_string(); + result += &indent.value(); + for (field_idx, field) in self.fields.iter().enumerate() { + result += field.to_compact_string(indent.deepen()).as_str(); + if field_idx < self.fields.len() - 1 { + result += ","; + } + result += indent.value().as_str(); + } + result += "}"; + result + } +} diff --git a/python/vendor/lance-arrow/src/stream.rs b/python/vendor/lance-arrow/src/stream.rs new file mode 100644 index 000000000..2d0c57bca --- /dev/null +++ b/python/vendor/lance-arrow/src/stream.rs @@ -0,0 +1,553 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Utilities for working with streams of [`RecordBatch`]. + +use arrow_array::RecordBatch; +use arrow_schema::{ArrowError, SchemaRef}; +use futures::stream::{self, Stream, StreamExt}; +use std::pin::Pin; + +use crate::deepcopy::deep_copy_batch_sliced; + +/// Rechunks a stream of [`RecordBatch`] so that each output batch has +/// approximately `target_bytes` of array data. +/// +/// Small input batches are accumulated (by concatenation) until at least +/// `min_bytes` of data has been collected. If the resulting batch exceeds +/// `max_bytes`, it is sliced into roughly equal pieces of ~`max_bytes` +/// (assuming uniform row sizes). +pub fn rechunk_stream_by_size( + input: S, + input_schema: SchemaRef, + min_bytes: usize, + max_bytes: usize, +) -> impl Stream> +where + S: Stream>, + E: From, +{ + rechunk_stream_by_size_inner(input, input_schema, min_bytes, max_bytes, false) +} + +/// Like [`rechunk_stream_by_size`] but deep-copies slices so that +/// `get_array_memory_size` reflects the true size of each output batch. +/// +/// After a normal `RecordBatch::slice`, the backing buffers are shared with +/// the original batch, so `get_array_memory_size` still reports the full +/// parent size. This variant deep-copies every slice produced during the +/// splitting phase, which allows the stream to detect and re-split slices +/// that still exceed `max_bytes` (e.g. because a single row is much larger +/// than average). +/// +/// The deep copy is a last resort and potentially expensive for large +/// batches. However, it is only performed when a batch actually needs to be +/// sliced — batches that are already within the target range pass through at +/// zero cost. Use this only when the hard cap on `max_bytes` is a +/// correctness requirement, not merely a performance hint. +pub fn rechunk_stream_by_size_deep_copy( + input: S, + input_schema: SchemaRef, + min_bytes: usize, + max_bytes: usize, +) -> impl Stream> +where + S: Stream>, + E: From, +{ + rechunk_stream_by_size_inner(input, input_schema, min_bytes, max_bytes, true) +} + +fn rechunk_stream_by_size_inner( + input: S, + input_schema: SchemaRef, + min_bytes: usize, + max_bytes: usize, + deep_copy: bool, +) -> impl Stream> +where + S: Stream>, + E: From, +{ + stream::try_unfold( + RechunkState { + input: Box::pin(input), + accumulated: Vec::new(), + acc_bytes: 0, + done: false, + input_schema, + min_bytes, + max_bytes, + deep_copy, + }, + |mut state| async move { + if state.done && state.accumulated.is_empty() { + return Ok(None); + } + + // Pull batches until we reach the byte target or exhaust input. + // Always pull at least one batch so that min_bytes=0 works. + while !state.done && (state.accumulated.is_empty() || state.acc_bytes < state.min_bytes) + { + match state.input.next().await { + Some(Ok(batch)) => { + state.acc_bytes += batch.get_array_memory_size(); + state.accumulated.push(batch); + } + Some(Err(e)) => return Err(e), + None => { + state.done = true; + } + } + } + + if state.accumulated.is_empty() { + return Ok(None); + } + + // Fast path: if the first accumulated batch already meets the + // byte threshold, deliver it directly instead of concatenating + // everything together (which would just get sliced back apart). + if state.accumulated.len() > 1 + && state.accumulated[0].get_array_memory_size() >= state.min_bytes + { + let b = state.accumulated.remove(0); + state.acc_bytes -= b.get_array_memory_size(); + return Ok(Some((b, state))); + } + + let batch = if state.accumulated.len() == 1 { + state.accumulated.pop().unwrap() + } else { + let b = + arrow_select::concat::concat_batches(&state.input_schema, &state.accumulated) + .map_err(E::from)?; + state.accumulated.clear(); + b + }; + state.acc_bytes = 0; + + // Slice the batch into ~max_bytes pieces assuming uniform row sizes. + let mut slices = + slice_batch(batch, state.max_bytes, state.deep_copy).map_err(E::from)?; + + if slices.len() == 1 { + Ok(Some((slices.pop().unwrap(), state))) + } else { + let first = slices.remove(0); + + // Stash leftover slices for subsequent iterations. + for a in &slices { + state.acc_bytes += a.get_array_memory_size(); + } + state.accumulated = slices; + + Ok(Some((first, state))) + } + }, + ) +} + +/// Slice a batch into pieces of at most `max_bytes`. +/// +/// When `deep_copy` is false, slices share buffers with the original batch +/// and `get_array_memory_size` will still report the parent buffer size. +/// This is fine when the caller only needs approximate sizing. +/// +/// When `deep_copy` is true, each slice is deep-copied so that +/// `get_array_memory_size` reflects the true size. If a deep-copied slice +/// still exceeds `max_bytes` (due to non-uniform row sizes), it is +/// recursively split until every piece is within budget or contains only a +/// single row. +fn slice_batch( + batch: RecordBatch, + max_bytes: usize, + deep_copy: bool, +) -> Result, ArrowError> { + let batch_bytes = batch.get_array_memory_size(); + let num_rows = batch.num_rows(); + + if batch_bytes <= max_bytes { + return Ok(vec![batch]); + } + + if num_rows <= 1 { + // A single row cannot be split further, but the size just measured may + // belong to its source buffer rather than to the row: + // `get_array_memory_size` reports whole buffers, and a slice shares + // them. Returning it uncopied hands the caller that inflated figure -- + // a 100 KB row inside a 200 MB buffer measures as 200 MB, and a caller + // budgeting on it concludes the row cannot fit anywhere. + if deep_copy { + return Ok(vec![deep_copy_batch_sliced(&batch)?]); + } + return Ok(vec![batch]); + } + + let rows_per_chunk = (max_bytes as u64 * num_rows as u64 / batch_bytes as u64).max(1) as usize; + + let mut result = Vec::new(); + let mut offset = 0; + while offset < num_rows { + let len = rows_per_chunk.min(num_rows - offset); + let slice = batch.slice(offset, len); + if deep_copy { + let copied = deep_copy_batch_sliced(&slice)?; + // Recurse: the deep-copied slice has accurate sizes, so if it + // still exceeds max_bytes we can split further. + result.extend(slice_batch(copied, max_bytes, true)?); + } else { + result.push(slice); + } + offset += len; + } + + Ok(result) +} + +/// Internal state for [`rechunk_stream`]. +/// +/// Kept as a named struct so the `try_unfold` closure stays readable. +struct RechunkState { + input: Pin>, + accumulated: Vec, + acc_bytes: usize, + done: bool, + input_schema: SchemaRef, + min_bytes: usize, + max_bytes: usize, + deep_copy: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::sync::Arc; + + use arrow_array::Int32Array; + use arrow_schema::{DataType, Field, Schema}; + use futures::executor::block_on; + + /// A single row that shares a large source buffer must report its own + /// size, not the buffer's. Callers budget on this figure, and an inflated + /// one makes a small row look impossible to place. + #[test] + fn a_single_row_slice_reports_its_own_size() { + use arrow_array::LargeStringArray; + + let row = "x".repeat(100 * 1024); + let values: Vec<&str> = (0..2000).map(|_| row.as_str()).collect(); + let schema = Arc::new(Schema::new(vec![Field::new( + "text", + DataType::LargeUtf8, + false, + )])); + let batch = + RecordBatch::try_new(schema, vec![Arc::new(LargeStringArray::from(values))]).unwrap(); + let whole = batch.get_array_memory_size(); + + let one_row = batch.slice(7, 1); + assert_eq!( + one_row.get_array_memory_size(), + whole, + "a slice shares its source buffer, which is the reason this test exists" + ); + + // Budget far below the row so the split recurses to the single-row floor. + let chunks = slice_batch(one_row, 1024, true).unwrap(); + assert_eq!(chunks.len(), 1); + let measured = chunks[0].get_array_memory_size(); + assert_eq!(chunks[0].num_rows(), 1); + assert!( + measured < whole / 100, + "single row still measured as {measured} bytes against a {whole} byte source" + ); + } + + fn make_batch(num_rows: usize) -> RecordBatch { + let schema = test_schema(); + let values: Vec = (0..num_rows as i32).collect(); + RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(values))]).unwrap() + } + + fn test_schema() -> SchemaRef { + Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])) + } + + fn collect_rechunked( + batches: Vec, + min_bytes: usize, + max_bytes: usize, + ) -> Vec { + let input = stream::iter(batches.into_iter().map(Ok::<_, ArrowError>)); + let rechunked = rechunk_stream_by_size(input, test_schema(), min_bytes, max_bytes); + block_on(rechunked.collect::>()) + .into_iter() + .map(|r| r.unwrap()) + .collect() + } + + fn total_rows(batches: &[RecordBatch]) -> usize { + batches.iter().map(|b| b.num_rows()).sum() + } + + #[test] + fn test_empty_stream() { + let result = collect_rechunked(vec![], 100, 200); + assert!(result.is_empty()); + } + + #[test] + fn test_single_batch_passthrough() { + let batch = make_batch(100); + let bytes = batch.get_array_memory_size(); + // Batch is between min and max — should pass through as-is. + let result = collect_rechunked(vec![batch], bytes / 2, bytes * 2); + assert_eq!(result.len(), 1); + assert_eq!(result[0].num_rows(), 100); + } + + #[test] + fn test_small_batches_concatenated() { + let one_batch_bytes = make_batch(10).get_array_memory_size(); + let batches: Vec<_> = (0..8).map(|_| make_batch(10)).collect(); + // min = 5 batches worth, max = 10 batches worth. + let result = collect_rechunked(batches, one_batch_bytes * 5, one_batch_bytes * 10); + assert_eq!(total_rows(&result), 80); + // Should have been concatenated into fewer batches than the 8 inputs. + assert!( + result.len() < 8, + "expected fewer output batches, got {}", + result.len() + ); + } + + #[test] + fn test_large_batch_sliced() { + let batch = make_batch(1000); + let bytes = batch.get_array_memory_size(); + let result = collect_rechunked(vec![batch], bytes / 8, bytes / 4); + assert_eq!(total_rows(&result), 1000); + assert!( + result.len() >= 4, + "expected at least 4 slices, got {}", + result.len() + ); + } + + #[test] + fn test_sliced_leftovers_are_not_recombined() { + // Key test for the fast-path optimisation. When a large batch is + // sliced, leftover slices should be delivered one-at-a-time without + // being concatenated back together. We verify this by checking that + // every output buffer pointer falls inside the original batch's + // allocation (i.e. they are all zero-copy slices, not fresh copies). + let batch = make_batch(1000); + let bytes = batch.get_array_memory_size(); + let orig_data = batch.column(0).to_data(); + let orig_buf = &orig_data.buffers()[0]; + let orig_start = orig_buf.as_ptr() as usize; + let orig_end = orig_start + orig_buf.len(); + + let result = collect_rechunked(vec![batch], bytes / 8, bytes / 4); + + assert_eq!(total_rows(&result), 1000); + assert!(result.len() >= 4); + + for (i, b) in result.iter().enumerate() { + let ptr = b.column(0).to_data().buffers()[0].as_ptr() as usize; + assert!( + ptr >= orig_start && ptr < orig_end, + "slice {i} buffer at {ptr:#x} is outside the original allocation \ + [{orig_start:#x}, {orig_end:#x}) — it was re-concatenated" + ); + } + } + + #[test] + fn test_flush_remainder_on_stream_end() { + // Data below min_bytes should still be flushed when the stream ends. + let batch = make_batch(10); + let bytes = batch.get_array_memory_size(); + let result = collect_rechunked(vec![batch], bytes * 100, bytes * 200); + assert_eq!(result.len(), 1); + assert_eq!(result[0].num_rows(), 10); + } + + #[test] + fn test_large_then_small_batches() { + // After a large batch is fully drained, subsequent small batches + // should be accumulated normally. + let large = make_batch(1000); + let small_bytes = make_batch(10).get_array_memory_size(); + let batches = vec![ + large, + make_batch(10), + make_batch(10), + make_batch(10), + make_batch(10), + make_batch(10), + ]; + let result = collect_rechunked(batches, small_bytes * 3, small_bytes * 100); + assert_eq!(total_rows(&result), 1050); + // The large batch should appear (possibly sliced) followed by + // concatenated small batches, so we should have fewer output batches + // than the 6 inputs. + assert!(result.len() < 6); + } + + #[test] + fn test_row_preservation_across_slicing() { + // Verify that every input row appears exactly once in the output + // and in the correct order after slicing. + let batch = make_batch(237); // odd count to exercise remainder slice + let bytes = batch.get_array_memory_size(); + let result = collect_rechunked(vec![batch], bytes / 8, bytes / 5); + + assert_eq!(total_rows(&result), 237); + + let values: Vec = result + .iter() + .flat_map(|b| { + b.column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .iter() + .copied() + }) + .collect(); + let expected: Vec = (0..237).collect(); + assert_eq!(values, expected); + } + + #[test] + fn test_min_bytes_zero_still_yields_all_rows() { + // When min_bytes=0, the stream should still yield every batch. + // This is the "chop only, don't coalesce" use case. + let batches: Vec<_> = (0..5).map(|_| make_batch(100)).collect(); + let batch_bytes = batches[0].get_array_memory_size(); + let result = collect_rechunked(batches, 0, batch_bytes * 2); + assert_eq!(total_rows(&result), 500); + } + + #[test] + fn test_min_bytes_zero_slices_oversized() { + // min_bytes=0 with a small max_bytes should still slice large batches. + let batch = make_batch(1000); + let bytes = batch.get_array_memory_size(); + let result = collect_rechunked(vec![batch], 0, bytes / 4); + assert_eq!(total_rows(&result), 1000); + assert!( + result.len() >= 4, + "expected at least 4 slices, got {}", + result.len() + ); + } + + /// Build a batch with one variable-length string column. + /// Every row is `small_size` bytes except the row at index `big_row_idx` + /// which is `big_size` bytes. + fn make_variable_batch( + num_rows: usize, + small_size: usize, + big_row_idx: usize, + big_size: usize, + ) -> RecordBatch { + let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)])); + let values: Vec = (0..num_rows) + .map(|i| { + if i == big_row_idx { + "X".repeat(big_size) + } else { + "x".repeat(small_size) + } + }) + .collect(); + let array = arrow_array::StringArray::from(values); + RecordBatch::try_new(schema, vec![Arc::new(array)]).unwrap() + } + + fn variable_schema() -> SchemaRef { + Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)])) + } + + fn collect_rechunked_variable( + batches: Vec, + min_bytes: usize, + max_bytes: usize, + ) -> Vec { + let input = stream::iter(batches.into_iter().map(Ok::<_, ArrowError>)); + let rechunked = + rechunk_stream_by_size_deep_copy(input, variable_schema(), min_bytes, max_bytes); + block_on(rechunked.collect::>()) + .into_iter() + .map(|r| r.unwrap()) + .collect() + } + + #[test] + fn test_oversized_row_at_end() { + // 100 rows: 99 small (64 bytes each) + 1 large (100KiB) at the end. + let batch = make_variable_batch(100, 64, 99, 100 * 1024); + let max_bytes = 64 * 1024; + let result = collect_rechunked_variable(vec![batch], 0, max_bytes); + assert_eq!(total_rows(&result), 100); + for (i, b) in result.iter().enumerate() { + let size = b.get_array_memory_size(); + assert!( + size <= max_bytes || b.num_rows() == 1, + "batch {i} has {size} bytes (max {max_bytes}) and {} rows", + b.num_rows() + ); + } + } + + #[test] + fn test_oversized_row_at_start() { + // 100 rows: 1 large (100KiB) at the start + 99 small (64 bytes each). + let batch = make_variable_batch(100, 64, 0, 100 * 1024); + let max_bytes = 64 * 1024; + let result = collect_rechunked_variable(vec![batch], 0, max_bytes); + assert_eq!(total_rows(&result), 100); + for (i, b) in result.iter().enumerate() { + let size = b.get_array_memory_size(); + assert!( + size <= max_bytes || b.num_rows() == 1, + "batch {i} has {size} bytes (max {max_bytes}) and {} rows", + b.num_rows() + ); + } + } + + #[test] + fn test_oversized_row_in_middle() { + // 100 rows: 1 large (100KiB) in the middle + 99 small (64 bytes each). + let batch = make_variable_batch(100, 64, 50, 100 * 1024); + let max_bytes = 64 * 1024; + let result = collect_rechunked_variable(vec![batch], 0, max_bytes); + assert_eq!(total_rows(&result), 100); + for (i, b) in result.iter().enumerate() { + let size = b.get_array_memory_size(); + assert!( + size <= max_bytes || b.num_rows() == 1, + "batch {i} has {size} bytes (max {max_bytes}) and {} rows", + b.num_rows() + ); + } + } + + #[test] + fn test_error_propagation() { + let input = stream::iter(vec![ + Ok(make_batch(10)), + Err(ArrowError::ComputeError("boom".into())), + Ok(make_batch(10)), + ]); + let rechunked = rechunk_stream_by_size(input, test_schema(), 1, usize::MAX); + let results: Vec> = block_on(rechunked.collect()); + assert!(results.iter().any(|r| r.is_err())); + } +} diff --git a/python/vendor/lance-arrow/src/struct.rs b/python/vendor/lance-arrow/src/struct.rs new file mode 100644 index 000000000..4dee5032b --- /dev/null +++ b/python/vendor/lance-arrow/src/struct.rs @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Extension to arrow struct arrays + +use arrow_array::{Array, StructArray, cast::AsArray, make_array}; +use arrow_buffer::NullBuffer; +use arrow_data::{ArrayData, ArrayDataBuilder}; +use arrow_schema::ArrowError; + +pub trait StructArrayExt { + /// Removes the offset / length of the struct array by pushing it into the children + /// + /// In arrow-rs when slice is called it recursively slices the children. + /// In arrow-cpp when slice is called it just sets the offset/length of + /// the struct array and leaves the children as-is + /// + /// Both are legal approaches (╥﹏╥) + /// + /// This method helps reduce complexity by folding into the arrow-rs approach + fn normalize_slicing(&self) -> Result + where + Self: Sized; + + /// Structs are allowed to mask valid items. For example, a struct array might be: + /// + /// [ {"items": [1, 2, 3]}, NULL, {"items": NULL}] + /// + /// However, the underlying items array might be: [[1, 2, 3], [4, 5], NULL] + /// + /// The [4, 5] list is masked out because the struct array is null. + /// + /// The struct validity would be [true, false, true] and the list validity would be [true, true, false] + /// + /// This method pushes nulls down into all children. In the above example the list validity would become + /// [true, false, false]. + /// + /// This method is not recursive. If a child is a struct array it will not push that child's nulls down. + /// + /// This method does not remove garbage lists. It only updates the validity so a future call to + /// [crate::list::ListArrayExt::filter_garbage_nulls] will remove the garbage lists (without + /// this pushdown it would not) + fn pushdown_nulls(&self) -> Result + where + Self: Sized; +} + +fn normalized_struct_array_data(data: ArrayData) -> Result { + let parent_offset = data.offset(); + let parent_len = data.len(); + let modified_children = data + .child_data() + .iter() + .map(|d| { + let d = normalized_struct_array_data(d.clone())?; + let offset = d.offset(); + let len = d.len(); + if len < parent_len + parent_offset { + return Err(ArrowError::InvalidArgumentError(format!( + "Child array {} has length {} which is less than the parent length {} plus the parent offset {}", + d.data_type(), + len, + parent_len, + parent_offset + ))); + } + let new_offset = offset + parent_offset; + d.into_builder().offset(new_offset) + .len(parent_len) + .build() + }) + .collect::, _>>()?; + ArrayDataBuilder::new(data.data_type().clone()) + .len(parent_len) + .offset(0) + .buffers(data.buffers().to_vec()) + .child_data(modified_children) + .build() +} + +impl StructArrayExt for StructArray { + fn normalize_slicing(&self) -> Result + where + Self: Sized, + { + if self.offset() == 0 && self.columns().iter().all(|c| c.len() == self.len()) { + return Ok(self.clone()); + } + + let data = normalized_struct_array_data(self.to_data())?; + Ok(Self::from(data)) + } + + fn pushdown_nulls(&self) -> Result + where + Self: Sized, + { + let Some(validity) = self.nulls() else { + return Ok(self.clone()); + }; + let data = self.to_data(); + let children = data + .child_data() + .iter() + .map(|c| { + if let Some(child_validity) = c.nulls() { + let new_validity = child_validity.inner() & validity.inner(); + c.clone() + .into_builder() + .nulls(Some(NullBuffer::from(new_validity))) + .build() + } else { + Ok(c.clone() + .into_builder() + .nulls(Some(validity.clone())) + .build()?) + } + }) + .collect::, _>>()?; + let arr = make_array(data.into_builder().child_data(children).build()?); + Ok(arr.as_struct().clone()) + } +} + +#[cfg(test)] +mod tests { + use arrow_array::{Array, Int32Array, StructArray, cast::AsArray, make_array}; + use arrow_schema::{DataType, Field, Fields}; + use std::sync::Arc; + + use crate::r#struct::StructArrayExt; + + #[test] + fn test_normalize_slicing_no_offset() { + let x = Int32Array::from(vec![1, 2, 3]); + let y = Int32Array::from(vec![4, 5, 6]); + let struct_array = StructArray::new( + Fields::from(vec![ + Field::new("x", DataType::Int32, true), + Field::new("y", DataType::Int32, true), + ]), + vec![Arc::new(x), Arc::new(y)], + None, + ); + + let normalized = struct_array.normalize_slicing().unwrap(); + assert_eq!(normalized, struct_array); + } + + #[test] + fn test_arrow_rs_slicing() { + let x = Int32Array::from(vec![1, 2, 3, 4]); + let y = Int32Array::from(vec![5, 6, 7, 8]); + let struct_array = StructArray::new( + Fields::from(vec![ + Field::new("x", DataType::Int32, true), + Field::new("y", DataType::Int32, true), + ]), + vec![Arc::new(x), Arc::new(y)], + None, + ); + + // Slicing with arrow-rs propagates the slicing to the children so there should + // be no change needed to the struct array + let sliced = struct_array.slice(1, 2); + let normalized = sliced.normalize_slicing().unwrap(); + + assert_eq!(normalized, sliced); + } + + #[test] + fn test_arrow_cpp_slicing() { + let x = Int32Array::from(vec![1, 2, 3, 4]); + let y = Int32Array::from(vec![5, 6, 7, 8]); + let struct_array = StructArray::new( + Fields::from(vec![ + Field::new("x", DataType::Int32, true), + Field::new("y", DataType::Int32, true), + ]), + vec![Arc::new(x), Arc::new(y)], + None, + ); + + let data = struct_array.to_data(); + let sliced = data.into_builder().offset(1).len(2).build().unwrap(); + let sliced = make_array(sliced); + let normalized = sliced.as_struct().clone().normalize_slicing().unwrap(); + + assert_eq!(normalized, struct_array.slice(1, 2)); + } +} diff --git a/rust/lancedb/tests/nullable_struct_merge.rs b/rust/lancedb/tests/nullable_struct_merge.rs new file mode 100644 index 000000000..c245b1c2c --- /dev/null +++ b/rust/lancedb/tests/nullable_struct_merge.rs @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +use std::sync::Arc; + +use arrow_array::{Array, ArrayRef, BooleanArray, Int32Array, RecordBatch, StructArray}; +use arrow_buffer::NullBuffer; +use arrow_schema::{DataType, Field, Schema}; +use lance_arrow::RecordBatchExt; + +#[test] +fn merge_nullable_struct_with_sliced_boolean_child() { + let left_struct = StructArray::new( + vec![Field::new("id", DataType::Int32, false)].into(), + vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4])) as ArrayRef], + None, + ); + + // BooleanArray retains its offset when sliced. The merged child bitmap must + // cover that offset as well as the four visible rows. + let boolean_child: ArrayRef = Arc::new(BooleanArray::from_iter((0..1030).map(|index| { + if index == 1027 { + None + } else { + Some(index % 2 == 1) + } + }))); + let boolean_child = boolean_child.slice(1025, 4); + assert_eq!(boolean_child.offset(), 1025); + let right_struct = StructArray::new( + vec![Field::new("flag", DataType::Boolean, true)].into(), + vec![boolean_child], + Some(NullBuffer::from(vec![true, false, true, true])), + ); + + let left_batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "nested", + left_struct.data_type().clone(), + true, + )])), + vec![Arc::new(left_struct)], + ) + .unwrap(); + let right_batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "nested", + right_struct.data_type().clone(), + true, + )])), + vec![Arc::new(right_struct)], + ) + .unwrap(); + + let merged = left_batch.merge(&right_batch).unwrap(); + let nested = merged.column_by_name("nested").unwrap(); + let nested = nested.as_any().downcast_ref::().unwrap(); + let flag = nested.column_by_name("flag").unwrap(); + let flag = flag.as_any().downcast_ref::().unwrap(); + assert_eq!( + flag.iter().collect::>(), + vec![Some(true), None, None, Some(false)] + ); +}