From 0559108fa94b29cc5db4d390d8b37794b9fcc41c Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 28 Aug 2026 17:23:49 +0800 Subject: [PATCH] feat: support blob computed column refresh (#4081) Computed-column planning currently sees Blob v2 storage descriptors, so expressions cannot consume payload bytes or preserve Blob semantics in their outputs. A computed declaration now derives its output field from its expression. A direct projection of a Blob v2 field inherits the source field's Blob metadata; other expressions retain their ordinary Arrow-inferred type. Declarations remain ordered, so the same rule applies across chained projections. Refresh materializes referenced Blob inputs as `LargeBinary` payload bytes and publishes inherited Blob outputs through Lance's Blob conversion path. Remote requests remain within the shared namespace contract as `{name, computed}`; the server planner is being updated in tandem to implement the same Blob-aware planning semantics, and remote enablement must be aligned with that server rollout. The existing null-as-unfilled contract remains unchanged. Row-level freshness and cell flags remain follow-up work. --- python/python/lancedb/remote/table.py | 10 +- python/python/lancedb/table.py | 15 +- python/python/tests/test_table.py | 23 + rust/lancedb/src/remote/table.rs | 8 +- rust/lancedb/src/table.rs | 4 +- rust/lancedb/src/table/computed_columns.rs | 289 ++++++++++-- rust/lancedb/src/table/refresh.rs | 522 ++++++++++++++++++++- 7 files changed, 821 insertions(+), 50 deletions(-) diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index 02748b9bc..55014a423 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -67,7 +67,15 @@ from ..query import ( LanceTakeQueryBuilder, LanceVectorQueryBuilder, ) -from ..table import AsyncTable, BlobMode, Branches, IndexStatistics, Query, Table, Tags +from ..table import ( + AsyncTable, + BlobMode, + Branches, + IndexStatistics, + Query, + Table, + Tags, +) from ..types import BaseTokenizerType diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 535ed7c0d..36c5b727d 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -2165,9 +2165,11 @@ class Table(ABC): Function columns are supported only on LanceDB Cloud and Enterprise. computed: Dict[str, str], optional - A map of column name to a SQL expression defining the column. The - column's type and inputs are derived from the expression, so no - data type is supplied. + A mapping from output column names to SQL expressions derives each + output field from its expression. A direct projection of a Blob v2 + field inherits Blob v2 semantics; other expressions derive their + ordinary Arrow type. Mapping order is declaration and dependency + order. Unlike ``transforms``, the expression is stored rather than evaluated now: the column is committed with no values, and rows get @@ -6268,8 +6270,11 @@ class AsyncTable: Function columns are supported only on LanceDB Cloud and Enterprise. computed: Dict[str, str], optional - A map of column name to a SQL expression defining the column. The - column's type and inputs are derived from the expression. + A mapping from output column names to SQL expressions derives each + output field from its expression. A direct projection of a Blob v2 + field inherits Blob v2 semantics; other expressions derive their + ordinary Arrow type. Mapping order is declaration and dependency + order. Unlike ``transforms``, the expression is stored rather than evaluated now: the column is committed with no values, and rows get diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 0be9e139d..fbdfac5d8 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -4087,6 +4087,29 @@ def test_computed_column_rejects_transforms_and_computed_together(tmp_path): table.add_columns({"a": "x + 1"}, computed={"b": "x * 2"}) +def test_computed_column_blob_projection_inherits_semantics(tmp_path): + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + db = lancedb.connect(tmp_path) + table = db.create_table("computed_column_blob", schema=schema) + table.add( + [ + {"id": 1, "image": b"hello"}, + {"id": 2, "image": b""}, + {"id": 3, "image": None}, + ] + ) + + table.add_columns(computed={"image_copy": "image", "second_copy": "image_copy"}) + assert table.refresh_column("image_copy").rows_filled == 2 + assert table.refresh_column("second_copy").rows_filled == 2 + assert table.blob_columns() == ["image", "image_copy", "second_copy"] + + hits = table.search().with_row_id(True).limit(10).to_arrow() + rows = sorted(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist())) + copied = table.fetch_blobs("second_copy", [row_id for _, row_id in rows]) + assert copied.to_pylist() == [b"hello", b"", None] + + @pytest.mark.asyncio async def test_computed_column_async(tmp_path): db = await lancedb.connect_async(tmp_path) diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 1afc2615a..57d2dc47d 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -3180,8 +3180,8 @@ impl BaseTable for RemoteTable { self.schema().await?.as_ref(), "schema evolution", )?; - // The server plans the declaration: expression validation, type - // inference and the persisted binding all happen there. + // The server plans the declaration against its table schema, including + // Blob v2 semantics inherited by a direct field projection. let entries = columns .iter() .map( @@ -7388,8 +7388,8 @@ mod tests { assert_eq!(result.version, if old_server { 0 } else { 43 }); } - /// A declaration is sent as `{name, computed}` entries for the server to - /// plan; the client never types the expression itself. + /// A declaration is sent as `{name, computed}` for the server to plan; the + /// client never types the expression itself. #[tokio::test] async fn test_add_computed_columns_sends_the_expression() { let table = Table::new_with_handler("my_table", |request| match request.url().path() { diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 8436657ca..70b86f3cf 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -750,8 +750,8 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { /// Declare computed columns, each defined by a SQL expression. /// /// Where the declaration is planned depends on the backend: a local table - /// validates and types the expression itself, a remote one sends the text - /// for the server to plan. + /// validates and types the expression itself, while a remote one sends the + /// expression for the server to plan. async fn add_computed_columns( &self, _columns: &[(String, String)], diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index 0f89ca612..4e8d8211e 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -9,29 +9,35 @@ //! refresh fills the rows. //! //! The rule is tagged by kind ([`ComputedColumnKind`]) because kinds differ in -//! where the column's type and inputs come from. A SQL expression is -//! self-describing -- both are derived from the expression, so a caller writes -//! neither -- while a kind resolved through a registry cannot be typed without -//! consulting it. Registered Functions use an exact remote version plus a -//! schema-level Function binding; unknown newer kinds remain readable and fail -//! closed before mutation. +//! where the column's type and inputs come from. A SQL expression determines +//! its inputs and physical result type. A direct projection of a Blob v2 field +//! also inherits that field's semantic type while execution continues to use +//! `LargeBinary`. A kind resolved through a registry cannot be typed without +//! consulting it. +//! Registered Functions use an exact remote version plus a schema-level +//! Function binding; unknown newer kinds remain readable and fail closed +//! before mutation. //! //! [`computed_columns`] and [`computed_column_from_field`] read declarations //! back off a schema. -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::sync::Arc; use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema, SchemaRef}; -use datafusion_common::tree_node::TreeNode; +use datafusion_common::{ScalarValue, tree_node::TreeNode}; +use datafusion_expr::Expr; use datafusion_physical_plan::PhysicalExpr; use lance::dataset::NewColumnTransform; +use lance_arrow::FieldExt; +use lance_core::datatypes::{BLOB_V2_DESC_FIELD, format_field_path_minimal, parse_field_path}; use lance_datafusion::planner::Planner; use lance_namespace::models::{JsonArrowDataType, JsonArrowField, JsonArrowSchema}; use serde::{Deserialize, Serialize}; use serde_json::Value; use crate::function::{FunctionApplication, FunctionBinding}; +use crate::utils::resolve_arrow_field_path; use crate::{Error, Result}; /// Field metadata key marking a column as computed. The value is `"true"`. @@ -1106,15 +1112,20 @@ pub(crate) fn ensure_no_foreign_declarations<'a>( fields: impl IntoIterator>, ) -> Result<()> { for field in fields { - if field.metadata().keys().any(|k| is_declaration_key(k)) { - return Err(Error::InvalidInput { - message: format!( - "field '{}' carries computed-column metadata; declare computed columns \ - with add_columns().computed()", - field.name() - ), - }); - } + ensure_no_foreign_declaration(field)?; + } + Ok(()) +} + +fn ensure_no_foreign_declaration(field: &ArrowField) -> Result<()> { + if field.metadata().keys().any(|k| is_declaration_key(k)) { + return Err(Error::InvalidInput { + message: format!( + "field '{}' carries computed-column metadata; declare computed columns \ + with add_columns().computed()", + field.name() + ), + }); } Ok(()) } @@ -1162,15 +1173,154 @@ pub(crate) struct BoundExpression { /// The columns the expression names, as written; nested inputs keep /// their dotted path. pub inputs: Vec, - /// The top-level columns evaluation reads, in [`Self::read_schema`] - /// order. A nested input appears through its root. + /// The top-level columns evaluation reads, in physical-expression order. + /// A nested input appears through its root. pub roots: Vec, - /// The projected schema evaluation runs against. - pub read_schema: SchemaRef, /// The compiled expression. pub physical: Arc, /// The type the expression yields. pub data_type: DataType, + /// Blob v2 leaves the scan must materialize as `LargeBinary`. + pub blob_paths: Vec, + /// A directly projected Blob v2 field whose semantics the output inherits. + projected_blob_field: Option, +} + +fn is_direct_field_projection(expr: &Expr) -> bool { + match expr { + Expr::Column(_) => true, + Expr::ScalarFunction(function) + if function.name() == "get_field" && function.args.len() == 2 => + { + is_direct_field_projection(&function.args[0]) + && matches!( + &function.args[1], + Expr::Literal(ScalarValue::Utf8(Some(_)), _) + ) + } + _ => false, + } +} + +fn projected_blob_field(schema: &ArrowSchema, expr: &Expr) -> Result> { + if !is_direct_field_projection(expr) { + return Ok(None); + } + let paths = Planner::column_names_in_expr(expr); + let [path] = paths.as_slice() else { + return Ok(None); + }; + let (_, field) = resolve_arrow_field_path(schema, path)?; + Ok(field.is_blob_v2().then_some(field)) +} + +fn collect_blob_paths(field: &ArrowField, parent: &[String], paths: &mut Vec>) { + let mut path = parent.to_vec(); + path.push(field.name().clone()); + if field.is_blob_v2() { + paths.push(path); + return; + } + match field.data_type() { + DataType::Struct(children) => { + for child in children { + collect_blob_paths(child, &path, paths); + } + } + DataType::List(child) + | DataType::LargeList(child) + | DataType::FixedSizeList(child, _) + | DataType::Map(child, _) => collect_blob_paths(child, &path, paths), + _ => {} + } +} + +fn schema_blob_paths(schema: &ArrowSchema) -> Vec> { + let mut paths = Vec::new(); + for field in schema.fields() { + collect_blob_paths(field, &[], &mut paths); + } + paths +} + +fn transform_blob_field( + field: &ArrowField, + parent: &[String], + materialized: &HashSet>, +) -> ArrowField { + let mut path = parent.to_vec(); + path.push(field.name().clone()); + if field.is_blob_v2() { + if materialized.contains(&path) { + return ArrowField::new(field.name(), DataType::LargeBinary, field.is_nullable()); + } + return ArrowField::new( + field.name(), + BLOB_V2_DESC_FIELD.data_type().clone(), + field.is_nullable(), + ) + .with_metadata(BLOB_V2_DESC_FIELD.metadata().clone()); + } + + let data_type = match field.data_type() { + DataType::Struct(children) => DataType::Struct( + children + .iter() + .map(|child| Arc::new(transform_blob_field(child, &path, materialized))) + .collect(), + ), + DataType::List(child) => { + DataType::List(Arc::new(transform_blob_field(child, &path, materialized))) + } + DataType::LargeList(child) => { + DataType::LargeList(Arc::new(transform_blob_field(child, &path, materialized))) + } + DataType::FixedSizeList(child, size) => DataType::FixedSizeList( + Arc::new(transform_blob_field(child, &path, materialized)), + *size, + ), + DataType::Map(child, sorted) => DataType::Map( + Arc::new(transform_blob_field(child, &path, materialized)), + *sorted, + ), + _ => return field.clone(), + }; + ArrowField::new(field.name(), data_type, field.is_nullable()) + .with_metadata(field.metadata().clone()) +} + +fn blob_runtime_schema(schema: &ArrowSchema, materialized: &HashSet>) -> SchemaRef { + Arc::new(ArrowSchema::new_with_metadata( + schema + .fields() + .iter() + .map(|field| Arc::new(transform_blob_field(field, &[], materialized))) + .collect::(), + schema.metadata().clone(), + )) +} + +fn referenced_blob_paths(schema: &ArrowSchema, inputs: &[String]) -> Result>> { + let input_paths = inputs + .iter() + .map(|input| { + parse_field_path(input).map_err(|error| Error::InvalidInput { + message: format!("invalid computed-column input path '{input}': {error}"), + }) + }) + .collect::>>()?; + Ok(schema_blob_paths(schema) + .into_iter() + .filter(|blob_path| { + input_paths.iter().any(|input_path| { + input_path.len() <= blob_path.len() + && input_path + .iter() + .zip(blob_path) + .all(|(input, blob)| input == blob) + }) + }) + .collect()) } /// Parse, resolve and compile `expression` against `schema`. @@ -1185,10 +1335,18 @@ pub(crate) fn bind(schema: SchemaRef, column: &str, expression: &str) -> Result< message, }; - let planner = Planner::new(schema.clone()); + // Blob v2 is a semantic type whose runtime expression ABI is + // `LargeBinary`. Parse against that ABI first so a direct Blob reference + // is not mistaken for its storage descriptor struct. + let all_blob_paths = schema_blob_paths(schema.as_ref()) + .into_iter() + .collect::>(); + let parsing_schema = blob_runtime_schema(schema.as_ref(), &all_blob_paths); + let planner = Planner::new(parsing_schema); let parsed = planner .parse_expr(expression) .map_err(|e| invalid(e.to_string()))?; + let projected_blob_field = projected_blob_field(schema.as_ref(), &parsed)?; // A declaration is evaluated more than once -- staging and writing are // separate passes, and a refresh years later replays the same text -- so @@ -1218,13 +1376,19 @@ pub(crate) fn bind(schema: SchemaRef, column: &str, expression: &str) -> Result< inputs.sort(); inputs.dedup(); + let blob_paths = referenced_blob_paths(schema.as_ref(), &inputs)?; + let runtime_schema = blob_runtime_schema( + schema.as_ref(), + &blob_paths.iter().cloned().collect::>(), + ); + // A nested input is recorded by its path but read through its root // column; Schema::index_of resolves top-level names only. Resolved here // rather than left to the planner so an unknown column names itself in // the error instead of surfacing as a plan failure. let mut indices = Vec::with_capacity(inputs.len()); for input in &inputs { - let index = schema + let index = runtime_schema .index_of(root(input)) .map_err(|_| invalid(format!("unknown column '{input}'")))?; if !indices.contains(&index) { @@ -1237,7 +1401,7 @@ pub(crate) fn bind(schema: SchemaRef, column: &str, expression: &str) -> Result< // compiles the expression has to be built on the projected schema // evaluation will actually read. let read_schema = Arc::new( - schema + runtime_schema .project(&indices) .map_err(|e| invalid(e.to_string()))?, ); @@ -1247,7 +1411,8 @@ pub(crate) fn bind(schema: SchemaRef, column: &str, expression: &str) -> Result< .map(|field| field.name().clone()) .collect(); - let optimized = planner + let runtime_planner = Planner::new(runtime_schema); + let optimized = runtime_planner .optimize_expr(parsed) .map_err(|e| invalid(e.to_string()))?; let physical = Planner::new(read_schema.clone()) @@ -1260,9 +1425,16 @@ pub(crate) fn bind(schema: SchemaRef, column: &str, expression: &str) -> Result< Ok(BoundExpression { inputs, roots, - read_schema, physical, data_type, + blob_paths: blob_paths + .iter() + .map(|path| { + let segments = path.iter().map(String::as_str).collect::>(); + format_field_path_minimal(&segments) + }) + .collect(), + projected_blob_field, }) } @@ -1278,7 +1450,7 @@ pub(crate) fn bind(schema: SchemaRef, column: &str, expression: &str) -> Result< /// batch may declare `a` and then `b = a + 1` in one commit. Refresh order /// then matters, and refresh enforces it: `b` is refused while `a` still has /// unfilled rows. -pub(crate) fn plan(schema: SchemaRef, columns: &[(String, String)]) -> Result> { +fn plan_declarations(schema: SchemaRef, columns: &[(String, String)]) -> Result> { if columns.is_empty() { return Err(Error::InvalidInput { message: "at least one computed column is required".into(), @@ -1290,15 +1462,28 @@ pub(crate) fn plan(schema: SchemaRef, columns: &[(String, String)]) -> Result { + let mut metadata = source.metadata().clone(); + metadata.retain(|key, _| !is_declaration_key(key)); + metadata.extend(computed_metadata); + source + .with_name(name) + .with_nullable(true) + .with_metadata(metadata) + } + None => ArrowField::new(name, bound.data_type, true).with_metadata(computed_metadata), + }; schema = Arc::new(ArrowSchema::new_with_metadata( schema .fields() @@ -1314,6 +1499,10 @@ pub(crate) fn plan(schema: SchemaRef, columns: &[(String, String)]) -> Result Result> { + plan_declarations(schema, columns) +} + /// Run the schema-level checks of /// [`AddColumnsBuilder::computed`](super::AddColumnsBuilder::computed) against /// `schema` without committing: the Function-binding guard and the planning of @@ -1352,7 +1541,7 @@ pub(crate) fn declare( schema: SchemaRef, columns: &[(String, String)], ) -> Result { - let fields = plan(schema, columns)?; + let fields = plan_declarations(schema, columns)?; Ok(NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new( fields, )))) @@ -1478,6 +1667,44 @@ mod tests { ); } + #[test] + fn test_direct_blob_projection_inherits_semantics() { + let schema = Arc::new(ArrowSchema::new(vec![crate::blob("image", false)])); + let fields = plan( + schema, + &[ + ("first".to_string(), "image".to_string()), + ("second".to_string(), "first".to_string()), + ], + ) + .unwrap(); + + for field in &fields { + assert!(field.is_blob_v2()); + assert!(field.is_nullable()); + } + assert_eq!( + fields[1] + .metadata() + .get(EXPRESSION_META_KEY) + .map(String::as_str), + Some("first") + ); + } + + #[test] + fn test_blob_expression_transformation_does_not_inherit_semantics() { + let schema = Arc::new(ArrowSchema::new(vec![crate::blob("image", true)])); + let fields = plan( + schema, + &[("payload".to_string(), "coalesce(image, image)".to_string())], + ) + .unwrap(); + + assert!(!fields[0].is_blob_v2()); + assert_eq!(fields[0].data_type(), &DataType::LargeBinary); + } + /// The binding reaches the schema only if `AllNulls` carries per-field /// metadata through the commit. The whole representation rests on it. #[tokio::test] diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index bc2cc38d1..511fce8ff 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -29,10 +29,14 @@ //! inputs masked to null first, so a poison value in a row nobody is filling //! cannot fail the refresh. +use std::collections::HashSet; use std::sync::Arc; -use arrow_array::{ArrayRef, BooleanArray, RecordBatch, RecordBatchOptions}; -use arrow_schema::Schema as ArrowSchema; +use arrow_array::{ + Array, ArrayRef, BooleanArray, LargeBinaryArray, RecordBatch, RecordBatchOptions, StructArray, + new_null_array, +}; +use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; use datafusion_expr::ColumnarValue; use futures::{Stream, StreamExt, TryStreamExt}; use lance::Dataset; @@ -40,7 +44,7 @@ use lance::dataset::WriteDestination; use lance::dataset::fragment::FileFragment; use lance::dataset::transaction::Operation; use lance_core::ROW_ID; -use lance_core::datatypes::Schema as LanceSchema; +use lance_core::datatypes::{BlobHandling, Schema as LanceSchema}; use serde::{Deserialize, Serialize}; use super::computed_columns::{BoundExpression, ComputedColumnKind, computed_column_from_field}; @@ -104,6 +108,7 @@ async fn execute_refresh_column_with_source( fields: vec![field.clone()], metadata: Default::default(), }; + let output_is_blob = field.is_blob_v2(); let mut rows_filled = 0u64; let mut replacements = Vec::new(); @@ -113,7 +118,8 @@ async fn execute_refresh_column_with_source( continue; } rows_filled += gained; - let values = fill_stream(&dataset, &fragment, bound.clone(), column).await?; + let values = + fill_stream(&dataset, &fragment, bound.clone(), column, output_is_blob).await?; replacements.push(fragment.write_columns(values, &column_schema).await?); } @@ -294,12 +300,15 @@ fn evaluation_batch( mask_out: Option<&BooleanArray>, ) -> lance_core::Result { let mut columns = Vec::with_capacity(bound.roots.len()); + let mut fields = Vec::with_capacity(bound.roots.len()); for name in &bound.roots { - let column = batch.column_by_name(name).ok_or_else(|| { + let index = batch.schema_ref().index_of(name).map_err(|_| { lance_core::Error::invalid_input(format!( "refreshing a computed column read no {name} column" )) })?; + let column = batch.column(index); + fields.push(batch.schema_ref().field(index).clone()); // Rows outside the mask must not reach the expression: a value in a // deleted or already-filled row can be one it would choke on. columns.push(match mask_out { @@ -308,7 +317,7 @@ fn evaluation_batch( }); } Ok(RecordBatch::try_new_with_options( - bound.read_schema.clone(), + Arc::new(ArrowSchema::new(fields)), columns, &RecordBatchOptions::new().with_row_count(Some(batch.num_rows())), )?) @@ -329,6 +338,99 @@ fn evaluate(bound: &BoundExpression, batch: &RecordBatch) -> lance_core::Result< } } +fn materialized_blob_ids(schema: &LanceSchema, paths: &[String]) -> Result> { + paths + .iter() + .map(|path| { + let field = schema + .resolve(path) + .and_then(|fields| fields.last().copied()) + .ok_or_else(|| Error::InvalidInput { + message: format!("computed Blob input '{path}' no longer exists"), + })?; + if !field.is_blob_v2() { + return Err(Error::InvalidInput { + message: format!("computed Blob input '{path}' is no longer Blob v2"), + }); + } + u32::try_from(field.id).map_err(|_| Error::InvalidInput { + message: format!( + "computed Blob input '{path}' has invalid field id {}", + field.id + ), + }) + }) + .collect() +} + +fn configure_blob_inputs( + scanner: &mut lance::dataset::scanner::Scanner, + schema: &LanceSchema, + bound: &BoundExpression, + extra_blob_id: Option, +) -> Result<()> { + let mut ids = materialized_blob_ids(schema, &bound.blob_paths)?; + ids.extend(extra_blob_id); + scanner.blob_handling(BlobHandling::SomeBlobsBinary(ids)); + Ok(()) +} + +fn blob_array_from_binary( + array: &ArrayRef, + target_field: &ArrowField, +) -> lance_core::Result { + let values = array + .as_any() + .downcast_ref::() + .ok_or_else(|| { + lance_core::Error::invalid_input(format!( + "a Blob v2 computed output produced {}, expected LargeBinary", + array.data_type() + )) + })?; + let mut builder = lance::blob::BlobArrayBuilder::new(values.len()); + for index in 0..values.len() { + if values.is_null(index) { + builder.push_null()?; + } else { + builder.push_bytes(values.value(index))?; + } + } + let minimal = builder.finish()?; + let minimal = minimal + .as_any() + .downcast_ref::() + .ok_or_else(|| lance_core::Error::internal("Blob builder returned a non-struct array"))?; + let DataType::Struct(target_fields) = target_field.data_type() else { + return Err(lance_core::Error::invalid_input(format!( + "Blob v2 output field '{}' has non-struct type {}", + target_field.name(), + target_field.data_type() + ))); + }; + let columns = target_fields + .iter() + .map(|field| match field.name().as_str() { + "data" | "uri" => minimal + .column_by_name(field.name()) + .cloned() + .ok_or_else(|| { + lance_core::Error::internal(format!("Blob builder omitted '{}'", field.name())) + }), + "position" | "size" => Ok(new_null_array(field.data_type(), minimal.len())), + name => Err(lance_core::Error::invalid_input(format!( + "Blob v2 output field '{}' has unsupported logical child '{name}'", + target_field.name() + ))), + }) + .collect::>>()?; + Ok(Arc::new(StructArray::try_new( + target_fields.clone(), + columns, + minimal.nulls().cloned(), + )?)) +} + /// How many rows of one fragment would gain a value. /// /// Scans only the unfilled live rows -- deleted rows never reach the @@ -347,6 +449,7 @@ async fn count_fragment_gains( .with_row_id() .filter(&format!("{} IS NULL", quote_identifier(column)))? .project(&bound.roots)?; + configure_blob_inputs(&mut scanner, dataset.schema(), bound, None)?; let mut gained = 0u64; let mut batches = scanner.try_into_stream().await?; @@ -368,6 +471,7 @@ async fn fill_stream( fragment: &FileFragment, bound: Arc, column: &str, + output_is_blob: bool, ) -> Result> + Send + use<>> { let mut projection: Vec = bound.roots.clone(); projection.push(column.to_string()); @@ -377,6 +481,20 @@ async fn fill_stream( .with_row_id() .include_deleted_rows() .project(&projection)?; + let output_blob_id = output_is_blob + .then(|| { + dataset + .schema() + .field(column) + .and_then(|field| u32::try_from(field.id).ok()) + }) + .flatten(); + configure_blob_inputs( + &mut scanner, + dataset.schema(), + bound.as_ref(), + output_blob_id, + )?; let projected = Arc::new(ArrowSchema::new(vec![ ArrowSchema::from(dataset.schema()) @@ -412,6 +530,11 @@ async fn fill_stream( let computed = evaluate(&bound, &evaluation_batch(&batch, &bound, Some(&keep))?)?; let merged = arrow_select::zip::zip(&fill, &computed, existing)?; + let merged = if output_is_blob { + blob_array_from_binary(&merged, projected.field(0))? + } else { + merged + }; Ok(RecordBatch::try_new(projected.clone(), vec![merged])?) })) } @@ -420,8 +543,12 @@ async fn fill_stream( mod tests { use std::sync::Arc; - use arrow_array::{Int32Array, record_batch}; + use arrow_array::{ + Array, ArrayRef, Int32Array, LargeBinaryArray, RecordBatch, StructArray, record_batch, + }; + use arrow_schema::Field as ArrowField; use futures::TryStreamExt; + use lance_core::ROW_ID; use crate::connect; use crate::query::{ExecutableQuery, QueryBase, Select}; @@ -477,6 +604,25 @@ mod tests { table.add(batch).execute().await.unwrap(); } + #[test] + fn test_blob_output_matches_complete_logical_field() { + let values: ArrayRef = Arc::new(LargeBinaryArray::from(vec![ + Some(b"hello".as_slice()), + None, + ])); + let field = ArrowField::new( + "image", + lance_core::datatypes::BLOB_V2_LOGICAL_TYPE.clone(), + true, + ); + + let output = super::blob_array_from_binary(&values, &field).unwrap(); + assert_eq!(output.data_type(), field.data_type()); + let output = output.as_any().downcast_ref::().unwrap(); + assert_eq!(output.column_by_name("position").unwrap().null_count(), 2); + assert_eq!(output.column_by_name("size").unwrap().null_count(), 2); + } + /// The gate's reproducer: `b = coalesce(a, 0)` refreshed before `a` /// must not bake zeros from `a`'s placeholder null. It is refused, and /// names the input, until `a` is filled -- after every append too. @@ -1164,4 +1310,366 @@ mod tests { let err = table.refresh_column("embedding").await.unwrap_err(); assert!(matches!(err, Error::NotSupported { message } if message.contains("udf"))); } + + fn blob_batch(ids: Vec, payloads: Vec>) -> RecordBatch { + use arrow_array::Int32Array; + use arrow_schema::{Field, Schema}; + + let mut builder = lance::blob::BlobArrayBuilder::new(payloads.len()); + for payload in payloads { + match payload { + Some(payload) => builder.push_bytes(payload).unwrap(), + None => builder.push_null().unwrap(), + } + } + RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", arrow_schema::DataType::Int32, false), + crate::blob("image", true), + ])), + vec![Arc::new(Int32Array::from(ids)), builder.finish().unwrap()], + ) + .unwrap() + } + + async fn create_blob_table(path: &std::path::Path, batch: RecordBatch) -> Table { + let conn = connect(path.to_str().unwrap()).execute().await.unwrap(); + conn.create_table("blobs", batch).execute().await.unwrap() + } + + #[tokio::test] + async fn test_refresh_inherits_and_publishes_blob_output() { + use arrow_array::UInt64Array; + use lance_arrow::{ + BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, BLOB_INLINE_SIZE_THRESHOLD_META_KEY, + }; + use lance_core::datatypes::BlobKind; + + use crate::table::schema_evolution::FieldMetadataUpdate; + + let tmp = tempfile::tempdir().unwrap(); + let table = create_blob_table( + tmp.path(), + blob_batch( + vec![1, 2, 3, 4], + vec![Some(b"hello"), Some(b"ab"), Some(b""), None], + ), + ) + .await; + table + .add_columns() + .computed("image_copy", "image") + .execute() + .await + .unwrap(); + table + .update_field_metadata(&[FieldMetadataUpdate::new("image_copy") + .set(BLOB_INLINE_SIZE_THRESHOLD_META_KEY, "1") + .set(BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, "4")]) + .await + .unwrap(); + + let first_refresh = table.refresh_column("image_copy").await.unwrap(); + assert_eq!(first_refresh.rows_filled, 3); + assert_eq!( + table.blob_columns().await.unwrap(), + vec!["image".to_string(), "image_copy".to_string()] + ); + + let batches = table + .query() + .with_row_id() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let batch = arrow_select::concat::concat_batches(&batches[0].schema(), &batches).unwrap(); + assert!( + batch + .column_by_name("image_copy") + .unwrap() + .as_any() + .is::() + ); + let row_ids = batch + .column_by_name(ROW_ID) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec(); + let original = table.fetch_blobs("image", &row_ids).await.unwrap(); + let copied = table.fetch_blobs("image_copy", &row_ids).await.unwrap(); + assert_eq!(original, copied); + let ids = batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let files = table + .fetch_blob_files("image_copy", &row_ids) + .await + .unwrap(); + let mut layouts = ids + .values() + .iter() + .copied() + .zip(files) + .map(|(id, file)| (id, file.and_then(|file| file.kind()))) + .collect::>(); + layouts.sort_by_key(|(id, _)| *id); + assert_eq!( + layouts, + vec![ + (1, Some(BlobKind::Dedicated)), + (2, Some(BlobKind::Packed)), + (3, Some(BlobKind::Inline)), + (4, None), + ] + ); + + table + .add(blob_batch(vec![5], vec![Some(b"appended")])) + .execute() + .await + .unwrap(); + table + .optimize(crate::table::OptimizeAction::Compact { + options: crate::table::CompactionOptions::default(), + remap_options: None, + }) + .await + .unwrap(); + assert_eq!( + table + .refresh_column("image_copy") + .await + .unwrap() + .rows_filled, + 1 + ); + assert_eq!( + table + .refresh_column("image_copy") + .await + .unwrap() + .rows_filled, + 0 + ); + + table.checkout(first_refresh.version).await.unwrap(); + assert_eq!(table.count_rows(None).await.unwrap(), 4); + assert_eq!( + table.blob_columns().await.unwrap(), + vec!["image".to_string(), "image_copy".to_string()] + ); + table.checkout_latest().await.unwrap(); + } + + #[tokio::test] + async fn test_refresh_inherits_nested_struct_blob_input() { + use arrow_array::{Int32Array, StructArray, UInt64Array}; + use arrow_schema::{DataType, Field, Fields, Schema}; + + let tmp = tempfile::tempdir().unwrap(); + let mut blob_builder = lance::blob::BlobArrayBuilder::new(2); + blob_builder.push_bytes(b"nested").unwrap(); + blob_builder.push_null().unwrap(); + let blob_field = crate::blob("image", true); + let metadata_fields = Fields::from(vec![blob_field.clone()]); + let metadata = StructArray::new( + metadata_fields.clone(), + vec![blob_builder.finish().unwrap()], + None, + ); + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("metadata", DataType::Struct(metadata_fields), true), + ])), + vec![Arc::new(Int32Array::from(vec![1, 2])), Arc::new(metadata)], + ) + .unwrap(); + let table = create_blob_table(tmp.path(), batch).await; + table + .add_columns() + .computed("payload_copy", "metadata.image") + .execute() + .await + .unwrap(); + + assert_eq!( + table + .refresh_column("payload_copy") + .await + .unwrap() + .rows_filled, + 1 + ); + assert_eq!( + table.blob_columns().await.unwrap(), + vec!["metadata.image".to_string(), "payload_copy".to_string()] + ); + let batches = table + .query() + .with_row_id() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let row_ids = batches[0] + .column_by_name(ROW_ID) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values(); + let payloads = table.fetch_blobs("payload_copy", row_ids).await.unwrap(); + assert_eq!(payloads.value(0), b"nested"); + assert!(payloads.is_null(1)); + } + + #[tokio::test] + async fn test_refresh_preserves_list_shape_when_materializing_blob_input() { + use arrow_array::{Int32Array, ListArray}; + use arrow_buffer::{OffsetBuffer, ScalarBuffer}; + use arrow_schema::{DataType, Field, Schema}; + + let tmp = tempfile::tempdir().unwrap(); + let mut blob_builder = lance::blob::BlobArrayBuilder::new(3); + blob_builder.push_bytes(b"a").unwrap(); + blob_builder.push_bytes(b"bb").unwrap(); + blob_builder.push_null().unwrap(); + let item = Arc::new(crate::blob("item", true)); + let images = ListArray::new( + item.clone(), + OffsetBuffer::new(ScalarBuffer::from(vec![0, 2, 3])), + blob_builder.finish().unwrap(), + None, + ); + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("images", DataType::List(item), true), + ])), + vec![Arc::new(Int32Array::from(vec![1, 2])), Arc::new(images)], + ) + .unwrap(); + let table = create_blob_table(tmp.path(), batch).await; + table + .add_columns() + .computed("image_payloads", "images") + .execute() + .await + .unwrap(); + + assert_eq!( + table + .refresh_column("image_payloads") + .await + .unwrap() + .rows_filled, + 2 + ); + let batches = table + .query() + .select(Select::columns(&["image_payloads"])) + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let output = batches[0] + .column_by_name("image_payloads") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(output.value_offsets(), &[0, 2, 3]); + assert!(output.values().as_any().is::()); + } + + #[tokio::test] + async fn test_refresh_inherits_external_blob_input() { + use arrow_array::{Int32Array, StringArray, UInt64Array}; + use arrow_schema::{DataType, Field, Schema}; + + let tmp = tempfile::tempdir().unwrap(); + let payload = b"external-payload"; + let path = tmp.path().join("payload.bin"); + std::fs::write(&path, payload).unwrap(); + let uri = url::Url::from_file_path(path).unwrap().to_string(); + let conn = connect(tmp.path().join("db").to_str().unwrap()) + .execute() + .await + .unwrap(); + let table = conn + .create_empty_table( + "external", + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + crate::blob("image", true), + ])), + ) + .execute() + .await + .unwrap(); + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("image", DataType::Utf8, true), + ])), + vec![ + Arc::new(Int32Array::from(vec![1])), + Arc::new(StringArray::from(vec![Some(uri)])), + ], + ) + .unwrap(); + table + .add(batch) + .allow_external_blob_outside_bases(true) + .execute() + .await + .unwrap(); + table + .add_columns() + .computed("payload_copy", "image") + .execute() + .await + .unwrap(); + + assert_eq!( + table + .refresh_column("payload_copy") + .await + .unwrap() + .rows_filled, + 1 + ); + let batches = table + .query() + .with_row_id() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let row_ids = batches[0] + .column_by_name(ROW_ID) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values(); + let payloads = table.fetch_blobs("payload_copy", row_ids).await.unwrap(); + assert_eq!(payloads.value(0), payload); + } }