From a487d4033ea34a657cabc8270b6731f239597600 Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Sun, 6 Sep 2026 13:43:35 -0700 Subject: [PATCH] chore: update lance dependency to v12.0.0-beta.14 (#4141) Update the Rust workspace Lance dependencies and Java lance-core from v12.0.0-beta.11 to [v12.0.0-beta.14](https://github.com/lance-format/lance/releases/tag/v12.0.0-beta.14), refreshing Cargo.lock. Resolve two Clippy diagnostics by making an internal Node.js helper private and using a byte string literal in a remote-table test fixture. Validation: `cargo clippy --quiet --workspace --tests --all-features -- -D warnings`, `cargo fmt --all --quiet`, `git diff --check`, and `pnpm build` in nodejs. --------- Co-authored-by: Jack Ye --- Cargo.lock | 90 ++++++++-------- Cargo.toml | 28 ++--- java/pom.xml | 2 +- nodejs/src/job.rs | 2 +- rust/lancedb/src/remote/table.rs | 2 +- .../src/table/datafusion/blob_coerce.rs | 100 +++++++++++++++++- 6 files changed, 161 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 081a64a5f..2d16a7259 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3526,8 +3526,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4886,8 +4886,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arc-swap", "arrow", @@ -4959,8 +4959,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arrow-array", "arrow-buffer", @@ -4982,7 +4982,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arrow-array", "arrow-buffer", @@ -4996,7 +4996,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arrow-array", "arrow-schema", @@ -5005,8 +5005,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arrayref", "crunchy", @@ -5016,8 +5016,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arrow-array", "arrow-buffer", @@ -5054,8 +5054,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arrow", "arrow-array", @@ -5085,8 +5085,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arrow", "arrow-array", @@ -5103,8 +5103,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "proc-macro2", "quote", @@ -5113,8 +5113,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arrow-arith", "arrow-array", @@ -5147,8 +5147,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arrow-arith", "arrow-array", @@ -5179,8 +5179,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arc-swap", "arrow", @@ -5244,8 +5244,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arrow-array", "arrow-schema", @@ -5267,8 +5267,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arrow", "arrow-array", @@ -5308,8 +5308,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arrow-array", "arrow-schema", @@ -5323,21 +5323,23 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arrow", "async-trait", "bytes", "lance-core", "lance-namespace-reqwest-client", + "serde", + "serde_json", "snafu 0.9.0", ] [[package]] name = "lance-namespace-impls" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arrow", "arrow-ipc", @@ -5376,9 +5378,9 @@ dependencies = [ [[package]] name = "lance-namespace-reqwest-client" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d06b1fbb5d41f93bc652b61e2872af92e8a6c5f6b4ce8839a8ecfa05365d359" +checksum = "d8d23e54b1634d5bbb434f8dd33dc3c05f6e58d876a9a27b3b4aef58ddbe11af" dependencies = [ "reqwest 0.12.28", "serde", @@ -5390,8 +5392,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arrow-array", "arrow-buffer", @@ -5405,8 +5407,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arrow", "arrow-array", @@ -5446,8 +5448,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "arrow-array", "arrow-schema", @@ -5460,8 +5462,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "12.0.0-beta.11" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472" +version = "12.0.0-beta.14" +source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index 63ebeb75a..5c5b0362d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=12.0.0-beta.14", default-features = false, "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=12.0.0-beta.14", default-features = false, "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=12.0.0-beta.14", default-features = false, "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index b8a34293a..4359c74cc 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 12.0.0-beta.11 + 12.0.0-beta.14 false 2.30.0 1.7 diff --git a/nodejs/src/job.rs b/nodejs/src/job.rs index 9c6559dfd..214e687b4 100644 --- a/nodejs/src/job.rs +++ b/nodejs/src/job.rs @@ -127,7 +127,7 @@ impl Job { } /// Serialise Arrow batches as a single IPC stream for the TypeScript layer. -pub(crate) fn batches_to_ipc_buffer(batches: &[RecordBatch]) -> napi::Result { +fn batches_to_ipc_buffer(batches: &[RecordBatch]) -> napi::Result { let Some(first) = batches.first() else { return Ok(Buffer::from(Vec::::new())); }; diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 89885061e..5c32e3cdd 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -8021,7 +8021,7 @@ mod tests { match request.url().path() { "/v1/table/my_table/backfill_column" => http::Response::builder() .status(202) - .body(r#"{"job_id": "j-42"}"#.as_bytes().to_vec()) + .body(br#"{"job_id": "j-42"}"#.to_vec()) .unwrap(), "/v1/jobs/describe" => http::Response::builder() .status(200) diff --git a/rust/lancedb/src/table/datafusion/blob_coerce.rs b/rust/lancedb/src/table/datafusion/blob_coerce.rs index 0596e7a2d..3b79ec022 100644 --- a/rust/lancedb/src/table/datafusion/blob_coerce.rs +++ b/rust/lancedb/src/table/datafusion/blob_coerce.rs @@ -5,12 +5,17 @@ //! //! [`super::cast::cast_to_table_schema`] calls [`coerce_blob_expr`]. +use std::fmt; +use std::hash::{Hash, Hasher}; use std::sync::Arc; -use arrow_schema::{DataType, Field, FieldRef, Fields}; +use arrow_array::{Array, BooleanArray, RecordBatch}; +use arrow_schema::{DataType, Field, FieldRef, Fields, Schema}; +use arrow_select::nullif::nullif; use datafusion::functions::core::{get_field, named_struct}; use datafusion_common::ScalarValue; use datafusion_common::config::ConfigOptions; +use datafusion_expr::ColumnarValue; use datafusion_physical_expr::ScalarFunctionExpr; use datafusion_physical_expr::expressions::{CastExpr, Literal}; use datafusion_physical_plan::PhysicalExpr; @@ -133,16 +138,102 @@ pub(super) fn coerce_blob_expr( ns_args.push(value); } - let expr: Arc = Arc::new(ScalarFunctionExpr::new( + let built: Arc = Arc::new(ScalarFunctionExpr::new( &format!("named_struct({})", table_field.name()), named_struct(), ns_args, table_field.clone(), config.clone(), )); + + // `named_struct` always yields a valid struct, so a null input would land + // as a row that set neither `data` nor `uri` -- not an absent blob but a + // malformed one, which Lance rejects on write. + let expr: Arc = Arc::new(AbsentBlobIsNull { + source: input_expr, + built, + field: table_field.clone(), + }); Ok((expr, table_field.clone())) } +/// Carries the source column's nullity onto the struct built for it. +/// +/// This is its own expression rather than a `CASE` because the projection +/// takes its output field from `return_field`, and the generic implementation +/// rebuilds a bare field -- which would drop the `lance.blob.v2` extension +/// metadata and stop the column being recognised as a blob at all. +#[derive(Debug, Clone)] +struct AbsentBlobIsNull { + source: Arc, + built: Arc, + field: FieldRef, +} + +impl fmt::Display for AbsentBlobIsNull { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "absent_blob_is_null({}, {})", self.source, self.built) + } +} + +impl PartialEq for AbsentBlobIsNull { + fn eq(&self, other: &Self) -> bool { + self.source.eq(&other.source) && self.built.eq(&other.built) && self.field == other.field + } +} + +impl Eq for AbsentBlobIsNull {} + +impl Hash for AbsentBlobIsNull { + fn hash(&self, state: &mut H) { + self.source.hash(state); + self.built.hash(state); + self.field.hash(state); + } +} + +impl PhysicalExpr for AbsentBlobIsNull { + fn return_field(&self, _input_schema: &Schema) -> datafusion_common::Result { + Ok(self.field.clone()) + } + + fn nullable(&self, _input_schema: &Schema) -> datafusion_common::Result { + Ok(true) + } + + fn evaluate(&self, batch: &RecordBatch) -> datafusion_common::Result { + let rows = batch.num_rows(); + let built = self.built.evaluate(batch)?.into_array(rows)?; + let source = self.source.evaluate(batch)?.into_array(rows)?; + let Some(nulls) = source.logical_nulls() else { + return Ok(ColumnarValue::Array(built)); + }; + // `nullif` nulls the rows the mask marks true, which is where the + // source had no value. + let absent = BooleanArray::new(!nulls.inner(), None); + Ok(ColumnarValue::Array(nullif(built.as_ref(), &absent)?)) + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.source, &self.built] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> datafusion_common::Result> { + Ok(Arc::new(Self { + source: children[0].clone(), + built: children[1].clone(), + field: self.field.clone(), + })) + } + + fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{self}") + } +} + enum BlobInputShape<'a> { Bytes, String, @@ -313,6 +404,11 @@ mod tests { let data = image.column_by_name("data").unwrap(); assert!(!data.is_null(0)); assert!(data.is_null(1)); + // The row itself has to be null, not merely a struct whose children + // are. A present-but-empty struct set neither `data` nor `uri`, which + // Lance rejects as malformed rather than reading as an absent blob. + assert!(!image.is_null(0)); + assert!(image.is_null(1)); } #[tokio::test]