From 05602ec7d563d08c063bf4e36c087911b39af138 Mon Sep 17 00:00:00 2001 From: LuQQiu Date: Fri, 31 Jul 2026 16:16:17 -0700 Subject: [PATCH] chore: update lance dependency to v10.1.0-beta.1 (#3757) Updates the Lance Rust workspace dependencies and Java lance-core version to v10.1.0-beta.1. Includes a compatibility fix for the Lance file writer API by using the explicit V2_1 writer creation path for permutation shuffle spill files. Triggered by https://github.com/lance-format/lance/releases/tag/v10.1.0-beta.1 --- Cargo.lock | 91 +++++++++---------- Cargo.toml | 28 +++--- java/pom.xml | 2 +- .../src/dataloader/permutation/shuffle.rs | 12 ++- 4 files changed, 67 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 76a15794b..57766d7e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3421,8 +3421,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4777,8 +4777,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arc-swap", "arrow", @@ -4852,8 +4852,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-array", "arrow-buffer", @@ -4875,7 +4875,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-array", "arrow-buffer", @@ -4889,7 +4889,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-array", "arrow-schema", @@ -4898,8 +4898,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrayref", "crunchy", @@ -4909,8 +4909,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-array", "arrow-buffer", @@ -4950,8 +4950,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow", "arrow-array", @@ -4981,8 +4981,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow", "arrow-array", @@ -4999,8 +4999,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "proc-macro2", "quote", @@ -5009,8 +5009,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-arith", "arrow-array", @@ -5045,12 +5045,13 @@ dependencies = [ [[package]] name = "lance-file" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-arith", "arrow-array", "arrow-buffer", + "arrow-cast", "arrow-data", "arrow-schema", "arrow-select", @@ -5076,8 +5077,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arc-swap", "arrow", @@ -5144,8 +5145,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-array", "arrow-schema", @@ -5167,18 +5168,12 @@ dependencies = [ [[package]] name = "lance-io" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow", - "arrow-arith", "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", "arrow-schema", - "arrow-select", - "async-recursion", "async-trait", "aws-config", "aws-credential-types", @@ -5211,8 +5206,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-array", "arrow-buffer", @@ -5228,8 +5223,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow", "async-trait", @@ -5241,8 +5236,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow", "arrow-ipc", @@ -5296,8 +5291,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-array", "arrow-buffer", @@ -5312,8 +5307,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow", "arrow-array", @@ -5352,8 +5347,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-array", "arrow-schema", @@ -5366,8 +5361,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "10.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "icu_segmenter", "jieba-rs", diff --git a/Cargo.toml b/Cargo.toml index 9638da17f..35370e58f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=10.0.0-beta.7", default-features = false, "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=10.0.0-beta.7", default-features = false, "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=10.0.0-beta.7", default-features = false, "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } ahash = "0.8" # Note that this one does not include pyarrow arrow = { version = "58.0.0", optional = false } diff --git a/java/pom.xml b/java/pom.xml index 7e5194159..1b96cbcab 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 10.0.0-beta.7 + 10.1.0-beta.1 false 2.30.0 1.7 diff --git a/rust/lancedb/src/dataloader/permutation/shuffle.rs b/rust/lancedb/src/dataloader/permutation/shuffle.rs index cf118a996..2bc06452c 100644 --- a/rust/lancedb/src/dataloader/permutation/shuffle.rs +++ b/rust/lancedb/src/dataloader/permutation/shuffle.rs @@ -11,7 +11,9 @@ use lance_core::{cache::LanceCache, utils::futures::FinallyStreamExt}; use lance_encoding::decoder::{DecoderPlugins, FilterExpression}; use lance_file::{ reader::{FileReader, FileReaderOptions}, - writer::{FileWriter, FileWriterOptions}, + version::ConcreteFileVersion, + versions, + writer::FileWriterOptions, }; use lance_io::{ ReadBatchParams, @@ -152,8 +154,12 @@ impl Shuffler { source: None, })?; let object_writer = object_store.create(&path).await?; - let writer = - FileWriter::try_new(object_writer, schema.clone(), FileWriterOptions::default())?; + let writer = versions::create_writer( + ConcreteFileVersion::V2_1, + object_writer, + schema.clone(), + FileWriterOptions::default(), + )?; file_writers.push(writer); }