From b36cdc72d9215995d58fb30be1bc4059c2183d79 Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Fri, 31 Jul 2026 16:47:25 -0400 Subject: [PATCH] chore: update lance dependency to v10.1.0-beta.1 --- Cargo.lock | 92 +++++++++---------- Cargo.toml | 28 +++--- java/pom.xml | 2 +- .../src/dataloader/permutation/shuffle.rs | 12 ++- 4 files changed, 68 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e80165a70..39711faa5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3421,8 +3421,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4777,8 +4777,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arc-swap", "arrow", @@ -4852,8 +4852,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-array", "arrow-buffer", @@ -4875,7 +4875,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-array", "arrow-buffer", @@ -4889,7 +4889,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-array", "arrow-schema", @@ -4898,8 +4898,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrayref", "crunchy", @@ -4909,14 +4909,15 @@ dependencies = [ [[package]] name = "lance-core" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "async-trait", + "blake3", "byteorder", "bytes", "datafusion-common", @@ -4949,8 +4950,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow", "arrow-array", @@ -4980,8 +4981,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow", "arrow-array", @@ -4998,8 +4999,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "proc-macro2", "quote", @@ -5008,8 +5009,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-arith", "arrow-array", @@ -5044,12 +5045,13 @@ dependencies = [ [[package]] name = "lance-file" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-arith", "arrow-array", "arrow-buffer", + "arrow-cast", "arrow-data", "arrow-schema", "arrow-select", @@ -5075,8 +5077,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arc-swap", "arrow", @@ -5143,8 +5145,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-array", "arrow-schema", @@ -5166,18 +5168,12 @@ dependencies = [ [[package]] name = "lance-io" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow", - "arrow-arith", "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", "arrow-schema", - "arrow-select", - "async-recursion", "async-trait", "aws-config", "aws-credential-types", @@ -5210,8 +5206,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-array", "arrow-buffer", @@ -5227,8 +5223,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow", "async-trait", @@ -5240,8 +5236,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow", "arrow-ipc", @@ -5295,8 +5291,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-array", "arrow-buffer", @@ -5311,8 +5307,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow", "arrow-array", @@ -5351,8 +5347,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "arrow-array", "arrow-schema", @@ -5365,8 +5361,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "10.0.0-beta.5" -source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8" +version = "10.1.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7" dependencies = [ "icu_segmenter", "jieba-rs", diff --git a/Cargo.toml b/Cargo.toml index fee01b732..35370e58f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } ahash = "0.8" # Note that this one does not include pyarrow arrow = { version = "58.0.0", optional = false } diff --git a/java/pom.xml b/java/pom.xml index ab8ec2cc8..c3a7111b8 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 10.0.0-beta.5 + 10.1.0-beta.1 false 2.30.0 1.7 diff --git a/rust/lancedb/src/dataloader/permutation/shuffle.rs b/rust/lancedb/src/dataloader/permutation/shuffle.rs index cf118a996..2bc06452c 100644 --- a/rust/lancedb/src/dataloader/permutation/shuffle.rs +++ b/rust/lancedb/src/dataloader/permutation/shuffle.rs @@ -11,7 +11,9 @@ use lance_core::{cache::LanceCache, utils::futures::FinallyStreamExt}; use lance_encoding::decoder::{DecoderPlugins, FilterExpression}; use lance_file::{ reader::{FileReader, FileReaderOptions}, - writer::{FileWriter, FileWriterOptions}, + version::ConcreteFileVersion, + versions, + writer::FileWriterOptions, }; use lance_io::{ ReadBatchParams, @@ -152,8 +154,12 @@ impl Shuffler { source: None, })?; let object_writer = object_store.create(&path).await?; - let writer = - FileWriter::try_new(object_writer, schema.clone(), FileWriterOptions::default())?; + let writer = versions::create_writer( + ConcreteFileVersion::V2_1, + object_writer, + schema.clone(), + FileWriterOptions::default(), + )?; file_writers.push(writer); }