chore: keep remote write types crate-visible

# Conflicts:
#	Cargo.lock
#	Cargo.toml
#	java/pom.xml

Cargo.lock

@@ -3421,8 +3421,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arrow-array",
  "rand 0.9.5",
@@ -4777,8 +4777,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 
 [[package]]
 name = "lance"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -4852,8 +4852,8 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4875,7 +4875,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4889,7 +4889,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -4898,8 +4898,8 @@ dependencies = [
 
 [[package]]
 name = "lance-bitpacking"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arrayref",
  "crunchy",
@@ -4909,8 +4909,8 @@ dependencies = [
 
 [[package]]
 name = "lance-core"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4950,8 +4950,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datafusion"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4981,8 +4981,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4999,8 +4999,8 @@ dependencies = [
 
 [[package]]
 name = "lance-derive"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5009,8 +5009,8 @@ dependencies = [
 
 [[package]]
 name = "lance-encoding"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -5045,12 +5045,13 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arrow-arith",
  "arrow-array",
  "arrow-buffer",
+ "arrow-cast",
  "arrow-data",
  "arrow-schema",
  "arrow-select",
@@ -5076,8 +5077,8 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -5144,8 +5145,8 @@ dependencies = [
 
 [[package]]
 name = "lance-index-core"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5167,18 +5168,12 @@ dependencies = [
 
 [[package]]
 name = "lance-io"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arrow",
- "arrow-arith",
  "arrow-array",
- "arrow-buffer",
- "arrow-cast",
- "arrow-data",
  "arrow-schema",
- "arrow-select",
- "async-recursion",
  "async-trait",
  "aws-config",
  "aws-credential-types",
@@ -5211,8 +5206,8 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5228,8 +5223,8 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arrow",
  "async-trait",
@@ -5241,8 +5236,8 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-impls"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -5296,8 +5291,8 @@ dependencies = [
 
 [[package]]
 name = "lance-select"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5312,8 +5307,8 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5352,8 +5347,8 @@ dependencies = [
 
 [[package]]
 name = "lance-testing"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5366,8 +5361,8 @@ dependencies = [
 
 [[package]]
 name = "lance-tokenizer"
-version = "10.0.0-beta.7"
+version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
 dependencies = [
  "icu_segmenter",
  "jieba-rs",

Cargo.toml

@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"
 
 [workspace.dependencies]
-lance = { "version" = "=10.0.0-beta.7", default-features = false, "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=10.0.0-beta.7", default-features = false, "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=10.0.0-beta.7", default-features = false, "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
 arrow = { version = "58.0.0", optional = false }

java/pom.xml

@@ -28,7 +28,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>10.0.0-beta.7</lance-core.version>
+        <lance-core.version>10.1.0-beta.1</lance-core.version>
         <spotless.skip>false</spotless.skip>
         <spotless.version>2.30.0</spotless.version>
         <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>

rust/lancedb/src/dataloader/permutation/shuffle.rs

@@ -11,7 +11,9 @@ use lance_core::{cache::LanceCache, utils::futures::FinallyStreamExt};
 use lance_encoding::decoder::{DecoderPlugins, FilterExpression};
 use lance_file::{
     reader::{FileReader, FileReaderOptions},
-    writer::{FileWriter, FileWriterOptions},
+    version::ConcreteFileVersion,
+    versions,
+    writer::FileWriterOptions,
 };
 use lance_io::{
     ReadBatchParams,
@@ -152,8 +154,12 @@ impl Shuffler {
                     source: None,
                 })?;
             let object_writer = object_store.create(&path).await?;
-            let writer =
+            let writer = versions::create_writer(
-                FileWriter::try_new(object_writer, schema.clone(), FileWriterOptions::default())?;
+                ConcreteFileVersion::V2_1,
+                object_writer,
+                schema.clone(),
+                FileWriterOptions::default(),
+            )?;
             file_writers.push(writer);
         }