fix(listing): paginate table listing instead of enumerating the database

`ListingDatabase::table_names` and `list_tables` listed every table directory under the database prefix before applying `limit` and `page_token`. The cost of a request was set by the size of the database rather than the size of the page, so listing one table out of ten thousand took ten S3 round trips instead of one. List through `ObjectStore::read_dir_stream`, which pushes the resume position and the page size into the store's list request. Stores with no paginated list API fall back to a full listing, which is what every store did before. Two behaviour changes come with it: - Names are reported in the order the store lists directories, which differs from sorting by name only between a name and one that extends it: `users-archive` now precedes `users`, because `-` sorts below the `.` of `users.lance`. Pagination cannot report an order other than the one it resumes in. - `list_tables` returned the first name of the *next* page as its `page_token`, and `page_token` resumes *after* the name it is given, so paging dropped one table per page boundary. The token is now the last name of the page. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
chore: point lance at the read_dir_stream branch
2026-09-04 12:38:38 +00:00 · 2026-08-03 16:24:44 -07:00 · 2026-08-03 16:24:44 -07:00
19 changed files with 308 additions and 801 deletions
@@ -3422,7 +3422,7 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 [[package]]
 name = "fsst"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-array",
 "rand 0.9.5",
@@ -4778,7 +4778,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 [[package]]
 name = "lance"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -4853,7 +4853,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4875,7 +4875,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4889,7 +4889,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -4899,7 +4899,7 @@ dependencies = [
 [[package]]
 name = "lance-bitpacking"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrayref",
 "crunchy",
@@ -4910,7 +4910,7 @@ dependencies = [
 [[package]]
 name = "lance-core"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4951,7 +4951,7 @@ dependencies = [
 [[package]]
 name = "lance-datafusion"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -4982,7 +4982,7 @@ dependencies = [
 [[package]]
 name = "lance-datagen"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5000,7 +5000,7 @@ dependencies = [
 [[package]]
 name = "lance-derive"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -5010,7 +5010,7 @@ dependencies = [
 [[package]]
 name = "lance-encoding"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5046,7 +5046,7 @@ dependencies = [
 [[package]]
 name = "lance-file"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5078,7 +5078,7 @@ dependencies = [
 [[package]]
 name = "lance-index"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -5146,7 +5146,7 @@ dependencies = [
 [[package]]
 name = "lance-index-core"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5169,7 +5169,7 @@ dependencies = [
 [[package]]
 name = "lance-io"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5207,7 +5207,7 @@ dependencies = [
 [[package]]
 name = "lance-linalg"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5224,7 +5224,7 @@ dependencies = [
 [[package]]
 name = "lance-namespace"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow",
 "async-trait",
@@ -5237,7 +5237,7 @@ dependencies = [
 [[package]]
 name = "lance-namespace-impls"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow",
 "arrow-ipc",
@@ -5292,7 +5292,7 @@ dependencies = [
 [[package]]
 name = "lance-select"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5308,7 +5308,7 @@ dependencies = [
 [[package]]
 name = "lance-table"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5348,7 +5348,7 @@ dependencies = [
 [[package]]
 name = "lance-testing"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5362,7 +5362,7 @@ dependencies = [
 [[package]]
 name = "lance-tokenizer"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "icu_segmenter",
 "jieba-rs",
@@ -7583,7 +7583,7 @@ version = "0.14.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
 dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
 "itertools 0.14.0",
 "log",
 "multimap",
@@ -8498,9 +8498,9 @@ dependencies = [

 [[package]]
 name = "rkyv"
-version = "0.8.17"
+version = "0.8.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "815cc8a37159a463064825246cadb07961e25cd9885908606f6d08a98d8f8874"
+checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3"
 dependencies = [
 "bytecheck",
 "bytes",
@@ -8517,9 +8517,9 @@ dependencies = [

 [[package]]
 name = "rkyv_derive"
-version = "0.8.17"
+version = "0.8.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0ed1a78a1b19d184b0daa629dd9a024573173ec7d485b287cb369fb3607cc1c"
+checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -9277,7 +9277,7 @@ version = "0.8.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
 dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
 "proc-macro2",
 "quote",
 "syn 2.0.117",
@@ -9289,7 +9289,7 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40"
 dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
 "proc-macro2",
 "quote",
 "syn 2.0.117",
@@ -9730,7 +9730,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
 "fastrand",
- "getrandom 0.3.4",
+ "getrandom 0.4.2",
 "once_cell",
 "rustix",
 "windows-sys 0.61.2",
@@ -13,20 +13,23 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"

 [workspace.dependencies]
-lance = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
+# TEMPORARY: `ObjectStore::read_dir_stream` is not in a lance release yet, so these point at
+# lance-format/lance#8120 cherry-picked onto the v10.1.0-beta.1 tag. Put the tag back once that
+# PR has merged and shipped in a release.
+lance = { "version" = "=10.1.0-beta.1", default-features = false, "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=10.1.0-beta.1", default-features = false, "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=10.1.0-beta.1", default-features = false, "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
 arrow = { version = "58.0.0", optional = false }
@@ -339,9 +339,7 @@ impl Table {
        let transforms = NewColumnTransform::SqlExpressions(transforms);
        let res = self
            .inner_ref()?
-            .add_columns()
-            .transform(transforms)
-            .execute()
+            .add_columns(transforms, None)
            .await
            .default_error()?;
        Ok(res.into())
@@ -358,9 +356,7 @@ impl Table {
        let transforms = NewColumnTransform::AllNulls(schema);
        let res = self
            .inner_ref()?
-            .add_columns()
-            .transform(transforms)
-            .execute()
+            .add_columns(transforms, None)
            .await
            .default_error()?;
        Ok(res.into())
@@ -707,9 +707,6 @@ class LanceDBConnection(DBConnection):
        self._namespace_client_properties = namespace_client_properties
        if _inner is not None:
            self._conn = _inner
-            # Native-derived wrappers resolve this in their async reconstruction
-            # path so construction never synchronously re-enters LOOP.
-            self._read_consistency_interval = read_consistency_interval
            self._cached_namespace_client = None
            return

@@ -759,14 +756,11 @@ class LanceDBConnection(DBConnection):
        # storage_options.  Also, this class really shouldn't be holding any state
        # beyond _conn.
        self._conn = AsyncConnection(LOOP.run(do_connect()))
-        # Keep property access synchronous so debugger introspection cannot wait on
-        # the background loop while that thread is suspended at a breakpoint.
-        self._read_consistency_interval = read_consistency_interval
        self._cached_namespace_client: Optional[LanceNamespace] = None

    @property
    def read_consistency_interval(self) -> Optional[timedelta]:
-        return self._read_consistency_interval
+        return LOOP.run(self._conn.get_read_consistency_interval())

    @property
    def session(self) -> Optional[Session]:
@@ -777,16 +771,8 @@ class LanceDBConnection(DBConnection):
        return self._conn.uri

    @classmethod
-    def from_inner(
-        cls,
-        inner: LanceDbConnection,
-        read_consistency_interval: Optional[timedelta],
-    ):
-        return cls(
-            None,
-            read_consistency_interval=read_consistency_interval,
-            _inner=inner,
-        )
+    def from_inner(cls, inner: LanceDbConnection):
+        return cls(None, _inner=inner)

    def __repr__(self) -> str:
        return f"{self.__class__.__name__}(uri={self._conn.uri!r})"
@@ -3,7 +3,6 @@


 from typing import List
-from urllib.parse import unquote, urlparse

 import numpy as np

@@ -126,20 +125,9 @@ class InstructorEmbeddingFunction(TextEmbeddingFunction):

    @weak_lru(maxsize=1)
    def get_model(self):
-        huggingface_hub = attempt_import_or_raise("huggingface_hub", "huggingface-hub")
-        missing = object()
-        original_cached_download = getattr(huggingface_hub, "cached_download", missing)
-        if original_cached_download is missing:
-            huggingface_hub.cached_download = _cached_download(huggingface_hub)
-
-        try:
-            instructor_embedding = attempt_import_or_raise(
-                "InstructorEmbedding", "InstructorEmbedding"
-            )
-        finally:
-            if original_cached_download is missing:
-                del huggingface_hub.cached_download
-
+        instructor_embedding = attempt_import_or_raise(
+            "InstructorEmbedding", "InstructorEmbedding"
+        )
        torch = attempt_import_or_raise("torch", "torch")

        model = instructor_embedding.INSTRUCTOR(self.name)
@@ -152,44 +140,3 @@ class InstructorEmbeddingFunction(TextEmbeddingFunction):
                model, {torch.nn.Linear}, dtype=torch.qint8
            )
        return model
-
-
-def _cached_download(huggingface_hub):
-    """Provide the legacy download API used by sentence-transformers 2.2.x."""
-
-    def cached_download(
-        *,
-        url,
-        cache_dir=None,
-        force_filename=None,
-        library_name=None,
-        library_version=None,
-        user_agent=None,
-        use_auth_token=None,
-        **_,
-    ):
-        path = urlparse(url).path.lstrip("/")
-        try:
-            repo_id, resolved_path = path.split("/resolve/", maxsplit=1)
-            revision, filename = resolved_path.split("/", maxsplit=1)
-        except ValueError as err:
-            raise ValueError(f"Unsupported Hugging Face Hub URL: {url}") from err
-
-        repo_id = unquote(repo_id)
-        revision = unquote(revision)
-        filename = unquote(filename)
-        # sentence-transformers derives force_filename from this Hub path with
-        # os.path.join. Using the URL path beneath local_dir produces the same
-        # local destination without sending Windows separators to the Hub.
-        return huggingface_hub.hf_hub_download(
-            repo_id=repo_id,
-            filename=filename,
-            revision=revision,
-            local_dir=cache_dir,
-            library_name=library_name,
-            library_version=library_version,
-            user_agent=user_agent,
-            token=use_auth_token,
-        )
-
-    return cached_download
@@ -226,7 +226,7 @@ class PermutationBuilder:

        async def do_execute():
            inner_tbl = await self._async.execute()
-            return await LanceTable.from_inner(inner_tbl)
+            return LanceTable.from_inner(inner_tbl)

        return LOOP.run(do_execute())

@@ -2182,15 +2182,11 @@ class LanceTable(Table):
        return self.name

    @classmethod
-    async def from_inner(cls, tbl: LanceDBTable):
-        from .db import AsyncConnection, LanceDBConnection
+    def from_inner(cls, tbl: LanceDBTable):
+        from .db import LanceDBConnection

        async_tbl = AsyncTable(tbl)
-        inner_conn = tbl.database()
-        read_consistency_interval = await AsyncConnection(
-            inner_conn
-        ).get_read_consistency_interval()
-        conn = LanceDBConnection.from_inner(inner_conn, read_consistency_interval)
+        conn = LanceDBConnection.from_inner(tbl.database())
        return cls(
            conn,
            async_tbl.name,
@@ -77,23 +77,6 @@ def test_sync_repr_does_not_use_background_loop(tmp_path, monkeypatch):
    assert repr(table) == f"LanceTable(name='test', _conn={db!r})"


-def test_read_consistency_interval_does_not_use_background_loop(tmp_path, monkeypatch):
-    from lancedb.background_loop import LOOP
-    from lancedb.db import LanceDBConnection
-
-    consistency_interval = timedelta(seconds=5)
-    db = lancedb.connect(tmp_path, read_consistency_interval=consistency_interval)
-    db_from_inner = LanceDBConnection.from_inner(db._inner, consistency_interval)
-
-    def fail_run(*args, **kwargs):
-        raise AssertionError("properties should not use the Python background loop")
-
-    monkeypatch.setattr(LOOP, "run", fail_run)
-
-    assert db.read_consistency_interval == consistency_interval
-    assert db_from_inner.read_consistency_interval == consistency_interval
-
-
 def test_ingest_pd(tmp_path):
    db = lancedb.connect(tmp_path)

@@ -1,11 +1,8 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors

-import ntpath
 import os
 import pickle
-import sys
-from types import ModuleType
 from typing import List, Optional, Union
 from unittest.mock import MagicMock, patch

@@ -525,59 +522,6 @@ def test_embedding_function_safe_model_dump(embedding_type):
            )


-def test_instructor_embedding_supports_huggingface_hub_without_cached_download(
-    tmp_path, monkeypatch
-):
-    from lancedb.embeddings.instructor import InstructorEmbeddingFunction
-
-    hub_download = MagicMock(return_value="/cache/1_Pooling/config.json")
-    huggingface_hub = ModuleType("huggingface_hub")
-    huggingface_hub.hf_hub_download = hub_download
-    torch = ModuleType("torch")
-    monkeypatch.setitem(sys.modules, "huggingface_hub", huggingface_hub)
-    monkeypatch.setitem(sys.modules, "torch", torch)
-    monkeypatch.delitem(sys.modules, "InstructorEmbedding", raising=False)
-    monkeypatch.syspath_prepend(str(tmp_path))
-
-    (tmp_path / "InstructorEmbedding.py").write_text(
-        "from huggingface_hub import cached_download\n\n"
-        "class INSTRUCTOR:\n"
-        "    def __init__(self, name):\n"
-        "        self.name = name\n"
-    )
-
-    embedding = InstructorEmbeddingFunction.create(show_progress_bar=False)
-    instructor_model = embedding.get_model()
-
-    assert instructor_model.name == "hkunlp/instructor-base"
-    assert not hasattr(huggingface_hub, "cached_download")
-
-    instructor_embedding = sys.modules["InstructorEmbedding"]
-    path = instructor_embedding.cached_download(
-        url=(
-            "https://huggingface.co/hkunlp/instructor-base/resolve/abc123/"
-            "1_Pooling/config.json"
-        ),
-        cache_dir="/cache",
-        force_filename=ntpath.join("1_Pooling", "config.json"),
-        library_name="sentence-transformers",
-        library_version="2.2.2",
-        use_auth_token="token",
-    )
-
-    assert path == "/cache/1_Pooling/config.json"
-    hub_download.assert_called_once_with(
-        repo_id="hkunlp/instructor-base",
-        filename="1_Pooling/config.json",
-        revision="abc123",
-        local_dir="/cache",
-        library_name="sentence-transformers",
-        library_version="2.2.2",
-        user_agent=None,
-        token="token",
-    )
-
-
@patch("time.sleep")
 def test_retry(mock_sleep):
    test_function = MagicMock(side_effect=[Exception] * 9 + ["result"])
@@ -6,7 +6,6 @@ import math
 import pytest

 from lancedb import DBConnection, Table, connect
-from lancedb.background_loop import LOOP
 from lancedb.permutation import Permutation, Permutations, permutation_builder


@@ -32,25 +31,6 @@ def test_split_random_ratios(mem_db):
    assert 65 <= split_1_count <= 75  # ~70% ± tolerance


-def test_execute_does_not_reenter_background_loop(tmp_path, monkeypatch):
-    import threading
-
-    db = connect(tmp_path)
-    tbl = db.create_table("test_table", pa.table({"x": range(10)}))
-    original_run = LOOP.run
-
-    def fail_on_reentry(future):
-        assert threading.current_thread() is not LOOP.thread
-        return original_run(future)
-
-    monkeypatch.setattr(LOOP, "run", fail_on_reentry)
-
-    permutation_tbl = permutation_builder(tbl).execute()
-
-    assert permutation_tbl.count_rows() == 10
-    assert permutation_tbl._conn.read_consistency_interval is None
-
-
 def test_split_random_counts(mem_db):
    """Test random splitting with absolute counts."""
    tbl = mem_db.create_table(
@@ -6,7 +6,6 @@ import os
 import sys
 import threading
 import warnings
-from concurrent.futures import ThreadPoolExecutor
 from datetime import date, datetime, timedelta
 from time import sleep
 from typing import List
@@ -2125,27 +2124,6 @@ def test_delete(mem_db: DBConnection):
    assert table.to_arrow()["id"].to_pylist() == [1]


-def test_concurrent_deletes_are_thread_safe(mem_db: DBConnection):
-    num_workers = 8
-    table = mem_db.create_table(
-        "my_table", data=[{"id": row_id} for row_id in range(num_workers)]
-    )
-    barrier = threading.Barrier(num_workers)
-
-    def delete(row_id: int):
-        barrier.wait()
-        return table.delete(f"id = {row_id}")
-
-    with ThreadPoolExecutor(max_workers=num_workers) as pool:
-        results = list(pool.map(delete, range(num_workers)))
-
-    assert all(result.num_deleted_rows == 1 for result in results)
-    assert sorted(result.version for result in results) == list(
-        range(2, num_workers + 2)
-    )
-    assert table.count_rows() == 0
-
-
 def test_delete_expr(mem_db: DBConnection):
    table = mem_db.create_table(
        "my_table",
@@ -745,9 +745,6 @@ impl Table {

    #[allow(private_interfaces)]
    pub fn delete(self_: PyRef<'_, Self>, condition: PredicateArg) -> PyResult<Bound<'_, PyAny>> {
-        // Do not hold the Python borrow across the await.  The cloned Rust table
-        // handle is thread-safe and allows deletes on the same Python table to
-        // run concurrently without PyO3 reporting "Already borrowed".
        let inner = self_.inner_ref()?.clone();
        future_into_py(self_.py(), async move {
            let result = match &condition {
@@ -1378,12 +1375,7 @@ impl Table {

        let inner = self_.inner_ref()?.clone();
        future_into_py(self_.py(), async move {
-            let result = inner
-                .add_columns()
-                .transform(definitions)
-                .execute()
-                .await
-                .infer_error()?;
+            let result = inner.add_columns(definitions, None).await.infer_error()?;
            Ok(AddColumnsResult::from(result))
        })
    }
@@ -1397,12 +1389,7 @@ impl Table {

        let inner = self_.inner_ref()?.clone();
        future_into_py(self_.py(), async move {
-            let result = inner
-                .add_columns()
-                .transform(transform)
-                .execute()
-                .await
-                .infer_error()?;
+            let result = inner.add_columns(transform, None).await.infer_error()?;
            Ok(AddColumnsResult::from(result))
        })
    }
@@ -8,12 +8,15 @@ use std::fs::create_dir_all;
 use std::path::Path;
 use std::{collections::HashMap, sync::Arc};

+use futures::TryStreamExt;
 use lance::dataset::refs::Ref;
 use lance::dataset::{ReadParams, WriteMode, builder::DatasetBuilder};
 use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore};
 use lance_datafusion::utils::StreamingWriteSource;
 use lance_encoding::version::LanceFileVersion;
-use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
+use lance_io::object_store::{
+    DirCursor, ReadDirOptions, StorageOptionsAccessor, StorageOptionsProvider,
+};
 use lance_table::io::commit::commit_handler_from_url;
 use object_store::local::LocalFileSystem;
 use snafu::ResultExt;
@@ -718,6 +721,54 @@ impl ListingDatabase {
        self.namespace_database.clone()
    }

+    /// List up to `limit` table names, resuming after the table named `start_after`.
+    ///
+    /// The cursor and the page size go into the object store's list request rather than
+    /// being applied to a full listing, so the cost of a page is set by the size of the
+    /// page and not by the size of the database. Stores with no paginated list API fall
+    /// back to a full listing, which is what this did for every store before.
+    ///
+    /// Names come back in the order the store lists the directories in, which is by key:
+    /// `foo-bar` precedes `foo`, because the `-` of `foo-bar.lance` sorts below the `.` of
+    /// `foo.lance`. Pagination has to follow the order the cursor is pushed down in, so
+    /// that is the order both listing methods report and the order `start_after` resumes
+    /// in. It matches sorting by name except between a name and one that extends it.
+    async fn list_table_dirs(
+        &self,
+        start_after: Option<&str>,
+        limit: Option<usize>,
+    ) -> Result<Vec<String>> {
+        let dir_suffix = format!(".{}", LANCE_EXTENSION);
+        let options = ReadDirOptions {
+            // An empty name means "from the start": that is how comparing names against it
+            // behaved, and how a client looping on a page token spells its first request.
+            // Built into a cursor it would instead sit after every name below `.lance`.
+            resume_from: start_after
+                .filter(|name| !name.is_empty())
+                .map(|name| DirCursor::after_directory(format!("{name}{dir_suffix}"))),
+            page_size: limit,
+        };
+        let mut entries = self
+            .object_store
+            .read_dir_stream(self.base_path.clone(), options);
+
+        let mut names = Vec::new();
+        while limit.is_none_or(|limit| names.len() < limit) {
+            let Some(entry) = entries.try_next().await? else {
+                break;
+            };
+            // A table is the directory `<name>.lance`; anything else under the database
+            // prefix belongs to something other than a table.
+            if !entry.is_dir() {
+                continue;
+            }
+            if let Some(name) = entry.name.strip_suffix(&dir_suffix) {
+                names.push(name.to_string());
+            }
+        }
+        Ok(names)
+    }
+
    async fn drop_tables(&self, names: Vec<String>) -> Result<()> {
        let object_store_params = ObjectStoreParams {
            storage_options_accessor: if self.storage_options.is_empty() {
@@ -959,80 +1010,37 @@ impl Database for ListingDatabase {
        if !request.namespace_path.is_empty() {
            return self.namespace_database().table_names(request).await;
        }
-        let mut f = self
-            .object_store
-            .read_dir(self.base_path.clone())
-            .await?
-            .iter()
-            .map(Path::new)
-            .filter(|path| {
-                let is_lance = path
-                    .extension()
-                    .and_then(|e| e.to_str())
-                    .map(|e| e == LANCE_EXTENSION);
-                is_lance.unwrap_or(false)
-            })
-            .filter_map(|p| p.file_stem().and_then(|s| s.to_str().map(String::from)))
-            .collect::<Vec<String>>();
-        f.sort();
-        if let Some(start_after) = request.start_after {
-            let index = f
-                .iter()
-                .position(|name| name.as_str() > start_after.as_str())
-                .unwrap_or(f.len());
-            f.drain(0..index);
-        }
-        if let Some(limit) = request.limit {
-            f.truncate(limit as usize);
-        }
-        Ok(f)
+        self.list_table_dirs(
+            request.start_after.as_deref(),
+            request.limit.map(|limit| limit as usize),
+        )
+        .await
    }

    async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
        if request.id.as_ref().map(|v| !v.is_empty()).unwrap_or(false) {
            return self.namespace_database().list_tables(request).await;
        }
-        let mut f = self
-            .object_store
-            .read_dir(self.base_path.clone())
-            .await?
-            .iter()
-            .map(Path::new)
-            .filter(|path| {
-                let is_lance = path
-                    .extension()
-                    .and_then(|e| e.to_str())
-                    .map(|e| e == LANCE_EXTENSION);
-                is_lance.unwrap_or(false)
-            })
-            .filter_map(|p| p.file_stem().and_then(|s| s.to_str().map(String::from)))
-            .collect::<Vec<String>>();
-        f.sort();
+        let limit = request.limit.map(|limit| limit as usize);
+        // Reading one past the page is how we learn whether another page follows, without
+        // a second request. The extra name is dropped before the response goes out.
+        let mut tables = self
+            .list_table_dirs(
+                request.page_token.as_deref(),
+                limit.map(|limit| limit.saturating_add(1)),
+            )
+            .await?;

-        // Handle pagination with page_token
-        if let Some(ref page_token) = request.page_token {
-            let index = f
-                .iter()
-                .position(|name| name.as_str() > page_token.as_str())
-                .unwrap_or(f.len());
-            f.drain(0..index);
-        }
-
-        // Determine if there's a next page
-        let next_page_token = if let Some(limit) = request.limit {
-            if f.len() > limit as usize {
-                let token = f[limit as usize].clone();
-                f.truncate(limit as usize);
-                Some(token)
-            } else {
-                None
+        let next_page_token = match limit {
+            Some(limit) if tables.len() > limit => {
+                tables.truncate(limit);
+                tables.last().cloned()
            }
-        } else {
-            None
+            _ => None,
        };

        Ok(ListTablesResponse {
-            tables: f,
+            tables,
            page_token: next_page_token,
        })
    }
@@ -1322,6 +1330,130 @@ mod tests {
        (tempdir, db)
    }

+    async fn create_tables(db: &ListingDatabase, names: &[&str]) {
+        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
+        for name in names {
+            db.create_table(CreateTableRequest {
+                name: name.to_string(),
+                namespace_path: vec![],
+                data: Box::new(RecordBatch::new_empty(schema.clone())) as Box<dyn Scannable>,
+                mode: CreateTableMode::Create,
+                write_options: Default::default(),
+                location: None,
+                namespace_client: None,
+            })
+            .await
+            .unwrap();
+        }
+    }
+
+    /// Paging with the returned token has to visit every table exactly once. The token is
+    /// the last name of the page, which is what `page_token` resumes after.
+    #[tokio::test]
+    async fn test_list_tables_pages_over_every_table_once() {
+        let (_tempdir, db) = setup_database().await;
+        create_tables(&db, &["a", "b", "c", "d", "e"]).await;
+
+        let mut seen = Vec::new();
+        let mut page_token = None;
+        loop {
+            let page = db
+                .list_tables(ListTablesRequest {
+                    limit: Some(2),
+                    page_token,
+                    ..Default::default()
+                })
+                .await
+                .unwrap();
+            seen.extend(page.tables);
+            match page.page_token {
+                Some(token) => page_token = Some(token),
+                None => break,
+            }
+        }
+
+        assert_eq!(seen, vec!["a", "b", "c", "d", "e"]);
+    }
+
+    /// The last page reports no token, so a caller paging by token knows to stop without
+    /// asking for an empty page.
+    #[tokio::test]
+    async fn test_list_tables_exhausted_page_has_no_token() {
+        let (_tempdir, db) = setup_database().await;
+        create_tables(&db, &["a", "b"]).await;
+
+        let page = db
+            .list_tables(ListTablesRequest {
+                limit: Some(2),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(page.tables, vec!["a", "b"]);
+        assert_eq!(page.page_token, None);
+    }
+
+    /// Listing follows the order the object store lists directories in, so a name that
+    /// extends another comes first: `-` sorts below the `.` of `.lance`. Pagination pushes
+    /// its cursor into the list request, so it cannot report a different order than the
+    /// one it resumes in.
+    #[tokio::test]
+    async fn test_listing_order_follows_the_store_not_the_table_name() {
+        let (_tempdir, db) = setup_database().await;
+        create_tables(&db, &["users", "users-archive", "users.old"]).await;
+
+        #[allow(deprecated)]
+        let names = db.table_names(TableNamesRequest::default()).await.unwrap();
+        assert_eq!(names, vec!["users-archive", "users", "users.old"]);
+
+        // Resuming after a name skips everything the store lists before it, which is what
+        // paging by the previous page's last name relies on.
+        #[allow(deprecated)]
+        let after = db
+            .table_names(TableNamesRequest {
+                start_after: Some("users-archive".to_string()),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        assert_eq!(after, vec!["users", "users.old"]);
+    }
+
+    /// An empty `start_after` means "from the start". A name that sorts below `.lance` is
+    /// what disappears if it is treated as a cursor instead.
+    #[tokio::test]
+    async fn test_empty_start_after_lists_from_the_start() {
+        let (_tempdir, db) = setup_database().await;
+        create_tables(&db, &["-dash", "alpha"]).await;
+
+        #[allow(deprecated)]
+        let names = db
+            .table_names(TableNamesRequest {
+                start_after: Some(String::new()),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(names, vec!["-dash", "alpha"]);
+    }
+
+    /// Only directories named `<name>.lance` are tables; loose files and other directories
+    /// under the database prefix are not.
+    #[tokio::test]
+    async fn test_listing_ignores_non_table_children() {
+        let (tempdir, db) = setup_database().await;
+        create_tables(&db, &["real"]).await;
+        std::fs::write(tempdir.path().join("loose.lance"), b"not a table").unwrap();
+        create_dir_all(tempdir.path().join("scratch")).unwrap();
+
+        #[allow(deprecated)]
+        let names = db.table_names(TableNamesRequest::default()).await.unwrap();
+
+        assert_eq!(names, vec!["real"]);
+    }
+
    #[tokio::test]
    async fn test_listing_database_root_ops_do_not_create_manifest() {
        let tempdir = tempdir().unwrap();
@@ -3089,12 +3089,10 @@ mod tests {
            Box::pin(table.delete("false").map_ok(|_| ())),
            Box::pin(
                table
-                    .add_columns()
-                    .transform(NewColumnTransform::SqlExpressions(vec![(
-                        "x".into(),
-                        "y".into(),
-                    )]))
-                    .execute()
+                    .add_columns(
+                        NewColumnTransform::SqlExpressions(vec![("x".into(), "y".into())]),
+                        None,
+                    )
                    .map_ok(|_| ()),
            ),
            Box::pin(async {
@@ -6390,12 +6388,13 @@ mod tests {
        });

        let result = table
-            .add_columns()
-            .transform(NewColumnTransform::SqlExpressions(vec![
-                ("b".into(), "a + 1".into()),
-                ("x".into(), "cast(NULL as int32)".into()),
-            ]))
-            .execute()
+            .add_columns(
+                NewColumnTransform::SqlExpressions(vec![
+                    ("b".into(), "a + 1".into()),
+                    ("x".into(), "cast(NULL as int32)".into()),
+                ]),
+                None,
+            )
            .await
            .unwrap();

@@ -7120,12 +7119,10 @@ mod tests {
            }
            "add_columns" => {
                let _ = table
-                    .add_columns()
-                    .transform(NewColumnTransform::SqlExpressions(vec![(
-                        "c".into(),
-                        "a + 1".into(),
-                    )]))
-                    .execute()
+                    .add_columns(
+                        NewColumnTransform::SqlExpressions(vec![("c".into(), "a + 1".into())]),
+                        None,
+                    )
                    .await;
            }
            "drop_columns" => {
@@ -9883,12 +9880,10 @@ mod tests {
            .await
            .unwrap();
        branch
-            .add_columns()
-            .transform(NewColumnTransform::SqlExpressions(vec![(
-                "b".into(),
-                "a + 1".into(),
-            )]))
-            .execute()
+            .add_columns(
+                NewColumnTransform::SqlExpressions(vec![("b".into(), "a + 1".into())]),
+                None,
+            )
            .await
            .unwrap();
        branch
@@ -65,7 +65,6 @@ use crate::utils::{PatchReadParam, PatchWriteParam, resolve_arrow_field_path};
 use self::dataset::DatasetConsistencyWrapper;
 use self::merge::MergeInsertBuilder;

-pub mod add_columns;
 mod add_data;
 pub mod branch_merge;
 mod create_index;
@@ -80,7 +79,6 @@ pub mod schema_evolution;
 pub mod update;
 pub mod write_progress;
 use crate::index::waiter::wait_for_index;
-pub use add_columns::AddColumnsBuilder;
 #[cfg(feature = "remote")]
 pub(crate) use add_data::PreprocessingOutput;
 pub use add_data::{AddDataBuilder, AddDataMode, AddResult, NaNVectorBehavior};
@@ -1622,8 +1620,12 @@ impl Table {
    }

    /// Add new columns to the table, providing values to fill in.
-    pub fn add_columns(&self) -> AddColumnsBuilder {
-        AddColumnsBuilder::new(self.inner.clone())
+    pub async fn add_columns(
+        &self,
+        transforms: NewColumnTransform,
+        read_columns: Option<Vec<String>>,
+    ) -> Result<AddColumnsResult> {
+        self.inner.add_columns(transforms, read_columns).await
    }

    /// Change a column's name or nullability.
@@ -1,161 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-//! Builder for adding columns to a table.
-
-use std::sync::Arc;
-
-use lance::dataset::NewColumnTransform;
-
-use super::BaseTable;
-use super::schema_evolution::AddColumnsResult;
-use crate::{Error, Result};
-
-/// Adds columns to a table. See [`Table::add_columns`](super::Table::add_columns).
-pub struct AddColumnsBuilder {
-    parent: Arc<dyn BaseTable>,
-    transform: Option<NewColumnTransform>,
-    read_columns: Option<Vec<String>>,
-}
-
-impl std::fmt::Debug for AddColumnsBuilder {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("AddColumnsBuilder")
-            .field("parent", &self.parent)
-            .field("has_transform", &self.transform.is_some())
-            .field("read_columns", &self.read_columns)
-            .finish()
-    }
-}
-
-impl AddColumnsBuilder {
-    pub(crate) fn new(parent: Arc<dyn BaseTable>) -> Self {
-        Self {
-            parent,
-            transform: None,
-            read_columns: None,
-        }
-    }
-
-    /// Set how the new columns' values are produced. Required.
-    pub fn transform(mut self, transform: NewColumnTransform) -> Self {
-        self.transform = Some(transform);
-        self
-    }
-
-    /// Limit which existing columns a [`NewColumnTransform::BatchUDF`] mapper
-    /// receives. Every other transform determines what it reads, so setting
-    /// this alongside one is an error rather than a silent no-op.
-    pub fn read_columns(mut self, columns: impl IntoIterator<Item = impl Into<String>>) -> Self {
-        self.read_columns = Some(columns.into_iter().map(Into::into).collect());
-        self
-    }
-
-    /// Add the columns.
-    pub async fn execute(self) -> Result<AddColumnsResult> {
-        let Self {
-            parent,
-            transform,
-            read_columns,
-        } = self;
-
-        let Some(transform) = transform else {
-            return Err(Error::InvalidInput {
-                message: "add_columns requires a transform".into(),
-            });
-        };
-
-        if read_columns.is_some() && !matches!(transform, NewColumnTransform::BatchUDF(_)) {
-            return Err(Error::InvalidInput {
-                message: "read_columns applies only to a BatchUDF transform; \
-                          every other transform determines what it reads"
-                    .into(),
-            });
-        }
-
-        parent.add_columns(transform, read_columns).await
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::sync::Arc;
-
-    use arrow_array::{Int32Array, RecordBatch, record_batch};
-    use arrow_schema::{DataType, Field, Schema};
-    use lance::dataset::{BatchUDF, NewColumnTransform};
-
-    use crate::Table;
-    use crate::connect;
-
-    async fn table_with_two_columns(name: &str) -> Table {
-        let conn = connect("memory://").execute().await.unwrap();
-        let batch = record_batch!(("x", Int32, [1, 2, 3]), ("y", Int32, [10, 20, 30])).unwrap();
-        conn.create_table(name, batch).execute().await.unwrap()
-    }
-
-    #[tokio::test]
-    async fn test_requires_a_transform() {
-        let table = table_with_two_columns("no_transform").await;
-        let err = table.add_columns().execute().await.unwrap_err();
-        assert!(
-            err.to_string().contains("requires a transform"),
-            "got: {err}"
-        );
-    }
-
-    #[tokio::test]
-    async fn test_read_columns_with_sql_expressions_is_rejected() {
-        let table = table_with_two_columns("read_cols_sql").await;
-        let err = table
-            .add_columns()
-            .transform(NewColumnTransform::SqlExpressions(vec![(
-                "doubled".into(),
-                "x * 2".into(),
-            )]))
-            .read_columns(["x"])
-            .execute()
-            .await
-            .unwrap_err();
-        assert!(err.to_string().contains("BatchUDF"), "got: {err}");
-
-        let schema = table.schema().await.unwrap();
-        assert!(
-            schema.field_with_name("doubled").is_err(),
-            "a rejected call must not commit"
-        );
-    }
-
-    #[tokio::test]
-    async fn test_read_columns_limits_what_a_batch_udf_sees() {
-        let table = table_with_two_columns("read_cols_udf").await;
-
-        let output_schema = Arc::new(Schema::new(vec![Field::new("sum", DataType::Int32, true)]));
-        let mapper_schema = output_schema.clone();
-        let udf = BatchUDF {
-            mapper: Box::new(move |batch: &RecordBatch| {
-                assert!(batch.column_by_name("x").is_some());
-                assert!(batch.column_by_name("y").is_none(), "y was not requested");
-                let x = batch["x"].as_any().downcast_ref::<Int32Array>().unwrap();
-                let doubled: Int32Array = x.iter().map(|v| v.map(|v| v * 2)).collect();
-                Ok(RecordBatch::try_new(
-                    mapper_schema.clone(),
-                    vec![Arc::new(doubled)],
-                )?)
-            }),
-            output_schema,
-            result_checkpoint: None,
-        };
-
-        table
-            .add_columns()
-            .transform(NewColumnTransform::BatchUDF(udf))
-            .read_columns(["x"])
-            .execute()
-            .await
-            .unwrap();
-
-        let schema = table.schema().await.unwrap();
-        assert!(schema.field_with_name("sum").is_ok());
-    }
-}
@@ -576,12 +576,10 @@ mod tests {

        // Add a new physical column AFTER the embedding column.
        table
-            .add_columns()
-            .transform(NewColumnTransform::SqlExpressions(vec![(
-                "score".into(),
-                "42.0".into(),
-            )]))
-            .execute()
+            .add_columns(
+                NewColumnTransform::SqlExpressions(vec![("score".into(), "42.0".into())]),
+                None,
+            )
            .await
            .unwrap();

@@ -685,9 +683,7 @@ mod tests {
            true,
        )]));
        table
-            .add_columns()
-            .transform(NewColumnTransform::AllNulls(nested_schema))
-            .execute()
+            .add_columns(NewColumnTransform::AllNulls(nested_schema), None)
            .await
            .unwrap();

@@ -193,12 +193,10 @@ mod tests {

        // Add a computed column
        let result = table
-            .add_columns()
-            .transform(NewColumnTransform::SqlExpressions(vec![(
-                "doubled".into(),
-                "id * 2".into(),
-            )]))
-            .execute()
+            .add_columns(
+                NewColumnTransform::SqlExpressions(vec![("doubled".into(), "id * 2".into())]),
+                None,
+            )
            .await
            .unwrap();

@@ -253,12 +251,13 @@ mod tests {

        // Add multiple columns at once
        table
-            .add_columns()
-            .transform(NewColumnTransform::SqlExpressions(vec![
-                ("y".into(), "x + 1".into()),
-                ("z".into(), "x * x".into()),
-            ]))
-            .execute()
+            .add_columns(
+                NewColumnTransform::SqlExpressions(vec![
+                    ("y".into(), "x + 1".into()),
+                    ("z".into(), "x * x".into()),
+                ]),
+                None,
+            )
            .await
            .unwrap();

@@ -284,12 +283,10 @@ mod tests {

        // Add a column with a constant value
        table
-            .add_columns()
-            .transform(NewColumnTransform::SqlExpressions(vec![(
-                "constant".into(),
-                "42".into(),
-            )]))
-            .execute()
+            .add_columns(
+                NewColumnTransform::SqlExpressions(vec![("constant".into(), "42".into())]),
+                None,
+            )
            .await
            .unwrap();

@@ -662,12 +659,10 @@ mod tests {

        // Add column increments version
        let add_result = table
-            .add_columns()
-            .transform(NewColumnTransform::SqlExpressions(vec![(
-                "c".into(),
-                "a + b".into(),
-            )]))
-            .execute()
+            .add_columns(
+                NewColumnTransform::SqlExpressions(vec![("c".into(), "a + b".into())]),
+                None,
+            )
            .await
            .unwrap();
        assert!(add_result.version > v1);
@@ -9,17 +9,14 @@ use arrow_array::{
 };
 use arrow_schema::{DataType, Field, Fields, Schema};
 use futures::TryStreamExt;
-use lance::Dataset;
 use lance_encoding::version::LanceFileVersion;
 use lancedb::{
    Connection, Error, Result, Table,
    blob::{BlobRangeRequest, blob},
    connect, connect_namespace,
-    database::listing::{
-        ListingDatabaseOptions, NewTableConfig, OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS,
-    },
+    database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS,
    query::{ExecutableQuery, QueryBase},
-    table::{AddDataMode, CompactionOptions, OptimizeAction, OptimizeStats},
+    table::{AddDataMode, CompactionOptions, OptimizeAction},
 };
 use tempfile::tempdir;

@@ -1078,252 +1075,3 @@ async fn fetch_blob_files_aligns_across_fragments_with_nulls_and_dups() -> Resul
    }
    Ok(())
 }
-
-/// Rows exercising the null/empty interleavings from
-/// <https://github.com/lancedb/lancedb/issues/3744>: a payload, a null, a valid
-/// empty value, then payloads whose descriptors a fragment rewrite used to zero.
-fn null_empty_input_batch() -> RecordBatch {
-    let owned = [
-        Some(dedicated_blob_bytes(1)),
-        None,
-        Some(Vec::new()),
-        Some(dedicated_blob_bytes(4)),
-        Some(dedicated_blob_bytes(5)),
-        Some(dedicated_blob_bytes(6)),
-    ];
-    let payloads: Vec<Option<&[u8]>> = owned.iter().map(|payload| payload.as_deref()).collect();
-    binary_input_batch(&[1, 2, 3, 4, 5, 6], &payloads)
-}
-
-/// One `(id, Some((payload length, first byte)))` per live row, or `(id, None)`
-/// for a null blob.  Comparing lengths and first bytes keeps failure output
-/// readable where comparing whole payloads would not.
-type BlobSummary = Vec<(i64, Option<(usize, Option<u8>)>)>;
-
-/// The rows [`null_empty_input_batch`] leaves behind after `id IN (1, 4)` is
-/// deleted: a null, a valid empty value, and the two payloads that follow them.
-fn expected_null_empty_survivors() -> BlobSummary {
-    vec![
-        (2, None),
-        (3, Some((0, None))),
-        (5, Some((DEDICATED_BLOB_LEN, Some(5)))),
-        (6, Some((DEDICATED_BLOB_LEN, Some(6)))),
-    ]
-}
-
-/// `optimize()` only rewrites a fragment when lance's compaction planner selects
-/// it — here because the delete pushes the fragment past
-/// `materialize_deletions_threshold` (0.1 by default; these tests delete 2 of 6
-/// rows). Without this check, a planner or threshold change upstream would leave
-/// both regression tests green while no rewrite happened at all.
-fn assert_compacted(stats: &OptimizeStats) {
-    let metrics = stats
-        .compaction
-        .as_ref()
-        .expect("OptimizeAction::All runs compaction");
-    assert!(
-        metrics.fragments_removed >= 1,
-        "optimize() rewrote no fragment, so this test proves nothing: {metrics:?}"
-    );
-}
-
-fn summarize(rows: &[(i64, Option<Vec<u8>>)]) -> BlobSummary {
-    rows.iter()
-        .map(|(id, payload)| {
-            (
-                *id,
-                payload
-                    .as_ref()
-                    .map(|bytes| (bytes.len(), bytes.first().copied())),
-            )
-        })
-        .collect()
-}
-
-async fn sorted_id_rowid(table: &Table) -> Result<Vec<(i64, u64)>> {
-    let mut pairs = collect_id_rowid(table).await?;
-    pairs.sort_by_key(|(id, _)| *id);
-    Ok(pairs)
-}
-
-/// `{position, size}` descriptors of a legacy v1 blob column, keyed by `id`.
-async fn v1_blob_descriptors(table: &Table) -> Result<Vec<(i64, Option<(u64, u64)>)>> {
-    let batches = table
-        .query()
-        .execute()
-        .await?
-        .try_collect::<Vec<_>>()
-        .await?;
-    let batch = arrow_select::concat::concat_batches(&batches[0].schema(), &batches).unwrap();
-    let ids = batch
-        .column_by_name("id")
-        .unwrap()
-        .as_any()
-        .downcast_ref::<Int64Array>()
-        .unwrap();
-    let descriptors = batch
-        .column_by_name("image")
-        .unwrap()
-        .as_any()
-        .downcast_ref::<StructArray>()
-        .expect("v1 blob column reads back as a descriptor struct");
-    let position = descriptors
-        .column_by_name("position")
-        .unwrap()
-        .as_any()
-        .downcast_ref::<UInt64Array>()
-        .unwrap();
-    let size = descriptors
-        .column_by_name("size")
-        .unwrap()
-        .as_any()
-        .downcast_ref::<UInt64Array>()
-        .unwrap();
-    let mut rows: Vec<(i64, Option<(u64, u64)>)> = (0..batch.num_rows())
-        .map(|row| {
-            let descriptor =
-                (!descriptors.is_null(row)).then(|| (position.value(row), size.value(row)));
-            (ids.value(row), descriptor)
-        })
-        .collect();
-    rows.sort_by_key(|(id, _)| *id);
-    Ok(rows)
-}
-
-/// Payload bytes of every live row of a legacy v1 blob column, keyed by `id`.
-/// [`Table::fetch_blobs`] rejects v1 columns, so read them through lance.
-async fn v1_blob_payloads(dataset_uri: &str, table: &Table) -> Result<Vec<(i64, Option<Vec<u8>>)>> {
-    let pairs = sorted_id_rowid(table).await?;
-    let row_ids: Vec<u64> = pairs.iter().map(|(_, row_id)| *row_id).collect();
-    let dataset = Arc::new(Dataset::open(dataset_uri).await?);
-    let files = dataset.take_blobs(&row_ids, "image").await?;
-    assert_eq!(
-        files.len(),
-        pairs.len(),
-        "take_blobs returned {} handles for {} live rows",
-        files.len(),
-        pairs.len()
-    );
-    let mut rows = Vec::with_capacity(pairs.len());
-    for ((id, _), file) in pairs.iter().zip(files) {
-        let payload = match file {
-            Some(file) => Some(file.read().await?.to_vec()),
-            None => None,
-        };
-        rows.push((*id, payload));
-    }
-    Ok(rows)
-}
-
-/// Length and first byte of every live blob v2 value, keyed by `id`.
-async fn blob_v2_values(table: &Table) -> Result<BlobSummary> {
-    let pairs = sorted_id_rowid(table).await?;
-    let row_ids: Vec<u64> = pairs.iter().map(|(_, row_id)| *row_id).collect();
-    let bytes = table.fetch_blobs("image", &row_ids).await?;
-    Ok(pairs
-        .iter()
-        .enumerate()
-        .map(|(slot, (id, _))| {
-            let value = (!bytes.is_null(slot))
-                .then(|| (bytes.value(slot).len(), bytes.value(slot).first().copied()));
-            (*id, value)
-        })
-        .collect())
-}
-
-/// Regression test for [#3744]: on storage 2.0 (legacy v1 descriptors),
-/// compaction rewrote every payload following a null or empty value in the same
-/// fragment as `{position: 0, size: 0}`, so the payload bytes read back as `b""`
-/// and the new fragment no longer referenced them at all.
-///
-/// [#3744]: https://github.com/lancedb/lancedb/issues/3744
-#[tokio::test]
-async fn optimize_preserves_v1_blob_payloads_with_null_and_empty() -> Result<()> {
-    let tmp = tempdir().unwrap();
-    let db_uri = tmp.path().to_str().unwrap().to_string();
-    let db = connect(&db_uri)
-        .database_options(&ListingDatabaseOptions {
-            new_table_config: NewTableConfig {
-                data_storage_version: Some(LanceFileVersion::V2_0),
-                ..Default::default()
-            },
-            ..Default::default()
-        })
-        .execute()
-        .await?;
-    let legacy = Field::new("image", DataType::LargeBinary, true).with_metadata(
-        std::collections::HashMap::from([("lance-encoding:blob".to_string(), "true".to_string())]),
-    );
-    let schema = Arc::new(Schema::new(vec![
-        Field::new("id", DataType::Int64, false),
-        legacy,
-    ]));
-    let table = db.create_empty_table("t", schema).execute().await?;
-    table.add(null_empty_input_batch()).execute().await?;
-    assert_eq!(
-        storage_format_version(&table).await,
-        LanceFileVersion::V2_0.resolve(),
-        "v1 blob descriptors only exist below storage 2.2"
-    );
-    let dataset_uri = table.uri().await?;
-
-    // Any rewrite triggers it; deleting rows is the shape from the issue.
-    table.delete("id IN (1, 4)").await?;
-    let descriptors_before = v1_blob_descriptors(&table).await?;
-    let before = v1_blob_payloads(&dataset_uri, &table).await?;
-    assert_eq!(
-        summarize(&before),
-        expected_null_empty_survivors(),
-        "test setup no longer produces the null/empty/payload mix"
-    );
-
-    let stats = table.optimize(OptimizeAction::All).await?;
-    assert_compacted(&stats);
-
-    let descriptors_after = v1_blob_descriptors(&table).await?;
-    let after = v1_blob_payloads(&dataset_uri, &table).await?;
-    assert_eq!(
-        summarize(&after),
-        summarize(&before),
-        "optimize() lost blob payloads; descriptors before={descriptors_before:?} after={descriptors_after:?}"
-    );
-    assert!(after == before, "optimize() changed blob payload bytes");
-    Ok(())
-}
-
-/// Regression test for the blob v2 half of [#3744]: compaction rewrote a valid
-/// empty value as null, destroying the null-vs-empty distinction.
-///
-/// [#3744]: https://github.com/lancedb/lancedb/issues/3744
-#[tokio::test]
-async fn optimize_preserves_blob_v2_null_and_empty_distinction() -> Result<()> {
-    let tmp = tempdir().unwrap();
-    let db = connect(tmp.path().to_str().unwrap()).execute().await?;
-    let table = db
-        .create_empty_table("t", blob_table_schema())
-        .execute()
-        .await?;
-    table.add(null_empty_input_batch()).execute().await?;
-    assert!(
-        storage_format_version(&table).await >= LanceFileVersion::V2_2,
-        "blob v2 columns require storage >= 2.2"
-    );
-
-    table.delete("id IN (1, 4)").await?;
-    let before = blob_v2_values(&table).await?;
-    assert_eq!(
-        before,
-        expected_null_empty_survivors(),
-        "test setup no longer produces the null/empty/payload mix"
-    );
-
-    let stats = table.optimize(OptimizeAction::All).await?;
-    assert_compacted(&stats);
-
-    assert_eq!(
-        blob_v2_values(&table).await?,
-        before,
-        "optimize() changed blob v2 values"
-    );
-    Ok(())
-}