fix(listing): paginate table listing instead of enumerating the database

`ListingDatabase::table_names` and `list_tables` listed every table directory under the database prefix before applying `limit` and `page_token`. The cost of a request was set by the size of the database rather than the size of the page, so listing one table out of ten thousand took ten S3 round trips instead of one. List through `ObjectStore::read_dir_stream`, which pushes the resume position and the page size into the store's list request. Stores with no paginated list API fall back to a full listing, which is what every store did before. Two behaviour changes come with it: - Names are reported in the order the store lists directories, which differs from sorting by name only between a name and one that extends it: `users-archive` now precedes `users`, because `-` sorts below the `.` of `users.lance`. Pagination cannot report an order other than the one it resumes in. - `list_tables` returned the first name of the *next* page as its `page_token`, and `page_token` resumes *after* the name it is given, so paging dropped one table per page boundary. The token is now the last name of the page. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
chore: point lance at the read_dir_stream branch
2026-09-04 20:48:50 +00:00 · 2026-08-03 16:24:44 -07:00 · 2026-08-03 16:24:44 -07:00
22 changed files with 327 additions and 852 deletions
@@ -3422,7 +3422,7 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 [[package]]
 name = "fsst"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-array",
 "rand 0.9.5",
@@ -4778,7 +4778,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 [[package]]
 name = "lance"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -4853,7 +4853,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4875,7 +4875,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4889,7 +4889,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -4899,7 +4899,7 @@ dependencies = [
 [[package]]
 name = "lance-bitpacking"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrayref",
 "crunchy",
@@ -4910,7 +4910,7 @@ dependencies = [
 [[package]]
 name = "lance-core"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4951,7 +4951,7 @@ dependencies = [
 [[package]]
 name = "lance-datafusion"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -4982,7 +4982,7 @@ dependencies = [
 [[package]]
 name = "lance-datagen"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5000,7 +5000,7 @@ dependencies = [
 [[package]]
 name = "lance-derive"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -5010,7 +5010,7 @@ dependencies = [
 [[package]]
 name = "lance-encoding"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5046,7 +5046,7 @@ dependencies = [
 [[package]]
 name = "lance-file"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5078,7 +5078,7 @@ dependencies = [
 [[package]]
 name = "lance-index"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -5146,7 +5146,7 @@ dependencies = [
 [[package]]
 name = "lance-index-core"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5169,7 +5169,7 @@ dependencies = [
 [[package]]
 name = "lance-io"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5207,7 +5207,7 @@ dependencies = [
 [[package]]
 name = "lance-linalg"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5224,7 +5224,7 @@ dependencies = [
 [[package]]
 name = "lance-namespace"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow",
 "async-trait",
@@ -5237,7 +5237,7 @@ dependencies = [
 [[package]]
 name = "lance-namespace-impls"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow",
 "arrow-ipc",
@@ -5292,7 +5292,7 @@ dependencies = [
 [[package]]
 name = "lance-select"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5308,7 +5308,7 @@ dependencies = [
 [[package]]
 name = "lance-table"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5348,7 +5348,7 @@ dependencies = [
 [[package]]
 name = "lance-testing"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5362,7 +5362,7 @@ dependencies = [
 [[package]]
 name = "lance-tokenizer"
 version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
 dependencies = [
 "icu_segmenter",
 "jieba-rs",
@@ -7583,7 +7583,7 @@ version = "0.14.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
 dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
 "itertools 0.14.0",
 "log",
 "multimap",
@@ -8498,9 +8498,9 @@ dependencies = [

 [[package]]
 name = "rkyv"
-version = "0.8.17"
+version = "0.8.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "815cc8a37159a463064825246cadb07961e25cd9885908606f6d08a98d8f8874"
+checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3"
 dependencies = [
 "bytecheck",
 "bytes",
@@ -8517,9 +8517,9 @@ dependencies = [

 [[package]]
 name = "rkyv_derive"
-version = "0.8.17"
+version = "0.8.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0ed1a78a1b19d184b0daa629dd9a024573173ec7d485b287cb369fb3607cc1c"
+checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -9277,7 +9277,7 @@ version = "0.8.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
 dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
 "proc-macro2",
 "quote",
 "syn 2.0.117",
@@ -9289,7 +9289,7 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40"
 dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
 "proc-macro2",
 "quote",
 "syn 2.0.117",
@@ -9730,7 +9730,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
 "fastrand",
- "getrandom 0.3.4",
+ "getrandom 0.4.2",
 "once_cell",
 "rustix",
 "windows-sys 0.61.2",
@@ -13,20 +13,23 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"

 [workspace.dependencies]
-lance = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
+# TEMPORARY: `ObjectStore::read_dir_stream` is not in a lance release yet, so these point at
+# lance-format/lance#8120 cherry-picked onto the v10.1.0-beta.1 tag. Put the tag back once that
+# PR has merged and shipped in a release.
+lance = { "version" = "=10.1.0-beta.1", default-features = false, "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=10.1.0-beta.1", default-features = false, "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=10.1.0-beta.1", default-features = false, "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
 arrow = { version = "58.0.0", optional = false }
@@ -587,9 +587,6 @@ Modeled after ``VACUUM`` in PostgreSQL.
 you have added or modified 100,000 or more records or run more than 20 data
 modification operations.

- Cleanup retention must exceed the longest expected write. Retention shorter
- than 10 minutes requires `deleteUnverified: true` and exclusive write access.
-
 #### Parameters

 * **options?**: `Partial`&lt;[`OptimizeOptions`](../interfaces/OptimizeOptions.md)&gt;
@@ -16,10 +16,7 @@ cleanupOlderThan: Date;

 If set then all versions older than the given date
 be removed.  The current version will never be removed.
-The default is 7 days. The resulting retention period must be longer than
-the longest expected write. Values shorter than 10 minutes require
-`deleteUnverified: true` and are only safe when no other process can write
-to the dataset.
+The default is 7 days

 #### Example

@@ -29,8 +26,8 @@ const olderThan = new Date();
 olderThan.setDate(olderThan.getDate() - 1));
 tbl.optimize({cleanupOlderThan: olderThan});

-// With exclusive access, delete all versions except the current version
-tbl.optimize({cleanupOlderThan: new Date(), deleteUnverified: true});
+// Delete all versions except the current version
+tbl.optimize({cleanupOlderThan: new Date()});
 ```

 ***
@@ -2196,15 +2196,7 @@ describe("when optimizing a dataset", () => {
  });

  it("cleanups old versions", async () => {
-    await expect(
-      table.optimize({ cleanupOlderThan: new Date() }),
-    ).rejects.toThrow("at least 10 minutes");
-    expect(await table.version()).toBe(2);
-
-    const stats = await table.optimize({
-      cleanupOlderThan: new Date(),
-      deleteUnverified: true,
-    });
+    const stats = await table.optimize({ cleanupOlderThan: new Date() });
    expect(stats.prune.bytesRemoved).toBeGreaterThan(0);
    expect(stats.prune.oldVersionsRemoved).toBe(3);
  });
@@ -129,18 +129,15 @@ export interface OptimizeOptions {
  /**
   * If set then all versions older than the given date
   * be removed.  The current version will never be removed.
-   * The default is 7 days. The resulting retention period must be longer than
-   * the longest expected write. Values shorter than 10 minutes require
-   * `deleteUnverified: true` and are only safe when no other process can write
-   * to the dataset.
+   * The default is 7 days
   * @example
   * // Delete all versions older than 1 day
   * const olderThan = new Date();
   * olderThan.setDate(olderThan.getDate() - 1));
   * tbl.optimize({cleanupOlderThan: olderThan});
   *
-   * // With exclusive access, delete all versions except the current version
-   * tbl.optimize({cleanupOlderThan: new Date(), deleteUnverified: true});
+   * // Delete all versions except the current version
+   * tbl.optimize({cleanupOlderThan: new Date()});
   */
  cleanupOlderThan: Date;
  /**
@@ -747,9 +744,6 @@ export abstract class Table {
   *  optimize should be run frequently.  A good rule of thumb is to run optimize if
   *  you have added or modified 100,000 or more records or run more than 20 data
   *  modification operations.
-   *
-   *  Cleanup retention must exceed the longest expected write. Retention shorter
-   *  than 10 minutes requires `deleteUnverified: true` and exclusive write access.
   */
  abstract optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
  /** List all indices that have been created with {@link Table.createIndex} */
@@ -6,7 +6,6 @@ use std::collections::HashMap;
 use chrono::{DateTime, Utc};

 use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
-use lancedb::table::optimize::validate_cleanup_options;
 use lancedb::table::{
    AddDataMode, ColumnAlteration as LanceColumnAlteration, Duration,
    FieldMetadataUpdate as LanceFieldMetadataUpdate, FtsToken as LanceDbFtsToken,
@@ -340,9 +339,7 @@ impl Table {
        let transforms = NewColumnTransform::SqlExpressions(transforms);
        let res = self
            .inner_ref()?
-            .add_columns()
-            .transform(transforms)
-            .execute()
+            .add_columns(transforms, None)
            .await
            .default_error()?;
        Ok(res.into())
@@ -359,9 +356,7 @@ impl Table {
        let transforms = NewColumnTransform::AllNulls(schema);
        let res = self
            .inner_ref()?
-            .add_columns()
-            .transform(transforms)
-            .execute()
+            .add_columns(transforms, None)
            .await
            .default_error()?;
        Ok(res.into())
@@ -560,7 +555,6 @@ impl Table {
        } else {
            None
        };
-        validate_cleanup_options(older_than, delete_unverified).default_error()?;

        let compaction_stats = inner
            .optimize(OptimizeAction::Compact {
@@ -707,9 +707,6 @@ class LanceDBConnection(DBConnection):
        self._namespace_client_properties = namespace_client_properties
        if _inner is not None:
            self._conn = _inner
-            # Native-derived wrappers resolve this in their async reconstruction
-            # path so construction never synchronously re-enters LOOP.
-            self._read_consistency_interval = read_consistency_interval
            self._cached_namespace_client = None
            return

@@ -759,14 +756,11 @@ class LanceDBConnection(DBConnection):
        # storage_options.  Also, this class really shouldn't be holding any state
        # beyond _conn.
        self._conn = AsyncConnection(LOOP.run(do_connect()))
-        # Keep property access synchronous so debugger introspection cannot wait on
-        # the background loop while that thread is suspended at a breakpoint.
-        self._read_consistency_interval = read_consistency_interval
        self._cached_namespace_client: Optional[LanceNamespace] = None

    @property
    def read_consistency_interval(self) -> Optional[timedelta]:
-        return self._read_consistency_interval
+        return LOOP.run(self._conn.get_read_consistency_interval())

    @property
    def session(self) -> Optional[Session]:
@@ -777,16 +771,8 @@ class LanceDBConnection(DBConnection):
        return self._conn.uri

    @classmethod
-    def from_inner(
-        cls,
-        inner: LanceDbConnection,
-        read_consistency_interval: Optional[timedelta],
-    ):
-        return cls(
-            None,
-            read_consistency_interval=read_consistency_interval,
-            _inner=inner,
-        )
+    def from_inner(cls, inner: LanceDbConnection):
+        return cls(None, _inner=inner)

    def __repr__(self) -> str:
        return f"{self.__class__.__name__}(uri={self._conn.uri!r})"
@@ -226,7 +226,7 @@ class PermutationBuilder:

        async def do_execute():
            inner_tbl = await self._async.execute()
-            return await LanceTable.from_inner(inner_tbl)
+            return LanceTable.from_inner(inner_tbl)

        return LOOP.run(do_execute())

@@ -117,23 +117,6 @@ _MODEL_BACKED_TOKENIZER_ERRORS = (
    "Failed to initialize default tokenizer",
 )

-_MIN_SAFE_CLEANUP_AGE = timedelta(minutes=10)
-
-
-def _validate_cleanup_options(
-    older_than: Optional[timedelta], delete_unverified: bool
-) -> None:
-    if (
-        older_than is not None
-        and older_than < _MIN_SAFE_CLEANUP_AGE
-        and not delete_unverified
-    ):
-        raise ValueError(
-            "cleanup age must be at least 10 minutes unless delete_unverified is "
-            "true; short cleanup windows can remove a manifest still needed by an "
-            "in-progress write"
-        )
-

 def _add_unique_note(exception: BaseException, note: str) -> None:
    existing_notes = getattr(exception, "__notes__", ()) or ()
@@ -1784,9 +1767,7 @@ class Table(ABC):
        ----------
        older_than: timedelta, default None
            The minimum age of the version to delete. If None, then this defaults
-            to two weeks. This must be longer than the longest expected write.
-            Values shorter than 10 minutes require `delete_unverified=True` and
-            are only safe when no other process can write to the dataset.
+            to two weeks.
        delete_unverified: bool, default False
            Because they may be part of an in-progress transaction, files newer
            than 7 days old are not deleted by default. If you are sure that
@@ -1854,11 +1835,9 @@ class Table(ABC):
        Parameters
        ----------
        cleanup_older_than: timedelta, optional default 7 days
-            All files belonging to versions older than this will be removed. The
-            latest version is never removed. This must be longer than the longest
-            expected write. Values shorter than 10 minutes require
-            `delete_unverified=True` and are only safe when no other process can
-            write to the dataset.
+            All files belonging to versions older than this will be removed.  Set
+            to 0 days to remove all versions except the latest.  The latest version
+            is never removed.
        delete_unverified: bool, default False
            Files leftover from a failed transaction may appear to be part of an
            in-progress operation (e.g. appending new data) and these files will not
@@ -2203,15 +2182,11 @@ class LanceTable(Table):
        return self.name

    @classmethod
-    async def from_inner(cls, tbl: LanceDBTable):
-        from .db import AsyncConnection, LanceDBConnection
+    def from_inner(cls, tbl: LanceDBTable):
+        from .db import LanceDBConnection

        async_tbl = AsyncTable(tbl)
-        inner_conn = tbl.database()
-        read_consistency_interval = await AsyncConnection(
-            inner_conn
-        ).get_read_consistency_interval()
-        conn = LanceDBConnection.from_inner(inner_conn, read_consistency_interval)
+        conn = LanceDBConnection.from_inner(tbl.database())
        return cls(
            conn,
            async_tbl.name,
@@ -3807,9 +3782,7 @@ class LanceTable(Table):
        ----------
        older_than: timedelta, default None
            The minimum age of the version to delete. If None, then this defaults
-            to two weeks. This must be longer than the longest expected write.
-            Values shorter than 10 minutes require `delete_unverified=True` and
-            are only safe when no other process can write to the dataset.
+            to two weeks.
        delete_unverified: bool, default False
            Because they may be part of an in-progress transaction, files newer
            than 7 days old are not deleted by default. If you are sure that
@@ -3822,7 +3795,6 @@ class LanceTable(Table):
            The stats of the cleanup operation, including how many bytes were
            freed.
        """
-        _validate_cleanup_options(older_than, delete_unverified)
        return self.to_lance().cleanup_old_versions(
            older_than, delete_unverified=delete_unverified
        )
@@ -3868,11 +3840,9 @@ class LanceTable(Table):
        Parameters
        ----------
        cleanup_older_than: timedelta, optional default 7 days
-            All files belonging to versions older than this will be removed. The
-            latest version is never removed. This must be longer than the longest
-            expected write. Values shorter than 10 minutes require
-            `delete_unverified=True` and are only safe when no other process can
-            write to the dataset.
+            All files belonging to versions older than this will be removed.  Set
+            to 0 days to remove all versions except the latest.  The latest version
+            is never removed.
        delete_unverified: bool, default False
            Files leftover from a failed transaction may appear to be part of an
            in-progress operation (e.g. appending new data) and these files will not
@@ -6064,11 +6034,9 @@ class AsyncTable:
        Parameters
        ----------
        cleanup_older_than: timedelta, optional default 7 days
-            All files belonging to versions older than this will be removed. The
-            latest version is never removed. This must be longer than the longest
-            expected write. Values shorter than 10 minutes require
-            `delete_unverified=True` and are only safe when no other process can
-            write to the dataset.
+            All files belonging to versions older than this will be removed.  Set
+            to 0 days to remove all versions except the latest.  The latest version
+            is never removed.
        delete_unverified: bool, default False
            Files leftover from a failed transaction may appear to be part of an
            in-progress operation (e.g. appending new data) and these files will not
@@ -77,23 +77,6 @@ def test_sync_repr_does_not_use_background_loop(tmp_path, monkeypatch):
    assert repr(table) == f"LanceTable(name='test', _conn={db!r})"


-def test_read_consistency_interval_does_not_use_background_loop(tmp_path, monkeypatch):
-    from lancedb.background_loop import LOOP
-    from lancedb.db import LanceDBConnection
-
-    consistency_interval = timedelta(seconds=5)
-    db = lancedb.connect(tmp_path, read_consistency_interval=consistency_interval)
-    db_from_inner = LanceDBConnection.from_inner(db._inner, consistency_interval)
-
-    def fail_run(*args, **kwargs):
-        raise AssertionError("properties should not use the Python background loop")
-
-    monkeypatch.setattr(LOOP, "run", fail_run)
-
-    assert db.read_consistency_interval == consistency_interval
-    assert db_from_inner.read_consistency_interval == consistency_interval
-
-
 def test_ingest_pd(tmp_path):
    db = lancedb.connect(tmp_path)

@@ -6,7 +6,6 @@ import math
 import pytest

 from lancedb import DBConnection, Table, connect
-from lancedb.background_loop import LOOP
 from lancedb.permutation import Permutation, Permutations, permutation_builder


@@ -32,25 +31,6 @@ def test_split_random_ratios(mem_db):
    assert 65 <= split_1_count <= 75  # ~70% ± tolerance


-def test_execute_does_not_reenter_background_loop(tmp_path, monkeypatch):
-    import threading
-
-    db = connect(tmp_path)
-    tbl = db.create_table("test_table", pa.table({"x": range(10)}))
-    original_run = LOOP.run
-
-    def fail_on_reentry(future):
-        assert threading.current_thread() is not LOOP.thread
-        return original_run(future)
-
-    monkeypatch.setattr(LOOP, "run", fail_on_reentry)
-
-    permutation_tbl = permutation_builder(tbl).execute()
-
-    assert permutation_tbl.count_rows() == 10
-    assert permutation_tbl._conn.read_consistency_interval is None
-
-
 def test_split_random_counts(mem_db):
    """Test random splitting with absolute counts."""
    tbl = mem_db.create_table(
@@ -6,7 +6,6 @@ import os
 import sys
 import threading
 import warnings
-from concurrent.futures import ThreadPoolExecutor
 from datetime import date, datetime, timedelta
 from time import sleep
 from typing import List
@@ -2125,27 +2124,6 @@ def test_delete(mem_db: DBConnection):
    assert table.to_arrow()["id"].to_pylist() == [1]


-def test_concurrent_deletes_are_thread_safe(mem_db: DBConnection):
-    num_workers = 8
-    table = mem_db.create_table(
-        "my_table", data=[{"id": row_id} for row_id in range(num_workers)]
-    )
-    barrier = threading.Barrier(num_workers)
-
-    def delete(row_id: int):
-        barrier.wait()
-        return table.delete(f"id = {row_id}")
-
-    with ThreadPoolExecutor(max_workers=num_workers) as pool:
-        results = list(pool.map(delete, range(num_workers)))
-
-    assert all(result.num_deleted_rows == 1 for result in results)
-    assert sorted(result.version for result in results) == list(
-        range(2, num_workers + 2)
-    )
-    assert table.count_rows() == 0
-
-
 def test_delete_expr(mem_db: DBConnection):
    table = mem_db.create_table(
        "my_table",
@@ -2980,9 +2958,6 @@ def test_compact_cleanup(tmp_db: DBConnection):
    stats = table.cleanup_old_versions()
    assert stats.bytes_removed == 0

-    with pytest.raises(ValueError, match="at least 10 minutes"):
-        table.cleanup_old_versions(older_than=timedelta(0))
-
    stats = table.cleanup_old_versions(older_than=timedelta(0), delete_unverified=True)
    assert stats.bytes_removed > 0
    assert table.version == 4
@@ -3399,14 +3374,7 @@ async def test_optimize(mem_db_async: AsyncConnection):
    assert stats.prune.bytes_removed == 0
    assert stats.prune.old_versions_removed == 0

-    version_before_rejected_cleanup = await table.version()
-    with pytest.raises(ValueError, match="at least 10 minutes"):
-        await table.optimize(cleanup_older_than=timedelta(seconds=0))
-    assert await table.version() == version_before_rejected_cleanup
-
-    stats = await table.optimize(
-        cleanup_older_than=timedelta(seconds=0), delete_unverified=True
-    )
+    stats = await table.optimize(cleanup_older_than=timedelta(seconds=0))
    assert stats.prune.bytes_removed > 0
    assert stats.prune.old_versions_removed == 3

@@ -19,7 +19,6 @@ use arrow::{
 };
 use lancedb::blob::{BlobFile, BlobRangeRequest};
 use lancedb::index::scalar::FtsIndexBuilder;
-use lancedb::table::optimize::validate_cleanup_options;
 use lancedb::table::{
    AddDataMode, ColumnAlteration, Duration, FieldMetadataUpdate, FtsToken as LanceDbFtsToken,
    NewColumnTransform, OptimizeAction, OptimizeOptions, Ref, Table as LanceDbTable,
@@ -746,9 +745,6 @@ impl Table {

    #[allow(private_interfaces)]
    pub fn delete(self_: PyRef<'_, Self>, condition: PredicateArg) -> PyResult<Bound<'_, PyAny>> {
-        // Do not hold the Python borrow across the await.  The cloned Rust table
-        // handle is thread-safe and allows deletes on the same Python table to
-        // run concurrently without PyO3 reporting "Already borrowed".
        let inner = self_.inner_ref()?.clone();
        future_into_py(self_.py(), async move {
            let result = match &condition {
@@ -1218,7 +1214,6 @@ impl Table {
        } else {
            None
        };
-        validate_cleanup_options(older_than, delete_unverified).infer_error()?;
        future_into_py(self_.py(), async move {
            let compaction_stats = inner
                .optimize(OptimizeAction::Compact {
@@ -1380,12 +1375,7 @@ impl Table {

        let inner = self_.inner_ref()?.clone();
        future_into_py(self_.py(), async move {
-            let result = inner
-                .add_columns()
-                .transform(definitions)
-                .execute()
-                .await
-                .infer_error()?;
+            let result = inner.add_columns(definitions, None).await.infer_error()?;
            Ok(AddColumnsResult::from(result))
        })
    }
@@ -1399,12 +1389,7 @@ impl Table {

        let inner = self_.inner_ref()?.clone();
        future_into_py(self_.py(), async move {
-            let result = inner
-                .add_columns()
-                .transform(transform)
-                .execute()
-                .await
-                .infer_error()?;
+            let result = inner.add_columns(transform, None).await.infer_error()?;
            Ok(AddColumnsResult::from(result))
        })
    }
@@ -8,12 +8,15 @@ use std::fs::create_dir_all;
 use std::path::Path;
 use std::{collections::HashMap, sync::Arc};

+use futures::TryStreamExt;
 use lance::dataset::refs::Ref;
 use lance::dataset::{ReadParams, WriteMode, builder::DatasetBuilder};
 use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore};
 use lance_datafusion::utils::StreamingWriteSource;
 use lance_encoding::version::LanceFileVersion;
-use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
+use lance_io::object_store::{
+    DirCursor, ReadDirOptions, StorageOptionsAccessor, StorageOptionsProvider,
+};
 use lance_table::io::commit::commit_handler_from_url;
 use object_store::local::LocalFileSystem;
 use snafu::ResultExt;
@@ -718,6 +721,54 @@ impl ListingDatabase {
        self.namespace_database.clone()
    }

+    /// List up to `limit` table names, resuming after the table named `start_after`.
+    ///
+    /// The cursor and the page size go into the object store's list request rather than
+    /// being applied to a full listing, so the cost of a page is set by the size of the
+    /// page and not by the size of the database. Stores with no paginated list API fall
+    /// back to a full listing, which is what this did for every store before.
+    ///
+    /// Names come back in the order the store lists the directories in, which is by key:
+    /// `foo-bar` precedes `foo`, because the `-` of `foo-bar.lance` sorts below the `.` of
+    /// `foo.lance`. Pagination has to follow the order the cursor is pushed down in, so
+    /// that is the order both listing methods report and the order `start_after` resumes
+    /// in. It matches sorting by name except between a name and one that extends it.
+    async fn list_table_dirs(
+        &self,
+        start_after: Option<&str>,
+        limit: Option<usize>,
+    ) -> Result<Vec<String>> {
+        let dir_suffix = format!(".{}", LANCE_EXTENSION);
+        let options = ReadDirOptions {
+            // An empty name means "from the start": that is how comparing names against it
+            // behaved, and how a client looping on a page token spells its first request.
+            // Built into a cursor it would instead sit after every name below `.lance`.
+            resume_from: start_after
+                .filter(|name| !name.is_empty())
+                .map(|name| DirCursor::after_directory(format!("{name}{dir_suffix}"))),
+            page_size: limit,
+        };
+        let mut entries = self
+            .object_store
+            .read_dir_stream(self.base_path.clone(), options);
+
+        let mut names = Vec::new();
+        while limit.is_none_or(|limit| names.len() < limit) {
+            let Some(entry) = entries.try_next().await? else {
+                break;
+            };
+            // A table is the directory `<name>.lance`; anything else under the database
+            // prefix belongs to something other than a table.
+            if !entry.is_dir() {
+                continue;
+            }
+            if let Some(name) = entry.name.strip_suffix(&dir_suffix) {
+                names.push(name.to_string());
+            }
+        }
+        Ok(names)
+    }
+
    async fn drop_tables(&self, names: Vec<String>) -> Result<()> {
        let object_store_params = ObjectStoreParams {
            storage_options_accessor: if self.storage_options.is_empty() {
@@ -959,80 +1010,37 @@ impl Database for ListingDatabase {
        if !request.namespace_path.is_empty() {
            return self.namespace_database().table_names(request).await;
        }
-        let mut f = self
-            .object_store
-            .read_dir(self.base_path.clone())
-            .await?
-            .iter()
-            .map(Path::new)
-            .filter(|path| {
-                let is_lance = path
-                    .extension()
-                    .and_then(|e| e.to_str())
-                    .map(|e| e == LANCE_EXTENSION);
-                is_lance.unwrap_or(false)
-            })
-            .filter_map(|p| p.file_stem().and_then(|s| s.to_str().map(String::from)))
-            .collect::<Vec<String>>();
-        f.sort();
-        if let Some(start_after) = request.start_after {
-            let index = f
-                .iter()
-                .position(|name| name.as_str() > start_after.as_str())
-                .unwrap_or(f.len());
-            f.drain(0..index);
-        }
-        if let Some(limit) = request.limit {
-            f.truncate(limit as usize);
-        }
-        Ok(f)
+        self.list_table_dirs(
+            request.start_after.as_deref(),
+            request.limit.map(|limit| limit as usize),
+        )
+        .await
    }

    async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
        if request.id.as_ref().map(|v| !v.is_empty()).unwrap_or(false) {
            return self.namespace_database().list_tables(request).await;
        }
-        let mut f = self
-            .object_store
-            .read_dir(self.base_path.clone())
-            .await?
-            .iter()
-            .map(Path::new)
-            .filter(|path| {
-                let is_lance = path
-                    .extension()
-                    .and_then(|e| e.to_str())
-                    .map(|e| e == LANCE_EXTENSION);
-                is_lance.unwrap_or(false)
-            })
-            .filter_map(|p| p.file_stem().and_then(|s| s.to_str().map(String::from)))
-            .collect::<Vec<String>>();
-        f.sort();
+        let limit = request.limit.map(|limit| limit as usize);
+        // Reading one past the page is how we learn whether another page follows, without
+        // a second request. The extra name is dropped before the response goes out.
+        let mut tables = self
+            .list_table_dirs(
+                request.page_token.as_deref(),
+                limit.map(|limit| limit.saturating_add(1)),
+            )
+            .await?;

-        // Handle pagination with page_token
-        if let Some(ref page_token) = request.page_token {
-            let index = f
-                .iter()
-                .position(|name| name.as_str() > page_token.as_str())
-                .unwrap_or(f.len());
-            f.drain(0..index);
-        }
-
-        // Determine if there's a next page
-        let next_page_token = if let Some(limit) = request.limit {
-            if f.len() > limit as usize {
-                let token = f[limit as usize].clone();
-                f.truncate(limit as usize);
-                Some(token)
-            } else {
-                None
+        let next_page_token = match limit {
+            Some(limit) if tables.len() > limit => {
+                tables.truncate(limit);
+                tables.last().cloned()
            }
-        } else {
-            None
+            _ => None,
        };

        Ok(ListTablesResponse {
-            tables: f,
+            tables,
            page_token: next_page_token,
        })
    }
@@ -1322,6 +1330,130 @@ mod tests {
        (tempdir, db)
    }

+    async fn create_tables(db: &ListingDatabase, names: &[&str]) {
+        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
+        for name in names {
+            db.create_table(CreateTableRequest {
+                name: name.to_string(),
+                namespace_path: vec![],
+                data: Box::new(RecordBatch::new_empty(schema.clone())) as Box<dyn Scannable>,
+                mode: CreateTableMode::Create,
+                write_options: Default::default(),
+                location: None,
+                namespace_client: None,
+            })
+            .await
+            .unwrap();
+        }
+    }
+
+    /// Paging with the returned token has to visit every table exactly once. The token is
+    /// the last name of the page, which is what `page_token` resumes after.
+    #[tokio::test]
+    async fn test_list_tables_pages_over_every_table_once() {
+        let (_tempdir, db) = setup_database().await;
+        create_tables(&db, &["a", "b", "c", "d", "e"]).await;
+
+        let mut seen = Vec::new();
+        let mut page_token = None;
+        loop {
+            let page = db
+                .list_tables(ListTablesRequest {
+                    limit: Some(2),
+                    page_token,
+                    ..Default::default()
+                })
+                .await
+                .unwrap();
+            seen.extend(page.tables);
+            match page.page_token {
+                Some(token) => page_token = Some(token),
+                None => break,
+            }
+        }
+
+        assert_eq!(seen, vec!["a", "b", "c", "d", "e"]);
+    }
+
+    /// The last page reports no token, so a caller paging by token knows to stop without
+    /// asking for an empty page.
+    #[tokio::test]
+    async fn test_list_tables_exhausted_page_has_no_token() {
+        let (_tempdir, db) = setup_database().await;
+        create_tables(&db, &["a", "b"]).await;
+
+        let page = db
+            .list_tables(ListTablesRequest {
+                limit: Some(2),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(page.tables, vec!["a", "b"]);
+        assert_eq!(page.page_token, None);
+    }
+
+    /// Listing follows the order the object store lists directories in, so a name that
+    /// extends another comes first: `-` sorts below the `.` of `.lance`. Pagination pushes
+    /// its cursor into the list request, so it cannot report a different order than the
+    /// one it resumes in.
+    #[tokio::test]
+    async fn test_listing_order_follows_the_store_not_the_table_name() {
+        let (_tempdir, db) = setup_database().await;
+        create_tables(&db, &["users", "users-archive", "users.old"]).await;
+
+        #[allow(deprecated)]
+        let names = db.table_names(TableNamesRequest::default()).await.unwrap();
+        assert_eq!(names, vec!["users-archive", "users", "users.old"]);
+
+        // Resuming after a name skips everything the store lists before it, which is what
+        // paging by the previous page's last name relies on.
+        #[allow(deprecated)]
+        let after = db
+            .table_names(TableNamesRequest {
+                start_after: Some("users-archive".to_string()),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        assert_eq!(after, vec!["users", "users.old"]);
+    }
+
+    /// An empty `start_after` means "from the start". A name that sorts below `.lance` is
+    /// what disappears if it is treated as a cursor instead.
+    #[tokio::test]
+    async fn test_empty_start_after_lists_from_the_start() {
+        let (_tempdir, db) = setup_database().await;
+        create_tables(&db, &["-dash", "alpha"]).await;
+
+        #[allow(deprecated)]
+        let names = db
+            .table_names(TableNamesRequest {
+                start_after: Some(String::new()),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(names, vec!["-dash", "alpha"]);
+    }
+
+    /// Only directories named `<name>.lance` are tables; loose files and other directories
+    /// under the database prefix are not.
+    #[tokio::test]
+    async fn test_listing_ignores_non_table_children() {
+        let (tempdir, db) = setup_database().await;
+        create_tables(&db, &["real"]).await;
+        std::fs::write(tempdir.path().join("loose.lance"), b"not a table").unwrap();
+        create_dir_all(tempdir.path().join("scratch")).unwrap();
+
+        #[allow(deprecated)]
+        let names = db.table_names(TableNamesRequest::default()).await.unwrap();
+
+        assert_eq!(names, vec!["real"]);
+    }
+
    #[tokio::test]
    async fn test_listing_database_root_ops_do_not_create_manifest() {
        let tempdir = tempdir().unwrap();
@@ -3089,12 +3089,10 @@ mod tests {
            Box::pin(table.delete("false").map_ok(|_| ())),
            Box::pin(
                table
-                    .add_columns()
-                    .transform(NewColumnTransform::SqlExpressions(vec![(
-                        "x".into(),
-                        "y".into(),
-                    )]))
-                    .execute()
+                    .add_columns(
+                        NewColumnTransform::SqlExpressions(vec![("x".into(), "y".into())]),
+                        None,
+                    )
                    .map_ok(|_| ()),
            ),
            Box::pin(async {
@@ -6390,12 +6388,13 @@ mod tests {
        });

        let result = table
-            .add_columns()
-            .transform(NewColumnTransform::SqlExpressions(vec![
-                ("b".into(), "a + 1".into()),
-                ("x".into(), "cast(NULL as int32)".into()),
-            ]))
-            .execute()
+            .add_columns(
+                NewColumnTransform::SqlExpressions(vec![
+                    ("b".into(), "a + 1".into()),
+                    ("x".into(), "cast(NULL as int32)".into()),
+                ]),
+                None,
+            )
            .await
            .unwrap();

@@ -7120,12 +7119,10 @@ mod tests {
            }
            "add_columns" => {
                let _ = table
-                    .add_columns()
-                    .transform(NewColumnTransform::SqlExpressions(vec![(
-                        "c".into(),
-                        "a + 1".into(),
-                    )]))
-                    .execute()
+                    .add_columns(
+                        NewColumnTransform::SqlExpressions(vec![("c".into(), "a + 1".into())]),
+                        None,
+                    )
                    .await;
            }
            "drop_columns" => {
@@ -9883,12 +9880,10 @@ mod tests {
            .await
            .unwrap();
        branch
-            .add_columns()
-            .transform(NewColumnTransform::SqlExpressions(vec![(
-                "b".into(),
-                "a + 1".into(),
-            )]))
-            .execute()
+            .add_columns(
+                NewColumnTransform::SqlExpressions(vec![("b".into(), "a + 1".into())]),
+                None,
+            )
            .await
            .unwrap();
        branch
@@ -65,7 +65,6 @@ use crate::utils::{PatchReadParam, PatchWriteParam, resolve_arrow_field_path};
 use self::dataset::DatasetConsistencyWrapper;
 use self::merge::MergeInsertBuilder;

-pub mod add_columns;
 mod add_data;
 pub mod branch_merge;
 mod create_index;
@@ -80,7 +79,6 @@ pub mod schema_evolution;
 pub mod update;
 pub mod write_progress;
 use crate::index::waiter::wait_for_index;
-pub use add_columns::AddColumnsBuilder;
 #[cfg(feature = "remote")]
 pub(crate) use add_data::PreprocessingOutput;
 pub use add_data::{AddDataBuilder, AddDataMode, AddResult, NaNVectorBehavior};
@@ -1622,8 +1620,12 @@ impl Table {
    }

    /// Add new columns to the table, providing values to fill in.
-    pub fn add_columns(&self) -> AddColumnsBuilder {
-        AddColumnsBuilder::new(self.inner.clone())
+    pub async fn add_columns(
+        &self,
+        transforms: NewColumnTransform,
+        read_columns: Option<Vec<String>>,
+    ) -> Result<AddColumnsResult> {
+        self.inner.add_columns(transforms, read_columns).await
    }

    /// Change a column's name or nullability.
@@ -1,161 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-//! Builder for adding columns to a table.
-
-use std::sync::Arc;
-
-use lance::dataset::NewColumnTransform;
-
-use super::BaseTable;
-use super::schema_evolution::AddColumnsResult;
-use crate::{Error, Result};
-
-/// Adds columns to a table. See [`Table::add_columns`](super::Table::add_columns).
-pub struct AddColumnsBuilder {
-    parent: Arc<dyn BaseTable>,
-    transform: Option<NewColumnTransform>,
-    read_columns: Option<Vec<String>>,
-}
-
-impl std::fmt::Debug for AddColumnsBuilder {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("AddColumnsBuilder")
-            .field("parent", &self.parent)
-            .field("has_transform", &self.transform.is_some())
-            .field("read_columns", &self.read_columns)
-            .finish()
-    }
-}
-
-impl AddColumnsBuilder {
-    pub(crate) fn new(parent: Arc<dyn BaseTable>) -> Self {
-        Self {
-            parent,
-            transform: None,
-            read_columns: None,
-        }
-    }
-
-    /// Set how the new columns' values are produced. Required.
-    pub fn transform(mut self, transform: NewColumnTransform) -> Self {
-        self.transform = Some(transform);
-        self
-    }
-
-    /// Limit which existing columns a [`NewColumnTransform::BatchUDF`] mapper
-    /// receives. Every other transform determines what it reads, so setting
-    /// this alongside one is an error rather than a silent no-op.
-    pub fn read_columns(mut self, columns: impl IntoIterator<Item = impl Into<String>>) -> Self {
-        self.read_columns = Some(columns.into_iter().map(Into::into).collect());
-        self
-    }
-
-    /// Add the columns.
-    pub async fn execute(self) -> Result<AddColumnsResult> {
-        let Self {
-            parent,
-            transform,
-            read_columns,
-        } = self;
-
-        let Some(transform) = transform else {
-            return Err(Error::InvalidInput {
-                message: "add_columns requires a transform".into(),
-            });
-        };
-
-        if read_columns.is_some() && !matches!(transform, NewColumnTransform::BatchUDF(_)) {
-            return Err(Error::InvalidInput {
-                message: "read_columns applies only to a BatchUDF transform; \
-                          every other transform determines what it reads"
-                    .into(),
-            });
-        }
-
-        parent.add_columns(transform, read_columns).await
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::sync::Arc;
-
-    use arrow_array::{Int32Array, RecordBatch, record_batch};
-    use arrow_schema::{DataType, Field, Schema};
-    use lance::dataset::{BatchUDF, NewColumnTransform};
-
-    use crate::Table;
-    use crate::connect;
-
-    async fn table_with_two_columns(name: &str) -> Table {
-        let conn = connect("memory://").execute().await.unwrap();
-        let batch = record_batch!(("x", Int32, [1, 2, 3]), ("y", Int32, [10, 20, 30])).unwrap();
-        conn.create_table(name, batch).execute().await.unwrap()
-    }
-
-    #[tokio::test]
-    async fn test_requires_a_transform() {
-        let table = table_with_two_columns("no_transform").await;
-        let err = table.add_columns().execute().await.unwrap_err();
-        assert!(
-            err.to_string().contains("requires a transform"),
-            "got: {err}"
-        );
-    }
-
-    #[tokio::test]
-    async fn test_read_columns_with_sql_expressions_is_rejected() {
-        let table = table_with_two_columns("read_cols_sql").await;
-        let err = table
-            .add_columns()
-            .transform(NewColumnTransform::SqlExpressions(vec![(
-                "doubled".into(),
-                "x * 2".into(),
-            )]))
-            .read_columns(["x"])
-            .execute()
-            .await
-            .unwrap_err();
-        assert!(err.to_string().contains("BatchUDF"), "got: {err}");
-
-        let schema = table.schema().await.unwrap();
-        assert!(
-            schema.field_with_name("doubled").is_err(),
-            "a rejected call must not commit"
-        );
-    }
-
-    #[tokio::test]
-    async fn test_read_columns_limits_what_a_batch_udf_sees() {
-        let table = table_with_two_columns("read_cols_udf").await;
-
-        let output_schema = Arc::new(Schema::new(vec![Field::new("sum", DataType::Int32, true)]));
-        let mapper_schema = output_schema.clone();
-        let udf = BatchUDF {
-            mapper: Box::new(move |batch: &RecordBatch| {
-                assert!(batch.column_by_name("x").is_some());
-                assert!(batch.column_by_name("y").is_none(), "y was not requested");
-                let x = batch["x"].as_any().downcast_ref::<Int32Array>().unwrap();
-                let doubled: Int32Array = x.iter().map(|v| v.map(|v| v * 2)).collect();
-                Ok(RecordBatch::try_new(
-                    mapper_schema.clone(),
-                    vec![Arc::new(doubled)],
-                )?)
-            }),
-            output_schema,
-            result_checkpoint: None,
-        };
-
-        table
-            .add_columns()
-            .transform(NewColumnTransform::BatchUDF(udf))
-            .read_columns(["x"])
-            .execute()
-            .await
-            .unwrap();
-
-        let schema = table.schema().await.unwrap();
-        assert!(schema.field_with_name("sum").is_ok());
-    }
-}
@@ -576,12 +576,10 @@ mod tests {

        // Add a new physical column AFTER the embedding column.
        table
-            .add_columns()
-            .transform(NewColumnTransform::SqlExpressions(vec![(
-                "score".into(),
-                "42.0".into(),
-            )]))
-            .execute()
+            .add_columns(
+                NewColumnTransform::SqlExpressions(vec![("score".into(), "42.0".into())]),
+                None,
+            )
            .await
            .unwrap();

@@ -685,9 +683,7 @@ mod tests {
            true,
        )]));
        table
-            .add_columns()
-            .transform(NewColumnTransform::AllNulls(nested_schema))
-            .execute()
+            .add_columns(NewColumnTransform::AllNulls(nested_schema), None)
            .await
            .unwrap();

@@ -18,33 +18,7 @@ pub use chrono::Duration;
 pub use lance::dataset::optimize::CompactionOptions;

 use super::NativeTable;
-use crate::error::{Error, Result};
-
-const MIN_SAFE_CLEANUP_AGE_MINUTES: i64 = 10;
-
-/// Validate version-cleanup options before starting an optimization operation.
-///
-/// This is public for the language bindings, which run compaction and cleanup as
-/// separate operations and must reject unsafe cleanup options before compaction
-/// changes the table.
-#[doc(hidden)]
-pub fn validate_cleanup_options(
-    older_than: Option<Duration>,
-    delete_unverified: Option<bool>,
-) -> Result<()> {
-    let minimum_age =
-        Duration::try_minutes(MIN_SAFE_CLEANUP_AGE_MINUTES).expect("minimum cleanup age is valid");
-    if older_than.is_some_and(|age| age < minimum_age) && delete_unverified != Some(true) {
-        return Err(Error::InvalidInput {
-            message: format!(
-                "cleanup age must be at least {MIN_SAFE_CLEANUP_AGE_MINUTES} minutes unless \
-                 delete_unverified is true; short cleanup windows can remove a manifest still \
-                 needed by an in-progress write"
-            ),
-        });
-    }
-    Ok(())
-}
+use crate::error::Result;

 /// Optimize the dataset.
 ///
@@ -86,9 +60,7 @@ pub enum OptimizeAction {
    ///
    /// Once a version is pruned it can no longer be checked out.
    Prune {
-        /// The duration of time to keep versions of the dataset. This should be longer than the
-        /// longest expected write. Values shorter than 10 minutes require `delete_unverified` to
-        /// be true and are only safe when no other process can write to the dataset.
+        /// The duration of time to keep versions of the dataset.
        older_than: Option<Duration>,
        /// Because they may be part of an in-progress transaction, files newer than 7 days old are not deleted by default.
        /// If you are sure that there are no in-progress transactions, then you can set this to True to delete all files older than `older_than`.
@@ -192,15 +164,6 @@ pub(crate) async fn execute_optimize(
    table: &NativeTable,
    action: OptimizeAction,
 ) -> Result<OptimizeStats> {
-    if let OptimizeAction::Prune {
-        older_than,
-        delete_unverified,
-        ..
-    } = &action
-    {
-        validate_cleanup_options(*older_than, *delete_unverified)?;
-    }
-
    let mut stats = OptimizeStats {
        compaction: None,
        prune: None,
@@ -259,7 +222,7 @@ mod tests {
    use crate::connect;
    use crate::index::{Index, scalar::BTreeIndexBuilder};
    use crate::query::ExecutableQuery;
-    use crate::table::{CompactionOptions, Duration, OptimizeAction, OptimizeStats};
+    use crate::table::{CompactionOptions, OptimizeAction, OptimizeStats};
    use futures::TryStreamExt;

    #[tokio::test]
@@ -417,48 +380,6 @@ mod tests {
        assert_eq!(all_values, expected);
    }

-    #[tokio::test]
-    async fn test_optimize_rejects_unsafe_cleanup_age() {
-        let conn = connect("memory://").execute().await.unwrap();
-        let batch = RecordBatch::try_new(
-            Arc::new(Schema::new(vec![Field::new("i", DataType::Int32, false)])),
-            vec![Arc::new(Int32Array::from_iter_values(0..10))],
-        )
-        .unwrap();
-        let table = conn
-            .create_table("test_unsafe_prune", batch.clone())
-            .execute()
-            .await
-            .unwrap();
-        table.add(batch).execute().await.unwrap();
-
-        let versions_before = table
-            .list_versions()
-            .await
-            .unwrap()
-            .into_iter()
-            .map(|version| version.version)
-            .collect::<Vec<_>>();
-        let err = table
-            .optimize(OptimizeAction::Prune {
-                older_than: Some(Duration::zero()),
-                delete_unverified: Some(false),
-                error_if_tagged_old_versions: None,
-            })
-            .await
-            .unwrap_err();
-
-        assert!(err.to_string().contains("at least 10 minutes"));
-        let versions_after = table
-            .list_versions()
-            .await
-            .unwrap()
-            .into_iter()
-            .map(|version| version.version)
-            .collect::<Vec<_>>();
-        assert_eq!(versions_after, versions_before);
-    }
-
    #[tokio::test]
    async fn test_optimize_index() {
        let conn = connect("memory://").execute().await.unwrap();
@@ -193,12 +193,10 @@ mod tests {

        // Add a computed column
        let result = table
-            .add_columns()
-            .transform(NewColumnTransform::SqlExpressions(vec![(
-                "doubled".into(),
-                "id * 2".into(),
-            )]))
-            .execute()
+            .add_columns(
+                NewColumnTransform::SqlExpressions(vec![("doubled".into(), "id * 2".into())]),
+                None,
+            )
            .await
            .unwrap();

@@ -253,12 +251,13 @@ mod tests {

        // Add multiple columns at once
        table
-            .add_columns()
-            .transform(NewColumnTransform::SqlExpressions(vec![
-                ("y".into(), "x + 1".into()),
-                ("z".into(), "x * x".into()),
-            ]))
-            .execute()
+            .add_columns(
+                NewColumnTransform::SqlExpressions(vec![
+                    ("y".into(), "x + 1".into()),
+                    ("z".into(), "x * x".into()),
+                ]),
+                None,
+            )
            .await
            .unwrap();

@@ -284,12 +283,10 @@ mod tests {

        // Add a column with a constant value
        table
-            .add_columns()
-            .transform(NewColumnTransform::SqlExpressions(vec![(
-                "constant".into(),
-                "42".into(),
-            )]))
-            .execute()
+            .add_columns(
+                NewColumnTransform::SqlExpressions(vec![("constant".into(), "42".into())]),
+                None,
+            )
            .await
            .unwrap();

@@ -662,12 +659,10 @@ mod tests {

        // Add column increments version
        let add_result = table
-            .add_columns()
-            .transform(NewColumnTransform::SqlExpressions(vec![(
-                "c".into(),
-                "a + b".into(),
-            )]))
-            .execute()
+            .add_columns(
+                NewColumnTransform::SqlExpressions(vec![("c".into(), "a + b".into())]),
+                None,
+            )
            .await
            .unwrap();
        assert!(add_result.version > v1);
@@ -9,17 +9,14 @@ use arrow_array::{
 };
 use arrow_schema::{DataType, Field, Fields, Schema};
 use futures::TryStreamExt;
-use lance::Dataset;
 use lance_encoding::version::LanceFileVersion;
 use lancedb::{
    Connection, Error, Result, Table,
    blob::{BlobRangeRequest, blob},
    connect, connect_namespace,
-    database::listing::{
-        ListingDatabaseOptions, NewTableConfig, OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS,
-    },
+    database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS,
    query::{ExecutableQuery, QueryBase},
-    table::{AddDataMode, CompactionOptions, OptimizeAction, OptimizeStats},
+    table::{AddDataMode, CompactionOptions, OptimizeAction},
 };
 use tempfile::tempdir;

@@ -1078,252 +1075,3 @@ async fn fetch_blob_files_aligns_across_fragments_with_nulls_and_dups() -> Resul
    }
    Ok(())
 }
-
-/// Rows exercising the null/empty interleavings from
-/// <https://github.com/lancedb/lancedb/issues/3744>: a payload, a null, a valid
-/// empty value, then payloads whose descriptors a fragment rewrite used to zero.
-fn null_empty_input_batch() -> RecordBatch {
-    let owned = [
-        Some(dedicated_blob_bytes(1)),
-        None,
-        Some(Vec::new()),
-        Some(dedicated_blob_bytes(4)),
-        Some(dedicated_blob_bytes(5)),
-        Some(dedicated_blob_bytes(6)),
-    ];
-    let payloads: Vec<Option<&[u8]>> = owned.iter().map(|payload| payload.as_deref()).collect();
-    binary_input_batch(&[1, 2, 3, 4, 5, 6], &payloads)
-}
-
-/// One `(id, Some((payload length, first byte)))` per live row, or `(id, None)`
-/// for a null blob.  Comparing lengths and first bytes keeps failure output
-/// readable where comparing whole payloads would not.
-type BlobSummary = Vec<(i64, Option<(usize, Option<u8>)>)>;
-
-/// The rows [`null_empty_input_batch`] leaves behind after `id IN (1, 4)` is
-/// deleted: a null, a valid empty value, and the two payloads that follow them.
-fn expected_null_empty_survivors() -> BlobSummary {
-    vec![
-        (2, None),
-        (3, Some((0, None))),
-        (5, Some((DEDICATED_BLOB_LEN, Some(5)))),
-        (6, Some((DEDICATED_BLOB_LEN, Some(6)))),
-    ]
-}
-
-/// `optimize()` only rewrites a fragment when lance's compaction planner selects
-/// it — here because the delete pushes the fragment past
-/// `materialize_deletions_threshold` (0.1 by default; these tests delete 2 of 6
-/// rows). Without this check, a planner or threshold change upstream would leave
-/// both regression tests green while no rewrite happened at all.
-fn assert_compacted(stats: &OptimizeStats) {
-    let metrics = stats
-        .compaction
-        .as_ref()
-        .expect("OptimizeAction::All runs compaction");
-    assert!(
-        metrics.fragments_removed >= 1,
-        "optimize() rewrote no fragment, so this test proves nothing: {metrics:?}"
-    );
-}
-
-fn summarize(rows: &[(i64, Option<Vec<u8>>)]) -> BlobSummary {
-    rows.iter()
-        .map(|(id, payload)| {
-            (
-                *id,
-                payload
-                    .as_ref()
-                    .map(|bytes| (bytes.len(), bytes.first().copied())),
-            )
-        })
-        .collect()
-}
-
-async fn sorted_id_rowid(table: &Table) -> Result<Vec<(i64, u64)>> {
-    let mut pairs = collect_id_rowid(table).await?;
-    pairs.sort_by_key(|(id, _)| *id);
-    Ok(pairs)
-}
-
-/// `{position, size}` descriptors of a legacy v1 blob column, keyed by `id`.
-async fn v1_blob_descriptors(table: &Table) -> Result<Vec<(i64, Option<(u64, u64)>)>> {
-    let batches = table
-        .query()
-        .execute()
-        .await?
-        .try_collect::<Vec<_>>()
-        .await?;
-    let batch = arrow_select::concat::concat_batches(&batches[0].schema(), &batches).unwrap();
-    let ids = batch
-        .column_by_name("id")
-        .unwrap()
-        .as_any()
-        .downcast_ref::<Int64Array>()
-        .unwrap();
-    let descriptors = batch
-        .column_by_name("image")
-        .unwrap()
-        .as_any()
-        .downcast_ref::<StructArray>()
-        .expect("v1 blob column reads back as a descriptor struct");
-    let position = descriptors
-        .column_by_name("position")
-        .unwrap()
-        .as_any()
-        .downcast_ref::<UInt64Array>()
-        .unwrap();
-    let size = descriptors
-        .column_by_name("size")
-        .unwrap()
-        .as_any()
-        .downcast_ref::<UInt64Array>()
-        .unwrap();
-    let mut rows: Vec<(i64, Option<(u64, u64)>)> = (0..batch.num_rows())
-        .map(|row| {
-            let descriptor =
-                (!descriptors.is_null(row)).then(|| (position.value(row), size.value(row)));
-            (ids.value(row), descriptor)
-        })
-        .collect();
-    rows.sort_by_key(|(id, _)| *id);
-    Ok(rows)
-}
-
-/// Payload bytes of every live row of a legacy v1 blob column, keyed by `id`.
-/// [`Table::fetch_blobs`] rejects v1 columns, so read them through lance.
-async fn v1_blob_payloads(dataset_uri: &str, table: &Table) -> Result<Vec<(i64, Option<Vec<u8>>)>> {
-    let pairs = sorted_id_rowid(table).await?;
-    let row_ids: Vec<u64> = pairs.iter().map(|(_, row_id)| *row_id).collect();
-    let dataset = Arc::new(Dataset::open(dataset_uri).await?);
-    let files = dataset.take_blobs(&row_ids, "image").await?;
-    assert_eq!(
-        files.len(),
-        pairs.len(),
-        "take_blobs returned {} handles for {} live rows",
-        files.len(),
-        pairs.len()
-    );
-    let mut rows = Vec::with_capacity(pairs.len());
-    for ((id, _), file) in pairs.iter().zip(files) {
-        let payload = match file {
-            Some(file) => Some(file.read().await?.to_vec()),
-            None => None,
-        };
-        rows.push((*id, payload));
-    }
-    Ok(rows)
-}
-
-/// Length and first byte of every live blob v2 value, keyed by `id`.
-async fn blob_v2_values(table: &Table) -> Result<BlobSummary> {
-    let pairs = sorted_id_rowid(table).await?;
-    let row_ids: Vec<u64> = pairs.iter().map(|(_, row_id)| *row_id).collect();
-    let bytes = table.fetch_blobs("image", &row_ids).await?;
-    Ok(pairs
-        .iter()
-        .enumerate()
-        .map(|(slot, (id, _))| {
-            let value = (!bytes.is_null(slot))
-                .then(|| (bytes.value(slot).len(), bytes.value(slot).first().copied()));
-            (*id, value)
-        })
-        .collect())
-}
-
-/// Regression test for [#3744]: on storage 2.0 (legacy v1 descriptors),
-/// compaction rewrote every payload following a null or empty value in the same
-/// fragment as `{position: 0, size: 0}`, so the payload bytes read back as `b""`
-/// and the new fragment no longer referenced them at all.
-///
-/// [#3744]: https://github.com/lancedb/lancedb/issues/3744
-#[tokio::test]
-async fn optimize_preserves_v1_blob_payloads_with_null_and_empty() -> Result<()> {
-    let tmp = tempdir().unwrap();
-    let db_uri = tmp.path().to_str().unwrap().to_string();
-    let db = connect(&db_uri)
-        .database_options(&ListingDatabaseOptions {
-            new_table_config: NewTableConfig {
-                data_storage_version: Some(LanceFileVersion::V2_0),
-                ..Default::default()
-            },
-            ..Default::default()
-        })
-        .execute()
-        .await?;
-    let legacy = Field::new("image", DataType::LargeBinary, true).with_metadata(
-        std::collections::HashMap::from([("lance-encoding:blob".to_string(), "true".to_string())]),
-    );
-    let schema = Arc::new(Schema::new(vec![
-        Field::new("id", DataType::Int64, false),
-        legacy,
-    ]));
-    let table = db.create_empty_table("t", schema).execute().await?;
-    table.add(null_empty_input_batch()).execute().await?;
-    assert_eq!(
-        storage_format_version(&table).await,
-        LanceFileVersion::V2_0.resolve(),
-        "v1 blob descriptors only exist below storage 2.2"
-    );
-    let dataset_uri = table.uri().await?;
-
-    // Any rewrite triggers it; deleting rows is the shape from the issue.
-    table.delete("id IN (1, 4)").await?;
-    let descriptors_before = v1_blob_descriptors(&table).await?;
-    let before = v1_blob_payloads(&dataset_uri, &table).await?;
-    assert_eq!(
-        summarize(&before),
-        expected_null_empty_survivors(),
-        "test setup no longer produces the null/empty/payload mix"
-    );
-
-    let stats = table.optimize(OptimizeAction::All).await?;
-    assert_compacted(&stats);
-
-    let descriptors_after = v1_blob_descriptors(&table).await?;
-    let after = v1_blob_payloads(&dataset_uri, &table).await?;
-    assert_eq!(
-        summarize(&after),
-        summarize(&before),
-        "optimize() lost blob payloads; descriptors before={descriptors_before:?} after={descriptors_after:?}"
-    );
-    assert!(after == before, "optimize() changed blob payload bytes");
-    Ok(())
-}
-
-/// Regression test for the blob v2 half of [#3744]: compaction rewrote a valid
-/// empty value as null, destroying the null-vs-empty distinction.
-///
-/// [#3744]: https://github.com/lancedb/lancedb/issues/3744
-#[tokio::test]
-async fn optimize_preserves_blob_v2_null_and_empty_distinction() -> Result<()> {
-    let tmp = tempdir().unwrap();
-    let db = connect(tmp.path().to_str().unwrap()).execute().await?;
-    let table = db
-        .create_empty_table("t", blob_table_schema())
-        .execute()
-        .await?;
-    table.add(null_empty_input_batch()).execute().await?;
-    assert!(
-        storage_format_version(&table).await >= LanceFileVersion::V2_2,
-        "blob v2 columns require storage >= 2.2"
-    );
-
-    table.delete("id IN (1, 4)").await?;
-    let before = blob_v2_values(&table).await?;
-    assert_eq!(
-        before,
-        expected_null_empty_survivors(),
-        "test setup no longer produces the null/empty/payload mix"
-    );
-
-    let stats = table.optimize(OptimizeAction::All).await?;
-    assert_compacted(&stats);
-
-    assert_eq!(
-        blob_v2_values(&table).await?,
-        before,
-        "optimize() changed blob v2 values"
-    );
-    Ok(())
-}