fix(listing): paginate table listing instead of enumerating the database

`ListingDatabase::table_names` and `list_tables` listed every table directory under the database prefix before applying `limit` and `page_token`. The cost of a request was set by the size of the database rather than the size of the page, so listing one table out of ten thousand took ten S3 round trips instead of one. List through `ObjectStore::read_dir_stream`, which pushes the resume position and the page size into the store's list request. Stores with no paginated list API fall back to a full listing, which is what every store did before. Two behaviour changes come with it: - Names are reported in the order the store lists directories, which differs from sorting by name only between a name and one that extends it: `users-archive` now precedes `users`, because `-` sorts below the `.` of `users.lance`. Pagination cannot report an order other than the one it resumes in. - `list_tables` returned the first name of the *next* page as its `page_token`, and `page_token` resumes *after* the name it is given, so paging dropped one table per page boundary. The token is now the last name of the page. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
chore: update lance dependency to v10.1.0-beta.2
2026-08-26 07:58:31 +00:00 · 2026-08-03 16:39:22 -07:00 · 2026-08-02 00:12:47 +00:00
7 changed files with 444 additions and 274 deletions
@@ -775,7 +775,7 @@ dependencies = [
 "http 0.2.12",
 "http 1.4.2",
 "http-body 1.1.0",
- "lru",
+ "lru 0.16.4",
 "percent-encoding",
 "regex-lite",
 "sha2 0.11.0",
@@ -1241,6 +1241,12 @@ version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"

+[[package]]
+name = "base64"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9"
+
 [[package]]
 name = "base64-simd"
 version = "0.8.0"
@@ -1964,15 +1970,6 @@ dependencies = [
 "spin 0.10.1",
 ]

-[[package]]
-name = "crc32c"
-version = "0.6.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47"
-dependencies = [
- "rustc_version",
-]
-
 [[package]]
 name = "crc32fast"
 version = "1.5.0"
@@ -2174,9 +2171,9 @@ dependencies = [

 [[package]]
 name = "ctor"
-version = "1.0.5"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "378f0974ae2468eaf63aa036dbe9c926b0dc7ea64c156f2ea618bc2f75b934f0"
+checksum = "2d83cb7e7a873830708d6b02a78cd36a592c6fa14bf267b68725103b85c0d77f"
 dependencies = [
 "link-section",
 "linktime-proc-macro",
@@ -2902,6 +2899,37 @@ dependencies = [
 "uuid",
 ]

+[[package]]
+name = "defmt"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1"
+dependencies = [
+ "bitflags 1.3.2",
+ "defmt-macros",
+]
+
+[[package]]
+name = "defmt-macros"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8"
+dependencies = [
+ "defmt-parser",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
+[[package]]
+name = "defmt-parser"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e"
+dependencies = [
+ "thiserror 2.0.18",
+]
+
 [[package]]
 name = "der"
 version = "0.6.1"
@@ -3421,8 +3449,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"

 [[package]]
 name = "fsst"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arrow-array",
 "rand 0.9.5",
@@ -3779,14 +3807,22 @@ dependencies = [

 [[package]]
 name = "goosefs-sdk"
-version = "0.1.5"
+version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f"
+checksum = "4a9bc9414e3b2cb0bd08dfe0eb315b177e86b119c7fa5e16179c92fc7b184860"
 dependencies = [
+ "arc-swap",
 "async-trait",
 "bytes",
 "dashmap",
+ "futures",
 "hostname",
+ "io-uring",
+ "itoa",
+ "libc",
+ "lru 0.12.5",
+ "memmap2 0.9.10",
+ "moka",
 "prost",
 "prost-types",
 "rand 0.9.5",
@@ -3799,6 +3835,7 @@ dependencies = [
 "tonic-prost",
 "tracing",
 "uuid",
+ "xxhash-rust",
 ]

 [[package]]
@@ -3888,6 +3925,8 @@ version = "0.15.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
 dependencies = [
+ "allocator-api2",
+ "equivalent",
 "foldhash 0.1.5",
 ]

@@ -4599,10 +4638,12 @@ dependencies = [

 [[package]]
 name = "jiff"
-version = "0.2.24"
+version = "0.2.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d"
+checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc"
 dependencies = [
+ "defmt",
+ "jiff-core",
 "jiff-static",
 "jiff-tzdb-platform",
 "js-sys",
@@ -4611,15 +4652,25 @@ dependencies = [
 "portable-atomic-util",
 "serde_core",
 "wasm-bindgen",
- "windows-sys 0.61.2",
+ "windows-link",
+]
+
+[[package]]
+name = "jiff-core"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09"
+dependencies = [
+ "defmt",
 ]

 [[package]]
 name = "jiff-static"
-version = "0.2.24"
+version = "0.2.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7"
+checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204"
 dependencies = [
+ "jiff-core",
 "proc-macro2",
 "quote",
 "syn 2.0.117",
@@ -4731,24 +4782,6 @@ dependencies = [
 "zmij",
 ]

-[[package]]
-name = "jsonwebtoken"
-version = "10.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc"
-dependencies = [
- "aws-lc-rs",
- "base64 0.22.1",
- "getrandom 0.2.17",
- "js-sys",
- "pem",
- "serde",
- "serde_json",
- "signature 2.2.0",
- "simple_asn1",
- "zeroize",
-]
-
 [[package]]
 name = "kanaria"
 version = "0.2.0"
@@ -4777,8 +4810,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"

 [[package]]
 name = "lance"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -4852,8 +4885,8 @@ dependencies = [

 [[package]]
 name = "lance-arrow"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4875,7 +4908,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4889,7 +4922,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -4898,8 +4931,8 @@ dependencies = [

 [[package]]
 name = "lance-bitpacking"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arrayref",
 "crunchy",
@@ -4909,8 +4942,8 @@ dependencies = [

 [[package]]
 name = "lance-core"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4950,8 +4983,8 @@ dependencies = [

 [[package]]
 name = "lance-datafusion"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -4981,8 +5014,8 @@ dependencies = [

 [[package]]
 name = "lance-datagen"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -4999,8 +5032,8 @@ dependencies = [

 [[package]]
 name = "lance-derive"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -5009,8 +5042,8 @@ dependencies = [

 [[package]]
 name = "lance-encoding"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5045,8 +5078,8 @@ dependencies = [

 [[package]]
 name = "lance-file"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5077,8 +5110,8 @@ dependencies = [

 [[package]]
 name = "lance-index"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -5145,8 +5178,8 @@ dependencies = [

 [[package]]
 name = "lance-index-core"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5168,8 +5201,8 @@ dependencies = [

 [[package]]
 name = "lance-io"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5181,7 +5214,6 @@ dependencies = [
 "bytes",
 "chrono",
 "futures",
- "goosefs-sdk",
 "http 1.4.2",
 "io-uring",
 "lance-arrow",
@@ -5206,8 +5238,8 @@ dependencies = [

 [[package]]
 name = "lance-linalg"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5223,8 +5255,8 @@ dependencies = [

 [[package]]
 name = "lance-namespace"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arrow",
 "async-trait",
@@ -5236,8 +5268,8 @@ dependencies = [

 [[package]]
 name = "lance-namespace-impls"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arrow",
 "arrow-ipc",
@@ -5291,8 +5323,8 @@ dependencies = [

 [[package]]
 name = "lance-select"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5307,8 +5339,8 @@ dependencies = [

 [[package]]
 name = "lance-table"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5347,8 +5379,8 @@ dependencies = [

 [[package]]
 name = "lance-testing"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5361,8 +5393,8 @@ dependencies = [

 [[package]]
 name = "lance-tokenizer"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "10.1.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.2#94cca93d13b1c731afd6660d46ad2f8d4b4345c4"
 dependencies = [
 "icu_segmenter",
 "jieba-rs",
@@ -5682,15 +5714,15 @@ dependencies = [

 [[package]]
 name = "link-section"
-version = "0.16.1"
+version = "0.19.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c8600ca3dbe044f07955b443ff606c50f45295b863289bbe7d0844d50cf11e4"
+checksum = "5ee1a0d6e252afe82e7bc2db42fba60e02ddf3b1accaf8cb21d96e34ba61f3d4"

 [[package]]
 name = "linktime-proc-macro"
-version = "0.1.0"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a44cd706ff0d503ee32b2071166510ca27e281228de10cd3aa8d35ff94560f81"
+checksum = "348d0075b1fc163b26d72a7f75fc5141daf2fd1bdf128d873cbaf6785d495bdf"

 [[package]]
 name = "linux-raw-sys"
@@ -5738,6 +5770,15 @@ dependencies = [
 "tracing-subscriber",
 ]

+[[package]]
+name = "lru"
+version = "0.12.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38"
+dependencies = [
+ "hashbrown 0.15.5",
+]
+
 [[package]]
 name = "lru"
 version = "0.16.4"
@@ -6067,7 +6108,7 @@ checksum = "de33522036981030a75c231829566bc63414e08101a6f5ff4ac6cef19c8e0941"
 dependencies = [
 "bitflags 2.11.1",
 "chrono",
- "ctor 1.0.5",
+ "ctor 1.0.12",
 "futures",
 "napi-build",
 "napi-sys",
@@ -6091,7 +6132,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a49c513341a61a16a10af6efcce46b30d0822ba2d4fb197d24d33dfc199c78d5"
 dependencies = [
 "convert_case",
- "ctor 1.0.5",
+ "ctor 1.0.12",
 "napi-derive-backend",
 "proc-macro2",
 "quote",
@@ -6384,9 +6425,9 @@ dependencies = [

 [[package]]
 name = "object_store_opendal"
-version = "0.57.0"
+version = "0.58.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8"
+checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac"
 dependencies = [
 "async-trait",
 "bytes",
@@ -6447,12 +6488,13 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"

 [[package]]
 name = "opendal"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1"
+checksum = "4f20562cc7447fcc915fc5c23df305a412ea80a733c9f2fd9e2d267e2815be6d"
 dependencies = [
- "ctor 1.0.5",
+ "ctor 1.0.12",
 "opendal-core",
+ "opendal-http-transport-reqwest",
 "opendal-layer-concurrent-limit",
 "opendal-layer-logging",
 "opendal-layer-retry",
@@ -6469,24 +6511,22 @@ dependencies = [

 [[package]]
 name = "opendal-core"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309"
+checksum = "ec75551ff4cf3e57da98979f6a937aaa9ddb3915bf68cc17d03df733be6646ed"
 dependencies = [
 "anyhow",
- "base64 0.22.1",
+ "base64 0.23.0",
 "bytes",
 "futures",
 "http 1.4.2",
- "http-body 1.1.0",
 "jiff",
 "log",
 "md-5 0.11.0",
 "mea",
 "percent-encoding",
- "quick-xml 0.39.4",
+ "quick-xml 0.41.0",
 "reqsign-core",
- "reqwest 0.13.3",
 "serde",
 "serde_json",
 "tokio",
@@ -6496,10 +6536,24 @@ dependencies = [
 ]

 [[package]]
-name = "opendal-layer-concurrent-limit"
-version = "0.57.0"
+name = "opendal-http-transport-reqwest"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f"
+checksum = "ad4d4f19c3ce01126a30611f8e544eaa217104a278c889ac17c9374fe4f9e4ef"
+dependencies = [
+ "bytes",
+ "futures",
+ "http 1.4.2",
+ "http-body 1.1.0",
+ "opendal-core",
+ "reqwest 0.13.4",
+]
+
+[[package]]
+name = "opendal-layer-concurrent-limit"
+version = "0.58.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "249ac5b0aa5a7a6c3737342d10456067937f9c9a6f3f02544271f7908ab91081"
 dependencies = [
 "futures",
 "http 1.4.2",
@@ -6509,9 +6563,9 @@ dependencies = [

 [[package]]
 name = "opendal-layer-logging"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2"
+checksum = "5c75411ab00f77851ff086b686c1e9ca8175ac18c15afa2cb75b9036436cb06c"
 dependencies = [
 "log",
 "opendal-core",
@@ -6519,9 +6573,9 @@ dependencies = [

 [[package]]
 name = "opendal-layer-retry"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7"
+checksum = "80b7738bd5f233ad8da39af9b9316b9b7a4eaddd91e8e32a1e19b7030688121d"
 dependencies = [
 "backon",
 "log",
@@ -6530,9 +6584,9 @@ dependencies = [

 [[package]]
 name = "opendal-layer-timeout"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3"
+checksum = "a704141924500f3803c05ed871b53305d2a2f11cb5ef20160c3ee688a1857f66"
 dependencies = [
 "opendal-core",
 "tokio",
@@ -6540,17 +6594,17 @@ dependencies = [

 [[package]]
 name = "opendal-service-azblob"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a"
+checksum = "b3310fbbb48f111c6f590473c2cd15e1b7f8e384444b0d4e328f0464c864d767"
 dependencies = [
- "base64 0.22.1",
+ "base64 0.23.0",
 "bytes",
 "http 1.4.2",
 "log",
 "opendal-core",
 "opendal-service-azure-common",
- "quick-xml 0.39.4",
+ "quick-xml 0.41.0",
 "reqsign-azure-storage",
 "reqsign-core",
 "reqsign-file-read-tokio",
@@ -6561,17 +6615,18 @@ dependencies = [

 [[package]]
 name = "opendal-service-azdls"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361"
+checksum = "2e3c406729935fe214ce574d68681a1ff7e0b322548f14094912bdbfe50e5c53"
 dependencies = [
- "base64 0.22.1",
+ "base64 0.23.0",
 "bytes",
 "http 1.4.2",
 "log",
+ "mea",
 "opendal-core",
 "opendal-service-azure-common",
- "quick-xml 0.39.4",
+ "quick-xml 0.41.0",
 "reqsign-azure-storage",
 "reqsign-core",
 "reqsign-file-read-tokio",
@@ -6581,9 +6636,9 @@ dependencies = [

 [[package]]
 name = "opendal-service-azure-common"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051"
+checksum = "7348c88edf15af435b7be930077746b569fac5e738c1bf6a363b675e7317c9df"
 dependencies = [
 "http 1.4.2",
 "opendal-core",
@@ -6591,15 +6646,15 @@ dependencies = [

 [[package]]
 name = "opendal-service-cos"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901"
+checksum = "d533d4582105d269c8aebeee5f0e8bcf960f41b8aab6197df7012254d9f39bf0"
 dependencies = [
 "bytes",
 "http 1.4.2",
 "log",
 "opendal-core",
- "quick-xml 0.39.4",
+ "quick-xml 0.41.0",
 "reqsign-core",
 "reqsign-file-read-tokio",
 "reqsign-tencent-cos",
@@ -6608,9 +6663,9 @@ dependencies = [

 [[package]]
 name = "opendal-service-gcs"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47"
+checksum = "007f3fba63c21e516c956b891e96ff9892d8175662bfb781cdada9d3766a11e6"
 dependencies = [
 "async-trait",
 "bytes",
@@ -6618,7 +6673,7 @@ dependencies = [
 "log",
 "opendal-core",
 "percent-encoding",
- "quick-xml 0.39.4",
+ "quick-xml 0.41.0",
 "reqsign-core",
 "reqsign-file-read-tokio",
 "reqsign-google",
@@ -6629,9 +6684,9 @@ dependencies = [

 [[package]]
 name = "opendal-service-goosefs"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4"
+checksum = "60871e6386f04d831e6a5bdbc032af4a91aeba49963252d0ef456a2cf36a9b78"
 dependencies = [
 "bytes",
 "goosefs-sdk",
@@ -6643,9 +6698,9 @@ dependencies = [

 [[package]]
 name = "opendal-service-hf"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848"
+checksum = "b41fd41eb7ed03c5e66cefda61e8e117808ffd2908f2916737cb020a6beb02c7"
 dependencies = [
 "bytes",
 "hf-xet",
@@ -6653,22 +6708,21 @@ dependencies = [
 "log",
 "opendal-core",
 "percent-encoding",
- "reqwest 0.13.3",
 "serde",
 "serde_json",
 ]

 [[package]]
 name = "opendal-service-oss"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace"
+checksum = "cd528ec2d49c5ca69e674ffed7b3e0686fb9cfcfea0596870de381467fda4f1b"
 dependencies = [
 "bytes",
 "http 1.4.2",
 "log",
 "opendal-core",
- "quick-xml 0.39.4",
+ "quick-xml 0.41.0",
 "reqsign-aliyun-oss",
 "reqsign-core",
 "reqsign-file-read-tokio",
@@ -6677,18 +6731,18 @@ dependencies = [

 [[package]]
 name = "opendal-service-s3"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb"
+checksum = "58e80cdf192d7eff05feed747894d64f81905ac4eaf132edf7ea270abdd2d663"
 dependencies = [
- "base64 0.22.1",
+ "base64 0.23.0",
 "bytes",
- "crc32c",
+ "crc-fast",
 "http 1.4.2",
 "log",
 "md-5 0.11.0",
 "opendal-core",
- "quick-xml 0.39.4",
+ "quick-xml 0.41.0",
 "reqsign-aws-v4",
 "reqsign-core",
 "reqsign-file-read-tokio",
@@ -7799,6 +7853,16 @@ dependencies = [
 "memchr",
 ]

+[[package]]
+name = "quick-xml"
+version = "0.41.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
 [[package]]
 name = "quick_cache"
 version = "0.6.24"
@@ -8220,9 +8284,9 @@ dependencies = [

 [[package]]
 name = "reqsign-aliyun-oss"
-version = "3.0.0"
+version = "3.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57ac2757f3140aa2e213b554148ae0b52733e624fc6723f0cc6bb3d440176c95"
+checksum = "9c0f9f69a519dd6958c4b43606bb8e1278cdc76d611fc8fed4b796eee548dc0f"
 dependencies = [
 "anyhow",
 "form_urlencoded",
@@ -8237,37 +8301,37 @@ dependencies = [

 [[package]]
 name = "reqsign-aws-v4"
-version = "3.0.0"
+version = "3.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44eaca382e94505a49f1a4849658d153aebf79d9c1a58e5dd3b10361511e9f43"
+checksum = "cc883bc56889f3e4a419265c87facea222a921debc5c6f15c7fd8b68ec4b36b2"
 dependencies = [
 "anyhow",
 "bytes",
 "form_urlencoded",
+ "hex",
 "http 1.4.2",
 "log",
 "percent-encoding",
- "quick-xml 0.39.4",
+ "quick-xml 0.41.0",
 "reqsign-core",
 "rust-ini",
 "serde",
 "serde_json",
 "serde_urlencoded",
- "sha1 0.10.6",
+ "sha1 0.11.0",
 ]

 [[package]]
 name = "reqsign-azure-storage"
-version = "3.0.0"
+version = "3.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a321980405d596bd34aaf95c4722a3de4128a67fd19e74a81a83aa3fdf082e6"
+checksum = "a6ebd8524185ce9c64063e3095f83968acfa90922f00c601a4a0f3aca15b077e"
 dependencies = [
 "anyhow",
 "base64 0.22.1",
 "bytes",
 "form_urlencoded",
 "http 1.4.2",
- "jsonwebtoken",
 "log",
 "pem",
 "percent-encoding",
@@ -8275,36 +8339,38 @@ dependencies = [
 "rsa",
 "serde",
 "serde_json",
- "sha1 0.10.6",
+ "sha1 0.11.0",
 ]

 [[package]]
 name = "reqsign-core"
-version = "3.0.0"
+version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b10302cf0a7d7e7352ba211fc92c3c5bebf1286153e49cc5aa87348078a8e102"
+checksum = "7e38b44697c60a823705ccef85cb04d8e0527c9d16ed7c58bf1c6395bdd24ceb"
 dependencies = [
 "anyhow",
 "base64 0.22.1",
 "bytes",
- "form_urlencoded",
 "futures",
 "hex",
- "hmac 0.12.1",
+ "hmac 0.13.0",
 "http 1.4.2",
 "jiff",
 "log",
 "percent-encoding",
- "sha1 0.10.6",
- "sha2 0.10.9",
+ "rsa",
+ "serde",
+ "serde_json",
+ "sha1 0.11.0",
+ "sha2 0.11.0",
 "windows-sys 0.61.2",
 ]

 [[package]]
 name = "reqsign-file-read-tokio"
-version = "3.0.0"
+version = "3.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2d89295b3d17abea31851cc8de55d843d89c52132c864963c38d41920613dc5"
+checksum = "688ff0ae421b8d4b92b53fdafaf53df2de28f428a9962edcf21702990b26f74b"
 dependencies = [
 "anyhow",
 "reqsign-core",
@@ -8313,13 +8379,12 @@ dependencies = [

 [[package]]
 name = "reqsign-google"
-version = "3.0.0"
+version = "3.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35cc609b49c69e76ecaceb775a03f792d1ed3e7755ab3548d4534fd801e3242e"
+checksum = "a96da0b579b846d358090cb06b9e3c2ad1375529efbe3e0c45f96bd7bcf043ea"
 dependencies = [
 "form_urlencoded",
 "http 1.4.2",
- "jsonwebtoken",
 "log",
 "percent-encoding",
 "reqsign-aws-v4",
@@ -8327,15 +8392,14 @@ dependencies = [
 "rsa",
 "serde",
 "serde_json",
- "sha2 0.10.9",
 "tokio",
 ]

 [[package]]
 name = "reqsign-tencent-cos"
-version = "3.0.0"
+version = "3.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e128f19525861dbded59e1e7c17653a8ed63d573ca04aed708d552dbef5bb32a"
+checksum = "f6497dd9f6e3d1349b420521484099b284f95e8d3a65f088fccef42493a7b644"
 dependencies = [
 "anyhow",
 "http 1.4.2",
@@ -8394,9 +8458,9 @@ dependencies = [

 [[package]]
 name = "reqwest"
-version = "0.13.3"
+version = "0.13.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0"
+checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
 dependencies = [
 "base64 0.22.1",
 "bytes",
@@ -8457,7 +8521,7 @@ dependencies = [
 "anyhow",
 "async-trait",
 "http 1.4.2",
- "reqwest 0.13.3",
+ "reqwest 0.13.4",
 "thiserror 2.0.18",
 "tower-service",
 ]
@@ -9206,18 +9270,6 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"

-[[package]]
-name = "simple_asn1"
-version = "0.6.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d"
-dependencies = [
- "num-bigint",
- "num-traits",
- "thiserror 2.0.18",
- "time",
-]
-
 [[package]]
 name = "siphasher"
 version = "1.0.3"
@@ -11161,7 +11213,7 @@ dependencies = [
 "more-asserts",
 "rand 0.10.1",
 "redb",
- "reqwest 0.13.3",
+ "reqwest 0.13.4",
 "reqwest-middleware",
 "serde",
 "serde_json",
@@ -11274,7 +11326,7 @@ dependencies = [
 "oneshot",
 "pin-project",
 "rand 0.10.1",
- "reqwest 0.13.3",
+ "reqwest 0.13.4",
 "serde",
 "serde_json",
 "shellexpand",
@@ -11370,20 +11422,6 @@ name = "zeroize"
 version = "1.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
-dependencies = [
- "zeroize_derive",
-]
-
-[[package]]
-name = "zeroize_derive"
-version = "1.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.117",
-]

 [[package]]
 name = "zerotrie"
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"

 [workspace.dependencies]
-lance = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=10.1.0-beta.2", default-features = false, "tag" = "v10.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=10.1.0-beta.2", "tag" = "v10.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=10.1.0-beta.2", "tag" = "v10.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=10.1.0-beta.2", "tag" = "v10.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=10.1.0-beta.2", default-features = false, "tag" = "v10.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=10.1.0-beta.2", "tag" = "v10.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=10.1.0-beta.2", "tag" = "v10.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=10.1.0-beta.2", "tag" = "v10.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=10.1.0-beta.2", default-features = false, "tag" = "v10.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=10.1.0-beta.2", "tag" = "v10.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=10.1.0-beta.2", "tag" = "v10.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=10.1.0-beta.2", "tag" = "v10.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=10.1.0-beta.2", "tag" = "v10.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=10.1.0-beta.2", "tag" = "v10.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
 arrow = { version = "58.0.0", optional = false }
@@ -28,7 +28,7 @@
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>10.1.0-beta.1</lance-core.version>
+        <lance-core.version>10.1.0-beta.2</lance-core.version>
        <spotless.skip>false</spotless.skip>
        <spotless.version>2.30.0</spotless.version>
        <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
@@ -49,8 +49,8 @@ lance-namespace = { workspace = true }
 lance-namespace-impls = { workspace = true }
 metrics = { workspace = true, optional = true }
 metrics-util = { workspace = true, optional = true }
-# Pin the transitive GooseFS SDK until the 0.1.6 compile break is fixed upstream.
-goosefs-sdk = { version = "=0.1.5", optional = true }
+# Keep the direct dependency aligned with the version required by OpenDAL.
+goosefs-sdk = { version = "=0.1.8", optional = true }
 moka = { workspace = true }
 pin-project = { workspace = true }
 tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
@@ -8,12 +8,15 @@ use std::fs::create_dir_all;
 use std::path::Path;
 use std::{collections::HashMap, sync::Arc};

+use futures::TryStreamExt;
 use lance::dataset::refs::Ref;
 use lance::dataset::{ReadParams, WriteMode, builder::DatasetBuilder};
 use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore};
 use lance_datafusion::utils::StreamingWriteSource;
 use lance_encoding::version::LanceFileVersion;
-use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
+use lance_io::object_store::{
+    DirCursor, ReadDirOptions, StorageOptionsAccessor, StorageOptionsProvider,
+};
 use lance_table::io::commit::commit_handler_from_url;
 use object_store::local::LocalFileSystem;
 use snafu::ResultExt;
@@ -718,6 +721,54 @@ impl ListingDatabase {
        self.namespace_database.clone()
    }

+    /// List up to `limit` table names, resuming after the table named `start_after`.
+    ///
+    /// The cursor and the page size go into the object store's list request rather than
+    /// being applied to a full listing, so the cost of a page is set by the size of the
+    /// page and not by the size of the database. Stores with no paginated list API fall
+    /// back to a full listing, which is what this did for every store before.
+    ///
+    /// Names come back in the order the store lists the directories in, which is by key:
+    /// `foo-bar` precedes `foo`, because the `-` of `foo-bar.lance` sorts below the `.` of
+    /// `foo.lance`. Pagination has to follow the order the cursor is pushed down in, so
+    /// that is the order both listing methods report and the order `start_after` resumes
+    /// in. It matches sorting by name except between a name and one that extends it.
+    async fn list_table_dirs(
+        &self,
+        start_after: Option<&str>,
+        limit: Option<usize>,
+    ) -> Result<Vec<String>> {
+        let dir_suffix = format!(".{}", LANCE_EXTENSION);
+        let options = ReadDirOptions {
+            // An empty name means "from the start": that is how comparing names against it
+            // behaved, and how a client looping on a page token spells its first request.
+            // Built into a cursor it would instead sit after every name below `.lance`.
+            resume_from: start_after
+                .filter(|name| !name.is_empty())
+                .map(|name| DirCursor::after_directory(format!("{name}{dir_suffix}"))),
+            page_size: limit,
+        };
+        let mut entries = self
+            .object_store
+            .read_dir_stream(self.base_path.clone(), options);
+
+        let mut names = Vec::new();
+        while limit.is_none_or(|limit| names.len() < limit) {
+            let Some(entry) = entries.try_next().await? else {
+                break;
+            };
+            // A table is the directory `<name>.lance`; anything else under the database
+            // prefix belongs to something other than a table.
+            if !entry.is_dir() {
+                continue;
+            }
+            if let Some(name) = entry.name.strip_suffix(&dir_suffix) {
+                names.push(name.to_string());
+            }
+        }
+        Ok(names)
+    }
+
    async fn drop_tables(&self, names: Vec<String>) -> Result<()> {
        let object_store_params = ObjectStoreParams {
            storage_options_accessor: if self.storage_options.is_empty() {
@@ -959,80 +1010,37 @@ impl Database for ListingDatabase {
        if !request.namespace_path.is_empty() {
            return self.namespace_database().table_names(request).await;
        }
-        let mut f = self
-            .object_store
-            .read_dir(self.base_path.clone())
-            .await?
-            .iter()
-            .map(Path::new)
-            .filter(|path| {
-                let is_lance = path
-                    .extension()
-                    .and_then(|e| e.to_str())
-                    .map(|e| e == LANCE_EXTENSION);
-                is_lance.unwrap_or(false)
-            })
-            .filter_map(|p| p.file_stem().and_then(|s| s.to_str().map(String::from)))
-            .collect::<Vec<String>>();
-        f.sort();
-        if let Some(start_after) = request.start_after {
-            let index = f
-                .iter()
-                .position(|name| name.as_str() > start_after.as_str())
-                .unwrap_or(f.len());
-            f.drain(0..index);
-        }
-        if let Some(limit) = request.limit {
-            f.truncate(limit as usize);
-        }
-        Ok(f)
+        self.list_table_dirs(
+            request.start_after.as_deref(),
+            request.limit.map(|limit| limit as usize),
+        )
+        .await
    }

    async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
        if request.id.as_ref().map(|v| !v.is_empty()).unwrap_or(false) {
            return self.namespace_database().list_tables(request).await;
        }
-        let mut f = self
-            .object_store
-            .read_dir(self.base_path.clone())
-            .await?
-            .iter()
-            .map(Path::new)
-            .filter(|path| {
-                let is_lance = path
-                    .extension()
-                    .and_then(|e| e.to_str())
-                    .map(|e| e == LANCE_EXTENSION);
-                is_lance.unwrap_or(false)
-            })
-            .filter_map(|p| p.file_stem().and_then(|s| s.to_str().map(String::from)))
-            .collect::<Vec<String>>();
-        f.sort();
+        let limit = request.limit.map(|limit| limit as usize);
+        // Reading one past the page is how we learn whether another page follows, without
+        // a second request. The extra name is dropped before the response goes out.
+        let mut tables = self
+            .list_table_dirs(
+                request.page_token.as_deref(),
+                limit.map(|limit| limit.saturating_add(1)),
+            )
+            .await?;

-        // Handle pagination with page_token
-        if let Some(ref page_token) = request.page_token {
-            let index = f
-                .iter()
-                .position(|name| name.as_str() > page_token.as_str())
-                .unwrap_or(f.len());
-            f.drain(0..index);
-        }
-
-        // Determine if there's a next page
-        let next_page_token = if let Some(limit) = request.limit {
-            if f.len() > limit as usize {
-                let token = f[limit as usize].clone();
-                f.truncate(limit as usize);
-                Some(token)
-            } else {
-                None
+        let next_page_token = match limit {
+            Some(limit) if tables.len() > limit => {
+                tables.truncate(limit);
+                tables.last().cloned()
            }
-        } else {
-            None
+            _ => None,
        };

        Ok(ListTablesResponse {
-            tables: f,
+            tables,
            page_token: next_page_token,
        })
    }
@@ -1322,6 +1330,130 @@ mod tests {
        (tempdir, db)
    }

+    async fn create_tables(db: &ListingDatabase, names: &[&str]) {
+        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
+        for name in names {
+            db.create_table(CreateTableRequest {
+                name: name.to_string(),
+                namespace_path: vec![],
+                data: Box::new(RecordBatch::new_empty(schema.clone())) as Box<dyn Scannable>,
+                mode: CreateTableMode::Create,
+                write_options: Default::default(),
+                location: None,
+                namespace_client: None,
+            })
+            .await
+            .unwrap();
+        }
+    }
+
+    /// Paging with the returned token has to visit every table exactly once. The token is
+    /// the last name of the page, which is what `page_token` resumes after.
+    #[tokio::test]
+    async fn test_list_tables_pages_over_every_table_once() {
+        let (_tempdir, db) = setup_database().await;
+        create_tables(&db, &["a", "b", "c", "d", "e"]).await;
+
+        let mut seen = Vec::new();
+        let mut page_token = None;
+        loop {
+            let page = db
+                .list_tables(ListTablesRequest {
+                    limit: Some(2),
+                    page_token,
+                    ..Default::default()
+                })
+                .await
+                .unwrap();
+            seen.extend(page.tables);
+            match page.page_token {
+                Some(token) => page_token = Some(token),
+                None => break,
+            }
+        }
+
+        assert_eq!(seen, vec!["a", "b", "c", "d", "e"]);
+    }
+
+    /// The last page reports no token, so a caller paging by token knows to stop without
+    /// asking for an empty page.
+    #[tokio::test]
+    async fn test_list_tables_exhausted_page_has_no_token() {
+        let (_tempdir, db) = setup_database().await;
+        create_tables(&db, &["a", "b"]).await;
+
+        let page = db
+            .list_tables(ListTablesRequest {
+                limit: Some(2),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(page.tables, vec!["a", "b"]);
+        assert_eq!(page.page_token, None);
+    }
+
+    /// Listing follows the order the object store lists directories in, so a name that
+    /// extends another comes first: `-` sorts below the `.` of `.lance`. Pagination pushes
+    /// its cursor into the list request, so it cannot report a different order than the
+    /// one it resumes in.
+    #[tokio::test]
+    async fn test_listing_order_follows_the_store_not_the_table_name() {
+        let (_tempdir, db) = setup_database().await;
+        create_tables(&db, &["users", "users-archive", "users.old"]).await;
+
+        #[allow(deprecated)]
+        let names = db.table_names(TableNamesRequest::default()).await.unwrap();
+        assert_eq!(names, vec!["users-archive", "users", "users.old"]);
+
+        // Resuming after a name skips everything the store lists before it, which is what
+        // paging by the previous page's last name relies on.
+        #[allow(deprecated)]
+        let after = db
+            .table_names(TableNamesRequest {
+                start_after: Some("users-archive".to_string()),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        assert_eq!(after, vec!["users", "users.old"]);
+    }
+
+    /// An empty `start_after` means "from the start". A name that sorts below `.lance` is
+    /// what disappears if it is treated as a cursor instead.
+    #[tokio::test]
+    async fn test_empty_start_after_lists_from_the_start() {
+        let (_tempdir, db) = setup_database().await;
+        create_tables(&db, &["-dash", "alpha"]).await;
+
+        #[allow(deprecated)]
+        let names = db
+            .table_names(TableNamesRequest {
+                start_after: Some(String::new()),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(names, vec!["-dash", "alpha"]);
+    }
+
+    /// Only directories named `<name>.lance` are tables; loose files and other directories
+    /// under the database prefix are not.
+    #[tokio::test]
+    async fn test_listing_ignores_non_table_children() {
+        let (tempdir, db) = setup_database().await;
+        create_tables(&db, &["real"]).await;
+        std::fs::write(tempdir.path().join("loose.lance"), b"not a table").unwrap();
+        create_dir_all(tempdir.path().join("scratch")).unwrap();
+
+        #[allow(deprecated)]
+        let names = db.table_names(TableNamesRequest::default()).await.unwrap();
+
+        assert_eq!(names, vec!["real"]);
+    }
+
    #[tokio::test]
    async fn test_listing_database_root_ops_do_not_create_manifest() {
        let tempdir = tempdir().unwrap();
@@ -2939,7 +2939,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
 }

 #[derive(Serialize, Clone, Debug)]
-pub(crate) struct MergeInsertRequest {
+pub struct MergeInsertRequest {
    on: String,
    when_matched_update_all: bool,
    when_matched_update_all_filt: Option<String>,
@@ -33,7 +33,7 @@ use crate::table::{AddResult, MergeResult};
 /// same Arrow-IPC streaming body and error side-channel; only the target
 /// endpoint, query parameters, and parsed result type differ.
 #[derive(Debug, Clone)]
-pub(crate) enum WriteOp {
+pub enum WriteOp {
    /// `add`: stream to `/v1/table/{id}/insert/`, optionally overwriting.
    Insert { overwrite: bool },
    /// `merge_insert`: stream to `/v1/table/{id}/merge_insert/` with the merge
@@ -49,7 +49,7 @@ pub(crate) enum WriteOp {
 /// The parsed server response for a completed write, discriminated by the
 /// operation that produced it.
 #[derive(Debug, Clone)]
-pub(crate) enum WriteResult {
+pub enum WriteResult {
    Add(AddResult),
    Merge(MergeResult),
 }