chore: align preview lockfile versions

chore: integrate Sophon Lance overlay with 0.38.0-beta.14
Bump version: 0.38.0-beta.13 → 0.38.0-beta.14
2026-08-31 10:38:31 +00:00 · 2026-08-30 14:40:15 -07:00 · 2026-08-30 09:21:00 -07:00 · 2026-08-30 15:16:00 +00:00 · 2026-08-29 13:11:44 -07:00 · 2026-08-29 12:59:51 -07:00
27 changed files with 197 additions and 846 deletions
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.38.0"
+current_version = "0.38.0-beta.14"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
@@ -3455,9 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"

 [[package]]
 name = "fsst"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f727719438dfdb74f358a347c91ff81b6e7084a6421f34de3e473ce271f10caa"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arrow-array",
 "rand 0.9.5",
@@ -4816,9 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"

 [[package]]
 name = "lance"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be781f40c7a75f9eae2188a2f71174acb7a360dca97163db40b041d0828dea48"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -4890,9 +4888,8 @@ dependencies = [

 [[package]]
 name = "lance-arrow"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb97fd9875f3036d7c2561aa5b16eb87b80ccabaa4eeb5e6099b19cc662f1cd8"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4914,8 +4911,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "771f68b04b47f3addf781116f65061808de94b05e1e9411c23c18f32d14ebe79"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4929,20 +4925,17 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd47ec33c90bf29f688fd02118e37d3a5ad5c339caa3163f89e417dc0867001f"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arrow-array",
 "arrow-schema",
- "half",
 "lance-arrow-scalar",
 ]

 [[package]]
 name = "lance-bitpacking"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f45658c5b2dc9aada41b66ee44b83af3fa888b7385ae414bae951b12a9f1cd3"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arrayref",
 "crunchy",
@@ -4952,9 +4945,8 @@ dependencies = [

 [[package]]
 name = "lance-core"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27af3df3a7d08897efccd04461df31cedf0880c4b86a055ddce48e423d27f967"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4991,9 +4983,8 @@ dependencies = [

 [[package]]
 name = "lance-datafusion"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c377f837df5296e92f9fad724c83c1bef4e74d5af6e5a9312e9307e1dead8614"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5022,9 +5013,8 @@ dependencies = [

 [[package]]
 name = "lance-datagen"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "778e1a5065fa4bc184e36e32681f10f8f4680ad8cedc9377b4c088dce8c5b8da"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5041,9 +5031,8 @@ dependencies = [

 [[package]]
 name = "lance-derive"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13e5e95e0fd3d74f7938f4bee623041421b323b5c61f242c8622a1f48a202527"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -5052,9 +5041,8 @@ dependencies = [

 [[package]]
 name = "lance-encoding"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1625653c55c65f3426e281f6e29b54c603f38a40bd4cebd707bd4f3ea48be6c5"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5087,9 +5075,8 @@ dependencies = [

 [[package]]
 name = "lance-file"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7e13c9266b478fc98f36ee19347c4658f7a6613fed77778b1a455fe1b88552e"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5120,9 +5107,8 @@ dependencies = [

 [[package]]
 name = "lance-index"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0e0cb95f2c4f341c4dd04ac60f6a89ea26a6f75e09570225cbda4854c8b088e"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -5186,9 +5172,8 @@ dependencies = [

 [[package]]
 name = "lance-index-core"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79ccd371977c1f7168da259d66ad37154f23146f093d46136bc7f79559f00f2c"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5210,9 +5195,8 @@ dependencies = [

 [[package]]
 name = "lance-io"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "414d50997391b1ac83dc183c1612fdff88f58b959806078dc4c5e465154566de"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5252,9 +5236,8 @@ dependencies = [

 [[package]]
 name = "lance-linalg"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ff55b152ef23a56d7ba7e4d1b2c9cf0cc79aef6ee607c115597557ea4059f41"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5268,9 +5251,8 @@ dependencies = [

 [[package]]
 name = "lance-namespace"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09991c13ab282b731e323619613914e08da9cc82b312f904e58c128b23f2f0e3"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arrow",
 "async-trait",
@@ -5282,9 +5264,8 @@ dependencies = [

 [[package]]
 name = "lance-namespace-impls"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ec0bc005f6bb8f120774eb4a9ba02e10463d8338167a46cbb1391d46680a174"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arrow",
 "arrow-ipc",
@@ -5337,9 +5318,8 @@ dependencies = [

 [[package]]
 name = "lance-select"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8f676a2a1837cc85b77feb5326d3296827da964e40d67144f646563302a6ce9"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5353,9 +5333,8 @@ dependencies = [

 [[package]]
 name = "lance-table"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd33054347395048b1d842dfb85a13f7801392c2da5f39425db62f00a481744b"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5395,9 +5374,8 @@ dependencies = [

 [[package]]
 name = "lance-testing"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ecc9ad9ae24f045dfddd538a39e55a28fa7e1ca6ad9f23e20d4087eaf2bb66f7"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5410,9 +5388,8 @@ dependencies = [

 [[package]]
 name = "lance-tokenizer"
-version = "11.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3bfa6f0164c8b7056150f5682ce4d415a335b59b04c479873fda04b200117d27"
+version = "12.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
 dependencies = [
 "frostem",
 "icu_segmenter",
@@ -5425,7 +5402,7 @@ dependencies = [

 [[package]]
 name = "lancedb"
-version = "0.38.0"
+version = "0.38.0-beta.14"
 dependencies = [
 "ahash",
 "anyhow",
@@ -5513,7 +5490,7 @@ dependencies = [

 [[package]]
 name = "lancedb-nodejs"
-version = "0.38.0"
+version = "0.38.0-beta.14"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5538,7 +5515,7 @@ dependencies = [

 [[package]]
 name = "lancedb-python"
-version = "0.38.0"
+version = "0.38.0-beta.14"
 dependencies = [
 "arrow",
 "async-trait",
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"

 [workspace.dependencies]
-lance = { "version" = "=11.0.0", default-features = false }
-lance-core = "=11.0.0"
-lance-datagen = "=11.0.0"
-lance-file = "=11.0.0"
-lance-io = { "version" = "=11.0.0", default-features = false }
-lance-index = "=11.0.0"
-lance-linalg = "=11.0.0"
-lance-namespace = "=11.0.0"
-lance-namespace-impls = { "version" = "=11.0.0", default-features = false }
-lance-table = "=11.0.0"
-lance-testing = "=11.0.0"
-lance-datafusion = "=11.0.0"
-lance-encoding = "=11.0.0"
-lance-arrow = "=11.0.0"
+lance = { "version" = "=12.0.0-beta.5", default-features = false, "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=12.0.0-beta.5", default-features = false, "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=12.0.0-beta.5", default-features = false, "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
 lancedb = { path = "rust/lancedb", default-features = false }
 ahash = "0.8"
 # Note that this one does not include pyarrow
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-core</artifactId>
-    <version>0.38.0</version>
+    <version>0.38.0-beta.14</version>
 </dependency>
 ```

@@ -8,7 +8,7 @@
    <parent>
      <groupId>com.lancedb</groupId>
      <artifactId>lancedb-parent</artifactId>
-      <version>0.38.0-final.0</version>
+      <version>0.38.0-beta.14</version>
      <relativePath>../pom.xml</relativePath>
    </parent>

@@ -6,7 +6,7 @@

    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-parent</artifactId>
-    <version>0.38.0-final.0</version>
+    <version>0.38.0-beta.14</version>
    <packaging>pom</packaging>
    <name>${project.artifactId}</name>
    <description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>11.0.0</lance-core.version>
+        <lance-core.version>12.0.0-beta.5</lance-core.version>
        <spotless.skip>false</spotless.skip>
        <spotless.version>2.30.0</spotless.version>
        <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.38.0"
+version = "0.38.0-beta.14"
 publish = false
 license.workspace = true
 description.workspace = true
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.38.0",
+	"version": "0.38.0-beta.14",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.38.0",
+	"version": "0.38.0-beta.14",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.38.0",
+	"version": "0.38.0-beta.14",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.38.0",
+	"version": "0.38.0-beta.14",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.38.0",
+	"version": "0.38.0-beta.14",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",
@@ -1,6 +1,6 @@
 {
  "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.38.0",
+  "version": "0.38.0-beta.14",
  "os": [
    "win32"
  ],
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.38.0",
+	"version": "0.38.0-beta.14",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",
@@ -11,7 +11,7 @@
    "ann"
  ],
  "private": false,
-  "version": "0.38.0",
+  "version": "0.38.0-beta.14",
  "main": "dist/index.js",
  "exports": {
    ".": "./dist/index.js",
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.38.0"
+version = "0.38.0-beta.14"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"
@@ -49,8 +49,6 @@ from pydantic import (
    model_validator,
 )

-from .schema import is_blob_v2_field as _is_blob_v2_field
-
 _Int32 = conint(strict=True, ge=-(2**31), le=2**31 - 1)
 _UInt32 = conint(strict=True, ge=0, le=2**32 - 1)
 _UInt64 = conint(strict=True, ge=0, le=2**64 - 1)
@@ -520,7 +518,6 @@ class RefreshColumnResult(_RemoteValue):


 _FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$")
-_FUNCTION_BLOB_V2_TYPE = "blob_v2"


 _GRAMMAR_PRIMITIVES = (
@@ -537,7 +534,6 @@ _GRAMMAR_PRIMITIVES = (
    (pa.float32(), "float32"),
    (pa.float64(), "float64"),
    (pa.string(), "utf8"),
-    (pa.large_string(), "large_utf8"),
    (pa.binary(), "binary"),
    (pa.date32(), "date32"),
    (pa.date64(), "date64"),
@@ -585,90 +581,20 @@ def _validate_exact_arrow_field(field: pa.Field) -> None:
            "unsupported Arrow type for Function signature: field names "
            "must not be empty"
        )
-    if _is_blob_v2_field(field):
-        if not _has_supported_blob_v2_layout(field):
-            raise TypeError(
-                "unsupported Arrow type for Function signature: lance.blob.v2 "
-                f"requires a supported Blob storage layout, got {field}"
-            )
-    elif field.metadata:
+    if field.metadata:
        raise TypeError(
            "unsupported Arrow type for Function signature: field metadata "
            f"is not supported, got {field}"
        )


-def _has_supported_blob_v2_layout(field: pa.Field) -> bool:
-    data_type = field.type
-    if isinstance(data_type, pa.ExtensionType):
-        data_type = data_type.storage_type
-    if not pa.types.is_struct(data_type):
-        return False
-
-    fields = tuple(data_type)
-
-    def matches(spec, compare_nullable) -> bool:
-        return len(fields) == len(spec) and all(
-            actual.name == name
-            and actual.type == expected_type
-            and (not check_nullable or actual.nullable == nullable)
-            for actual, (name, expected_type, nullable), check_nullable in zip(
-                fields, spec, compare_nullable
-            )
-        )
-
-    logical_minimal = (
-        ("data", pa.large_binary(), True),
-        ("uri", pa.utf8(), True),
-    )
-    logical_full = logical_minimal + (
-        ("position", pa.uint64(), True),
-        ("size", pa.uint64(), True),
-    )
-    prepared = (
-        ("kind", pa.uint8(), True),
-        ("data", pa.large_binary(), True),
-        ("uri", pa.utf8(), True),
-        ("blob_id", pa.uint32(), True),
-        ("blob_size", pa.uint64(), True),
-        ("position", pa.uint64(), True),
-    )
-    descriptor = (
-        ("kind", pa.uint8(), False),
-        ("position", pa.uint64(), False),
-        ("size", pa.uint64(), False),
-        ("blob_id", pa.uint32(), False),
-        ("blob_uri", pa.utf8(), False),
-    )
-    return (
-        matches(logical_minimal, (True, True))
-        or matches(logical_full, (True, True, False, False))
-        or matches(prepared, (True,) * len(prepared))
-        or matches(descriptor, (False,) * len(descriptor))
-    )
-
-
-def _canonical_arrow_field(field: pa.Field) -> str:
-    _validate_exact_arrow_field(field)
-    if _is_blob_v2_field(field):
-        return _FUNCTION_BLOB_V2_TYPE
-    return _canonical_arrow_type(field.type)
-
-
 def _exact_arrow_field(field: pa.Field) -> dict[str, Any]:
    _validate_exact_arrow_field(field)
-    if _is_blob_v2_field(field):
-        raise TypeError(
-            "unsupported Arrow type for Function signature: nested Blob v2 "
-            "fields are not supported; declare Blob parameters or named result "
-            "fields directly"
-        )
-    value = {
+    return {
        "name": field.name,
        "nullable": field.nullable,
        "type": _exact_arrow_type(field.type),
    }
-    return value


 def _exact_arrow_type(data_type: pa.DataType) -> dict[str, Any]:
@@ -792,11 +718,7 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
        if output.metadata:
            raise TypeError("Function output schema metadata is not supported")
        fields = tuple(output)
-    elif (
-        isinstance(output, pa.Field)
-        and not _is_blob_v2_field(output)
-        and pa.types.is_struct(output.type)
-    ):
+    elif isinstance(output, pa.Field) and pa.types.is_struct(output.type):
        _validate_exact_arrow_field(output)
        if output.nullable:
            raise ValueError("Function output must be non-nullable")
@@ -818,7 +740,7 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
            raise ValueError("Function output must be non-nullable")
        return FunctionOutput(
            kind="scalar",
-            arrow_type=_canonical_arrow_field(field),
+            arrow_type=_canonical_arrow_type(field.type),
            nullable=False,
        )

@@ -836,7 +758,7 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
        fields=tuple(
            FunctionResultField(
                name=field.name,
-                arrow_type=_canonical_arrow_field(field),
+                arrow_type=_canonical_arrow_type(field.type),
                nullable=False,
            )
            for field in fields
@@ -870,7 +792,7 @@ def _infer_signature(
        inputs = tuple(
            FunctionParameter(
                name=field.name,
-                arrow_type=_canonical_arrow_field(field),
+                arrow_type=_canonical_arrow_type(field.type),
                nullable=field.nullable,
            )
            for field in input_schema
@@ -893,9 +815,7 @@ def _infer_signature(
        inputs.append(
            FunctionParameter(
                name=parameter.name,
-                arrow_type=_canonical_arrow_field(
-                    pa.field(parameter.name, data_type, nullable=nullable)
-                ),
+                arrow_type=_canonical_arrow_type(data_type),
                nullable=nullable,
            )
        )
@@ -19,13 +19,7 @@ import pyarrow as pa
 import pytest

 import lancedb
-from lancedb.functions import (
-    PythonRuntimeSpec,
-    UdfDefinition,
-    _canonical_arrow_type,
-    _GRAMMAR_PRIMITIVES,
-    udf,
-)
+from lancedb.functions import PythonRuntimeSpec, UdfDefinition, udf

 THRESHOLD = 20
 _CACHE = None
@@ -227,6 +221,8 @@ def test_udf_resolves_module_globals_before_builtins(tmp_path):


 def test_canonical_arrow_type_prefers_the_compact_grammar():
+    from lancedb.functions import _GRAMMAR_PRIMITIVES, _canonical_arrow_type
+
    golden = json.loads(
        (
            Path(__file__).parents[3]
@@ -247,6 +243,7 @@ def test_canonical_arrow_type_prefers_the_compact_grammar():
    for outside in [
        pa.timestamp("us"),
        pa.decimal128(10, 2),
+        pa.large_string(),
        pa.large_binary(),
        pa.binary(4),
        pa.duration("s"),
@@ -440,6 +437,8 @@ def test_udf_recursion_versus_a_rebound_module_name(tmp_path):


 def test_canonical_arrow_type_uses_exact_json_for_list_child_properties():
+    from lancedb.functions import _canonical_arrow_type
+
    nullable = pa.list_(pa.float32())
    assert json.loads(_canonical_arrow_type(nullable)) == {
        "type": "list",
@@ -529,7 +528,6 @@ def _arrow_type_from_golden(spec: dict) -> pa.DataType:
        "null": pa.null(),
        "bool": pa.bool_(),
        "utf8": pa.string(),
-        "large_utf8": pa.large_string(),
        "binary": pa.binary(),
        "float16": pa.float16(),
        "float32": pa.float32(),
@@ -546,6 +544,8 @@ def test_arrow_type_grammar_matches_the_shared_golden():
            / "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json"
        ).read_text()
    )
+    from lancedb.functions import _canonical_arrow_type
+
    emitted = {
        case["arrow_type"]: _canonical_arrow_type(_arrow_type_from_golden(case["json"]))
        for case in golden["valid"]
@@ -578,151 +578,6 @@ def test_explicit_arrow_schema_is_deterministic():
    assert signature.output.nullable is False


-def test_blob_fields_use_the_scalar_function_semantic_type():
-    @udf(
-        input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
-        output_schema=lancedb.blob("result", nullable=False),
-    )
-    def copy_blob(image):
-        return image
-
-    signature = copy_blob.registration_request.signature
-    assert signature.inputs[0].arrow_type == "blob_v2"
-    assert signature.output.kind == "scalar"
-    assert signature.output.arrow_type == "blob_v2"
-
-
-def test_named_struct_function_can_include_a_blob_result_field():
-    @udf(
-        input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
-        output_schema=pa.schema(
-            [
-                lancedb.blob("thumbnail", nullable=False),
-                pa.field("width", pa.int32(), nullable=False),
-            ]
-        ),
-    )
-    def inspect_blob(image):
-        return {"thumbnail": image, "width": 1}
-
-    output = inspect_blob.registration_request.signature.output
-    assert output.kind == "named_struct"
-    assert [(field.name, field.arrow_type) for field in output.fields] == [
-        ("thumbnail", "blob_v2"),
-        ("width", "int32"),
-    ]
-
-
-def test_metadata_marked_blob_field_uses_the_semantic_type():
-    extension = lancedb.blob("image", nullable=False).type
-    storage = (
-        extension.storage_type if isinstance(extension, pa.ExtensionType) else extension
-    )
-    metadata_blob = pa.field(
-        "image",
-        storage,
-        nullable=False,
-        metadata={"ARROW:extension:name": "lance.blob.v2"},
-    )
-
-    @udf(
-        input_schema=pa.schema([metadata_blob]),
-        output_schema=pa.field("size", pa.int64(), nullable=False),
-    )
-    def blob_size(image):
-        return len(image)
-
-    assert blob_size.registration_request.signature.inputs[0].arrow_type == "blob_v2"
-
-
-def test_blob_marker_rejects_invalid_storage_layout():
-    malformed = pa.field(
-        "image",
-        pa.int64(),
-        nullable=False,
-        metadata={"ARROW:extension:name": "lance.blob.v2"},
-    )
-
-    with pytest.raises(TypeError, match="requires a supported Blob storage layout"):
-
-        @udf(
-            input_schema=pa.schema([malformed]),
-            output_schema=pa.field("size", pa.int64(), nullable=False),
-        )
-        def blob_size(image):
-            return len(image)
-
-
-def test_nested_blob_signature_field_has_a_clear_error():
-    nested = pa.field(
-        "value",
-        pa.struct([lancedb.blob("image", nullable=False)]),
-        nullable=False,
-    )
-    with pytest.raises(TypeError, match="nested Blob v2 fields are not supported"):
-
-        @udf(
-            input_schema=pa.schema([nested]),
-            output_schema=pa.field("size", pa.int64(), nullable=False),
-        )
-        def blob_size(value):
-            return len(value["image"])
-
-
-def test_nested_non_blob_extension_is_not_silently_unwrapped():
-    class TestExtension(pa.ExtensionType):
-        def __init__(self):
-            super().__init__(pa.int64(), "test.function.extension")
-
-        def __arrow_ext_serialize__(self):
-            return b""
-
-        @classmethod
-        def __arrow_ext_deserialize__(cls, storage_type, serialized):
-            return cls()
-
-    nested = pa.field(
-        "value",
-        pa.struct([pa.field("extended", TestExtension(), nullable=False)]),
-        nullable=False,
-    )
-    with pytest.raises(TypeError, match="unsupported Arrow type"):
-
-        @udf(
-            input_schema=pa.schema([nested]),
-            output_schema=pa.field("result", pa.int64(), nullable=False),
-        )
-        def extension_value(value):
-            return value["extended"]
-
-
-def test_explicit_large_utf8_schemas_use_the_canonical_function_name():
-    input_schema = pa.schema([pa.field("text", pa.large_string(), nullable=True)])
-    output_schema = pa.field("result", pa.large_string(), nullable=False)
-
-    @udf(input_schema=input_schema, output_schema=output_schema)
-    def preserve(text):
-        return text
-
-    signature = preserve.registration_request.signature
-    assert signature.inputs[0].arrow_type == "large_utf8"
-    assert signature.inputs[0].nullable is True
-    assert signature.output.arrow_type == "large_utf8"
-    assert signature.output.nullable is False
-
-    nested = pa.struct([pa.field("text", pa.large_string(), nullable=True)])
-    assert json.loads(_canonical_arrow_type(nested)) == {
-        "type": "struct",
-        "fields": [
-            {
-                "name": "text",
-                "nullable": True,
-                "type": {"type": "large_utf8"},
-            }
-        ],
-    }
-
-
 def test_nested_struct_output_uses_canonical_exact_json():
    token = pa.struct(
        [
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.38.0"
+version = "0.38.0-beta.14"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
@@ -13,7 +13,7 @@ use lance::dataset::{ReadParams, WriteMode, builder::DatasetBuilder};
 use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore};
 use lance_datafusion::utils::StreamingWriteSource;
 use lance_file::version::LanceFileVersion;
-use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
+use lance_io::object_store::{ReadDirOptions, StorageOptionsAccessor, StorageOptionsProvider};
 use lance_table::io::commit::commit_handler_from_url;
 use object_store::local::LocalFileSystem;
 use snafu::ResultExt;
@@ -282,11 +282,14 @@ impl std::fmt::Display for ListingDatabase {

 const LANCE_EXTENSION: &str = "lance";

-/// The table a listed child directory holds, or `None` if it is not a table at all.
+/// The table a listed child of the database names, or `None` if the child is not a table.
 ///
 /// A table is the directory `<name>.lance`; a loose file or any other directory under the
 /// database prefix belongs to something else. `dir_suffix` is `.lance`, built once by the
 /// caller rather than per child.
+/// The table a listed child directory holds, or `None` if it is not a table at all.
+///
+/// Only directories are considered, so a loose object named like a table is not one.
 fn table_name(location: &object_store::path::Path, dir_suffix: &str) -> Option<String> {
    location
        .filename()?
@@ -294,75 +297,6 @@ fn table_name(location: &object_store::path::Path, dir_suffix: &str) -> Option<S
        .map(String::from)
        .filter(|name| !name.is_empty())
 }
-
-/// One page of the table directories under the database directory, in key order.
-struct DirPage {
-    /// The table directories the page holds, as the store lists them.
-    common_prefixes: Vec<object_store::path::Path>,
-    /// Resumes after this page, or `None` when the page reached the end of the level.
-    page_token: Option<String>,
-}
-
-/// Where a listed location sits inside the database directory — the space page tokens live
-/// in — or `None` if it is not a child of that directory at all. Matching both halves of the
-/// prefix drops a location that merely starts with the directory's name (`dbx/y` against
-/// `db/`) as well as the marker object some stores keep for the directory itself.
-fn relative_key<'a>(prefix: Option<&str>, location: &'a str) -> Option<&'a str> {
-    let relative = match prefix {
-        Some(prefix) => location.strip_prefix(prefix)?,
-        None => location,
-    };
-    (!relative.is_empty()).then_some(relative)
-}
-
-/// One page of the table directories under `base_path`, one directory level deep.
-///
-/// Lance 11 exposes no paginated directory listing, so the level is listed in full and paged
-/// locally: table directories go into key order (a directory's key keeps its trailing `/`,
-/// so a token is never a table name), the page is the smallest `limit` of them past
-/// `page_token`, and the token handed back is the key of the last directory the page took —
-/// so a page that took nothing ends the listing rather than resuming from a position no page
-/// ever reached. Only `<name>.lance/` directories enter the page: loose objects, other
-/// directories, and a bare `.lance/` never take a page slot or name a token, which keeps a
-/// page to exactly one listing of the level. Correct on every store, at the cost of that one
-/// full-level listing per page.
-async fn read_dir_page(
-    object_store: &ObjectStore,
-    base_path: &object_store::path::Path,
-    page_token: Option<String>,
-    limit: Option<usize>,
-) -> Result<DirPage> {
-    let listed = object_store.list_with_delimiter(Some(base_path)).await?;
-    let prefix = {
-        let base = base_path.as_ref();
-        (!base.is_empty()).then(|| format!("{base}/"))
-    };
-    let table_dir_suffix = format!(".{LANCE_EXTENSION}/");
-    let mut children: Vec<(String, object_store::path::Path)> = listed
-        .common_prefixes
-        .into_iter()
-        .filter_map(|location| {
-            let key = format!("{}/", relative_key(prefix.as_deref(), location.as_ref())?);
-            (key.len() > table_dir_suffix.len() && key.ends_with(&table_dir_suffix))
-                .then_some((key, location))
-        })
-        .collect();
-    children.sort_unstable_by(|(left, _), (right, _)| left.cmp(right));
-    if let Some(resume) = &page_token {
-        children.retain(|(key, _)| key > resume);
-    }
-    let total = children.len();
-    children.truncate(limit.unwrap_or(total).min(total));
-    let page_token = match children.last() {
-        Some((last, _)) if children.len() < total => Some(last.clone()),
-        _ => None,
-    };
-    Ok(DirPage {
-        common_prefixes: children.into_iter().map(|(_, location)| location).collect(),
-        page_token,
-    })
-}
-
 const ENGINE: &str = "engine";
 const MIRRORED_STORE: &str = "mirroredStore";

@@ -1048,7 +982,8 @@ impl Database for ListingDatabase {
        let mut tables = Vec::new();
        let mut page_token = request.page_token.filter(|token| !token.is_empty());

-        // A page of nothing: no table was handed over for a token to resume after.
+        // A page of nothing: the store rejects a limit of zero, and no table was handed over
+        // for a token to resume after.
        if limit == Some(0) {
            return Ok(ListTablesResponse {
                context: None,
@@ -1057,21 +992,35 @@ impl Database for ListingDatabase {
            });
        }

-        // The page holds only table directories, so one call — and the one full-level
-        // listing behind it — fills it.
-        let page = read_dir_page(
-            &self.object_store,
-            &self.base_path,
-            page_token.take(),
-            limit,
-        )
-        .await?;
-        page_token = page.page_token;
-        tables.extend(
-            page.common_prefixes
-                .iter()
-                .filter_map(|location| table_name(location, &dir_suffix)),
-        );
+        loop {
+            // Ask only for what the page still has room for, so a database holding more
+            // than one page costs one request per page rather than one per table.
+            let listing = self
+                .object_store
+                .read_dir_page(
+                    self.base_path.clone(),
+                    ReadDirOptions {
+                        page_token: page_token.take(),
+                        limit: limit.map(|limit| limit - tables.len()),
+                    },
+                )
+                .await?;
+            page_token = listing.page_token;
+            // Only child directories can be tables, and the store already separates them
+            // out, so the objects in the page are not looked at.
+            tables.extend(
+                listing
+                    .result
+                    .common_prefixes
+                    .iter()
+                    .filter_map(|location| table_name(location, &dir_suffix)),
+            );
+            // Children that are not tables leave the page short of the limit, so keep
+            // going until the page is full or the database runs out.
+            if page_token.is_none() || limit.is_none_or(|limit| tables.len() >= limit) {
+                break;
+            }
+        }

        Ok(ListTablesResponse {
            context: None,
@@ -1717,8 +1666,8 @@ mod tests {
    }

    /// Only directories named `<name>.lance` are tables; loose files and other directories
-    /// under the database prefix are not. They never take a page slot, so even a `limit`
-    /// smaller than the clutter ahead of the first table returns that table.
+    /// under the database prefix are not. A page spent on them is filled from the next one,
+    /// so a page holding only non-tables does not read as an empty database.
    #[tokio::test]
    async fn test_listing_ignores_non_table_children() {
        let (tempdir, db) = setup_database().await;
@@ -1737,37 +1686,6 @@ mod tests {
        assert_eq!(page.tables, vec!["real"]);
    }

-    /// The Lance 11 fallback pages locally over one full-level listing, so a bounded page
-    /// costs exactly one listing call — clutter ahead of the first table must not buy extra
-    /// round trips.
-    #[tokio::test]
-    async fn test_one_full_listing_per_public_page() {
-        use crate::io::object_store::io_tracking::IoStatsHolder;
-        use lance_io::object_store::WrappingObjectStore;
-
-        let (tempdir, mut db) = setup_database().await;
-        create_tables(&db, &["real"]).await;
-        std::fs::write(tempdir.path().join("aaa-loose.lance"), b"not a table").unwrap();
-        create_dir_all(tempdir.path().join("aaa-scratch")).unwrap();
-
-        let io_stats = IoStatsHolder::default();
-        let mut tracked_store = (*db.object_store).clone();
-        tracked_store.inner =
-            io_stats.wrap(&tracked_store.store_prefix, tracked_store.inner.clone());
-        db.object_store = Arc::new(tracked_store);
-
-        let page = db
-            .list_tables(ListTablesRequest {
-                limit: Some(1),
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-
-        assert_eq!(page.tables, vec!["real"]);
-        assert_eq!(io_stats.incremental_stats().read_iops, 1);
-    }
-
    #[tokio::test]
    async fn listing_ignores_empty_table_name() {
        let (tempdir, db) = setup_database().await;
@@ -15,9 +15,6 @@ use serde_json::Value;

 use crate::{Error, Result};

-/// Semantic Function type for a Blob v2 value.
-pub const FUNCTION_BLOB_V2_TYPE: &str = "blob_v2";
-
 fn invalid_json(error: impl std::fmt::Display) -> Error {
    Error::InvalidInput {
        message: format!("invalid remote Function JSON: {error}"),
@@ -10,7 +10,7 @@ use lance::io::WrappingObjectStore;
 use object_store::{
    CopyOptions, Error, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta,
    ObjectStore, ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result,
-    UploadPart, path::Path,
+    UploadPart, list::PaginatedListStore, path::Path,
 };

 use async_trait::async_trait;
@@ -187,6 +187,14 @@ impl WrappingObjectStore for MirroringObjectStoreWrapper {
            secondary: self.secondary.clone(),
        })
    }
+
+    fn wrap_paginated(
+        &self,
+        _store_prefix: &str,
+        original: Arc<dyn PaginatedListStore>,
+    ) -> Option<Arc<dyn PaginatedListStore>> {
+        Some(original)
+    }
 }

 // windows pathing can't be simply concatenated
@@ -12,7 +12,7 @@ use lance::io::WrappingObjectStore;
 use object_store::{
    CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
    PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result as OSResult,
-    UploadPart, path::Path,
+    UploadPart, list::PaginatedListStore, path::Path,
 };

 #[derive(Debug, Default)]
@@ -57,6 +57,14 @@ impl WrappingObjectStore for IoStatsHolder {
            stats: self.0.clone(),
        })
    }
+
+    fn wrap_paginated(
+        &self,
+        _store_prefix: &str,
+        original: Arc<dyn PaginatedListStore>,
+    ) -> Option<Arc<dyn PaginatedListStore>> {
+        Some(original)
+    }
 }

 impl IoTrackingStore {
@@ -4183,6 +4183,14 @@ mod tests {
                parent_list_calls: self.parent_list_calls.clone(),
            })
        }
+
+        fn wrap_paginated(
+            &self,
+            _store_prefix: &str,
+            _original: Arc<dyn object_store::list::PaginatedListStore>,
+        ) -> Option<Arc<dyn object_store::list::PaginatedListStore>> {
+            None
+        }
    }

    #[tokio::test]
@@ -4286,6 +4294,14 @@ mod tests {
            self.called.store(true, Ordering::Relaxed);
            original
        }
+
+        fn wrap_paginated(
+            &self,
+            _store_prefix: &str,
+            original: Arc<dyn object_store::list::PaginatedListStore>,
+        ) -> Option<Arc<dyn object_store::list::PaginatedListStore>> {
+            Some(original)
+        }
    }

    #[tokio::test]
@@ -29,16 +29,14 @@ use datafusion_common::{ScalarValue, tree_node::TreeNode};
 use datafusion_expr::Expr;
 use datafusion_physical_plan::PhysicalExpr;
 use lance::dataset::NewColumnTransform;
-use lance_arrow::{ARROW_EXT_NAME_KEY, BLOB_V2_EXT_NAME, FieldExt};
-use lance_core::datatypes::{
-    BLOB_V2_DESC_FIELD, BlobV2Layout, format_field_path_minimal, parse_field_path,
-};
+use lance_arrow::FieldExt;
+use lance_core::datatypes::{BLOB_V2_DESC_FIELD, format_field_path_minimal, parse_field_path};
 use lance_datafusion::planner::Planner;
 use lance_namespace::models::{JsonArrowDataType, JsonArrowField, JsonArrowSchema};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;

-use crate::function::{FUNCTION_BLOB_V2_TYPE, FunctionApplication, FunctionBinding};
+use crate::function::{FunctionApplication, FunctionBinding};
 use crate::utils::resolve_arrow_field_path;
 use crate::{Error, Result};

@@ -583,23 +581,6 @@ fn resolve_field_path<'a>(schema: &'a ArrowSchema, path: &str) -> Result<Resolve
 }

 fn canonical_input_arrow_type(field: &JsonArrowField) -> Result<String> {
-    let is_blob_v2 = field
-        .metadata
-        .as_ref()
-        .and_then(|metadata| metadata.get(ARROW_EXT_NAME_KEY))
-        .map(String::as_str)
-        == Some(BLOB_V2_EXT_NAME);
-    if is_blob_v2 {
-        let arrow_field = lance_namespace::schema::convert_json_arrow_field(field)
-            .map_err(|e| invalid_function(format!("invalid Function input field: {e}")))?;
-        if !has_supported_blob_v2_layout(&arrow_field) {
-            return Err(invalid_function(format!(
-                "Function input '{}' has an invalid Blob v2 storage layout",
-                arrow_field.name()
-            )));
-        }
-        return Ok(FUNCTION_BLOB_V2_TYPE.to_string());
-    }
    if field.r#type.fields.is_none() && field.r#type.length.is_none() {
        Ok(field.r#type.r#type.clone())
    } else {
@@ -609,14 +590,6 @@ fn canonical_input_arrow_type(field: &JsonArrowField) -> Result<String> {
    }
 }

-fn has_supported_blob_v2_layout(field: &ArrowField) -> bool {
-    field.is_blob_v2()
-        && matches!(
-            field.data_type(),
-            DataType::Struct(fields) if BlobV2Layout::classify(fields).is_some()
-        )
-}
-
 /// `fixed_size_list<item, size>` -> (`item`, `size`); the comma must sit outside
 /// any nested `<...>`.
 fn split_fixed_size_list(raw: &str) -> Option<(&str, i32)> {
@@ -696,76 +669,6 @@ fn parse_output_arrow_type(raw: &str) -> Result<JsonArrowDataType> {
    Ok(data_type)
 }

-fn function_output_field(name: &str, nullable: bool, raw: &str) -> Result<JsonArrowField> {
-    if raw == FUNCTION_BLOB_V2_TYPE {
-        return lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![
-            crate::blob(name, nullable),
-        ]))
-        .map_err(|e| invalid_function(format!("could not encode Blob v2 output field: {e}")))?
-        .fields
-        .into_iter()
-        .next()
-        .ok_or_else(|| invalid_function("Blob v2 output field is missing"));
-    }
-    Ok(JsonArrowField::new(
-        name.to_string(),
-        nullable,
-        parse_output_arrow_type(raw)?,
-    ))
-}
-
-fn function_output_field_matches(expected: &ArrowField, actual: &ArrowField) -> bool {
-    expected.name() == actual.name()
-        && expected.is_nullable() == actual.is_nullable()
-        && if expected.is_blob_v2() {
-            has_supported_blob_v2_layout(expected) && has_supported_blob_v2_layout(actual)
-        } else {
-            function_output_type_matches(expected.data_type(), actual.data_type())
-        }
-}
-
-fn function_output_type_matches(expected: &DataType, actual: &DataType) -> bool {
-    if expected == actual {
-        return true;
-    }
-    match (expected, actual) {
-        (DataType::Struct(expected), DataType::Struct(actual)) => {
-            expected.len() == actual.len()
-                && expected
-                    .iter()
-                    .zip(actual)
-                    .all(|(expected, actual)| function_output_field_matches(expected, actual))
-        }
-        (DataType::List(expected), DataType::List(actual))
-        | (DataType::LargeList(expected), DataType::LargeList(actual)) => {
-            function_output_field_matches(expected, actual)
-        }
-        (
-            DataType::FixedSizeList(expected, expected_size),
-            DataType::FixedSizeList(actual, actual_size),
-        ) => expected_size == actual_size && function_output_field_matches(expected, actual),
-        (DataType::Map(expected, expected_sorted), DataType::Map(actual, actual_sorted)) => {
-            expected_sorted == actual_sorted && function_output_field_matches(expected, actual)
-        }
-        _ => false,
-    }
-}
-
-fn function_output_type_has_blob(data_type: &DataType) -> bool {
-    match data_type {
-        DataType::Struct(fields) => fields
-            .iter()
-            .any(|field| field.is_blob_v2() || function_output_type_has_blob(field.data_type())),
-        DataType::List(field)
-        | DataType::LargeList(field)
-        | DataType::FixedSizeList(field, _)
-        | DataType::Map(field, _) => {
-            field.is_blob_v2() || function_output_type_has_blob(field.data_type())
-        }
-        _ => false,
-    }
-}
-
 fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding) -> Result<()> {
    let mut input_fields = Vec::with_capacity(binding.inputs().len());
    for input in binding.inputs() {
@@ -846,18 +749,10 @@ fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding
                binding.binding_id()
            )));
        }
-        let (type_matches, has_semantic_blob) = if output.arrow_type == FUNCTION_BLOB_V2_TYPE {
-            (has_supported_blob_v2_layout(field), true)
-        } else {
-            let expected_type = parse_output_arrow_type(&output.arrow_type)?;
-            let expected_type = lance_namespace::schema::convert_json_arrow_type(&expected_type)
-                .map_err(|e| invalid_function(format!("invalid Function output type: {e}")))?;
-            (
-                function_output_type_matches(&expected_type, field.data_type()),
-                function_output_type_has_blob(&expected_type),
-            )
-        };
-        if !type_matches {
+        let expected_type = parse_output_arrow_type(&output.arrow_type)?;
+        let expected_type = lance_namespace::schema::convert_json_arrow_type(&expected_type)
+            .map_err(|e| invalid_function(format!("invalid Function output type: {e}")))?;
+        if field.data_type() != &expected_type {
            return Err(invalid_function(format!(
                "Function output '{}' type no longer matches binding '{}'",
                output.output_name,
@@ -886,21 +781,15 @@ fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding
                binding.binding_id()
            )));
        }
-        if has_semantic_blob {
-            output_fields.push(function_output_field(
-                field.name(),
-                true,
-                &output.arrow_type,
-            )?);
-        } else {
-            let json = lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![
-                ArrowField::new(field.name().clone(), field.data_type().clone(), true),
-            ]))
-            .map_err(|e| invalid_function(format!("invalid Function output schema: {e}")))?;
-            output_fields.push(json.fields.into_iter().next().unwrap());
-        }
+        output_fields.push(ArrowField::new(
+            field.name().clone(),
+            field.data_type().clone(),
+            true,
+        ));
    }
-    let output_schema = JsonArrowSchema::new(output_fields);
+    let output_schema =
+        lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(output_fields))
+            .map_err(|e| invalid_function(format!("invalid Function output schema: {e}")))?;
    let output_schema = serde_json::to_value(output_schema).map_err(|e| {
        invalid_function(format!(
            "could not encode exact Function output schema: {e}"
@@ -1029,15 +918,16 @@ pub(crate) fn plan_function_application(
                    "Function logical outputs must be non-nullable during NULL assignment",
                ));
            }
-            let arrow_type = output.arrow_type.as_deref().ok_or_else(|| {
-                invalid_function("scalar Function output is missing its Arrow type")
-            })?;
+            let data_type =
+                parse_output_arrow_type(output.arrow_type.as_deref().ok_or_else(|| {
+                    invalid_function("scalar Function output is missing its Arrow type")
+                })?)?;
            outputs.push(FunctionOutputTarget {
                result_field: WHOLE_RESULT_FIELD.to_string(),
                output_name: name.to_string(),
                output_ordinal: 0,
            });
-            output_fields.push(function_output_field(name, true, arrow_type)?);
+            output_fields.push(JsonArrowField::new(name.to_string(), true, data_type));
        }
        "named_struct" => {
            if output.fields.is_empty() {
@@ -1081,7 +971,13 @@ pub(crate) fn plan_function_application(
                let fields = output
                    .fields
                    .iter()
-                    .map(|field| function_output_field(&field.name, false, &field.arrow_type))
+                    .map(|field| {
+                        Ok(JsonArrowField::new(
+                            field.name.clone(),
+                            false,
+                            parse_output_arrow_type(&field.arrow_type)?,
+                        ))
+                    })
                    .collect::<Result<Vec<_>>>()?;
                let mut data_type = JsonArrowDataType::new("struct".to_string());
                data_type.fields = Some(fields);
@@ -1108,7 +1004,11 @@ pub(crate) fn plan_function_application(
                        output_name: name.clone(),
                        output_ordinal: ordinal as u32,
                    });
-                    output_fields.push(function_output_field(name, true, &field.arrow_type)?);
+                    output_fields.push(JsonArrowField::new(
+                        name.clone(),
+                        true,
+                        parse_output_arrow_type(&field.arrow_type)?,
+                    ));
                }
            }
        }
@@ -1745,7 +1645,7 @@ mod tests {
    }

    use arrow_array::record_batch;
-    use arrow_schema::{DataType, TimeUnit};
+    use arrow_schema::DataType;
    use futures::TryStreamExt;
    use lance::dataset::ColumnAlteration;

@@ -2706,73 +2606,6 @@ mod tests {
        .unwrap()
    }

-    fn blob_application(output: &str) -> FunctionApplication {
-        FunctionApplication::from_json(&format!(
-            r#"{{
-                "function":{{"name":"blob_features","version":"fv_blob"}},
-                "inputs":[
-                    {{"parameter":"image","kind":"column","value":{{"path":"image"}}}}
-                ],
-                "output":{output}
-            }}"#
-        ))
-        .unwrap()
-    }
-
-    fn binding_from_plan(plan: &FunctionDeclarationPlan) -> FunctionBinding {
-        let inputs = plan
-            .input_bindings
-            .iter()
-            .enumerate()
-            .map(|(index, input)| {
-                serde_json::json!({
-                    "parameter": input.parameter,
-                    "field_id": index,
-                    "field_path": input.field_path,
-                    "arrow_type": input.arrow_type,
-                    "nullable": input.nullable,
-                })
-            })
-            .collect::<Vec<_>>();
-        let outputs = plan
-            .outputs
-            .iter()
-            .zip(&plan.output_schema.fields)
-            .enumerate()
-            .map(|(index, (output, field))| {
-                serde_json::json!({
-                    "result_field": output.result_field,
-                    "output_name": output.output_name,
-                    "output_field_id": 100 + index,
-                    "output_ordinal": output.output_ordinal,
-                    "arrow_type": canonical_input_arrow_type(field).unwrap(),
-                    "nullable": false,
-                })
-            })
-            .collect::<Vec<_>>();
-        FunctionBinding::from_json(
-            &serde_json::json!({
-                "binding_id": "fb_blob",
-                "function": plan.application.function(),
-                "inputs": inputs,
-                "outputs": outputs,
-                "input_schema": plan.input_schema,
-                "output_schema": plan.output_schema,
-            })
-            .to_string(),
-        )
-        .unwrap()
-    }
-
-    fn full_blob_field(name: &str, nullable: bool) -> ArrowField {
-        ArrowField::new(
-            name,
-            DataType::Struct(lance_core::datatypes::BLOB_V2_LOGICAL_FIELDS.clone()),
-            nullable,
-        )
-        .with_metadata(crate::blob(name, nullable).metadata().clone())
-    }
-
    fn function_binding_schema(title_nullable: bool, body_nullable: bool) -> ArrowSchema {
        ArrowSchema::new(vec![
            ArrowField::new("title", DataType::Utf8, title_nullable),
@@ -3046,151 +2879,6 @@ mod tests {
        );
    }

-    #[test]
-    fn test_blob_function_plans_semantic_input_and_scalar_output() {
-        let schema = ArrowSchema::new(vec![crate::blob("image", false)]);
-        let application =
-            blob_application(r#"{"kind":"scalar","arrow_type":"blob_v2","nullable":false}"#);
-        let plan = plan_function_application(&schema, &application, Some("thumbnail")).unwrap();
-
-        assert_eq!(plan.input_bindings[0].arrow_type, FUNCTION_BLOB_V2_TYPE);
-        let input_schema =
-            lance_namespace::schema::convert_json_arrow_schema(&plan.input_schema).unwrap();
-        assert!(input_schema.field(0).is_blob_v2());
-        let output_schema =
-            lance_namespace::schema::convert_json_arrow_schema(&plan.output_schema).unwrap();
-        assert!(output_schema.field(0).is_blob_v2());
-    }
-
-    #[test]
-    fn test_blob_scalar_binding_accepts_full_logical_layout() {
-        let input = crate::blob("image", false);
-        let application =
-            blob_application(r#"{"kind":"scalar","arrow_type":"blob_v2","nullable":false}"#);
-        let plan = plan_function_application(
-            &ArrowSchema::new(vec![input.clone()]),
-            &application,
-            Some("thumbnail"),
-        )
-        .unwrap();
-        let binding = binding_from_plan(&plan);
-        let mut metadata = full_blob_field("thumbnail", true).metadata().clone();
-        metadata.extend(function_computed_column_metadata(
-            binding.binding_id(),
-            0,
-            &["image".into()],
-        ));
-        let output = full_blob_field("thumbnail", true).with_metadata(metadata);
-
-        ensure_binding_matches_schema(&ArrowSchema::new(vec![input, output]), &binding).unwrap();
-    }
-
-    #[test]
-    fn test_blob_binding_rejects_marker_on_invalid_storage_layout() {
-        let input = crate::blob("image", false);
-        let application =
-            blob_application(r#"{"kind":"scalar","arrow_type":"blob_v2","nullable":false}"#);
-        let plan = plan_function_application(
-            &ArrowSchema::new(vec![input.clone()]),
-            &application,
-            Some("thumbnail"),
-        )
-        .unwrap();
-        let binding = binding_from_plan(&plan);
-        let malformed = ArrowField::new("thumbnail", DataType::Int64, true)
-            .with_metadata(crate::blob("thumbnail", true).metadata().clone());
-
-        ensure_binding_matches_schema(&ArrowSchema::new(vec![input, malformed]), &binding)
-            .unwrap_err();
-    }
-
-    #[test]
-    fn test_blob_input_rejects_marker_on_invalid_storage_layout() {
-        let malformed = ArrowField::new("image", DataType::Int64, false)
-            .with_metadata(crate::blob("image", false).metadata().clone());
-        let application =
-            blob_application(r#"{"kind":"scalar","arrow_type":"blob_v2","nullable":false}"#);
-
-        plan_function_application(
-            &ArrowSchema::new(vec![malformed]),
-            &application,
-            Some("thumbnail"),
-        )
-        .unwrap_err();
-    }
-
-    #[test]
-    fn test_non_blob_input_does_not_require_json_round_trip() {
-        let json = lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![
-            ArrowField::new("event_time", DataType::Time64(TimeUnit::Microsecond), false),
-        ]))
-        .unwrap();
-
-        assert_eq!(
-            canonical_input_arrow_type(&json.fields[0]).unwrap(),
-            "time64"
-        );
-    }
-
-    #[test]
-    fn test_blob_named_struct_plans_expanded_and_whole_outputs() {
-        let schema = ArrowSchema::new(vec![crate::blob("image", false)]);
-        let application = blob_application(
-            r#"{"kind":"named_struct","fields":[
-                {"name":"thumbnail","arrow_type":"blob_v2","nullable":false},
-                {"name":"width","arrow_type":"int32","nullable":false}
-            ]}"#,
-        );
-
-        let expanded = plan_function_application(&schema, &application, None).unwrap();
-        let expanded_schema =
-            lance_namespace::schema::convert_json_arrow_schema(&expanded.output_schema).unwrap();
-        assert!(expanded_schema.field(0).is_blob_v2());
-        assert_eq!(expanded_schema.field(1).data_type(), &DataType::Int32);
-
-        let whole = plan_function_application(&schema, &application, Some("analysis")).unwrap();
-        let whole_schema =
-            lance_namespace::schema::convert_json_arrow_schema(&whole.output_schema).unwrap();
-        let DataType::Struct(fields) = whole_schema.field(0).data_type() else {
-            panic!("whole Function output should be a struct");
-        };
-        assert!(fields[0].is_blob_v2());
-        assert_eq!(fields[1].data_type(), &DataType::Int32);
-    }
-
-    #[test]
-    fn test_blob_whole_struct_binding_accepts_full_logical_layout() {
-        let input = crate::blob("image", false);
-        let application = blob_application(
-            r#"{"kind":"named_struct","fields":[
-                {"name":"thumbnail","arrow_type":"blob_v2","nullable":false},
-                {"name":"width","arrow_type":"int32","nullable":false}
-            ]}"#,
-        );
-        let plan = plan_function_application(
-            &ArrowSchema::new(vec![input.clone()]),
-            &application,
-            Some("analysis"),
-        )
-        .unwrap();
-        let binding = binding_from_plan(&plan);
-
-        let output = ArrowField::new(
-            "analysis",
-            DataType::Struct(Fields::from(vec![
-                full_blob_field("thumbnail", false),
-                ArrowField::new("width", DataType::Int32, false),
-            ])),
-            true,
-        )
-        .with_metadata(function_computed_column_metadata(
-            binding.binding_id(),
-            0,
-            &["image".into()],
-        ));
-        ensure_binding_matches_schema(&ArrowSchema::new(vec![input, output]), &binding).unwrap();
-    }
-
    #[test]
    fn test_function_mapping_and_sibling_collisions_fail_before_request() {
        let unknown = named_struct_application(r#"{"missing":"renamed"}"#);
@@ -300,7 +300,7 @@ async fn build_read_context(
    for shard_id in shard_ids {
        let manifest_store =
            ShardManifestStore::new(store.clone(), &base_path, shard_id, scan_batch_size);
-        if let Some(manifest) = manifest_store.read_latest().await? {
+        if let Some(manifest) = manifest_store.latest().await? {
            snapshots.push(snapshot_from_manifest(shard_id, &manifest, &exclude));
        }
    }
@@ -78,12 +78,6 @@
        "type": "utf8"
      }
    },
-    {
-      "arrow_type": "large_utf8",
-      "json": {
-        "type": "large_utf8"
-      }
-    },
    {
      "arrow_type": "binary",
      "json": {
@@ -177,21 +171,6 @@
        ]
      }
    },
-    {
-      "arrow_type": "list<large_utf8>",
-      "json": {
-        "type": "list",
-        "fields": [
-          {
-            "name": "item",
-            "nullable": false,
-            "type": {
-              "type": "large_utf8"
-            }
-          }
-        ]
-      }
-    },
    {
      "arrow_type": "large_list<utf8>",
      "json": {
@@ -207,21 +186,6 @@
        ]
      }
    },
-    {
-      "arrow_type": "large_list<large_utf8>",
-      "json": {
-        "type": "large_list",
-        "fields": [
-          {
-            "name": "item",
-            "nullable": false,
-            "type": {
-              "type": "large_utf8"
-            }
-          }
-        ]
-      }
-    },
    {
      "arrow_type": "fixed_size_list<float32, 384>",
      "json": {
@@ -366,4 +330,4 @@
    "timestamp[us]",
    "struct<a: int32>"
  ]
-}
+}
Author	SHA1	Message	Date
Jack Ye	77f0027d35	chore: align preview lockfile versions	2026-08-30 14:40:15 -07:00
Jack Ye	582a260c2d	chore: integrate Sophon Lance overlay with 0.38.0-beta.14	2026-08-30 09:21:00 -07:00
Lance Release	cad46fc683	Bump version: 0.38.0-beta.13 → 0.38.0-beta.14	2026-08-30 15:16:00 +00:00
Jack Ye	a3c6926943	test: query Lance overlay support directly	2026-08-29 13:11:44 -07:00
Jack Ye	47c10304b3	test: respect unstable overlay feature gate	2026-08-29 12:59:51 -07:00
lancedb automation	5f80dcb335	chore: update lance dependency to v12.0.0-beta.5	2026-08-29 19:09:18 +00:00