Bump version: 0.38.0-beta.7 → 0.38.0-beta.8

fix: package ordinary @udf bodies and emit only the V1 type grammar (#4044 )
Registering a real (embedding) Function failed on the client for three reasons: - `_package_source` treated `inspect.getclosurevars().unbound` as "unresolved globals"; CPython puts attribute names there, so any body with `np.linalg.norm(...)` or `body.split()` was rejected. Module-scope references now come from Python's own scope analysis (`symtable`) over the function source, recursively, and each is resolved the way the interpreter would: the function's globals first (a module global may shadow a builtin), then builtins. Free variables of nested scopes stay lexical; postponed annotations are not runtime loads. A genuinely missing global still fails. - `_canonical_arrow_type` emitted spellings the server's frozen grammar rejects (`fixed_size_list<T>[n]`, `timestamp[us]`, `struct<...>`, zero-sized lists). It now emits exactly the grammar, with the server's `fixed_size_list<item, size>` form, and the Rust declaration planner parses that form too. A shared golden (`tests/fixtures/first_class_functions/v1/arrow_types.json`) enumerates every grammar type, nested forms and rejected spellings; the Python emitter and Rust parser are tested against it, and the same file is under test in sophon. Packaging tests execute the shipped artifact in a fresh namespace. Contract changes (hence `breaking-change`): - `@udf` now rejects namespace acquisition structurally (`globals()`/`eval`/... by name, plus `import sys`/`builtins`/`importlib`/`inspect` inside the body), requires the function's captured `__builtins__` to be the standard mapping itself (identity, so neither lookups nor implicit hooks such as `__import__` can differ), rejects module globals that are namespace-bearing modules (`builtins`, `sys`, ...), and treats the function's own name as recursion only when the module binds it to the function or to the exact `UdfDefinition` the decorator produced; it resolves module globals through the function's real namespace (a module global may shadow a builtin) and ships importable classes/functions as imports. - List outputs must declare a non-nullable, metadata-free child named `item` (`pa.list_(pa.field("item", t, nullable=False))`); that is what the grammar means, and pyarrow's default nullable child was being silently collapsed into it. Contract, stated in the `udf` docstring: the artifact is a snapshot of the function source plus exactly the module names it references. Reaching the module namespace by another route is rejected where a static packager can see it and is otherwise unsupported; there is no dynamic-access detection beyond that.
2026-09-03 20:18:54 +00:00 · 2026-08-25 06:26:01 +00:00 · 2026-08-25 14:15:07 +08:00 · 2026-08-24 17:16:42 -07:00 · 2026-08-24 16:41:05 -07:00 · 2026-08-24 16:25:14 -07:00
69 changed files with 5819 additions and 801 deletions
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.38.0-beta.3"
+current_version = "0.38.0-beta.8"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
@@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"

 [[package]]
 name = "fsst"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arrow-array",
 "rand 0.9.5",
@@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"

 [[package]]
 name = "lance"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -4888,8 +4888,8 @@ dependencies = [

 [[package]]
 name = "lance-arrow"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4911,7 +4911,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4925,7 +4925,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -4934,8 +4934,8 @@ dependencies = [

 [[package]]
 name = "lance-bitpacking"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arrayref",
 "crunchy",
@@ -4945,8 +4945,8 @@ dependencies = [

 [[package]]
 name = "lance-core"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4983,8 +4983,8 @@ dependencies = [

 [[package]]
 name = "lance-datafusion"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5013,8 +5013,8 @@ dependencies = [

 [[package]]
 name = "lance-datagen"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5031,8 +5031,8 @@ dependencies = [

 [[package]]
 name = "lance-derive"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -5041,8 +5041,8 @@ dependencies = [

 [[package]]
 name = "lance-encoding"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5075,8 +5075,8 @@ dependencies = [

 [[package]]
 name = "lance-file"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5107,8 +5107,8 @@ dependencies = [

 [[package]]
 name = "lance-index"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -5172,8 +5172,8 @@ dependencies = [

 [[package]]
 name = "lance-index-core"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5195,8 +5195,8 @@ dependencies = [

 [[package]]
 name = "lance-io"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5222,7 +5222,11 @@ dependencies = [
 "pin-project",
 "prost",
 "rand 0.9.5",
+ "reqsign-core",
+ "reqsign-file-read-tokio",
+ "reqsign-google",
 "serde",
+ "serde_json",
 "tempfile",
 "tokio",
 "tracing",
@@ -5232,8 +5236,8 @@ dependencies = [

 [[package]]
 name = "lance-linalg"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5247,8 +5251,8 @@ dependencies = [

 [[package]]
 name = "lance-namespace"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arrow",
 "async-trait",
@@ -5260,8 +5264,8 @@ dependencies = [

 [[package]]
 name = "lance-namespace-impls"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arrow",
 "arrow-ipc",
@@ -5314,8 +5318,8 @@ dependencies = [

 [[package]]
 name = "lance-select"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5329,8 +5333,8 @@ dependencies = [

 [[package]]
 name = "lance-table"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5370,8 +5374,8 @@ dependencies = [

 [[package]]
 name = "lance-testing"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5384,8 +5388,8 @@ dependencies = [

 [[package]]
 name = "lance-tokenizer"
-version = "11.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
 "frostem",
 "icu_segmenter",
@@ -5398,7 +5402,7 @@ dependencies = [

 [[package]]
 name = "lancedb"
-version = "0.38.0-beta.3"
+version = "0.38.0-beta.7"
 dependencies = [
 "ahash",
 "anyhow",
@@ -5486,7 +5490,7 @@ dependencies = [

 [[package]]
 name = "lancedb-nodejs"
-version = "0.38.0-beta.3"
+version = "0.38.0-beta.7"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5511,7 +5515,7 @@ dependencies = [

 [[package]]
 name = "lancedb-python"
-version = "0.38.0-beta.3"
+version = "0.38.0-beta.7"
 dependencies = [
 "arrow",
 "async-trait",
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"

 [workspace.dependencies]
-lance = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
 lancedb = { path = "rust/lancedb", default-features = false }
 ahash = "0.8"
 # Note that this one does not include pyarrow
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-core</artifactId>
-    <version>0.38.0-beta.3</version>
+    <version>0.38.0-beta.8</version>
 </dependency>
 ```

@@ -584,6 +584,70 @@ Child namespace names and

 ***

+### listTables()
+
+#### listTables(options)
+
+```ts
+abstract listTables(options?): Promise<ListTablesResponse>
+```
+
+List a page of the tables in this database.
+
+To retrieve the tables after the page, pass the `pageToken` the response
+carries back in. A page can be shorter than `limit` without being the last
+one, so walk until a response carries no page token:
+
+```ts
+const names = [];
+let pageToken = undefined;
+do {
+  const page = await conn.listTables({ pageToken, limit: 100 });
+  names.push(...page.tables);
+  pageToken = page.pageToken;
+} while (pageToken);
+```
+
+##### Parameters
+
+* **options?**: `Partial`&lt;[`ListTablesOptions`](../interfaces/ListTablesOptions.md)&gt;
+    Pagination options
+    (`pageToken`, `limit`).
+
+##### Returns
+
+`Promise`&lt;[`ListTablesResponse`](../interfaces/ListTablesResponse.md)&gt;
+
+A page of table names and an
+  optional token for the tables after it.
+
+#### listTables(namespacePath, options)
+
+```ts
+abstract listTables(namespacePath?, options?): Promise<ListTablesResponse>
+```
+
+List a page of the tables in this database.
+
+##### Parameters
+
+* **namespacePath?**: `string`[]
+    The namespace path to list tables from
+    (defaults to root namespace)
+
+* **options?**: `Partial`&lt;[`ListTablesOptions`](../interfaces/ListTablesOptions.md)&gt;
+    Pagination options
+    (`pageToken`, `limit`).
+
+##### Returns
+
+`Promise`&lt;[`ListTablesResponse`](../interfaces/ListTablesResponse.md)&gt;
+
+A page of table names and an
+  optional token for the tables after it.
+
+***
+
 ### openMaterializedView()

 ```ts
@@ -660,7 +724,7 @@ a "not supported" error.

 ***

-### tableNames()
+### ~~tableNames()~~

 #### tableNames(options)

@@ -682,6 +746,10 @@ Tables will be returned in lexicographical order.

 `Promise`&lt;`string`[]&gt;

+##### Deprecated
+
+Use [Connection.listTables](Connection.md#listtables) instead.
+
 #### tableNames(namespacePath, options)

 ```ts
@@ -704,3 +772,7 @@ Tables will be returned in lexicographical order.
 ##### Returns

 `Promise`&lt;`string`[]&gt;
+
+##### Deprecated
+
+Use [Connection.listTables](Connection.md#listtables) instead.
@@ -100,6 +100,8 @@
 - [JobInfo](interfaces/JobInfo.md)
 - [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
 - [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
+- [ListTablesOptions](interfaces/ListTablesOptions.md)
+- [ListTablesResponse](interfaces/ListTablesResponse.md)
 - [LsmStats](interfaces/LsmStats.md)
 - [LsmWriteSpec](interfaces/LsmWriteSpec.md)
 - [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md)
@@ -0,0 +1,34 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ListTablesOptions
+
+# Interface: ListTablesOptions
+
+## Properties
+
+### limit?
+
+```ts
+optional limit: number;
+```
+
+An upper bound on how many tables to return.
+
+A page may hold fewer than this and still not be the last one, so keep
+going while the response carries a page token rather than while pages are
+full.
+
+***
+
+### pageToken?
+
+```ts
+optional pageToken: string;
+```
+
+Token from a previous response, to resume listing where it left off.
+
+The token is opaque: it carries whatever the database needs to resume, and
+callers should not construct or interpret one.
@@ -0,0 +1,23 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ListTablesResponse
+
+# Interface: ListTablesResponse
+
+## Properties
+
+### pageToken?
+
+```ts
+optional pageToken: string;
+```
+
+***
+
+### tables
+
+```ts
+tables: string[];
+```
@@ -4,11 +4,16 @@

 [@lancedb/lancedb](../globals.md) / TableNamesOptions

-# Interface: TableNamesOptions
+# Interface: ~~TableNamesOptions~~
+
+## Deprecated
+
+Use [ListTablesOptions](ListTablesOptions.md) with [Connection.listTables](../classes/Connection.md#listtables)
+instead.

 ## Properties

-### limit?
+### ~~limit?~~

 ```ts
 optional limit: number;
@@ -18,7 +23,7 @@ An optional limit to the number of results to return.

 ***

-### startAfter?
+### ~~startAfter?~~

 ```ts
 optional startAfter: string;
@@ -261,6 +261,8 @@ instead of being materialized with the rest of the row.

 ::: lancedb.streaming.StreamingDataset

+::: lancedb.streaming.StreamingDataLoader
+
 ::: lancedb.permutation.permutation_builder

 ::: lancedb.permutation.PermutationBuilder
@@ -8,7 +8,7 @@
    <parent>
      <groupId>com.lancedb</groupId>
      <artifactId>lancedb-parent</artifactId>
-      <version>0.38.0-beta.3</version>
+      <version>0.38.0-beta.8</version>
      <relativePath>../pom.xml</relativePath>
    </parent>

@@ -6,7 +6,7 @@

    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-parent</artifactId>
-    <version>0.38.0-beta.3</version>
+    <version>0.38.0-beta.8</version>
    <packaging>pom</packaging>
    <name>${project.artifactId}</name>
    <description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>11.0.0-beta.19</lance-core.version>
+        <lance-core.version>11.0.0-beta.22</lance-core.version>
        <spotless.skip>false</spotless.skip>
        <spotless.version>2.30.0</spotless.version>
        <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.38.0-beta.3"
+version = "0.38.0-beta.8"
 publish = false
 license.workspace = true
 description.workspace = true
@@ -515,6 +515,137 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
        );
      });

+      it("will allow matching inferred types across records", function () {
+        expect(() =>
+          makeArrowTable([{ value: 1 }, { value: 2 }]),
+        ).not.toThrow();
+      });
+
+      it("will reject mismatched inferred types across records", function () {
+        expect(() => makeArrowTable([{ value: 1 }, { value: "two" }])).toThrow(
+          "Failed to infer schema for data. Previously inferred type Float64 but found Utf8 for field value at row 1. Consider providing an explicit schema.",
+        );
+      });
+
+      it("will ignore generated dictionary IDs when comparing inferred types", function () {
+        const table = makeArrowTable([{ str: "a" }, { str: "b" }], {
+          dictionaryEncodeStrings: true,
+        });
+
+        expect(table.getChild("str")?.toJSON()).toEqual(["a", "b"]);
+      });
+
+      it("will preserve null values without treating them as type mismatches", function () {
+        for (const records of [
+          [{ vector: [1, 2, 3] }, { vector: null }],
+          [{ vector: null }, { vector: [1, 2, 3] }],
+        ]) {
+          const table = makeArrowTable(records);
+
+          expect(table.numRows).toBe(2);
+          expect(table.getChild("vector")?.nullCount).toBe(1);
+        }
+      });
+
+      it("will preserve empty variable-size lists", function () {
+        for (const records of [
+          [{ items: [1] }, { items: [] }],
+          [{ items: [] }, { items: [1] }],
+        ]) {
+          const table = makeArrowTable(records);
+          expect(
+            table
+              .getChild("items")
+              ?.toJSON()
+              .map((value) => value.toJSON()),
+          ).toEqual(records.map((record) => record.items));
+        }
+      });
+
+      it("will propagate deferred evidence through nested lists", function () {
+        for (const records of [
+          [{ items: [1] }, { items: [null] }],
+          [{ items: [null] }, { items: [1] }],
+          [{ items: [null, 1] }, { items: [2, null] }],
+        ]) {
+          const table = makeArrowTable(records);
+          expect(
+            table
+              .getChild("items")
+              ?.toJSON()
+              .map((value) => value.toJSON()),
+          ).toEqual(records.map((record) => record.items));
+        }
+
+        const nestedRecords = [{ items: [[1]] }, { items: [[null]] }];
+        const nestedTable = makeArrowTable(nestedRecords);
+        expect(
+          nestedTable
+            .getChild("items")
+            ?.toJSON()
+            .map((value) =>
+              value
+                .toJSON()
+                .map((nestedValue: { toJSON: () => unknown[] }) =>
+                  nestedValue.toJSON(),
+                ),
+            ),
+        ).toEqual(nestedRecords.map((record) => record.items));
+      });
+
+      it("will reject incompatible deferred evidence within a list", function () {
+        for (const items of [
+          [[], 1],
+          [1, []],
+          [[null], 1],
+          [1, [null]],
+        ]) {
+          expect(() => makeArrowTable([{ items }])).toThrow(
+            "Failed to infer data type for field items at row 0.",
+          );
+        }
+      });
+
+      it("will reject empty fixed-size lists", function () {
+        expect(() =>
+          makeArrowTable([{ vector: [1, 2, 3] }, { vector: [] }]),
+        ).toThrow(
+          "Failed to infer schema for data. Previously inferred type FixedSizeList[3]<Float32> but found List[0] for field vector at row 1.",
+        );
+      });
+
+      it("will reject inferred leaf and branch shape changes", function () {
+        expect(() =>
+          makeArrowTable([{ value: 1 }, { value: { nested: 2 } }]),
+        ).toThrow(
+          "Failed to infer schema for data. Previously inferred type Float64 but found Struct for field value at row 1.",
+        );
+        expect(() =>
+          makeArrowTable([{ value: { nested: 1 } }, { value: 2 }]),
+        ).toThrow(
+          "Failed to infer schema for data. Previously inferred type Struct but found Float64 for field value at row 1.",
+        );
+      });
+
+      it("will allow null values around inferred struct values", function () {
+        for (const { records, nullIndex } of [
+          {
+            records: [{ value: null }, { value: { nested: 2 } }],
+            nullIndex: 0,
+          },
+          {
+            records: [{ value: { nested: 1 } }, { value: null }],
+            nullIndex: 1,
+          },
+        ]) {
+          const table = makeArrowTable(records);
+          const values = table.getChild("value");
+
+          expect(values?.nullCount).toBe(1);
+          expect(values?.get(nullIndex)).toBeNull();
+        }
+      });
+
      it("will allow a schema to be provided", async function () {
        await checkTableCreation(
          async (records, _, schema) =>
@@ -4,7 +4,13 @@
 import { readdirSync } from "fs";
 import { Field, Float64, Schema } from "apache-arrow";
 import * as tmp from "tmp";
-import { Connection, Table, connect, connectNamespace } from "../lancedb";
+import {
+  Connection,
+  ListTablesResponse,
+  Table,
+  connect,
+  connectNamespace,
+} from "../lancedb";
 import { LocalTable } from "../lancedb/table";

 describe("when connecting", () => {
@@ -47,6 +53,7 @@ describe("given a connection", () => {
    await db.close();
    expect(db.isOpen()).toBe(false);
    await expect(db.tableNames()).rejects.toThrow("Connection is closed");
+    await expect(db.listTables()).rejects.toThrow("Connection is closed");
    await expect(db.renameTable("a", "b")).rejects.toThrow(
      "Connection is closed",
    );
@@ -129,6 +136,66 @@ describe("given a connection", () => {
    expect(tables).toEqual(["b", "c"]);
  });

+  it("should respect limit and page token when listing tables", async () => {
+    const db = await connect(tmpDir.name);
+
+    await db.createTable("b", [{ id: 1 }]);
+    await db.createTable("a", [{ id: 1 }]);
+    await db.createTable("c", [{ id: 1 }]);
+
+    const all = await db.listTables();
+    expect(all.tables).toEqual(["a", "b", "c"]);
+    expect(all.pageToken).toBeUndefined();
+
+    const first = await db.listTables({ limit: 1 });
+    expect(first.tables).toEqual(["a"]);
+    expect(first.pageToken).toBeDefined();
+
+    const second = await db.listTables({
+      limit: 1,
+      pageToken: first.pageToken,
+    });
+    expect(second.tables).toEqual(["b"]);
+  });
+
+  it("should visit every table exactly once when walking pages", async () => {
+    const db = await connect(tmpDir.name);
+
+    const created = ["a", "b", "c", "d", "e"];
+    for (const name of created) {
+      await db.createTable(name, [{ id: 1 }]);
+    }
+
+    const seen: string[] = [];
+    let pageToken: string | undefined = undefined;
+    do {
+      const page: ListTablesResponse = await db.listTables({
+        limit: 2,
+        pageToken,
+      });
+      seen.push(...page.tables);
+      pageToken = page.pageToken;
+    } while (pageToken);
+
+    expect(seen).toEqual(created);
+  });
+
+  it("should list tables in a namespace", async () => {
+    const db = await connect(tmpDir.name, {
+      // biome-ignore lint/style/useNamingConvention: opaque backend property key, must match Rust
+      namespaceClientProperties: { manifest_enabled: "true" },
+    });
+    await db.createNamespace(["child"]);
+    await db.createTable("nested", [{ id: 1 }], ["child"]);
+
+    await expect(db.listTables(["child"])).resolves.toEqual(
+      expect.objectContaining({ tables: ["nested"] }),
+    );
+    await expect(db.listTables()).resolves.toEqual(
+      expect.objectContaining({ tables: [] }),
+    );
+  });
+
  it("should create tables in v2 mode", async () => {
    const db = await connect(tmpDir.name);
    const data = [...Array(10000).keys()].map((i) => ({ id: i }));
@@ -5,7 +5,6 @@ import {
  Data as ArrowData,
  Table as ArrowTable,
  Binary,
-  Bool,
  BufferType,
  DataType,
  DateUnit,
@@ -18,12 +17,7 @@ import {
  FixedSizeList,
  Float,
  Float32,
-  Float64,
  Int,
-  Int8,
-  Int16,
-  Int32,
-  Int64,
  LargeBinary,
  List,
  Null,
@@ -36,17 +30,16 @@ import {
  Struct,
  Timestamp,
  Type,
-  Uint8,
-  Uint16,
-  Uint32,
  Utf8,
  Vector,
  makeVector as arrowMakeVector,
+  util as arrowUtil,
  vectorFromArray as badVectorFromArray,
  makeBuilder,
  makeData,
 } from "apache-arrow";
 import { Buffers } from "apache-arrow/data";
+import { typedArrayToArrowType } from "./arrow_type";
 import { type EmbeddingFunction } from "./embedding/embedding_function";
 import {
  EmbeddingFunctionConfig,
@@ -59,14 +52,7 @@ import {
  sanitizeTable,
  sanitizeType,
 } from "./sanitize";
-
-/**
- * Check if a field name indicates a vector column.
- */
-function nameSuggestsVectorColumn(fieldName: string): boolean {
-  const nameLower = fieldName.toLowerCase();
-  return nameLower.includes("vector") || nameLower.includes("embedding");
-}
+import { inferSchema } from "./schema";

 export * from "apache-arrow";
 export type SchemaLike =
@@ -459,110 +445,6 @@ export function makeArrowTable(
  return new ArrowTable(inferredSchema, finalColumns);
 }

-function inferSchema(
-  data: Array<Record<string, unknown>>,
-  schema: Schema | undefined,
-  opts: MakeArrowTableOptions,
-): Schema {
-  // We will collect all fields we see in the data.
-  const pathTree = new PathTree<DataType>();
-
-  for (const [rowI, row] of data.entries()) {
-    for (const [path, value] of rowPathsAndValues(row)) {
-      if (!pathTree.has(path)) {
-        // First time seeing this field.
-        if (schema !== undefined) {
-          const field = getFieldForPath(schema, path);
-          if (field === undefined) {
-            throw new Error(
-              `Found field not in schema: ${path.join(".")} at row ${rowI}`,
-            );
-          } else {
-            pathTree.set(path, field.type);
-          }
-        } else {
-          const inferredType = inferType(value, path, opts);
-          if (inferredType === undefined) {
-            throw new Error(`Failed to infer data type for field ${path.join(
-              ".",
-            )} at row ${rowI}. \
-                             Consider providing an explicit schema.`);
-          }
-          pathTree.set(path, inferredType);
-        }
-      } else if (schema === undefined) {
-        const currentType = pathTree.get(path);
-        const newType = inferType(value, path, opts);
-        if (currentType !== newType) {
-          new Error(`Failed to infer schema for data. Previously inferred type \
-                     ${currentType} but found ${newType} at row ${rowI}. Consider \
-                     providing an explicit schema.`);
-        }
-      }
-    }
-  }
-
-  if (schema === undefined) {
-    function fieldsFromPathTree(pathTree: PathTree<DataType>): Field[] {
-      const fields = [];
-      for (const [name, value] of pathTree.map.entries()) {
-        if (value instanceof PathTree) {
-          const children = fieldsFromPathTree(value);
-          fields.push(new Field(name, new Struct(children), true));
-        } else {
-          fields.push(new Field(name, value, true));
-        }
-      }
-      return fields;
-    }
-    const fields = fieldsFromPathTree(pathTree);
-    return new Schema(fields);
-  } else {
-    function takeMatchingFields(
-      fields: Field[],
-      pathTree: PathTree<DataType>,
-    ): Field[] {
-      const outFields = [];
-      for (const field of fields) {
-        if (pathTree.map.has(field.name)) {
-          const value = pathTree.get([field.name]);
-          if (value instanceof PathTree) {
-            const struct = field.type as Struct;
-            const children = takeMatchingFields(struct.children, value);
-            outFields.push(
-              new Field(field.name, new Struct(children), field.nullable),
-            );
-          } else {
-            outFields.push(
-              new Field(field.name, value as DataType, field.nullable),
-            );
-          }
-        }
-      }
-      return outFields;
-    }
-    const fields = takeMatchingFields(schema.fields, pathTree);
-    return new Schema(fields);
-  }
-}
-
-function* rowPathsAndValues(
-  row: Record<string, unknown>,
-  basePath: string[] = [],
-): Generator<[string[], unknown]> {
-  for (const [key, value] of Object.entries(row)) {
-    if (isObject(value)) {
-      yield* rowPathsAndValues(value, [...basePath, key]);
-    } else {
-      // Skip undefined values - they should be treated the same as missing fields
-      // for embedding function purposes
-      if (value !== undefined) {
-        yield [[...basePath, key], value];
-      }
-    }
-  }
-}
-
 function isObject(value: unknown): value is Record<string, unknown> {
  return (
    typeof value === "object" &&
@@ -577,146 +459,19 @@ function isObject(value: unknown): value is Record<string, unknown> {
  );
 }

-function getFieldForPath(schema: Schema, path: string[]): Field | undefined {
-  let current: Field | Schema = schema;
+function valueAtPath(datum: Record<string, unknown>, path: string[]): unknown {
+  let current: unknown = datum;
  for (const key of path) {
-    if (current instanceof Schema) {
-      const field: Field | undefined = current.fields.find(
-        (f) => f.name === key,
-      );
-      if (field === undefined) {
-        return undefined;
-      }
-      current = field;
-    } else if (current instanceof Field && DataType.isStruct(current.type)) {
-      const struct: Struct = current.type;
-      const field = struct.children.find((f) => f.name === key);
-      if (field === undefined) {
-        return undefined;
-      }
-      current = field;
+    if (current == null) {
+      return null;
+    }
+    if (isObject(current) && (Object.hasOwn(current, key) || key in current)) {
+      current = current[key];
    } else {
      return undefined;
    }
  }
-  if (current instanceof Field) {
-    return current;
-  } else {
-    return undefined;
-  }
-}
-
-/**
- * Try to infer which Arrow type to use for a given value.
- *
- * May return undefined if the type cannot be inferred.
- */
-function inferType(
-  value: unknown,
-  path: string[],
-  opts: MakeArrowTableOptions,
-): DataType | undefined {
-  if (typeof value === "bigint") {
-    return new Int64();
-  } else if (typeof value === "number") {
-    // Even if it's an integer, it's safer to assume Float64. Users can
-    // always provide an explicit schema or use BigInt if they mean integer.
-    return new Float64();
-  } else if (typeof value === "string") {
-    if (opts.dictionaryEncodeStrings) {
-      return new Dictionary(new Utf8(), new Int32());
-    } else {
-      return new Utf8();
-    }
-  } else if (typeof value === "boolean") {
-    return new Bool();
-  } else if (value instanceof Buffer) {
-    return new Binary();
-  } else if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
-    const info = typedArrayToArrowType(value);
-    if (info !== undefined) {
-      const child = new Field("item", info.elementType, true);
-      return new FixedSizeList(info.length, child);
-    }
-    return undefined;
-  } else if (Array.isArray(value)) {
-    if (value.length === 0) {
-      return undefined; // Without any values we can't infer the type
-    }
-    if (path.length === 1 && Object.hasOwn(opts.vectorColumns, path[0])) {
-      const floatType = sanitizeType(opts.vectorColumns[path[0]].type);
-      return new FixedSizeList(
-        value.length,
-        new Field("item", floatType, true),
-      );
-    }
-    const valueType = inferType(value[0], path, opts);
-    if (valueType === undefined) {
-      return undefined;
-    }
-    // Try to automatically detect embedding columns.
-    if (nameSuggestsVectorColumn(path[path.length - 1])) {
-      // Check if value is a Uint8Array for integer vector type determination
-      if (value instanceof Uint8Array) {
-        // For integer vectors, we default to Uint8 (matching Python implementation)
-        const child = new Field("item", new Uint8(), true);
-        return new FixedSizeList(value.length, child);
-      } else {
-        // For float vectors, we default to Float32
-        const child = new Field("item", new Float32(), true);
-        return new FixedSizeList(value.length, child);
-      }
-    } else {
-      const child = new Field("item", valueType, true);
-      return new List(child);
-    }
-  } else {
-    // TODO: timestamp
-    return undefined;
-  }
-}
-
-class PathTree<V> {
-  map: Map<string, V | PathTree<V>>;
-
-  constructor(entries?: [string[], V][]) {
-    this.map = new Map();
-    if (entries !== undefined) {
-      for (const [path, value] of entries) {
-        this.set(path, value);
-      }
-    }
-  }
-  has(path: string[]): boolean {
-    let ref: PathTree<V> = this;
-    for (const part of path) {
-      if (!(ref instanceof PathTree) || !ref.map.has(part)) {
-        return false;
-      }
-      ref = ref.map.get(part) as PathTree<V>;
-    }
-    return true;
-  }
-  get(path: string[]): V | undefined {
-    let ref: PathTree<V> = this;
-    for (const part of path) {
-      if (!(ref instanceof PathTree) || !ref.map.has(part)) {
-        return undefined;
-      }
-      ref = ref.map.get(part) as PathTree<V>;
-    }
-    return ref as V;
-  }
-  set(path: string[], value: V): void {
-    let ref: PathTree<V> = this;
-    for (const part of path.slice(0, path.length - 1)) {
-      if (!ref.map.has(part)) {
-        ref.map.set(part, new PathTree<V>());
-      }
-      ref = ref.map.get(part) as PathTree<V>;
-    }
-    ref.map.set(path[path.length - 1], value);
-  }
+  return current;
 }

 function transposeData(
@@ -724,37 +479,26 @@ function transposeData(
  field: Field,
  path: string[] = [],
 ): Vector {
+  const valuesPath = [...path, field.name];
+  const values = data.map((datum) => valueAtPath(datum, valuesPath));
  if (field.type instanceof Struct) {
    const childFields = field.type.children;
-    const fullPath = [...path, field.name];
    const childVectors = childFields.map((child) => {
-      return transposeData(data, child, fullPath);
+      return transposeData(data, child, valuesPath);
    });
+    const nullCount = values.filter((value) => value === null).length;
    const structData = makeData({
      type: field.type,
+      length: values.length,
+      nullCount,
+      nullBitmap:
+        nullCount > 0
+          ? arrowUtil.packBools(values.map((value) => value !== null))
+          : undefined,
      children: childVectors as unknown as ArrowData<DataType>[],
    });
    return arrowMakeVector(structData);
  } else {
-    const valuesPath = [...path, field.name];
-    const values = data.map((datum) => {
-      let current: unknown = datum;
-      for (const key of valuesPath) {
-        if (current == null) {
-          return null;
-        }
-
-        if (
-          isObject(current) &&
-          (Object.hasOwn(current, key) || key in current)
-        ) {
-          current = current[key];
-        } else {
-          return null;
-        }
-      }
-      return current;
-    });
    return makeVector(values, field.type, undefined, field.nullable);
  }
 }
@@ -797,32 +541,6 @@ function makeListVector(lists: unknown[][]): Vector<unknown> {
  return listBuilder.finish().toVector();
 }

-/**
- * Map a JS TypedArray instance to the corresponding Arrow element DataType
- * and its length. Returns undefined if the value is not a recognized TypedArray.
- */
-function typedArrayToArrowType(
-  value: ArrayBufferView,
-): { elementType: DataType; length: number } | undefined {
-  if (value instanceof Float32Array)
-    return { elementType: new Float32(), length: value.length };
-  if (value instanceof Float64Array)
-    return { elementType: new Float64(), length: value.length };
-  if (value instanceof Uint8Array)
-    return { elementType: new Uint8(), length: value.length };
-  if (value instanceof Uint16Array)
-    return { elementType: new Uint16(), length: value.length };
-  if (value instanceof Uint32Array)
-    return { elementType: new Uint32(), length: value.length };
-  if (value instanceof Int8Array)
-    return { elementType: new Int8(), length: value.length };
-  if (value instanceof Int16Array)
-    return { elementType: new Int16(), length: value.length };
-  if (value instanceof Int32Array)
-    return { elementType: new Int32(), length: value.length };
-  return undefined;
-}
-
 /** Helper function to convert an Array of JS values to an Arrow Vector */
 function makeVector(
  values: unknown[],
@@ -1462,8 +1180,12 @@ export function ensureNestedFieldsExist(
          completeRow[field.name] = row[field.name];
        }
      } else {
-        // Field is missing from the data - set to null
-        completeRow[field.name] = null;
+        // Keep a missing struct valid while filling each of its children with
+        // null. This is distinct from an explicitly null struct value.
+        completeRow[field.name] =
+          field.type.constructor.name === "Struct"
+            ? ensureStructFieldsExist({}, field.type as Struct)
+            : null;
      }
    }

@@ -1498,8 +1220,12 @@ function ensureStructFieldsExist(
        completeStruct[childField.name] = data[childField.name];
      }
    } else {
-      // Field is missing - set to null
-      completeStruct[childField.name] = null;
+      // Keep a missing struct valid while filling each of its children with
+      // null. This is distinct from an explicitly null struct value.
+      completeStruct[childField.name] =
+        childField.type.constructor.name === "Struct"
+          ? ensureStructFieldsExist({}, childField.type as Struct)
+          : null;
    }
  }

@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import {
+  type DataType,
+  Float32,
+  Float64,
+  Int8,
+  Int16,
+  Int32,
+  Uint8,
+  Uint16,
+  Uint32,
+} from "apache-arrow";
+
+/**
+ * Map a JS TypedArray instance to the corresponding Arrow element type and
+ * length. Returns undefined when the view is not a supported TypedArray.
+ */
+export function typedArrayToArrowType(
+  value: ArrayBufferView,
+): { elementType: DataType; length: number } | undefined {
+  if (value instanceof Float32Array)
+    return { elementType: new Float32(), length: value.length };
+  if (value instanceof Float64Array)
+    return { elementType: new Float64(), length: value.length };
+  if (value instanceof Uint8Array)
+    return { elementType: new Uint8(), length: value.length };
+  if (value instanceof Uint16Array)
+    return { elementType: new Uint16(), length: value.length };
+  if (value instanceof Uint32Array)
+    return { elementType: new Uint32(), length: value.length };
+  if (value instanceof Int8Array)
+    return { elementType: new Int8(), length: value.length };
+  if (value instanceof Int16Array)
+    return { elementType: new Int16(), length: value.length };
+  if (value instanceof Int32Array)
+    return { elementType: new Int32(), length: value.length };
+  return undefined;
+}
@@ -31,12 +31,14 @@ import type {
  JobDescription,
  JobInfo,
  ListNamespacesResponse,
+  ListTablesResponse,
 } from "./native";
 export type {
  CreateNamespaceResponse,
  DescribeNamespaceResponse,
  DropNamespaceResponse,
  ListNamespacesResponse,
+  ListTablesResponse,
 };
 import { sanitizeTable } from "./sanitize";
 import { LocalTable, Table } from "./table";
@@ -134,6 +136,10 @@ export interface OpenTableOptions {
  indexCacheSize?: number;
 }

+/**
+ * @deprecated Use {@link ListTablesOptions} with {@link Connection.listTables}
+ * instead.
+ */
 export interface TableNamesOptions {
  /**
   * If present, only return names that come lexicographically after the
@@ -147,6 +153,24 @@ export interface TableNamesOptions {
  limit?: number;
 }

+export interface ListTablesOptions {
+  /**
+   * Token from a previous response, to resume listing where it left off.
+   *
+   * The token is opaque: it carries whatever the database needs to resume, and
+   * callers should not construct or interpret one.
+   */
+  pageToken?: string;
+  /**
+   * An upper bound on how many tables to return.
+   *
+   * A page may hold fewer than this and still not be the last one, so keep
+   * going while the response carries a page token rather than while pages are
+   * full.
+   */
+  limit?: number;
+}
+
 export interface ListNamespacesOptions {
  /** Token from a previous response for pagination. */
  pageToken?: string;
@@ -231,6 +255,7 @@ export abstract class Connection {
   * @param {Partial<TableNamesOptions>} options - options to control the
   * paging / start point (backwards compatibility)
   *
+   * @deprecated Use {@link Connection.listTables} instead.
   */
  abstract tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
  /**
@@ -241,12 +266,53 @@ export abstract class Connection {
   * @param {Partial<TableNamesOptions>} options - options to control the
   * paging / start point
   *
+   * @deprecated Use {@link Connection.listTables} instead.
   */
  abstract tableNames(
    namespacePath?: string[],
    options?: Partial<TableNamesOptions>,
  ): Promise<string[]>;

+  /**
+   * List a page of the tables in this database.
+   *
+   * To retrieve the tables after the page, pass the `pageToken` the response
+   * carries back in. A page can be shorter than `limit` without being the last
+   * one, so walk until a response carries no page token:
+   *
+   * ```ts
+   * const names = [];
+   * let pageToken = undefined;
+   * do {
+   *   const page = await conn.listTables({ pageToken, limit: 100 });
+   *   names.push(...page.tables);
+   *   pageToken = page.pageToken;
+   * } while (pageToken);
+   * ```
+   *
+   * @param {Partial<ListTablesOptions>} options - Pagination options
+   *   (`pageToken`, `limit`).
+   * @returns {Promise<ListTablesResponse>} A page of table names and an
+   *   optional token for the tables after it.
+   */
+  abstract listTables(
+    options?: Partial<ListTablesOptions>,
+  ): Promise<ListTablesResponse>;
+  /**
+   * List a page of the tables in this database.
+   *
+   * @param {string[]} namespacePath - The namespace path to list tables from
+   *   (defaults to root namespace)
+   * @param {Partial<ListTablesOptions>} options - Pagination options
+   *   (`pageToken`, `limit`).
+   * @returns {Promise<ListTablesResponse>} A page of table names and an
+   *   optional token for the tables after it.
+   */
+  abstract listTables(
+    namespacePath?: string[],
+    options?: Partial<ListTablesOptions>,
+  ): Promise<ListTablesResponse>;
+
  /**
   * Open a table in the database.
   * @param {string} name - The name of the table
@@ -601,6 +667,25 @@ export class LocalConnection extends Connection {
    return await this.inner.listMaterializedViews();
  }

+  async listTables(
+    namespacePathOrOptions?: string[] | Partial<ListTablesOptions>,
+    options?: Partial<ListTablesOptions>,
+  ): Promise<ListTablesResponse> {
+    // Detect if first argument is namespacePath array or options object
+    const namespacePath = Array.isArray(namespacePathOrOptions)
+      ? namespacePathOrOptions
+      : undefined;
+    const listTablesOptions = Array.isArray(namespacePathOrOptions)
+      ? options
+      : namespacePathOrOptions;
+
+    return this.inner.listTables(
+      namespacePath ?? [],
+      listTablesOptions?.pageToken,
+      listTablesOptions?.limit,
+    );
+  }
+
  async openTable(
    name: string,
    namespacePath?: string[],
@@ -81,11 +81,13 @@ export {
  Connection,
  CreateTableOptions,
  TableNamesOptions,
+  ListTablesOptions,
  OpenTableOptions,
  ListNamespacesOptions,
  CreateNamespaceOptions,
  DropNamespaceOptions,
  ListNamespacesResponse,
+  ListTablesResponse,
  CreateNamespaceResponse,
  DropNamespaceResponse,
  DescribeNamespaceResponse,
@@ -0,0 +1,566 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import {
+  Binary,
+  Bool,
+  DataType,
+  Dictionary,
+  Field,
+  FixedSizeList,
+  Float32,
+  Float64,
+  Int32,
+  Int64,
+  List,
+  Schema,
+  Struct,
+  Utf8,
+  util as arrowUtil,
+} from "apache-arrow";
+import { typedArrayToArrowType } from "./arrow_type";
+import { sanitizeType } from "./sanitize";
+
+type InferenceOptions = {
+  dictionaryEncodeStrings: boolean;
+  vectorColumns: Record<string, { type: unknown }>;
+};
+
+/**
+ * Infer the Arrow schema represented by a set of records.
+ *
+ * This is the intentionally small interface to schema inference. The stateful
+ * details of combining partial type evidence are encapsulated below so callers
+ * only need to provide records, an optional schema, and inference options.
+ */
+export function inferSchema(
+  data: Array<Record<string, unknown>>,
+  schema: Schema | undefined,
+  options: InferenceOptions,
+): Schema {
+  return new SchemaInferrer(schema, options).infer(data);
+}
+
+class SchemaInferrer {
+  private readonly fields = new FieldTree();
+
+  constructor(
+    private readonly providedSchema: Schema | undefined,
+    private readonly options: InferenceOptions,
+  ) {}
+
+  infer(data: Array<Record<string, unknown>>): Schema {
+    for (const [row, record] of data.entries()) {
+      for (const [path, value] of recordPathsAndValues(record)) {
+        this.observe(path, value, row);
+      }
+    }
+
+    return this.providedSchema === undefined
+      ? new Schema(fieldsFromTree(this.fields))
+      : new Schema(matchingFields(this.providedSchema.fields, this.fields));
+  }
+
+  private observe(path: string[], value: unknown, row: number): void {
+    const current = this.fields.get(path);
+    if (current === undefined) {
+      this.addField(path, value, row);
+    } else if (this.providedSchema === undefined) {
+      this.updateInferredField(path, value, row, current);
+    }
+  }
+
+  private addField(path: string[], value: unknown, row: number): void {
+    if (this.providedSchema !== undefined) {
+      this.addSchemaField(this.providedSchema, path, row);
+      return;
+    }
+
+    const evidence =
+      this.inferType(value, path) ?? DeferredTypeEvidence.from(value, row);
+    if (evidence === undefined) {
+      throw typeInferenceError(path, row);
+    }
+
+    const conflict = this.fields.set(
+      path,
+      evidence,
+      (existing) =>
+        existing instanceof DeferredTypeEvidence && existing.isOnlyNulls(),
+    );
+    if (conflict !== undefined) {
+      throw branchConflictError(conflict, row, "Struct");
+    }
+  }
+
+  private addSchemaField(schema: Schema, path: string[], row: number): void {
+    const field = fieldAtPath(schema, path);
+    if (field === undefined) {
+      throw new Error(
+        `Found field not in schema: ${path.join(".")} at row ${row}`,
+      );
+    }
+
+    const conflict = this.fields.set(path, field.type);
+    if (conflict !== undefined) {
+      throw branchConflictError(conflict, row, "Struct");
+    }
+  }
+
+  private updateInferredField(
+    path: string[],
+    value: unknown,
+    row: number,
+    current: FieldNode,
+  ): void {
+    const newType = this.inferType(value, path);
+    const deferred = DeferredTypeEvidence.from(value, row);
+
+    if (current instanceof FieldTree) {
+      if (deferred?.isOnlyNulls()) {
+        return;
+      }
+      throw schemaInferenceError(
+        path,
+        row,
+        "Struct",
+        describeEvidence(newType ?? deferred),
+      );
+    }
+
+    if (current instanceof DeferredTypeEvidence) {
+      this.resolveDeferredField(path, row, current, newType, deferred);
+      return;
+    }
+
+    if (newType !== undefined) {
+      if (!inferredTypesEqual(current, newType)) {
+        throw schemaInferenceError(
+          path,
+          row,
+          describeEvidence(current),
+          describeEvidence(newType),
+        );
+      }
+      return;
+    }
+
+    if (deferred === undefined || !deferred.matches(current)) {
+      throw schemaInferenceError(
+        path,
+        row,
+        describeEvidence(current),
+        describeEvidence(deferred),
+      );
+    }
+  }
+
+  private resolveDeferredField(
+    path: string[],
+    row: number,
+    current: DeferredTypeEvidence,
+    newType: DataType | undefined,
+    deferred: DeferredTypeEvidence | undefined,
+  ): void {
+    if (newType !== undefined) {
+      if (!current.matches(newType)) {
+        throw schemaInferenceError(
+          path,
+          row,
+          current.describe(),
+          describeEvidence(newType),
+        );
+      }
+      this.fields.set(path, newType);
+      return;
+    }
+
+    if (deferred !== undefined) {
+      this.fields.set(path, current.merge(deferred));
+      return;
+    }
+
+    throw schemaInferenceError(
+      path,
+      row,
+      current.describe(),
+      describeEvidence(newType),
+    );
+  }
+
+  private inferType(value: unknown, path: string[]): DataType | undefined {
+    if (typeof value === "bigint") {
+      return new Int64();
+    }
+    if (typeof value === "number") {
+      return new Float64();
+    }
+    if (typeof value === "string") {
+      return this.options.dictionaryEncodeStrings
+        ? new Dictionary(new Utf8(), new Int32())
+        : new Utf8();
+    }
+    if (typeof value === "boolean") {
+      return new Bool();
+    }
+    if (value instanceof Buffer) {
+      return new Binary();
+    }
+    if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
+      const typedArray = typedArrayToArrowType(value);
+      return typedArray === undefined
+        ? undefined
+        : new FixedSizeList(
+            typedArray.length,
+            new Field("item", typedArray.elementType, true),
+          );
+    }
+    if (!Array.isArray(value) || value.length === 0) {
+      return undefined;
+    }
+
+    const configuredVector =
+      path.length === 1 ? this.options.vectorColumns[path[0]] : undefined;
+    if (configuredVector !== undefined) {
+      return new FixedSizeList(
+        value.length,
+        new Field("item", sanitizeType(configuredVector.type), true),
+      );
+    }
+
+    const itemType = this.inferArrayItemType(value, path);
+    if (itemType === undefined) {
+      return undefined;
+    }
+
+    return nameSuggestsVectorColumn(path[path.length - 1])
+      ? new FixedSizeList(value.length, new Field("item", new Float32(), true))
+      : new List(new Field("item", itemType, true));
+  }
+
+  private inferArrayItemType(
+    values: unknown[],
+    path: string[],
+  ): DataType | undefined {
+    let itemType: DataType | undefined;
+    const deferredItems: unknown[] = [];
+
+    for (const value of values) {
+      const candidate = this.inferType(value, path);
+      if (candidate === undefined) {
+        if (!isDeferredValue(value)) {
+          return undefined;
+        }
+        deferredItems.push(value);
+      } else if (itemType === undefined) {
+        itemType = candidate;
+      } else if (!inferredTypesEqual(itemType, candidate)) {
+        return undefined;
+      }
+    }
+
+    if (itemType === undefined) {
+      return undefined;
+    }
+    return deferredItems.every((value) =>
+      deferredValueMatchesType(value, itemType),
+    )
+      ? itemType
+      : undefined;
+  }
+}
+
+/** Nulls and empty/all-null lists that do not determine a type by themselves. */
+class DeferredTypeEvidence {
+  private constructor(
+    private readonly values: Array<{ value: unknown; row: number }>,
+  ) {}
+
+  static from(value: unknown, row: number): DeferredTypeEvidence | undefined {
+    return isDeferredValue(value)
+      ? new DeferredTypeEvidence([{ value, row }])
+      : undefined;
+  }
+
+  isOnlyNulls(): boolean {
+    return this.values.every(({ value }) => value == null);
+  }
+
+  matches(type: DataType): boolean {
+    return this.values.every(({ value }) =>
+      deferredValueMatchesType(value, type),
+    );
+  }
+
+  merge(other: DeferredTypeEvidence): DeferredTypeEvidence {
+    return new DeferredTypeEvidence([...this.values, ...other.values]);
+  }
+
+  describe(): string {
+    const list = this.values.find(({ value }) => Array.isArray(value));
+    return list === undefined
+      ? "null"
+      : `List[${(list.value as unknown[]).length}]`;
+  }
+
+  firstRow(): number {
+    return this.values[0].row;
+  }
+}
+
+type FieldNode = DataType | DeferredTypeEvidence | FieldTree;
+type LeafNode = Exclude<FieldNode, FieldTree>;
+type FieldConflict = { path: string[]; value: FieldNode };
+
+/** Nested field state, kept separate from Arrow's eventual Struct types. */
+class FieldTree {
+  private readonly children = new Map<string, FieldNode>();
+
+  get(path: string[]): FieldNode | undefined {
+    let current: FieldNode = this;
+    for (const part of path) {
+      if (!(current instanceof FieldTree)) {
+        return undefined;
+      }
+      const child = current.children.get(part);
+      if (child === undefined) {
+        return undefined;
+      }
+      current = child;
+    }
+    return current;
+  }
+
+  set(
+    path: string[],
+    value: LeafNode,
+    canReplaceLeaf: (value: LeafNode) => boolean = () => false,
+  ): FieldConflict | undefined {
+    let branch: FieldTree = this;
+    for (const [index, part] of path.slice(0, -1).entries()) {
+      const child = branch.children.get(part);
+      if (child === undefined || (isLeaf(child) && canReplaceLeaf(child))) {
+        const nextBranch = new FieldTree();
+        branch.children.set(part, nextBranch);
+        branch = nextBranch;
+      } else if (child instanceof FieldTree) {
+        branch = child;
+      } else {
+        return { path: path.slice(0, index + 1), value: child };
+      }
+    }
+
+    const name = path[path.length - 1];
+    const current = branch.children.get(name);
+    if (current instanceof FieldTree) {
+      return { path, value: current };
+    }
+    branch.children.set(name, value);
+    return undefined;
+  }
+
+  entries(): IterableIterator<[string, FieldNode]> {
+    return this.children.entries();
+  }
+
+  has(name: string): boolean {
+    return this.children.has(name);
+  }
+}
+
+function isLeaf(value: FieldNode): value is LeafNode {
+  return !(value instanceof FieldTree);
+}
+
+function fieldsFromTree(tree: FieldTree, path: string[] = []): Field[] {
+  const fields: Field[] = [];
+  for (const [name, value] of tree.entries()) {
+    if (value instanceof FieldTree) {
+      fields.push(
+        new Field(
+          name,
+          new Struct(fieldsFromTree(value, [...path, name])),
+          true,
+        ),
+      );
+    } else if (value instanceof DeferredTypeEvidence) {
+      throw typeInferenceError([...path, name], value.firstRow());
+    } else {
+      fields.push(new Field(name, value, true));
+    }
+  }
+  return fields;
+}
+
+function matchingFields(fields: Field[], tree: FieldTree): Field[] {
+  const matches: Field[] = [];
+  for (const field of fields) {
+    if (!tree.has(field.name)) {
+      continue;
+    }
+    const value = tree.get([field.name]);
+    if (value instanceof FieldTree) {
+      const struct = field.type as Struct;
+      matches.push(
+        new Field(
+          field.name,
+          new Struct(matchingFields(struct.children, value)),
+          field.nullable,
+        ),
+      );
+    } else {
+      matches.push(new Field(field.name, value as DataType, field.nullable));
+    }
+  }
+  return matches;
+}
+
+function* recordPathsAndValues(
+  record: Record<string, unknown>,
+  path: string[] = [],
+): Generator<[string[], unknown]> {
+  for (const [name, value] of Object.entries(record)) {
+    if (isRecord(value)) {
+      yield* recordPathsAndValues(value, [...path, name]);
+    } else if (value !== undefined) {
+      yield [[...path, name], value];
+    }
+  }
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    !Array.isArray(value) &&
+    !(value instanceof RegExp) &&
+    !(value instanceof Date) &&
+    !(value instanceof Set) &&
+    !(value instanceof Map) &&
+    !(value instanceof Buffer) &&
+    !ArrayBuffer.isView(value)
+  );
+}
+
+function fieldAtPath(schema: Schema, path: string[]): Field | undefined {
+  let fields = schema.fields;
+  let field: Field | undefined;
+  for (const [index, name] of path.entries()) {
+    field = fields.find((candidate) => candidate.name === name);
+    if (field === undefined || index === path.length - 1) {
+      return field;
+    }
+    if (!DataType.isStruct(field.type)) {
+      return undefined;
+    }
+    fields = field.type.children;
+  }
+  return field;
+}
+
+function isDeferredValue(value: unknown): boolean {
+  return (
+    value == null || (Array.isArray(value) && value.every(isDeferredValue))
+  );
+}
+
+function deferredValueMatchesType(value: unknown, type: DataType): boolean {
+  if (value == null) {
+    return true;
+  }
+  if (!Array.isArray(value)) {
+    return false;
+  }
+  if (DataType.isList(type)) {
+    return value.every((item) =>
+      deferredValueMatchesType(item, type.valueType),
+    );
+  }
+  if (DataType.isFixedSizeList(type)) {
+    return (
+      value.length === type.listSize &&
+      value.every((item) => deferredValueMatchesType(item, type.valueType))
+    );
+  }
+  return false;
+}
+
+function inferredTypesEqual(current: DataType, candidate: DataType): boolean {
+  if (DataType.isDictionary(current)) {
+    return (
+      DataType.isDictionary(candidate) &&
+      current.isOrdered === candidate.isOrdered &&
+      inferredTypesEqual(current.indices, candidate.indices) &&
+      inferredTypesEqual(current.dictionary, candidate.dictionary)
+    );
+  }
+  if (DataType.isList(current)) {
+    return (
+      DataType.isList(candidate) &&
+      current.valueField.name === candidate.valueField.name &&
+      current.valueField.nullable === candidate.valueField.nullable &&
+      inferredTypesEqual(current.valueType, candidate.valueType)
+    );
+  }
+  if (DataType.isFixedSizeList(current)) {
+    return (
+      DataType.isFixedSizeList(candidate) &&
+      current.listSize === candidate.listSize &&
+      current.valueField.name === candidate.valueField.name &&
+      current.valueField.nullable === candidate.valueField.nullable &&
+      inferredTypesEqual(current.valueType, candidate.valueType)
+    );
+  }
+  return arrowUtil.compareTypes(current, candidate);
+}
+
+function describeEvidence(
+  evidence: DataType | DeferredTypeEvidence | undefined,
+): string {
+  if (evidence === undefined) {
+    return "an unsupported value";
+  }
+  return evidence instanceof DeferredTypeEvidence
+    ? evidence.describe()
+    : evidence.toString();
+}
+
+function branchConflictError(
+  conflict: FieldConflict,
+  row: number,
+  candidate: string,
+): Error {
+  return schemaInferenceError(
+    conflict.path,
+    row,
+    conflict.value instanceof FieldTree
+      ? "Struct"
+      : describeEvidence(conflict.value),
+    candidate,
+  );
+}
+
+function schemaInferenceError(
+  path: string[],
+  row: number,
+  currentType: string,
+  newType: string,
+): Error {
+  return new Error(
+    `Failed to infer schema for data. Previously inferred type ${currentType} ` +
+      `but found ${newType} for field ${path.join(".")} at row ${row}. ` +
+      "Consider providing an explicit schema.",
+  );
+}
+
+function typeInferenceError(path: string[], row: number): Error {
+  return new Error(
+    `Failed to infer data type for field ${path.join(".")} at row ${row}. ` +
+      "Consider providing an explicit schema.",
+  );
+}
+
+function nameSuggestsVectorColumn(name: string): boolean {
+  const normalized = name.toLowerCase();
+  return normalized.includes("vector") || normalized.includes("embedding");
+}
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.38.0-beta.3",
+	"version": "0.38.0-beta.8",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.38.0-beta.3",
+	"version": "0.38.0-beta.8",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.38.0-beta.3",
+	"version": "0.38.0-beta.8",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.38.0-beta.3",
+	"version": "0.38.0-beta.8",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.38.0-beta.3",
+	"version": "0.38.0-beta.8",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",
@@ -1,6 +1,6 @@
 {
  "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.38.0-beta.3",
+  "version": "0.38.0-beta.8",
  "os": [
    "win32"
  ],
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.38.0-beta.3",
+	"version": "0.38.0-beta.8",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",
@@ -1,12 +1,12 @@
 {
  "name": "@lancedb/lancedb",
-  "version": "0.38.0-beta.3",
+  "version": "0.38.0-beta.7",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "@lancedb/lancedb",
-      "version": "0.38.0-beta.3",
+      "version": "0.38.0-beta.7",
      "cpu": [
        "x64",
        "arm64"
@@ -11,7 +11,7 @@
    "ann"
  ],
  "private": false,
-  "version": "0.38.0-beta.3",
+  "version": "0.38.0-beta.8",
  "main": "dist/index.js",
  "exports": {
    ".": "./dist/index.js",
@@ -17,6 +17,7 @@ use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection, conne

 use lance_namespace::models::{
    CreateNamespaceRequest, DescribeNamespaceRequest, DropNamespaceRequest, ListNamespacesRequest,
+    ListTablesRequest,
 };
 use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};

@@ -36,6 +37,12 @@ pub struct ListNamespacesResponse {
    pub page_token: Option<String>,
 }

+#[napi(object)]
+pub struct ListTablesResponse {
+    pub tables: Vec<String>,
+    pub page_token: Option<String>,
+}
+
 #[napi(object)]
 pub struct CreateNamespaceResponse {
    pub properties: Option<HashMap<String, String>>,
@@ -206,6 +213,33 @@ impl Connection {
        op.execute().await.default_error()
    }

+    /// List a page of tables in the database.
+    #[napi(catch_unwind)]
+    pub async fn list_tables(
+        &self,
+        namespace_path: Option<Vec<String>>,
+        page_token: Option<String>,
+        limit: Option<u32>,
+    ) -> napi::Result<ListTablesResponse> {
+        let request = ListTablesRequest {
+            // The root namespace is an empty path, not an absent one: a namespace-backed
+            // database rejects a request that names no namespace.
+            id: Some(namespace_path.unwrap_or_default()),
+            page_token,
+            limit: limit.map(|limit| i32::try_from(limit).unwrap_or(i32::MAX)),
+            ..Default::default()
+        };
+        let response = self
+            .get_inner()?
+            .list_tables(request)
+            .await
+            .default_error()?;
+        Ok(ListTablesResponse {
+            tables: response.tables,
+            page_token: response.page_token,
+        })
+    }
+
    /// Create table from a Apache Arrow IPC (file) buffer.
    ///
    /// Parameters:
@@ -14,9 +14,12 @@ pub struct Job {
 }

 impl Job {
-    pub(crate) fn new(inner: lancedb::Job) -> Self {
+    pub(crate) fn new<T>(inner: lancedb::Job<T>) -> Self
+    where
+        T: Clone + Send + Sync + 'static,
+    {
        Self {
-            inner: Arc::new(inner),
+            inner: Arc::new(inner.map(|_| ())),
        }
    }
 }
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.38.0-beta.3"
+version = "0.38.0-beta.8"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"
@@ -29,6 +29,7 @@ from .functions import (
    FunctionRegistrationRequest as FunctionRegistrationRequest,
    FunctionVersion as FunctionVersion,
    PythonRuntimeSpec as PythonRuntimeSpec,
+    RefreshColumnResult as RefreshColumnResult,
    UdfDefinition as UdfDefinition,
    udf as udf,
 )
@@ -147,7 +147,7 @@ class Connection(object):
        limit: Optional[int],
    ) -> list[str]: ...  # Deprecated: Use list_tables instead
    def job(self, job_id: str) -> Job: ...
-    async def create_function_async(self, request_json: str) -> FunctionJob: ...
+    async def create_function_async(self, request_json: str) -> Job: ...
    async def get_function(self, name: str, version: str) -> str: ...
    async def list_jobs(self) -> List[JobInfo]: ...
    async def get_job(self, job_id: str) -> Optional[JobDescription]: ...
@@ -234,14 +234,7 @@ class Job:
    @property
    def id(self) -> Optional[str]: ...
    async def status(self) -> str: ...
-    async def wait(self) -> None: ...
-    async def cancel(self) -> None: ...
-
-class FunctionJob:
-    @property
-    def id(self) -> Optional[str]: ...
-    async def status(self) -> str: ...
-    async def wait(self) -> str: ...
+    async def wait(self) -> Optional[str]: ...
    async def cancel(self) -> None: ...

 class JobInfo:
@@ -46,7 +46,7 @@ from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
 from . import __version__
 from ._lancedb import connect as lancedb_connect  # type: ignore
 from .functions import FunctionVersion, UdfDefinition
-from .job import AsyncJob, Job, _function_job
+from .job import AsyncJob, Job, _typed_job
 from .materialized_view import (
    AsyncMaterializedView,
    MaterializedView,
@@ -2237,7 +2237,7 @@ class AsyncConnection(object):
        inner = await self._inner.create_function_async(
            definition.registration_request.to_canonical_json()
        )
-        return _function_job(inner)
+        return _typed_job(inner, FunctionVersion.from_json)

    async def get_function(self, name: str, *, version: str) -> FunctionVersion:
        """Open one exact immutable Function version from the remote catalog."""
@@ -1,19 +1,24 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors

-"""Canonical values exchanged with LanceDB Enterprise Function services.
+"""Canonical Function values exchanged with LanceDB Enterprise services.

 These immutable models contain client/wire state only. Catalog persistence,
 environment bake, secret resolution, and execution are owned by Sophon.
+``RefreshColumnResult`` is also the backend-neutral result of a local
+expression-backed refresh job.
 """

 from __future__ import annotations

 import ast
+import builtins
 import base64
 import functools
 import hashlib
+import importlib
 import inspect
+import symtable
 import json
 import math
 import re
@@ -460,7 +465,11 @@ class FunctionBinding(_RemoteValue):


 class RefreshColumnResult(_RemoteValue):
-    """Terminal result of a remote Function-column refresh Job."""
+    """Terminal result of an expression-backed or Function-backed refresh Job.
+
+    Local jobs produce this value in process. LanceDB Cloud and Enterprise
+    decode the same value from the durable server-job terminal payload.
+    """

    rows_assigned: _UInt64
    rows_failed: _UInt64
@@ -483,59 +492,58 @@ _FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$")
 _SECRET_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")


+_GRAMMAR_PRIMITIVES = (
+    (pa.bool_(), "bool"),
+    (pa.int8(), "int8"),
+    (pa.int16(), "int16"),
+    (pa.int32(), "int32"),
+    (pa.int64(), "int64"),
+    (pa.uint8(), "uint8"),
+    (pa.uint16(), "uint16"),
+    (pa.uint32(), "uint32"),
+    (pa.uint64(), "uint64"),
+    (pa.float16(), "float16"),
+    (pa.float32(), "float32"),
+    (pa.float64(), "float64"),
+    (pa.string(), "utf8"),
+    (pa.binary(), "binary"),
+    (pa.date32(), "date32"),
+    (pa.date64(), "date64"),
+)
+
+
 def _canonical_arrow_type(data_type: pa.DataType) -> str:
-    primitive_types = (
-        (pa.bool_(), "bool"),
-        (pa.int8(), "int8"),
-        (pa.int16(), "int16"),
-        (pa.int32(), "int32"),
-        (pa.int64(), "int64"),
-        (pa.uint8(), "uint8"),
-        (pa.uint16(), "uint16"),
-        (pa.uint32(), "uint32"),
-        (pa.uint64(), "uint64"),
-        (pa.float16(), "float16"),
-        (pa.float32(), "float32"),
-        (pa.float64(), "float64"),
-        (pa.string(), "utf8"),
-        (pa.large_utf8(), "large_utf8"),
-        (pa.binary(), "binary"),
-        (pa.large_binary(), "large_binary"),
-        (pa.date32(), "date32"),
-        (pa.date64(), "date64"),
-    )
-    for candidate, name in primitive_types:
+    """The server's V1 Function type grammar. Anything outside it is rejected
+    here rather than at registration."""
+    for candidate, name in _GRAMMAR_PRIMITIVES:
        if data_type == candidate:
            return name
-    if pa.types.is_fixed_size_binary(data_type):
-        return f"fixed_size_binary[{data_type.byte_width}]"
-    if pa.types.is_list(data_type):
-        return f"list<{_canonical_arrow_type(data_type.value_type)}>"
-    if pa.types.is_large_list(data_type):
-        return f"large_list<{_canonical_arrow_type(data_type.value_type)}>"
-    if pa.types.is_fixed_size_list(data_type):
+    if pa.types.is_list(data_type) or pa.types.is_large_list(data_type):
+        prefix = "list" if pa.types.is_list(data_type) else "large_list"
+        return f"{prefix}<{_canonical_list_item(data_type)}>"
+    if pa.types.is_fixed_size_list(data_type) and data_type.list_size > 0:
        return (
-            f"fixed_size_list<{_canonical_arrow_type(data_type.value_type)}>"
-            f"[{data_type.list_size}]"
+            f"fixed_size_list<{_canonical_list_item(data_type)}, {data_type.list_size}>"
        )
-    if pa.types.is_struct(data_type):
-        fields = ",".join(
-            f"{field.name}:{_canonical_arrow_type(field.type)}" for field in data_type
-        )
-        return f"struct<{fields}>"
-    if pa.types.is_timestamp(data_type):
-        timezone = f",tz={data_type.tz}" if data_type.tz is not None else ""
-        return f"timestamp[{data_type.unit}{timezone}]"
-    if pa.types.is_time32(data_type) or pa.types.is_time64(data_type):
-        return f"time[{data_type.unit}]"
-    if pa.types.is_duration(data_type):
-        return f"duration[{data_type.unit}]"
-    if pa.types.is_decimal(data_type):
-        bit_width = data_type.bit_width
-        return f"decimal{bit_width}({data_type.precision},{data_type.scale})"
    raise TypeError(f"unsupported Arrow type for Function signature: {data_type}")


+def _canonical_list_item(data_type: pa.DataType) -> str:
+    """The grammar names only the item type; it always means a non-nullable
+    child called `item`, so any other child metadata cannot be represented."""
+    child = data_type.value_field
+    if child.name != "item" or child.nullable or child.metadata:
+        raise TypeError(
+            "unsupported Arrow type for Function signature: list items must be a "
+            f"non-nullable field named 'item', got {child}"
+        )
+    return _canonical_arrow_type(child.type)
+
+
+def _list_of(item: pa.DataType) -> pa.DataType:
+    return pa.list_(pa.field("item", item, nullable=False))
+
+
 def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]:
    nullable = False
    origin = get_origin(annotation)
@@ -583,7 +591,7 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]:
        value_type, value_nullable = _annotation_type(arguments[0])
        if value_nullable:
            raise TypeError("nullable Function list elements are not supported")
-        return pa.list_(value_type), nullable
+        return _list_of(value_type), nullable
    raise TypeError(f"unsupported Function annotation: {annotation!r}")


@@ -730,6 +738,104 @@ def _literal_source(value: Any) -> str:
    )


+_DYNAMIC_NAMESPACE_ACCESS = frozenset(
+    {"globals", "locals", "vars", "eval", "exec", "compile", "__import__"}
+)
+# Modules that hand out namespaces (`sys.modules`, `builtins`, importers,
+# introspection). The artifact's module namespace holds only the names it was
+# packaged with, so reaching around it cannot be represented.
+_NAMESPACE_MODULES = frozenset(
+    {"sys", "builtins", "importlib", "inspect", "gc", "ctypes", "types"}
+)
+
+
+def _namespace_acquisition(
+    definition: ast.FunctionDef, references: set[str]
+) -> list[str]:
+    found = set(references & _DYNAMIC_NAMESPACE_ACCESS)
+    for node in ast.walk(definition):
+        if isinstance(node, ast.Import):
+            found.update(
+                alias.name
+                for alias in node.names
+                if alias.name.split(".")[0] in _NAMESPACE_MODULES
+            )
+        elif isinstance(node, ast.ImportFrom) and node.module:
+            if node.module.split(".")[0] in _NAMESPACE_MODULES:
+                found.add(node.module)
+    return sorted(found)
+
+
+def _module_references(module_source: str) -> set[str]:
+    """Names any scope in `module_source` binds or loads at module scope.
+    Python's own scope analysis on the exact text that ships: free variables
+    belong to an enclosing scope inside the function, and postponed
+    annotations are not runtime loads."""
+
+    def visit(table: symtable.SymbolTable, found: set[str]) -> None:
+        for symbol in table.get_symbols():
+            if symbol.is_global() and (
+                symbol.is_referenced() or symbol.is_declared_global()
+            ):
+                found.add(symbol.get_name())
+        for child in table.get_children():
+            visit(child, found)
+
+    found: set[str] = set()
+    for table in symtable.symtable(module_source, "<udf>", "exec").get_children():
+        visit(table, found)
+    return found
+
+
+def _global_source(name: str, value: Any) -> str:
+    """One module-level line that rebinds `name` to `value` in the artifact:
+    an import for modules and importable classes/functions, a literal otherwise."""
+    if isinstance(value, types.ModuleType):
+        if value.__name__.split(".")[0] in _NAMESPACE_MODULES:
+            raise ValueError(
+                f"@udf cannot package dynamic namespace access: {value.__name__!r}"
+            )
+        try:
+            imported = importlib.import_module(value.__name__)
+        except ImportError:
+            imported = None
+        if imported is not value:
+            raise TypeError(
+                f"Function source references module {name!r} that does not import "
+                f"as {value.__name__!r}"
+            )
+        return f"import {value.__name__} as {name}"
+    module_name = getattr(value, "__module__", None)
+    qualname = getattr(value, "__qualname__", None)
+    if (
+        isinstance(module_name, str)
+        and isinstance(qualname, str)
+        and module_name != "__main__"
+        and "." not in qualname
+        and "<" not in qualname
+    ):
+        try:
+            imported = getattr(importlib.import_module(module_name), qualname)
+        except (ImportError, AttributeError):
+            imported = None
+        if imported is value:
+            return f"from {module_name} import {qualname} as {name}"
+    return f"{name} = {_literal_source(value)}"
+
+
+def _is_recursive_reference(function: Callable[..., Any], name: str) -> bool:
+    """`name` inside the body means the function itself unless the module has
+    since bound it to something else."""
+    if name != function.__name__:
+        return False
+    bound = function.__globals__.get(name, function)
+    if bound is function:
+        return True
+    # The decorator's own result is the one wrapper known to call `function`
+    # unchanged; any other binding may behave differently from a self-call.
+    return type(bound) is UdfDefinition and bound._function is function
+
+
 def _package_source(function: Callable[..., Any]) -> bytes:
    if not inspect.isfunction(function) or inspect.iscoroutinefunction(function):
        raise TypeError("@udf requires a synchronous Python function")
@@ -754,23 +860,46 @@ def _package_source(function: Callable[..., Any]) -> bytes:
    closure = inspect.getclosurevars(function)
    if closure.nonlocals:
        raise ValueError("@udf cannot package functions that capture closure values")
-    if closure.unbound:
-        raise ValueError(
-            f"@udf source contains unresolved global names: {sorted(closure.unbound)!r}"
-        )
-    globals_source = []
-    for name, value in sorted(closure.globals.items()):
-        if isinstance(value, types.ModuleType):
-            globals_source.append(f"import {value.__name__} as {name}")
-        else:
-            globals_source.append(f"{name} = {_literal_source(value)}")
-
    function_source = ast.unparse(definition)
-    parts = ["from __future__ import annotations"]
+    module_header = "from __future__ import annotations"
+    references = _module_references(f"{module_header}\n\n{function_source}\n")
+    dynamic = _namespace_acquisition(definition, references)
+    if dynamic:
+        raise ValueError(f"@udf cannot package dynamic namespace access: {dynamic!r}")
+    # Resolve every module-scope reference the way the interpreter would: the
+    # function's own globals first (a module global may shadow a builtin, and
+    # nested scopes are not visible to getclosurevars), then its builtins.
+    # The artifact runs under the standard builtins; only the exact mapping is
+    # provably equivalent (a subclass or copy can change lookups and hooks).
+    if function.__builtins__ is not vars(builtins):
+        raise ValueError("@udf cannot package a non-standard builtins environment")
+    globals_source = []
+    unresolved = []
+    for name in sorted(references):
+        if name == function.__name__:
+            if not _is_recursive_reference(function, name):
+                raise ValueError(
+                    f"@udf cannot package {name!r}: the module binds that name to "
+                    "another value, which the artifact's own definition would shadow"
+                )
+            continue
+        if name in function.__globals__:
+            globals_source.append(_global_source(name, function.__globals__[name]))
+        elif hasattr(builtins, name):
+            pass
+        else:
+            unresolved.append(name)
+    if unresolved:
+        raise ValueError(
+            f"@udf source contains unresolved global names: {unresolved!r}"
+        )
+
+    parts = [module_header]
    if globals_source:
        parts.extend(["", *globals_source])
    parts.extend(["", function_source, ""])
-    return "\n".join(parts).encode("utf-8")
+    packaged = "\n".join(parts)
+    return packaged.encode("utf-8")


 class UdfDefinition:
@@ -917,6 +1046,13 @@ def udf(
    python_version : str, optional
        Remote Python major/minor version. Defaults to the client version.

+    The packaged artifact is a snapshot: the function source plus exactly
+    the module-level names it references (modules as imports, importable
+    classes and functions as imports, literals inline). Code that reaches the
+    module namespace another way -- ``globals()``/``eval``, ``sys.modules``,
+    ``builtins`` -- is rejected where it can be seen and otherwise
+    unsupported; closures and a non-standard ``__builtins__`` are rejected.
+
    Returns
    -------
    UdfDefinition
@@ -5,12 +5,11 @@

 import asyncio
 from datetime import timedelta
-from typing import Any, Generic, Optional, TypeVar, cast
+from typing import Any, Callable, Generic, Optional, TypeVar, cast

 from lancedb.background_loop import LOOP

 from . import _lancedb
-from .functions import FunctionVersion

 T = TypeVar("T")

@@ -18,11 +17,18 @@ T = TypeVar("T")
 class AsyncJob(Generic[T]):
    """A handle to an operation that may still be running.

-    The operation may already be complete when the handle is created.
+    The operation may already be complete when the handle is created. ``T``
+    is the endpoint's terminal result type; unit-result jobs resolve to
+    ``None``.
    """

-    def __init__(self, inner: Optional[Any]):
+    def __init__(
+        self,
+        inner: Optional[Any],
+        result_decoder: Optional[Callable[[Any], T]] = None,
+    ):
        self._inner = inner
+        self._result_decoder = result_decoder

    @property
    def id(self) -> Optional[str]:
@@ -50,17 +56,21 @@ class AsyncJob(Generic[T]):
    async def wait(self, timeout: Optional[timedelta] = None) -> T:
        """Wait until the operation reaches a terminal state.

+        Returns the endpoint's typed result, or ``None`` for a unit-result
+        job.
+
        Raises `JobFailedError` if the operation failed, `JobCancelledError`
        if it was cancelled, and `TimeoutError` if `timeout` elapses first.
        """
        if self._inner is None:
            return cast(T, None)
        if timeout is None:
-            return cast(T, await self._inner.wait())
-        return cast(
-            T,
-            await asyncio.wait_for(self._inner.wait(), timeout.total_seconds()),
-        )
+            result = await self._inner.wait()
+        else:
+            result = await asyncio.wait_for(self._inner.wait(), timeout.total_seconds())
+        if self._result_decoder is not None:
+            return self._result_decoder(result)
+        return cast(T, result)

    async def cancel(self):
        """Request cancellation. Cancelling a finished operation is a no-op."""
@@ -70,7 +80,7 @@ class AsyncJob(Generic[T]):


 class Job(Generic[T]):
-    """Synchronous counterpart of `AsyncJob`."""
+    """Synchronous counterpart of `AsyncJob` with the same result type."""

    def __init__(self, inner: Optional[AsyncJob[T]]):
        self._inner = inner
@@ -96,6 +106,9 @@ class Job(Generic[T]):
    def wait(self, timeout: Optional[timedelta] = None) -> T:
        """Block until the operation reaches a terminal state.

+        Returns the endpoint's typed result, or ``None`` for a unit-result
+        job.
+
        Raises `JobFailedError` if the operation failed, `JobCancelledError`
        if it was cancelled, and `TimeoutError` if `timeout` elapses first.
        """
@@ -110,23 +123,8 @@ class Job(Generic[T]):
        LOOP.run(self._inner.cancel())


-class _FunctionJobAdapter:
-    def __init__(self, inner: "_lancedb.FunctionJob"):
-        self._inner = inner
-
-    @property
-    def id(self) -> Optional[str]:
-        return self._inner.id
-
-    async def status(self) -> str:
-        return await self._inner.status()
-
-    async def wait(self) -> FunctionVersion:
-        return FunctionVersion.from_json(await self._inner.wait())
-
-    async def cancel(self):
-        await self._inner.cancel()
-
-
-def _function_job(inner: "_lancedb.FunctionJob") -> AsyncJob[FunctionVersion]:
-    return AsyncJob(_FunctionJobAdapter(inner))
+def _typed_job(
+    inner: "_lancedb.Job", result_decoder: Callable[[str], T]
+) -> AsyncJob[T]:
+    """Bind an internal JSON-producing job to its public result model."""
+    return AsyncJob(inner, result_decoder)
@@ -391,6 +391,15 @@ def _table_to_pickle_state(table: Table) -> dict[str, Any]:
    }


+def _drop_base_version(permutation_data: pa.Table) -> pa.Table:
+    """Strip the recorded base version so the reader leaves the base table unpinned."""
+    metadata = dict(permutation_data.schema.metadata or {})
+    if metadata.pop(b"base_version", None) is None:
+        return permutation_data
+    metadata.pop(b"base_branch", None)
+    return permutation_data.replace_schema_metadata(metadata)
+
+
 def _table_from_pickle_state(state: dict[str, Any]) -> Table:
    from . import connect

@@ -679,11 +688,15 @@ class Permutation:
        from . import connect

        connection_factory = state["connection_factory"]
+        rebuilt_base = False
        if connection_factory is not None:
            base_table = connection_factory(state["base_table_name"])
        elif "base_table_state" in state:
-            base_table = _table_from_pickle_state(state["base_table_state"])
+            base_state = state["base_table_state"]
+            rebuilt_base = base_state["kind"] == "memory"
+            base_table = _table_from_pickle_state(base_state)
        elif "base_table_data" in state:
+            rebuilt_base = True
            # In-memory base table inlined into the pickle; rebuild the same
            # way we rebuild the in-memory permutation table.
            mem_db = connect("memory://")
@@ -701,11 +714,14 @@ class Permutation:
            )

        permutation_table: Optional[Table] = None
-        if state["permutation_data"] is not None:
+        permutation_data = state["permutation_data"]
+        if permutation_data is not None:
+            if rebuilt_base:
+                # The base table was materialized from Arrow, so it is a fresh
+                # single-version dataset and the recorded pin cannot resolve on it.
+                permutation_data = _drop_base_version(permutation_data)
            mem_db = connect("memory://")
-            permutation_table = mem_db.create_table(
-                "permutation", state["permutation_data"]
-            )
+            permutation_table = mem_db.create_table("permutation", permutation_data)

        self.base_table = base_table
        self.permutation_table = permutation_table
@@ -49,7 +49,7 @@ from lancedb.index import (
    LabelList,
 )
 from lancedb.job import Job
-from lancedb.functions import FunctionApplication
+from lancedb.functions import FunctionApplication, RefreshColumnResult
 from lancedb.remote.db import LOOP
 from lancedb.table import IndexConfigType, KNOWN_METRICS
 import pyarrow as pa
@@ -972,7 +972,7 @@ class RemoteTable(Table):
    def refresh_column(self, column: str):
        return LOOP.run(self._table.refresh_column(column))

-    def refresh_column_async(self, column: str) -> Job:
+    def refresh_column_async(self, column: str) -> Job[RefreshColumnResult]:
        return Job(LOOP.run(self._table.refresh_column_async(column)))

    def alter_columns(
@@ -40,7 +40,7 @@ from ._blob import (
 from .types import BlobMode
 from lancedb.arrow import peek_reader
 from lancedb.background_loop import LOOP, embedding_executor
-from lancedb.job import AsyncJob, Job
+from lancedb.job import AsyncJob, Job, _typed_job
 from .dependencies import (
    _check_for_hugging_face,
    _check_for_lance,
@@ -72,7 +72,10 @@ from .index import (
    FTS,
 )
 from .expr import Expr
-from .functions import FunctionApplication
+from .functions import (
+    FunctionApplication,
+    RefreshColumnResult as RefreshColumnJobResult,
+)
 from .merge import LanceMergeInsertBuilder
 from .pydantic import LanceModel, model_to_dict
 from .query import (
@@ -2039,7 +2042,7 @@ class Table(ABC):
        """

    @abstractmethod
-    def refresh_column_async(self, column: str) -> Job:
+    def refresh_column_async(self, column: str) -> Job[RefreshColumnJobResult]:
        """
        Like :meth:`refresh_column`, but returns a handle to the refresh job
        instead of blocking until it completes.
@@ -2050,6 +2053,12 @@ class Table(ABC):
        than failing the job. On local tables the job runs in-process; on
        LanceDB Cloud and Enterprise it is the server's backfill job.

+        Returns
+        -------
+        Job[RefreshColumnResult]
+            A job whose successful ``wait`` returns row counts plus the source
+            and published table versions.
+
        Examples
        --------
        >>> import lancedb
@@ -2058,7 +2067,9 @@ class Table(ABC):
        >>> table.add_columns(computed={"doubled": "x * 2"})
        AddColumnsResult(version=2)
        >>> job = table.refresh_column_async("doubled")
-        >>> job.wait()
+        >>> result = job.wait()
+        >>> result.rows_assigned
+        2
        >>> job.status()
        'finished'
        """
@@ -4082,7 +4093,7 @@ class LanceTable(Table):
        [`AsyncTable.refresh_column`][lancedb.AsyncTable.refresh_column]."""
        return LOOP.run(self._table.refresh_column(column))

-    def refresh_column_async(self, column: str) -> Job:
+    def refresh_column_async(self, column: str) -> Job[RefreshColumnJobResult]:
        """Fill a computed column's unfilled rows, returning a handle to the
        refresh job. See
        [`Table.refresh_column_async`][lancedb.table.Table.refresh_column_async].
@@ -6122,7 +6133,9 @@ class AsyncTable:
        """
        return await self._inner.refresh_column(column)

-    async def refresh_column_async(self, column: str) -> AsyncJob:
+    async def refresh_column_async(
+        self, column: str
+    ) -> AsyncJob[RefreshColumnJobResult]:
        """
        Like :meth:`refresh_column`, but returns a handle to the refresh job
        instead of blocking until it completes.
@@ -6134,6 +6147,12 @@ class AsyncTable:
        in-process; on LanceDB Cloud and Enterprise it is the server's
        backfill job.

+        Returns
+        -------
+        AsyncJob[RefreshColumnResult]
+            A job whose successful ``wait`` returns row counts plus the source
+            and published table versions.
+
        Examples
        --------
        >>> import asyncio
@@ -6143,12 +6162,16 @@ class AsyncTable:
        ...     table = await db.create_table("computed_job_async_demo", [{"x": 1}])
        ...     await table.add_columns(computed={"doubled": "x * 2"})
        ...     job = await table.refresh_column_async("doubled")
-        ...     await job.wait()
+        ...     result = await job.wait()
+        ...     assert result.rows_assigned == 1
        ...     return await job.status()
        >>> asyncio.run(refresh_in_background())
        'finished'
        """
-        return AsyncJob(await self._inner.refresh_column_async(column))
+        return _typed_job(
+            await self._inner.refresh_column_async(column),
+            RefreshColumnJobResult.from_json,
+        )

    async def alter_columns(
        self, *alterations: Iterable[dict[str, Any]]
@@ -774,7 +774,7 @@ def test_drop_table_async(tmp_db: lancedb.DBConnection):
    job = tmp_db.drop_table_async("test")
    assert job.id is None
    assert job.status() == "finished"
-    job.wait()
+    assert job.wait() is None
    assert tmp_db.table_names() == []

    tmp_db.create_table("test", data=data)
@@ -790,7 +790,7 @@ async def test_drop_table_async_connection(tmp_db_async: lancedb.AsyncConnection
    job = await tmp_db_async.drop_table_async("test")
    assert job.id is None
    assert await job.status() == "finished"
-    await job.wait()
+    assert await job.wait() is None
    assert await tmp_db_async.table_names() == []


@@ -3,7 +3,12 @@

 from __future__ import annotations

+import base64
 import contextlib
+import functools
+import importlib.util
+import types
+from datetime import date
 import http.server
 import json
 from pathlib import Path
@@ -16,6 +21,9 @@ import pytest
 import lancedb
 from lancedb.functions import UdfDefinition, udf

+THRESHOLD = 20
+_CACHE = None
+

 FIXTURES = (
    Path(__file__).parents[3]
@@ -71,9 +79,396 @@ def test_scalar_udf_matches_shared_registration_golden_and_remains_callable():
    _assert_no_secret_values(request)


+def _run_packaged(definition, *args):
+    """Execute the shipped artifact in a fresh namespace, as a worker would."""
+    source = base64.b64decode(definition.registration_request.artifact.content.data)
+    namespace: dict = {}
+    exec(compile(source, "<udf>", "exec"), namespace)
+    return namespace[definition.registration_request.artifact.entrypoint](*args)
+
+
+def test_udf_packages_attribute_access_and_body_imports():
+    @udf
+    def word_norm(body: str) -> float:
+        import numpy as np
+
+        try:
+            words = body.split()
+        except AttributeError as error:
+            raise ValueError(str(error)) from error
+        return float(np.linalg.norm([len(w) for w in words]))
+
+    assert _run_packaged(word_norm, "aa bb") == pytest.approx(8**0.5)
+
+
+def test_udf_packages_module_globals_and_global_caches():
+    @udf
+    def label(value: int) -> str:
+        return "big" if value >= THRESHOLD else "small"
+
+    assert _run_packaged(label, 21) == "big"
+
+    @udf
+    def cached(value: int) -> int:
+        global _CACHE
+        if _CACHE is None:
+            _CACHE = 40
+        return _CACHE + value
+
+    assert _run_packaged(cached, 2) == 42
+
+
+def test_udf_annotations_are_not_runtime_names():
+    @udf
+    def identity(value: date) -> date:
+        return value
+
+    assert _run_packaged(identity, date(2026, 8, 25)) == date(2026, 8, 25)
+
+
+def test_udf_nested_scopes_resolve_lexically():
+    @udf
+    def score(value: int) -> int:
+        offset = 2
+
+        def add_offset() -> int:
+            return value + offset
+
+        return add_offset() + sum(v for v in [0])
+
+    assert _run_packaged(score, 3) == 5
+
+
+def test_udf_resolves_module_globals_before_builtins(tmp_path):
+    module_path = tmp_path / "shadowing_udfs.py"
+    module_path.write_text(
+        "max = 7\n"
+        "len = lambda _: 99\n"
+        "\n"
+        "def uses_literal_shadow(value: int) -> int:\n"
+        "    def nested() -> int:\n"
+        "        return max\n"
+        "    return nested() + value\n"
+        "\n"
+        "def uses_callable_shadow(value: int) -> int:\n"
+        "    def nested() -> int:\n"
+        "        return len([1])\n"
+        "    return nested() + value\n"
+    )
+    spec = importlib.util.spec_from_file_location("shadowing_udfs", module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+
+    # The module's `max = 7` is what the interpreter would use, so it ships.
+    assert _run_packaged(udf(module.uses_literal_shadow), 1) == 8
+    # A callable global cannot ship; it must not be silently swapped for the builtin.
+    with pytest.raises(TypeError, match="unsupported global value of type function"):
+        udf(module.uses_callable_shadow)
+
+
+def test_canonical_arrow_type_is_exactly_the_grammar():
+    from lancedb.functions import _GRAMMAR_PRIMITIVES, _canonical_arrow_type
+
+    golden = json.loads(
+        (
+            Path(__file__).parents[3]
+            / "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json"
+        ).read_text()
+    )
+    primitives = [
+        case["arrow_type"] for case in golden["valid"] if "<" not in case["arrow_type"]
+    ]
+    assert [name for _, name in _GRAMMAR_PRIMITIVES] == primitives
+    for outside in [
+        pa.timestamp("us"),
+        pa.decimal128(10, 2),
+        pa.large_string(),
+        pa.large_binary(),
+        pa.binary(4),
+        pa.duration("s"),
+        pa.struct([pa.field("a", pa.int32())]),
+        pa.list_(pa.float32(), 0),
+        pa.list_(pa.timestamp("us")),
+    ]:
+        with pytest.raises(TypeError, match="unsupported Arrow type"):
+            _canonical_arrow_type(outside)
+
+
+def test_udf_nested_annotations_are_postponed_in_the_artifact():
+    @udf
+    def score(value: int) -> int:
+        def identity(item: date) -> date:
+            return item
+
+        identity(date(2026, 8, 25))
+        return value
+
+    assert _run_packaged(score, 3) == 3
+
+
+def test_udf_ships_globals_the_body_deletes():
+    @udf
+    def clear(value: int) -> int:
+        global _CACHE
+        del _CACHE
+        return value
+
+    assert _run_packaged(clear, 3) == 3
+
+
+def test_udf_rejects_a_module_global_that_does_not_import_as_itself(tmp_path):
+    module_path = tmp_path / "fake_module_udfs.py"
+    module_path.write_text(
+        "import types\n"
+        "np = types.ModuleType('numpy')\n"
+        "np.sqrt = lambda x: 0\n"
+        "\n"
+        "def score(value: int) -> int:\n"
+        "    return int(np.sqrt(value))\n"
+    )
+    spec = importlib.util.spec_from_file_location("fake_module_udfs", module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    with pytest.raises(TypeError, match="does not import as 'numpy'"):
+        udf(module.score)
+
+
+def test_udf_rejects_a_module_level_namespace_alias(tmp_path):
+    module_path = tmp_path / "aliasing_udfs.py"
+    module_path.write_text(
+        "import builtins as b\n"
+        "THRESHOLD = 5\n"
+        "\n"
+        "def score(value: int) -> int:\n"
+        "    return value + b.vars(b.__import__('aliasing_udfs'))['THRESHOLD']\n"
+    )
+    spec = importlib.util.spec_from_file_location("aliasing_udfs", module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    with pytest.raises(ValueError, match="dynamic namespace access"):
+        udf(module.score)
+
+
+@pytest.mark.parametrize(
+    "access",
+    [
+        "globals()['THRESHOLD']",
+        "eval('THRESHOLD')",
+        "(lambda g: g()['THRESHOLD'])(globals)",
+        "__import__('sys').modules[__name__].THRESHOLD",
+        "sys.modules[__name__].THRESHOLD",
+    ],
+)
+def test_udf_rejects_dynamic_namespace_access(access):
+    namespace: dict = {}
+    exec(
+        f"def score(value: int) -> int:\n    return value + {access}\n",
+        {"THRESHOLD": 5},
+        namespace,
+    )
+    with pytest.raises(ValueError, match="dynamic namespace access"):
+        _package_from_text(
+            "def score(value: int) -> int:\n"
+            "    import sys\n"
+            f"    return value + {access}\n"
+        )
+
+
+def _package_from_text(source: str, module_globals: dict | None = None):
+    """Load `source` as a real module file so the packager can inspect it."""
+    import tempfile
+
+    directory = tempfile.mkdtemp()
+    path = Path(directory) / "generated_udf_module.py"
+    path.write_text(source)
+    spec = importlib.util.spec_from_file_location(f"generated_udf_{id(source)}", path)
+    module = importlib.util.module_from_spec(spec)
+    if module_globals:
+        module.__dict__.update(module_globals)
+    spec.loader.exec_module(module)
+    functions = [
+        value
+        for value in vars(module).values()
+        if callable(value) and getattr(value, "__module__", None) == module.__name__
+    ]
+    return udf(functions[0])
+
+
+def test_udf_rejects_a_non_standard_builtins_environment():
+    def score(value: int) -> int:
+        return len([1]) + value
+
+    score.__globals__  # noqa: B018 -- real function, real globals
+    import builtins
+
+    patched = types.FunctionType(
+        score.__code__,
+        {"__builtins__": {**vars(builtins), "len": lambda _: 99}},
+        "score",
+    )
+    patched.__annotations__ = score.__annotations__
+    assert patched(3) == 102
+    with pytest.raises(ValueError, match="non-standard builtins environment"):
+        udf(patched)
+
+    class ReportingDict(dict):  # reports standard entries, resolves differently
+        def __missing__(self, key):
+            return vars(builtins)[key]
+
+    disguised = types.FunctionType(
+        score.__code__, {"__builtins__": ReportingDict(len=lambda _: 99)}, "score"
+    )
+    disguised.__annotations__ = score.__annotations__
+    assert disguised(3) == 102
+    with pytest.raises(ValueError, match="non-standard builtins environment"):
+        udf(disguised)
+
+    hooked = types.FunctionType(
+        score.__code__,
+        {"__builtins__": {**vars(builtins), "__import__": lambda *a, **k: None}},
+        "score",
+    )
+    hooked.__annotations__ = score.__annotations__
+    with pytest.raises(ValueError, match="non-standard builtins environment"):
+        udf(hooked)
+
+
+def test_udf_recursion_versus_a_rebound_module_name(tmp_path):
+    module_path = tmp_path / "rebound_udfs.py"
+    module_path.write_text(
+        "def fact(value: int) -> int:\n"
+        "    return 1 if value <= 1 else value * fact(value - 1)\n"
+        "\n"
+        "def score(value: int) -> int:\n"
+        "    return score + value\n"
+    )
+    spec = importlib.util.spec_from_file_location("rebound_udfs", module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    assert _run_packaged(udf(module.fact), 5) == 120
+    raw = module.score
+    module.score = 10
+    with pytest.raises(ValueError, match="binds that name to another value"):
+        udf(raw)
+    # A wrapper that merely exposes __wrapped__ is not the function.
+    module.score = functools.wraps(raw)(lambda value: 41)
+    with pytest.raises(ValueError, match="binds that name to another value"):
+        udf(raw)
+    # The decorator's own result is; a subclass of it is not.
+    module.fact = udf(module.fact)
+    assert _run_packaged(module.fact, 4) == 24
+
+    class Twisted(UdfDefinition):
+        def __call__(self, *args, **kwargs):
+            return 41
+
+    raw_fact = module.fact._function
+    module.fact = Twisted(
+        raw_fact,
+        name=None,
+        input_schema=None,
+        output_schema=None,
+        pip=(),
+        env={},
+        secrets=(),
+        python_version=None,
+    )
+    with pytest.raises(ValueError, match="binds that name to another value"):
+        udf(raw_fact)
+
+
+def test_canonical_arrow_type_rejects_unrepresentable_list_children():
+    from lancedb.functions import _canonical_arrow_type
+
+    for outside in [
+        pa.list_(pa.float32()),  # pyarrow default: nullable child
+        pa.list_(pa.field("custom", pa.float32(), nullable=False)),
+        pa.list_(pa.field("item", pa.float32(), nullable=False, metadata={"k": "v"})),
+        pa.list_(pa.field("item", pa.float32(), nullable=False), 0),
+    ]:
+        with pytest.raises(TypeError, match="unsupported Arrow type"):
+            _canonical_arrow_type(outside)
+    assert (
+        _canonical_arrow_type(
+            pa.list_(pa.field("item", pa.float32(), nullable=False), 3)
+        )
+        == "fixed_size_list<float32, 3>"
+    )
+
+
+def _calls_missing(value: int) -> int:
+    return missing(value)  # noqa: F821
+
+
+def _shadows_missing_in_a_comprehension(value: int) -> int:
+    return missing(value) + sum(missing for missing in ())  # noqa: F821
+
+
+def _shadows_missing_in_a_lambda(value: int) -> int:
+    return (lambda missing: missing)(value) + missing  # noqa: F821
+
+
+@pytest.mark.parametrize(
+    "function",
+    [_calls_missing, _shadows_missing_in_a_comprehension, _shadows_missing_in_a_lambda],
+)
+def test_udf_rejects_a_truly_unresolved_global(function):
+    with pytest.raises(ValueError, match=r"unresolved global names: \['missing'\]"):
+        udf(function)
+
+
+def _arrow_type_from_golden(spec: dict) -> pa.DataType:
+    kind = spec["type"]
+    if kind in ("list", "large_list", "fixed_size_list"):
+        item = _arrow_type_from_golden(spec["fields"][0]["type"])
+        field = pa.field("item", item, nullable=False)
+        if kind == "list":
+            return pa.list_(field)
+        if kind == "large_list":
+            return pa.large_list(field)
+        return pa.list_(field, spec["length"])
+    return {
+        "null": pa.null(),
+        "bool": pa.bool_(),
+        "utf8": pa.string(),
+        "binary": pa.binary(),
+        "float16": pa.float16(),
+        "float32": pa.float32(),
+        "float64": pa.float64(),
+        "date32": pa.date32(),
+        "date64": pa.date64(),
+    }.get(kind) or getattr(pa, kind)()
+
+
+def test_arrow_type_grammar_matches_the_shared_golden():
+    golden = json.loads(
+        (
+            Path(__file__).parents[3]
+            / "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json"
+        ).read_text()
+    )
+    from lancedb.functions import _canonical_arrow_type
+
+    emitted = {
+        case["arrow_type"]: _canonical_arrow_type(_arrow_type_from_golden(case["json"]))
+        for case in golden["valid"]
+    }
+    assert emitted == {
+        case["arrow_type"]: case["arrow_type"] for case in golden["valid"]
+    }
+    assert not set(emitted) & set(golden["invalid"])
+    for case in golden["server_only"]:
+        with pytest.raises(TypeError, match="unsupported Arrow type"):
+            _canonical_arrow_type(_arrow_type_from_golden(case["json"]))
+
+
 def test_explicit_arrow_schema_is_deterministic():
    input_schema = pa.schema([pa.field("value", pa.float32(), nullable=True)])
-    output_schema = pa.field("embedding", pa.list_(pa.float32(), 3), nullable=False)
+    output_schema = pa.field(
+        "embedding",
+        pa.list_(pa.field("item", pa.float32(), nullable=False), 3),
+        nullable=False,
+    )

    @udf(input_schema=input_schema, output_schema=output_schema)
    def explicit(value):
@@ -82,7 +477,7 @@ def test_explicit_arrow_schema_is_deterministic():
    signature = explicit.registration_request.signature
    assert signature.inputs[0].arrow_type == "float32"
    assert signature.inputs[0].nullable is True
-    assert signature.output.arrow_type == "fixed_size_list<float32>[3]"
+    assert signature.output.arrow_type == "fixed_size_list<float32, 3>"
    assert signature.output.nullable is False


@@ -187,7 +582,7 @@ def _mock_remote_function_catalog():
                    "job_state": "DONE",
                    "result": state["version"],
                }
-            elif self.path == "/v1/functions/get":
+            elif self.path == "/v1/functions/describe":
                assert body == {
                    "name": "normalize_score",
                    "version": "fv_exact",
@@ -88,7 +88,7 @@ async def binary_table(db_async):
 async def test_create_index_async_returns_done_job(some_table: AsyncTable):
    job = await some_table.create_index_async("id", config=BTree())
    assert job.id is None
-    await job.wait()
+    assert await job.wait() is None
    assert len(await some_table.list_indices()) == 1
    await job.cancel()

@@ -56,6 +56,31 @@ def test_execute_does_not_reenter_background_loop(tmp_path, monkeypatch):
    assert permutation_tbl._conn.read_consistency_interval is None


+def test_pickled_permutation_reads_pinned_version(tmp_path):
+    """An unpickled copy must still read the pinned version, which also covers the
+    version surviving the ``to_arrow()`` round trip in ``__getstate__``."""
+    import pickle
+
+    db = connect(tmp_path)
+    tbl = db.create_table("base", pa.table({"idx": range(20)}))
+    permutation_tbl = permutation_builder(tbl).execute()
+    perm = Permutation.from_tables(tbl, permutation_tbl)
+
+    payload = pickle.dumps(perm)
+
+    # Compact so the stored row addresses no longer describe these rows at latest.
+    tbl.delete("true")
+    tbl.optimize()
+    assert tbl.count_rows() == 0
+
+    # Unpickle after the mutation: __setstate__ reopens at latest, so this only
+    # passes if the recorded version is applied on reopen.
+    restored = pickle.loads(payload)
+    assert len(restored) == 20
+    rows = restored.__getitems__(list(range(20)))
+    assert sorted(row["idx"] for row in rows) == list(range(20))
+
+
 def test_split_random_counts(mem_db):
    """Test random splitting with absolute counts."""
    tbl = mem_db.create_table(
@@ -875,11 +875,85 @@ def test_remote_create_index_async_returns_job():
        table = db.create_table("test", [{"id": 1}])
        job = table.create_index_async("id", config=BTree())
        assert job.id == "job-1"
-        job.wait(timeout=timedelta(seconds=30))
+        assert job.wait(timeout=timedelta(seconds=30)) is None
        assert len(describe_calls) == 2
        job.cancel()


+def test_remote_refresh_async_returns_typed_terminal_result():
+    terminal_result = {
+        "rows_assigned": 12,
+        "rows_failed": 0,
+        "rows_remaining": 0,
+        "source_version": 7,
+        "published_version": 8,
+    }
+
+    def handler(request):
+        content_len = int(request.headers.get("Content-Length", 0))
+        body = request.rfile.read(content_len) if content_len > 0 else b""
+        if request.path == "/v1/table/test/backfill_column":
+            assert json.loads(body)["column"] == "derived"
+            request.send_response(202)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(b'{"job_id": "refresh-1"}')
+        elif request.path == "/v1/jobs/describe":
+            assert json.loads(body)["job_id"] == "refresh-1"
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(
+                json.dumps(
+                    {
+                        "job_id": "refresh-1",
+                        "job_type": "function_refresh",
+                        "job_state": "DONE",
+                        "result": terminal_result,
+                    }
+                ).encode()
+            )
+        elif request.path == "/v1/table/test/create/?mode=create":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(b"{}")
+        elif request.path == "/v1/table/test/describe/":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(
+                json.dumps(
+                    {
+                        "version": 1,
+                        "schema": {
+                            "fields": [
+                                {
+                                    "name": "id",
+                                    "type": {"type": "int64"},
+                                    "nullable": False,
+                                }
+                            ]
+                        },
+                    }
+                ).encode()
+            )
+        else:
+            request.send_response(404)
+            request.end_headers()
+
+    with mock_lancedb_connection(handler) as db:
+        table = db.create_table("test", [{"id": 1}])
+        job = table.refresh_column_async("derived")
+        assert job.id == "refresh-1"
+        result = job.wait(timeout=timedelta(seconds=30))
+
+    assert isinstance(result, lancedb.RefreshColumnResult)
+    assert result.model_dump() == terminal_result
+    assert result.rows_filled == 12
+    assert result.version == 8
+
+
 def test_remote_job_wait_raises_on_failure():
    from lancedb.exceptions import JobFailedError
    from lancedb.index import BTree
@@ -1467,7 +1467,7 @@ def test_create_index_async_returns_done_job(mem_db: DBConnection):
    table = mem_db.create_table("job_test", [{"id": i} for i in range(10)])
    job = table.create_index_async("id", config=BTree())
    assert job.id is None
-    job.wait()
+    assert job.wait() is None
    assert len(table.list_indices()) == 1
    job.cancel()

@@ -3947,10 +3947,21 @@ def test_refresh_column_async_returns_job(tmp_path):

    job = table.refresh_column_async("doubled")
    assert job.id is None  # in-process jobs have no server id
-    assert job.wait() is None
+    result = job.wait()
+    assert isinstance(result, lancedb.RefreshColumnResult)
+    assert result.rows_assigned == 2
+    assert result.rows_failed == 0
+    assert result.rows_remaining == 0
+    assert result.source_version == 2
+    assert result.published_version == 3
    assert job.status() == "finished"
    assert sorted(table.to_arrow()["doubled"].to_pylist()) == [2, 4]

+    no_op = table.refresh_column_async("doubled").wait()
+    assert no_op.rows_assigned == 0
+    assert no_op.source_version == 3
+    assert no_op.published_version is None
+
    # Bad input raises at the call, not through the job.
    with pytest.raises(Exception, match="not a computed column"):
        table.refresh_column_async("x")
@@ -3963,6 +3974,10 @@ async def test_refresh_column_async_job_async_table(tmp_path):
    await table.add_columns(computed={"tripled": "x * 3"})

    job = await table.refresh_column_async("tripled")
-    assert await job.wait() is None
+    result = await job.wait()
+    assert isinstance(result, lancedb.RefreshColumnResult)
+    assert result.rows_assigned == 1
+    assert result.source_version == 2
+    assert result.published_version == 3
    assert await job.status() == "finished"
    assert (await table.to_arrow())["tripled"].to_pylist() == [9]
@@ -609,7 +609,7 @@ impl Connection {
                .create_function_async(request)
                .await
                .infer_error()
-                .map(crate::job::FunctionJob::new)
+                .map(crate::job::Job::new_typed)
        })
    }

@@ -5,72 +5,33 @@ use std::sync::Arc;

 use crate::runtime::future_into_py;
 use pyo3::{Bound, PyAny, PyRef, PyResult, pyclass, pymethods};
+use serde::Serialize;

 use crate::error::PythonErrorExt;

 #[pyclass]
 pub struct Job {
-    inner: Arc<lancedb::Job>,
-}
-
-/// Python bridge for a typed remote Function registration job.
-///
-/// The public Python layer decodes the canonical JSON returned by `wait`
-/// into its immutable `FunctionVersion` model.
-#[pyclass]
-pub struct FunctionJob {
-    inner: Arc<lancedb::Job<lancedb::function::FunctionVersion>>,
-}
-
-impl FunctionJob {
-    pub(crate) fn new(inner: lancedb::Job<lancedb::function::FunctionVersion>) -> Self {
-        Self {
-            inner: Arc::new(inner),
-        }
-    }
+    inner: Arc<lancedb::Job<std::result::Result<Option<String>, String>>>,
 }

 impl Job {
    pub(crate) fn new(inner: lancedb::Job) -> Self {
        Self {
-            inner: Arc::new(inner),
+            inner: Arc::new(inner.map(|()| Ok(None))),
        }
    }
-}

-#[pymethods]
-impl FunctionJob {
-    #[getter]
-    pub fn id(&self) -> Option<String> {
-        self.inner.id().map(str::to_string)
-    }
-
-    pub fn status(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.inner.clone();
-        future_into_py(
-            self_.py(),
-            async move { inner.status().await.infer_error() },
-        )
-    }
-
-    pub fn wait(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.inner.clone();
-        future_into_py(self_.py(), async move {
-            inner
-                .wait()
-                .await
-                .infer_error()?
-                .to_canonical_json()
-                .infer_error()
-        })
-    }
-
-    pub fn cancel(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.inner.clone();
-        future_into_py(self_.py(), async move {
-            inner.cancel().await.infer_error()?;
-            Ok(())
-        })
+    pub(crate) fn new_typed<T>(inner: lancedb::Job<T>) -> Self
+    where
+        T: Clone + Serialize + Send + Sync + 'static,
+    {
+        Self {
+            inner: Arc::new(inner.map(|result| {
+                serde_json::to_string(&result)
+                    .map(Some)
+                    .map_err(|error| format!("failed to serialize typed job result: {error}"))
+            })),
+        }
    }
 }

@@ -92,8 +53,10 @@ impl Job {
    pub fn wait(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
        let inner = self_.inner.clone();
        future_into_py(self_.py(), async move {
-            inner.wait().await.infer_error()?;
-            Ok(None::<()>)
+            let result = inner.wait().await.infer_error()?;
+            result
+                .map_err(|message| lancedb::Error::Runtime { message })
+                .infer_error()
        })
    }

@@ -47,7 +47,6 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
    m.add_class::<Session>()?;
    m.add_class::<Table>()?;
    m.add_class::<crate::job::Job>()?;
-    m.add_class::<crate::job::FunctionJob>()?;
    m.add_class::<crate::job::JobInfo>()?;
    m.add_class::<crate::job::JobDescription>()?;
    m.add_class::<crate::job::JobFailureInfo>()?;
@@ -1619,7 +1619,7 @@ impl Table {
        let inner = self_.inner_ref()?.clone();
        future_into_py(self_.py(), async move {
            let job = inner.refresh_column_async(column).await.infer_error()?;
-            Ok(crate::job::Job::new(job))
+            Ok(crate::job::Job::new_typed(job))
        })
    }

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.38.0-beta.3"
+version = "0.38.0-beta.8"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
@@ -1679,6 +1679,50 @@ mod tests {
        assert_eq!(tables, names[..7]);
    }

+    #[tokio::test]
+    async fn test_list_tables_walks_page_boundaries() {
+        let tc = new_test_connection().await.unwrap();
+        if tc.is_remote {
+            // What resumes a page is the server's to decide, and asserting it here would be
+            // asserting the server's contract rather than this one.
+            return;
+        }
+        let db = tc.connection;
+        let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)]));
+        let mut names = Vec::with_capacity(5);
+        for _ in 0..5 {
+            let name = uuid::Uuid::new_v4().to_string();
+            names.push(name.clone());
+            db.create_empty_table(name, schema.clone())
+                .execute()
+                .await
+                .unwrap();
+        }
+        names.sort();
+
+        // Walking in pages has to reach every table exactly once, with nothing lost at a
+        // page boundary.
+        let mut seen = Vec::with_capacity(names.len());
+        let mut page_token = None;
+        loop {
+            let page = db
+                .list_tables(ListTablesRequest {
+                    id: Some(Vec::new()),
+                    limit: Some(2),
+                    page_token,
+                    ..Default::default()
+                })
+                .await
+                .unwrap();
+            seen.extend(page.tables);
+            page_token = page.page_token.filter(|token| !token.is_empty());
+            if page_token.is_none() {
+                break;
+            }
+        }
+        assert_eq!(seen, names);
+    }
+
    #[tokio::test]
    async fn test_open_table() {
        let tc = new_test_connection().await.unwrap();
@@ -974,17 +974,15 @@ impl Database for ListingDatabase {
            f.drain(0..index);
        }

-        // Determine if there's a next page
-        let next_page_token = if let Some(limit) = request.limit {
-            if f.len() > limit as usize {
-                let token = f[limit as usize].clone();
+        // Determine if there's a next page. The token is the last name of this page,
+        // not the first of the next one: the next page resumes strictly after the
+        // token, so naming the next page's first entry would skip it.
+        let next_page_token = match request.limit {
+            Some(limit) if f.len() > limit as usize => {
                f.truncate(limit as usize);
-                Some(token)
-            } else {
-                None
+                f.last().cloned()
            }
-        } else {
-            None
+            _ => None,
        };

        Ok(ListTablesResponse {
@@ -27,6 +27,12 @@ pub const SRC_ROW_ID_COL: &str = "row_id";

 pub const SPLIT_NAMES_CONFIG_KEY: &str = "split_names";

+/// Base table version the permutation was built against.
+pub const BASE_VERSION_CONFIG_KEY: &str = "base_version";
+
+/// Base table branch the permutation was built against.  Absent means main.
+pub const BASE_BRANCH_CONFIG_KEY: &str = "base_branch";
+
 pub const DEFAULT_MEMORY_LIMIT: usize = 100 * 1024 * 1024;

 /// Where to store the permutation table
@@ -214,21 +220,11 @@ impl PermutationBuilder {
        Ok(Box::pin(SimpleRecordBatchStream { schema, stream }))
    }

-    fn add_split_names(
+    fn add_config_metadata(
        data: SendableRecordBatchStream,
-        split_names: &[String],
+        metadata: HashMap<String, String>,
    ) -> Result<SendableRecordBatchStream> {
-        let schema = data
-            .schema()
-            .as_ref()
-            .clone()
-            .with_metadata(HashMap::from([(
-                SPLIT_NAMES_CONFIG_KEY.to_string(),
-                serde_json::to_string(split_names).map_err(|e| Error::Other {
-                    message: format!("Failed to serialize split names: {}", e),
-                    source: Some(e.into()),
-                })?,
-            )]));
+        let schema = data.schema().as_ref().clone().with_metadata(metadata);
        let schema = Arc::new(schema);
        let schema_clone = schema.clone();
        let stream = data.map_ok(move |batch| batch.with_schema(schema.clone()).unwrap());
@@ -239,7 +235,20 @@ impl PermutationBuilder {
    }

    /// Builds the permutation table and stores it in the given database.
-    pub async fn build(self) -> Result<Table> {
+    pub async fn build(mut self) -> Result<Table> {
+        // Remote tables resolve latest independently for each request. Use a
+        // separate pinned handle so count, projection, and scan all refer to one
+        // snapshot without changing the caller's table checkout state. Native
+        // tables return `None` here and retain their existing behavior.
+        if let Some(snapshot) = self
+            .base_table
+            .base_table()
+            .snapshot_at_current_version()
+            .await?
+        {
+            self.base_table = Table::from(snapshot);
+        }
+
        // Unflushed rows have no row id, so a permutation cannot address them.
        match self.base_table.base_table().get_lsm_write_spec().await {
            Ok(Some(_)) => {
@@ -256,9 +265,14 @@ impl PermutationBuilder {
            Err(err) => return Err(err),
        }

+        // The handle above is already pinned to one version.  Record which one, so a
+        // reader -- in a DataLoader worker, against a table that has since moved --
+        // resolves these row addresses against the same snapshot.
+        let base_version = self.base_table.version().await?;
+        let base_branch = self.base_table.current_branch();
+
        // First pass, apply filter and load row ids.  `Shuffler` permutes positions, so
        // every rank must scan the rows in the same order to build the same permutation.
-        // TODO: pin the version resolved here; remote does not implement Lazy.
        let mut rows = self.base_table.query().select(Select::columns(&[ROW_ID]));

        if let Some(filter) = &self.config.filter {
@@ -318,11 +332,24 @@ impl PermutationBuilder {
        // Rename _rowid to row_id
        let renamed = rename_column(sorted, ROW_ID, SRC_ROW_ID_COL)?;

-        let streaming_data = if let Some(split_names) = &self.config.split_names {
-            Self::add_split_names(renamed, split_names)?
-        } else {
-            renamed
-        };
+        let mut metadata = HashMap::from([(
+            BASE_VERSION_CONFIG_KEY.to_string(),
+            base_version.to_string(),
+        )]);
+        // Version numbers are per-branch, so the branch is part of the coordinate.
+        if let Some(branch) = &base_branch {
+            metadata.insert(BASE_BRANCH_CONFIG_KEY.to_string(), branch.clone());
+        }
+        if let Some(split_names) = &self.config.split_names {
+            metadata.insert(
+                SPLIT_NAMES_CONFIG_KEY.to_string(),
+                serde_json::to_string(split_names).map_err(|e| Error::Other {
+                    message: format!("Failed to serialize split names: {}", e),
+                    source: Some(e.into()),
+                })?,
+            );
+        }
+        let streaming_data = Self::add_config_metadata(renamed, metadata)?;

        let (name, database) = match &self.config.destination {
            PermutationDestination::Permanent(database, table_name) => {
@@ -409,6 +436,253 @@ mod tests {
        assert!(table.base_table().scan_order_is_deterministic());
    }

+    #[cfg(feature = "remote")]
+    #[tokio::test]
+    async fn test_remote_permutation_builder_pins_snapshot() {
+        use std::sync::{
+            Mutex,
+            atomic::{AtomicU64, Ordering},
+        };
+
+        use arrow_array::{RecordBatch, UInt64Array};
+        use arrow_schema::{DataType, Field, Schema};
+
+        let row_ids = RecordBatch::try_new(
+            Arc::new(Schema::new(vec![Field::new(
+                ROW_ID,
+                DataType::UInt64,
+                false,
+            )])),
+            vec![Arc::new(UInt64Array::from(vec![100]))],
+        )
+        .unwrap();
+        let mut query_body = Vec::new();
+        {
+            let mut writer =
+                arrow_ipc::writer::FileWriter::try_new(&mut query_body, &row_ids.schema()).unwrap();
+            writer.write(&row_ids).unwrap();
+            writer.finish().unwrap();
+        }
+
+        let latest = Arc::new(AtomicU64::new(7));
+        let expected_snapshot = Arc::new(AtomicU64::new(7));
+        let planning_versions = Arc::new(Mutex::new(Vec::new()));
+        let latest_ref = latest.clone();
+        let expected_snapshot_ref = expected_snapshot.clone();
+        let planning_versions_ref = planning_versions.clone();
+        let table = Table::new_with_handler("remote_base", move |request| {
+            let path = request.url().path();
+            let body = request
+                .body()
+                .and_then(|body| body.as_bytes())
+                .map(|body| serde_json::from_slice::<serde_json::Value>(body).unwrap());
+
+            match path {
+                "/v1/table/remote_base/describe/" => {
+                    let requested = body.as_ref().and_then(|body| body["version"].as_u64());
+                    let version = requested.unwrap_or_else(|| latest_ref.load(Ordering::SeqCst));
+                    http::Response::builder()
+                        .status(200)
+                        .body(
+                            format!(r#"{{"version":{version},"schema":{{"fields":[]}}}}"#)
+                                .into_bytes(),
+                        )
+                        .unwrap()
+                }
+                "/v1/table/remote_base/get_lsm_write_spec/" => http::Response::builder()
+                    .status(200)
+                    .body(br#"{"lsm_write_spec":null}"#.to_vec())
+                    .unwrap(),
+                "/v1/table/remote_base/count_rows/" => {
+                    let body = body.unwrap();
+                    let version = body["version"].as_u64().unwrap();
+                    assert_eq!(version, expected_snapshot_ref.load(Ordering::SeqCst));
+                    assert_eq!(body["predicate"], "value > 0");
+                    planning_versions_ref.lock().unwrap().push(version);
+
+                    // Simulate a concurrent append after count_rows. An unpinned
+                    // scan would now resolve version 8 and include different rows.
+                    latest_ref.store(8, Ordering::SeqCst);
+                    http::Response::builder()
+                        .status(200)
+                        .body(b"1".to_vec())
+                        .unwrap()
+                }
+                "/v1/table/remote_base/query/" => {
+                    let body = body.unwrap();
+                    let version = body["version"].as_u64().unwrap();
+                    assert_eq!(version, expected_snapshot_ref.load(Ordering::SeqCst));
+                    assert_eq!(body["filter"], "value > 0");
+                    assert_eq!(body["columns"], serde_json::json!([ROW_ID]));
+                    planning_versions_ref.lock().unwrap().push(version);
+                    http::Response::builder()
+                        .status(200)
+                        .header("content-type", "application/vnd.apache.arrow.file")
+                        .body(query_body.clone())
+                        .unwrap()
+                }
+                _ => panic!("unexpected request: {path}"),
+            }
+        });
+
+        let permutation = PermutationBuilder::new(table.clone())
+            .with_filter("value > 0".to_string())
+            .build()
+            .await
+            .unwrap();
+        assert_eq!(permutation.count_rows(None).await.unwrap(), 1);
+
+        // Building uses a separate handle and must not pin the caller's table.
+        assert_eq!(table.version().await.unwrap(), 8);
+
+        // An explicit checkout is copied as-is and remains checked out afterward.
+        expected_snapshot.store(6, Ordering::SeqCst);
+        table.checkout(6).await.unwrap();
+        let permutation = PermutationBuilder::new(table.clone())
+            .with_filter("value > 0".to_string())
+            .build()
+            .await
+            .unwrap();
+        assert_eq!(permutation.count_rows(None).await.unwrap(), 1);
+        assert_eq!(table.version().await.unwrap(), 6);
+        assert_eq!(*planning_versions.lock().unwrap(), vec![7, 7, 6, 6]);
+    }
+
+    #[tokio::test]
+    async fn test_permutation_records_base_version() {
+        let temp_dir = tempfile::tempdir().unwrap();
+
+        let db = connect(temp_dir.path().to_str().unwrap())
+            .execute()
+            .await
+            .unwrap();
+
+        let initial_data = lance_datagen::gen_batch()
+            .col("col_a", lance_datagen::array::step::<Int32Type>())
+            .into_ldb_stream(RowCount::from(100), BatchCount::from(2));
+        let data_table = db
+            .create_table("base_tbl", initial_data)
+            .execute()
+            .await
+            .unwrap();
+
+        let build_version = data_table.version().await.unwrap();
+        let permutation_table = PermutationBuilder::new(data_table.clone())
+            .build()
+            .await
+            .unwrap();
+
+        let recorded = permutation_table
+            .schema()
+            .await
+            .unwrap()
+            .metadata
+            .get(BASE_VERSION_CONFIG_KEY)
+            .expect("permutation should record the base version")
+            .parse::<u64>()
+            .unwrap();
+        assert_eq!(recorded, build_version);
+
+        // Advancing the base table must not move the recorded version.
+        let more_data = lance_datagen::gen_batch()
+            .col("col_a", lance_datagen::array::step::<Int32Type>())
+            .into_ldb_stream(RowCount::from(50), BatchCount::from(1));
+        data_table.add(more_data).execute().await.unwrap();
+        assert!(data_table.version().await.unwrap() > recorded);
+        assert_eq!(
+            permutation_table
+                .schema()
+                .await
+                .unwrap()
+                .metadata
+                .get(BASE_VERSION_CONFIG_KEY)
+                .unwrap()
+                .parse::<u64>()
+                .unwrap(),
+            recorded,
+        );
+    }
+
+    /// Version numbers are per-branch, so a permutation built on a branch must record
+    /// it -- a worker reopens by name and lands on main at the same number.
+    #[tokio::test]
+    async fn test_permutation_records_base_branch() {
+        let temp_dir = tempfile::tempdir().unwrap();
+        let db = connect(temp_dir.path().to_str().unwrap())
+            .execute()
+            .await
+            .unwrap();
+
+        let initial_data = lance_datagen::gen_batch()
+            .col("col_a", lance_datagen::array::step::<Int32Type>())
+            .into_ldb_stream(RowCount::from(10), BatchCount::from(1));
+        let data_table = db
+            .create_table("base_tbl", initial_data)
+            .execute()
+            .await
+            .unwrap();
+
+        let branch = data_table
+            .create_branch("exp", lance::dataset::refs::Ref::from(("main", 1)))
+            .await
+            .unwrap();
+        let permutation_table = PermutationBuilder::new(branch.clone())
+            .build()
+            .await
+            .unwrap();
+
+        let metadata = permutation_table.schema().await.unwrap().metadata.clone();
+        assert_eq!(
+            metadata.get(BASE_BRANCH_CONFIG_KEY).map(String::as_str),
+            Some("exp")
+        );
+
+        // Main records nothing, so an absent key keeps meaning main.
+        let main_permutation = PermutationBuilder::new(data_table.clone())
+            .build()
+            .await
+            .unwrap();
+        assert!(
+            !main_permutation
+                .schema()
+                .await
+                .unwrap()
+                .metadata
+                .contains_key(BASE_BRANCH_CONFIG_KEY)
+        );
+    }
+
+    #[tokio::test]
+    async fn test_build_does_not_pin_the_callers_table() {
+        let temp_dir = tempfile::tempdir().unwrap();
+
+        let db = connect(temp_dir.path().to_str().unwrap())
+            .execute()
+            .await
+            .unwrap();
+
+        let initial_data = lance_datagen::gen_batch()
+            .col("col_a", lance_datagen::array::step::<Int32Type>())
+            .into_ldb_stream(RowCount::from(100), BatchCount::from(1));
+        let data_table = db
+            .create_table("base_tbl", initial_data)
+            .execute()
+            .await
+            .unwrap();
+
+        PermutationBuilder::new(data_table.clone())
+            .build()
+            .await
+            .unwrap();
+
+        // The builder pins its own handle; the caller's must still track latest.
+        let more_data = lance_datagen::gen_batch()
+            .col("col_a", lance_datagen::array::step::<Int32Type>())
+            .into_ldb_stream(RowCount::from(50), BatchCount::from(1));
+        data_table.add(more_data).execute().await.unwrap();
+        assert_eq!(data_table.count_rows(None).await.unwrap(), 150);
+    }
+
    #[tokio::test]
    async fn test_permutation_builder() {
        let temp_dir = tempfile::tempdir().unwrap();
@@ -8,7 +8,9 @@
 //! the rows from a source table that correspond to row IDs stored in a separate table.

 use crate::arrow::{SendableRecordBatchStream, SimpleRecordBatchStream};
-use crate::dataloader::permutation::builder::SRC_ROW_ID_COL;
+use crate::dataloader::permutation::builder::{
+    BASE_BRANCH_CONFIG_KEY, BASE_VERSION_CONFIG_KEY, SRC_ROW_ID_COL,
+};
 use crate::dataloader::permutation::split::SPLIT_ID_COLUMN;
 use crate::error::Error;
 use crate::query::{
@@ -23,6 +25,7 @@ use arrow_array::{RecordBatch, UInt64Array};
 use arrow_schema::SchemaRef;
 use datafusion_expr::{Expr, col, lit};
 use futures::{StreamExt, TryStreamExt};
+use lance::dataset::refs::MAIN_BRANCH;
 use lance::dataset::scanner::DatasetRecordBatchStream;
 use lance::io::RecordBatchStream;
 use lance_arrow::RecordBatchExt;
@@ -69,6 +72,10 @@ impl PermutationReader {
        permutation_table: Option<Arc<dyn BaseTable>>,
        split: u64,
    ) -> Result<Self> {
+        let base_table = match &permutation_table {
+            Some(permutation_table) => Self::pin_base_table(base_table, permutation_table).await?,
+            None => base_table,
+        };
        let mut slf = Self {
            base_table,
            permutation_table,
@@ -89,6 +96,34 @@ impl PermutationReader {
        Ok(slf)
    }

+    /// Pins the base table to the version the permutation was built against.
+    /// Permutations written before that was recorded carry no key and stay unpinned.
+    async fn pin_base_table(
+        base_table: Arc<dyn BaseTable>,
+        permutation_table: &Arc<dyn BaseTable>,
+    ) -> Result<Arc<dyn BaseTable>> {
+        let schema = permutation_table.schema().await?;
+        let Some(raw) = schema.metadata.get(BASE_VERSION_CONFIG_KEY) else {
+            return Ok(base_table);
+        };
+        let version = raw.parse::<u64>().map_err(|e| Error::InvalidInput {
+            message: format!(
+                "Permutation table has an unreadable {} of {:?}: {}",
+                BASE_VERSION_CONFIG_KEY, raw, e
+            ),
+        })?;
+        // The recorded branch, not the handle's: a worker reopens by name and lands
+        // on main, and version numbers are per-branch.
+        let branch = schema
+            .metadata
+            .get(BASE_BRANCH_CONFIG_KEY)
+            .map(String::as_str)
+            .unwrap_or(MAIN_BRANCH);
+        base_table
+            .checkout_branch_version(branch, Some(version))
+            .await
+    }
+
    pub async fn try_from_tables(
        base_table: Arc<dyn BaseTable>,
        permutation_table: Arc<dyn BaseTable>,
@@ -511,9 +546,13 @@ mod tests {
    use lance_datagen::{BatchCount, RowCount};
    use rand::seq::SliceRandom;

+    // Aliased: `test_utils::datagen` exports a trait of the same name.
+    use crate::arrow::LanceDbDatagenExt as _;
    use crate::{
        Table,
        arrow::SendableRecordBatchStream,
+        connect,
+        dataloader::permutation::builder::PermutationBuilder,
        query::{ExecutableQuery, QueryBase},
        test_utils::datagen::{LanceDbDatagenExt, virtual_table},
    };
@@ -545,6 +584,58 @@ mod tests {
        .await
    }

+    /// Compaction moves row addresses, so the reader must read the pinned version.
+    #[tokio::test]
+    async fn test_reader_pins_base_version() {
+        let temp_dir = tempfile::tempdir().unwrap();
+        let db = connect(temp_dir.path().to_str().unwrap())
+            .execute()
+            .await
+            .unwrap();
+
+        let data = lance_datagen::gen_batch()
+            .col("idx", lance_datagen::array::step::<Int32Type>())
+            .into_ldb_stream(RowCount::from(20), BatchCount::from(1));
+        let base_table = db.create_table("base_tbl", data).execute().await.unwrap();
+
+        let permutation_table = PermutationBuilder::new(base_table.clone())
+            .build()
+            .await
+            .unwrap();
+
+        base_table.delete("true").await.unwrap();
+        base_table
+            .optimize(crate::table::OptimizeAction::All)
+            .await
+            .unwrap();
+        assert_eq!(base_table.count_rows(None).await.unwrap(), 0);
+
+        let reader = PermutationReader::try_from_tables(
+            base_table.base_table().clone(),
+            permutation_table.base_table().clone(),
+            0,
+        )
+        .await
+        .unwrap();
+
+        let values = collect_from_stream::<Int32Type>(
+            reader
+                .read(
+                    Select::Columns(vec!["idx".to_string()]),
+                    QueryExecutionOptions::default(),
+                )
+                .await
+                .unwrap(),
+            "idx",
+        )
+        .await;
+        assert_eq!(
+            values.len(),
+            20,
+            "reader should still see the pinned version"
+        );
+    }
+
    #[tokio::test]
    async fn test_permutation_reader() {
        let base_table = lance_datagen::gen_batch()
@@ -1,7 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors

-//! Canonical values exchanged with the Enterprise Function service.
+//! Canonical Function values exchanged with the Enterprise service, plus the
+//! backend-neutral terminal result of a computed-column refresh.
 //!
 //! This module contains client/wire values only. Catalog persistence,
 //! environment bake, secret resolution, and execution are owned by Sophon.
@@ -580,13 +581,22 @@ impl FunctionBinding {

 impl_json!(FunctionBinding);

-/// Stable terminal result of a remote Function-column refresh Job.
+/// Stable terminal result of an expression-backed or Function-backed column
+/// refresh [`crate::Job`].
+///
+/// Local refresh jobs produce this value in process. LanceDB Cloud and
+/// Enterprise decode the same value from the durable job's terminal payload.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct RefreshColumnResult {
+    /// Rows assigned a value by this refresh.
    pub rows_assigned: u64,
+    /// Rows whose computation failed.
    pub rows_failed: u64,
+    /// Rows that still need a value when the job completes.
    pub rows_remaining: u64,
+    /// Exact table version the refresh read.
    pub source_version: u64,
+    /// Table version made visible by the refresh, when one was published.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub published_version: Option<u64>,
 }
@@ -6,7 +6,7 @@
 use std::sync::Arc;

 use async_trait::async_trait;
-use serde::de::DeserializeOwned;
+use serde::{Serialize, de::DeserializeOwned};
 use serde_json::Value;
 use tokio::sync::watch;
 use tokio::task::{AbortHandle, JoinHandle};
@@ -26,20 +26,16 @@ pub(crate) trait JobHandle: Send + Sync {
 }

 /// A backend-neutral successful terminal result.
-///
-/// Local operations do not carry a value. Remote operations may carry JSON
-/// that the public [`Job`] decodes according to its result type.
+#[derive(Clone)]
 pub(crate) struct TerminalResult {
-    #[allow(dead_code)] // Typed remote submit endpoints consume this after Slice 1.
    value: Option<Value>,
-    #[allow(dead_code)] // Preserved so typed decode errors retain request correlation.
    request_id: Option<String>,
 }

 impl TerminalResult {
-    pub(crate) fn local() -> Self {
+    fn local(value: Value) -> Self {
        Self {
-            value: None,
+            value: Some(value),
            request_id: None,
        }
    }
@@ -51,23 +47,31 @@ impl TerminalResult {
        }
    }

-    #[allow(dead_code)] // Exercised by the remote typed-result fixtures in Slice 1.
    fn decode<T: DeserializeOwned>(self) -> Result<T> {
-        let request_id = self.request_id.unwrap_or_default();
-        let value = self.value.ok_or_else(|| Error::Http {
-            source: "successful typed job response did not contain a result".into(),
-            request_id: request_id.clone(),
-            status_code: None,
+        let value = self.value.ok_or_else(|| match &self.request_id {
+            Some(request_id) => Error::Http {
+                source: "successful typed job response did not contain a result".into(),
+                request_id: request_id.clone(),
+                status_code: None,
+            },
+            None => Error::Runtime {
+                message: "successful typed job did not contain a result".to_string(),
+            },
        })?;
-        serde_json::from_value(value).map_err(|error| Error::Http {
-            source: format!("failed to parse typed job result: {error}").into(),
-            request_id,
-            status_code: None,
+        serde_json::from_value(value).map_err(|error| match self.request_id {
+            Some(request_id) => Error::Http {
+                source: format!("failed to parse typed job result: {error}").into(),
+                request_id,
+                status_code: None,
+            },
+            None => Error::Runtime {
+                message: format!("failed to parse typed job result: {error}"),
+            },
        })
    }
 }

-type ResultDecoder<T> = fn(TerminalResult) -> Result<T>;
+type ResultDecoder<T> = Arc<dyn Fn(TerminalResult) -> Result<T> + Send + Sync>;

 enum JobInner<T> {
    Handle {
@@ -79,7 +83,9 @@ enum JobInner<T> {

 /// A handle to an operation that may still be running.
 ///
-/// The operation may already be complete when the handle is created.
+/// The operation may already be complete when the handle is created. `T` is
+/// the endpoint's successful terminal result; unit-result operations use the
+/// default `Job<()>`.
 pub struct Job<T = ()>
 where
    T: Clone + Send + Sync + 'static,
@@ -111,15 +117,10 @@ impl Job<()> {
        Self {
            inner: JobInner::Handle {
                handle,
-                decode: |_| Ok(()),
+                decode: Arc::new(|_| Ok(())),
            },
        }
    }
-
-    /// A unit-result job running as a task in this process.
-    pub(crate) fn spawned(task: JoinHandle<Result<()>>) -> Self {
-        Self::new(Box::new(SpawnedJob::new(task)))
-    }
 }

 impl<T> Job<T>
@@ -131,12 +132,22 @@ where
        Self {
            inner: JobInner::Handle {
                handle,
-                decode: TerminalResult::decode::<T>,
+                decode: Arc::new(TerminalResult::decode::<T>),
            },
        }
    }
 }

+impl<T> Job<T>
+where
+    T: Clone + Serialize + DeserializeOwned + Send + Sync + 'static,
+{
+    /// A typed job running as a task in this process.
+    pub(crate) fn spawned(task: JoinHandle<Result<T>>) -> Self {
+        Self::new_typed(Box::new(SpawnedJob::new(task)))
+    }
+}
+
 impl<T> Job<T>
 where
    T: Clone + Send + Sync + 'static,
@@ -169,11 +180,13 @@ where

    /// Waits until the operation reaches a terminal state.
    ///
+    /// Returns the endpoint's typed result. Unit-result jobs return `()`.
+    ///
    /// Returns [`crate::Error::JobFailed`] if the operation failed and
    /// [`crate::Error::JobCancelled`] if it was cancelled.
    pub async fn wait(&self) -> Result<T> {
        match &self.inner {
-            JobInner::Handle { handle, decode } => decode(handle.wait().await?),
+            JobInner::Handle { handle, decode } => (decode)(handle.wait().await?),
            JobInner::Completed(result) => Ok(result.clone()),
        }
    }
@@ -187,21 +200,53 @@ where
            JobInner::Completed(_) => Ok(()),
        }
    }
+
+    /// Maps a successful terminal result without changing the job lifecycle.
+    /// The mapping may run once for each call to [`Job::wait`], so it should
+    /// be deterministic and free of externally visible side effects.
+    ///
+    /// ```
+    /// use lancedb::{Job, function::RefreshColumnResult};
+    ///
+    /// # async fn rows_assigned(
+    /// #     job: Job<RefreshColumnResult>,
+    /// # ) -> lancedb::Result<u64> {
+    /// let job = job.map(|result| result.rows_assigned);
+    /// job.wait().await
+    /// # }
+    /// ```
+    pub fn map<U, F>(self, map: F) -> Job<U>
+    where
+        U: Clone + Send + Sync + 'static,
+        F: Fn(T) -> U + Send + Sync + 'static,
+    {
+        match self.inner {
+            JobInner::Handle { handle, decode } => Job {
+                inner: JobInner::Handle {
+                    handle,
+                    decode: Arc::new(move |result| Ok(map((decode)(result)?))),
+                },
+            },
+            JobInner::Completed(result) => Job {
+                inner: JobInner::Completed(map(result)),
+            },
+        }
+    }
 }

 /// How an in-process operation ended. Cloneable so every waiter can be given
 /// the outcome; [`Error`] is not, so failures share one behind an [`Arc`].
 #[derive(Clone)]
 enum Outcome {
-    Succeeded,
+    Succeeded(TerminalResult),
    Failed(Arc<Error>),
    Cancelled,
 }

 impl Outcome {
-    fn into_result(self) -> Result<()> {
+    fn into_result(self) -> Result<TerminalResult> {
        match self {
-            Self::Succeeded => Ok(()),
+            Self::Succeeded(result) => Ok(result),
            Self::Failed(source) => Err(Error::JobFailed {
                job_id: None,
                failure: JobFailure::from_source(source),
@@ -220,12 +265,20 @@ struct SpawnedJob {
 }

 impl SpawnedJob {
-    fn new(task: JoinHandle<Result<()>>) -> Self {
+    fn new<T>(task: JoinHandle<Result<T>>) -> Self
+    where
+        T: Serialize + Send + 'static,
+    {
        let abort = task.abort_handle();
        let (tx, outcome) = watch::channel(None);
        tokio::spawn(async move {
            let outcome = match task.await {
-                Ok(Ok(())) => Outcome::Succeeded,
+                Ok(Ok(result)) => match serde_json::to_value(result) {
+                    Ok(value) => Outcome::Succeeded(TerminalResult::local(value)),
+                    Err(err) => Outcome::Failed(Arc::new(Error::Runtime {
+                        message: format!("failed to serialize job result: {err}"),
+                    })),
+                },
                Ok(Err(err)) => Outcome::Failed(Arc::new(err)),
                Err(err) if err.is_cancelled() => Outcome::Cancelled,
                Err(err) => Outcome::Failed(Arc::new(Error::Runtime {
@@ -243,7 +296,7 @@ impl JobHandle for SpawnedJob {
    async fn status(&self) -> Result<String> {
        let label = match &*self.outcome.borrow() {
            None => "running",
-            Some(Outcome::Succeeded) => "finished",
+            Some(Outcome::Succeeded(_)) => "finished",
            Some(Outcome::Failed(_)) => "failed",
            Some(Outcome::Cancelled) => "cancelled",
        };
@@ -256,12 +309,11 @@ impl JobHandle for SpawnedJob {
            .wait_for(|outcome| outcome.is_some())
            .await
            .map_err(|_| Error::Runtime {
-                message: "index job outcome was dropped before it completed".to_string(),
+                message: "job outcome was dropped before it completed".to_string(),
            })?
            .clone()
            .expect("wait_for returns once an outcome is set");
-        settled.into_result()?;
-        Ok(TerminalResult::local())
+        settled.into_result()
    }

    async fn cancel(&self) -> Result<()> {
@@ -269,3 +321,29 @@ impl JobHandle for SpawnedJob {
        Ok(())
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::future::pending;
+
+    use super::*;
+
+    #[tokio::test]
+    async fn mapped_spawned_job_reuses_outcome() {
+        let job = Job::spawned(tokio::spawn(async { Ok(41_u64) })).map(|value| value + 1);
+
+        assert_eq!(job.wait().await.unwrap(), 42);
+        assert_eq!(job.wait().await.unwrap(), 42);
+        assert_eq!(job.status().await.unwrap(), "finished");
+    }
+
+    #[tokio::test]
+    async fn mapped_spawned_job_preserves_cancellation() {
+        let job = Job::spawned(tokio::spawn(async { pending::<Result<u64>>().await }))
+            .map(|value| value.to_string());
+
+        job.cancel().await.unwrap();
+        assert!(matches!(job.wait().await, Err(Error::JobCancelled { .. })));
+        assert_eq!(job.status().await.unwrap(), "cancelled");
+    }
+}
@@ -37,6 +37,13 @@ pub use refresh::{RefreshMaterializedViewResult, RefreshMode};
 /// Schema metadata key holding the view definition, as kind-tagged JSON.
 pub const DEFINITION_META_KEY: &str = "mv.definition";

+/// Schema metadata key holding the view's incarnation: a token minted at each
+/// physical creation of a view table, so a view dropped and recreated under
+/// the same name and definition is still told apart from the one a caller
+/// captured. A view whose metadata was replaced wholesale, or one declared
+/// before tokens existed, carries none until its next refresh mints one.
+pub const INCARNATION_META_KEY: &str = "mv.incarnation";
+
 /// Schema metadata key holding the source table version the view was last
 /// refreshed to. Absent until the first refresh.
 pub const SOURCE_VERSION_META_KEY: &str = "mv.source_version";
@@ -612,8 +619,17 @@ impl PreparedDeclaration {
    pub async fn create(self, name: &str) -> Result<MaterializedView> {
        let empty: Vec<std::result::Result<arrow_array::RecordBatch, arrow_schema::ArrowError>> =
            vec![];
+        // Minted here, not at preparation: a declaration can be cloned and
+        // create more than one physical table, and each needs its own token.
+        let incarnation = uuid::Uuid::new_v4().to_string();
+        let mut metadata = self.schema.metadata().clone();
+        metadata.insert(INCARNATION_META_KEY.to_string(), incarnation.clone());
+        let schema = Arc::new(ArrowSchema::new_with_metadata(
+            self.schema.fields().clone(),
+            metadata,
+        ));
        let reader: Box<dyn arrow_array::RecordBatchReader + Send> =
-            Box::new(arrow_array::RecordBatchIterator::new(empty, self.schema));
+            Box::new(arrow_array::RecordBatchIterator::new(empty, schema));
        let mut request = CreateTableRequest::new(name.to_string(), Box::new(reader));
        let write_params = request
            .write_options
@@ -648,6 +664,7 @@ impl PreparedDeclaration {
        Ok(MaterializedView {
            table,
            definition: self.definition,
+            incarnation: Some(incarnation),
        })
    }
 }
@@ -878,6 +895,7 @@ impl CreateMaterializedViewBuilder {
 pub struct MaterializedView {
    table: Table,
    definition: MaterializedViewDefinition,
+    incarnation: Option<String>,
 }

 impl MaterializedView {
@@ -893,8 +911,13 @@ impl MaterializedView {
            });
        }
        let schema = table.schema().await?;
+        let incarnation = schema.metadata().get(INCARNATION_META_KEY).cloned();
        match materialized_view_kind(schema.metadata())? {
-            Some(MaterializedViewKind::Select(definition)) => Ok(Self { table, definition }),
+            Some(MaterializedViewKind::Select(definition)) => Ok(Self {
+                table,
+                definition,
+                incarnation,
+            }),
            Some(MaterializedViewKind::Unrecognized { kind }) => Err(Error::NotSupported {
                message: format!(
                    "materialized view '{}' is defined by '{kind}', which this version of \
@@ -923,6 +946,13 @@ impl MaterializedView {
        &self.definition
    }

+    /// The view's incarnation token as of when this handle was opened; see
+    /// [`RefreshMaterializedViewBuilder::expect_incarnation`]. `None` for a
+    /// view that has none yet (see [`INCARNATION_META_KEY`]).
+    pub fn incarnation(&self) -> Option<&str> {
+        self.incarnation.as_deref()
+    }
+
    /// Recompute the view from its source.
    ///
    /// By default the refresh is incremental when the source's changes can be
@@ -943,6 +973,7 @@ impl MaterializedView {
            view: self.clone(),
            full: false,
            source_version: None,
+            expected_incarnation: None,
        }
    }
 }
@@ -952,6 +983,7 @@ pub struct RefreshMaterializedViewBuilder {
    view: MaterializedView,
    full: bool,
    source_version: Option<u64>,
+    expected_incarnation: Option<String>,
 }

 impl RefreshMaterializedViewBuilder {
@@ -967,8 +999,28 @@ impl RefreshMaterializedViewBuilder {
        self
    }

+    /// Refresh only if the view is still the incarnation that minted `token`
+    /// (see [`MaterializedView::incarnation`]): a refresh requested against
+    /// one declaration must not land in a view dropped and recreated since,
+    /// even under the same name and definition.
+    ///
+    /// Best effort. The token is read from the latest stored manifest before
+    /// planning and again immediately before every commit, but it is not part
+    /// of the commit's own condition, so a recreation that lands between that
+    /// final read and the commit is not caught.
+    pub fn expect_incarnation(mut self, token: impl Into<String>) -> Self {
+        self.expected_incarnation = Some(token.into());
+        self
+    }
+
    pub async fn execute(self) -> Result<RefreshMaterializedViewResult> {
-        refresh::execute_refresh(&self.view.table, self.full, self.source_version).await
+        refresh::execute_refresh(
+            &self.view.table,
+            self.full,
+            self.source_version,
+            self.expected_incarnation.as_deref(),
+        )
+        .await
    }
 }

@@ -46,8 +46,8 @@ use lance_table::format::Fragment;
 use serde::{Deserialize, Serialize};

 use super::{
-    MaterializedViewDefinition, REFRESHED_AT_MS_META_KEY, SOURCE_ROW_ID_COLUMN,
-    SOURCE_VERSION_META_KEY,
+    INCARNATION_META_KEY, MaterializedViewDefinition, REFRESHED_AT_MS_META_KEY,
+    SOURCE_ROW_ID_COLUMN, SOURCE_VERSION_META_KEY,
 };
 use crate::database::OpenTableRequest;
 use crate::table::{NativeTable, NativeTableExt, Table};
@@ -108,6 +108,7 @@ pub(crate) async fn execute_refresh(
    view: &Table,
    full: bool,
    pinned: Option<u64>,
+    expected_incarnation: Option<&str>,
 ) -> Result<RefreshMaterializedViewResult> {
    let view_native = view.as_native().ok_or_else(|| Error::NotSupported {
        message: "materialized views are supported only on local tables".into(),
@@ -122,6 +123,8 @@ pub(crate) async fn execute_refresh(
    view_native.dataset.reload().await?;
    let view_ds = view_native.dataset.get().await?.as_ref().clone();

+    ensure_incarnation(&view_ds, expected_incarnation, view.name()).await?;
+
    // The definition a handle cached at open may since have been replaced;
    // what refresh executes and what it stamps must be one generation.
    let definition = match super::materialized_view_kind(&view_ds.schema().metadata)? {
@@ -240,6 +243,7 @@ pub(crate) async fn execute_refresh(
                increment,
                definition,
                watermark,
+                expected_incarnation,
            )
            .await?;
            match reconciled {
@@ -253,6 +257,7 @@ pub(crate) async fn execute_refresh(
                        source_version,
                        source_ts,
                        definition,
+                        expected_incarnation,
                    )
                    .await
                }
@@ -266,6 +271,7 @@ pub(crate) async fn execute_refresh(
                source_version,
                source_ts,
                definition,
+                expected_incarnation,
            )
            .await
        }
@@ -595,6 +601,7 @@ async fn incremental(
    increment: Increment,
    definition: &MaterializedViewDefinition,
    watermark: Option<u64>,
+    expected_incarnation: Option<&str>,
 ) -> Result<Option<RefreshMaterializedViewResult>> {
    let new_fragments = increment.appended;
    let watermark_version = watermark.unwrap_or(0);
@@ -671,15 +678,35 @@ async fn incremental(
    };
    let nothing_to_add = (new_fragments.is_empty() && !updated_rows) || remaining == Some(0);
    if nothing_to_add && eviction.is_none() {
-        result.version =
-            stamp_watermark(view_native, view_ds.clone(), source_version, source_ts).await?;
+        result.version = stamp_watermark(
+            view_native,
+            view_ds.clone(),
+            source_version,
+            source_ts,
+            expected_incarnation,
+        )
+        .await?;
        return Ok(Some(result));
    }
    // Rows left but none arrive: the removals still have to be published.
    if nothing_to_add {
        let filter = refresh_filter(&empty_keys(view_ds)?)?;
-        let published = publish(view_ds, eviction, Vec::new(), Some(filter)).await?;
-        result.version = stamp_watermark(view_native, published, source_version, source_ts).await?;
+        let published = publish(
+            view_ds,
+            eviction,
+            Vec::new(),
+            Some(filter),
+            expected_incarnation,
+        )
+        .await?;
+        result.version = stamp_watermark(
+            view_native,
+            published,
+            source_version,
+            source_ts,
+            expected_incarnation,
+        )
+        .await?;
        return Ok(Some(result));
    }

@@ -737,12 +764,20 @@ async fn incremental(
                eviction,
                Vec::new(),
                Some(refresh_filter(&empty_keys(view_ds)?)?),
+                expected_incarnation,
            )
            .await?
        } else {
            view_ds.clone()
        };
-        result.version = stamp_watermark(view_native, published, source_version, source_ts).await?;
+        result.version = stamp_watermark(
+            view_native,
+            published,
+            source_version,
+            source_ts,
+            expected_incarnation,
+        )
+        .await?;
        return Ok(Some(result));
    };
    let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(
@@ -775,9 +810,23 @@ async fn incremental(
        });
    };
    let filter = refresh_filter(&keys)?;
-    let appended = publish(view_ds, eviction, new_fragments, Some(filter)).await?;
+    let appended = publish(
+        view_ds,
+        eviction,
+        new_fragments,
+        Some(filter),
+        expected_incarnation,
+    )
+    .await?;
    result.rows_written = rows_written.load(Ordering::Relaxed);
-    result.version = stamp_watermark(view_native, appended, source_version, source_ts).await?;
+    result.version = stamp_watermark(
+        view_native,
+        appended,
+        source_version,
+        source_ts,
+        expected_incarnation,
+    )
+    .await?;
    Ok(Some(result))
 }

@@ -788,6 +837,7 @@ async fn rebuild(
    source_version: u64,
    source_ts: u128,
    definition: &MaterializedViewDefinition,
+    expected_incarnation: Option<&str>,
 ) -> Result<RefreshMaterializedViewResult> {
    let rows_written = Arc::new(AtomicU64::new(0));
    let schema = Arc::new(ArrowSchema::from(view_ds.schema()));
@@ -810,8 +860,16 @@ async fn rebuild(
    // carries no schema metadata, so it cannot erase a definition update
    // that raced in the way an overwrite (which adopts its stream's schema)
    // durably would -- and it must land on the planned generation or abort.
-    let replaced = replace_retaining_indices(view_ds.clone(), stream, keys).await?;
-    let version = stamp_watermark(view_native, replaced, source_version, source_ts).await?;
+    let replaced =
+        replace_retaining_indices(view_ds.clone(), stream, keys, expected_incarnation).await?;
+    let version = stamp_watermark(
+        view_native,
+        replaced,
+        source_version,
+        source_ts,
+        expected_incarnation,
+    )
+    .await?;
    Ok(RefreshMaterializedViewResult {
        mode: RefreshMode::Rebuild,
        rows_written: rows_written.load(Ordering::Relaxed),
@@ -828,11 +886,13 @@ async fn replace_retaining_indices(
    view_ds: Dataset,
    stream: SendableRecordBatchStream,
    keys: Arc<StdMutex<KeyExistenceFilterBuilder>>,
+    expected_incarnation: Option<&str>,
 ) -> Result<Dataset> {
    let ds = Arc::new(view_ds);
    let read_version = ds.version().version;
    #[cfg(test)]
    tests::hold_before_publish(ds.uri()).await;
+    ensure_incarnation(&ds, expected_incarnation, ds.uri()).await?;
    let removed_fragment_ids: Vec<u64> = ds.get_fragments().iter().map(|f| f.id() as u64).collect();

    let write_txn = InsertBuilder::new(WriteDestination::Dataset(ds.clone()))
@@ -886,6 +946,32 @@ async fn replace_retaining_indices(
 }

 /// Record that the view now reflects `source_version`, including the view
+/// Refuse to act on a view that is not `expected`'s incarnation, judged from
+/// the latest stored manifest. Not a commit condition; see
+/// `RefreshMaterializedViewBuilder::expect_incarnation`.
+async fn ensure_incarnation(view_ds: &Dataset, expected: Option<&str>, what: &str) -> Result<()> {
+    let Some(expected) = expected else {
+        return Ok(());
+    };
+    let mut latest = view_ds.clone();
+    latest.checkout_latest().await?;
+    match latest.schema().metadata.get(INCARNATION_META_KEY) {
+        Some(actual) if actual == expected => Ok(()),
+        Some(_) => Err(Error::Runtime {
+            message: format!(
+                "materialized view '{what}' is not the incarnation this refresh was \
+                 requested for: it was dropped and recreated"
+            ),
+        }),
+        None => Err(Error::Runtime {
+            message: format!(
+                "materialized view '{what}' carries no incarnation token: its schema \
+                 metadata was replaced since the token was captured"
+            ),
+        }),
+    }
+}
+
 /// version this very commit produces. The version is predicted and then
 /// verified; on a mismatch another commit raced in between, and the stamp
 /// ABORTS rather than certify that commit as the refresh's own generation.
@@ -895,10 +981,21 @@ async fn stamp_watermark(
    mut dataset: Dataset,
    source_version: u64,
    source_ts: u128,
+    expected_incarnation: Option<&str>,
 ) -> Result<u64> {
+    ensure_incarnation(&dataset, expected_incarnation, dataset.uri()).await?;
    let predicted = dataset.version().version + 1;
+    // A view with no token (declared before tokens existed, or its metadata
+    // replaced wholesale) starts a new incarnation here.
+    let incarnation = dataset
+        .schema()
+        .metadata
+        .get(INCARNATION_META_KEY)
+        .cloned()
+        .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
    dataset
        .update_schema_metadata([
+            (INCARNATION_META_KEY.to_string(), Some(incarnation)),
            (
                SOURCE_VERSION_META_KEY.to_string(),
                Some(source_version.to_string()),
@@ -1052,12 +1149,14 @@ async fn publish(
    eviction: Option<(Vec<Fragment>, Vec<u64>)>,
    new_fragments: Vec<Fragment>,
    keys: Option<KeyExistenceFilter>,
+    expected_incarnation: Option<&str>,
 ) -> Result<Dataset> {
    let planned = view_ds.version().version;
    #[cfg(test)]
    tests::hold_before_publish(view_ds.uri()).await;
    #[cfg(test)]
    tests::hold_until_peers_planned();
+    ensure_incarnation(view_ds, expected_incarnation, view_ds.uri()).await?;
    let (updated_fragments, removed_fragment_ids) = eviction.unwrap_or_default();
    let committed = CommitBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone())))
        .execute(Transaction::new(
@@ -1830,7 +1929,9 @@ mod tests {
        );
        let staged = eviction.finish().await.unwrap();
        assert!(staged.is_some(), "four ids over a chunk of two stage twice");
-        publish(&view_ds, staged, Vec::new(), None).await.unwrap();
+        publish(&view_ds, staged, Vec::new(), None, None)
+            .await
+            .unwrap();
        native.dataset.reload().await.unwrap();

        assert_eq!(read(view.table(), "x").await, vec![5, 6]);
@@ -2486,6 +2587,144 @@ mod tests {
        assert_eq!(read(view.table(), "twice").await, vec![14]);
    }

+    /// A refresh bound to an incarnation refuses a view dropped and recreated
+    /// since, even under the same name and definition; the recreated view's
+    /// own token is accepted, and the token survives a refresh's stamp.
+    #[tokio::test]
+    async fn test_refresh_refuses_a_recreated_view_incarnation() {
+        let (conn, _, view) = refreshed_doubled(vec![1]).await;
+        let token = view.incarnation().unwrap().to_string();
+        view.refresh()
+            .expect_incarnation(&token)
+            .execute()
+            .await
+            .unwrap();
+        let reopened = conn.open_materialized_view("doubled").await.unwrap();
+        assert_eq!(reopened.incarnation(), Some(token.as_str()));
+
+        conn.drop_table("doubled", &[]).await.unwrap();
+        let recreated = doubled_view(&conn).await;
+        assert_ne!(recreated.incarnation(), Some(token.as_str()));
+
+        let err = recreated
+            .refresh()
+            .expect_incarnation(&token)
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("dropped and recreated"), "{err}");
+        assert_eq!(read(recreated.table(), "twice").await, Vec::<i32>::new());
+
+        recreated
+            .refresh()
+            .expect_incarnation(recreated.incarnation().unwrap())
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(read(recreated.table(), "twice").await, vec![2]);
+    }
+
+    /// A cloned declaration creates two physical tables; each gets its own
+    /// token.
+    #[tokio::test]
+    async fn test_cloned_declaration_mints_a_fresh_incarnation_per_create() {
+        let (conn, source) = db_with_source(vec![1]).await;
+        let prepared = crate::materialized_view::prepare_declaration(
+            &source,
+            &[("x".into(), "x".into()), ("twice".into(), "x * 2".into())],
+            None,
+            None,
+        )
+        .await
+        .unwrap();
+        let replacement = prepared.clone();
+        let first = prepared.create("cloned").await.unwrap();
+        let first_token = first.incarnation().unwrap().to_string();
+
+        conn.drop_table("cloned", &[]).await.unwrap();
+        let second = replacement.create("cloned").await.unwrap();
+        assert_ne!(second.incarnation(), Some(first_token.as_str()));
+    }
+
+    /// A recreation that lands after planning but before publication is
+    /// caught by the pre-commit read: the stale refresh fails and the
+    /// replacement stays empty under its own token.
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_bound_refresh_cannot_publish_into_a_raced_recreation() {
+        let _serial = DRIFT_LOCK.lock().await;
+        let (conn, _) = db_with_source(vec![1]).await;
+        let view = doubled_view(&conn).await;
+        let token = view.incarnation().unwrap().to_string();
+        let uri = view
+            .table()
+            .as_native()
+            .unwrap()
+            .dataset
+            .get()
+            .await
+            .unwrap()
+            .uri()
+            .to_string();
+
+        *DRIFT_TARGET.lock().unwrap() = Some(uri);
+        let refreshing =
+            tokio::spawn(async move { view.refresh().expect_incarnation(token).execute().await });
+        tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified())
+            .await
+            .expect("refresh never reached publication");
+
+        conn.drop_table("doubled", &[]).await.unwrap();
+        let replacement = doubled_view(&conn).await;
+        let replacement_token = replacement.incarnation().unwrap().to_string();
+        DRIFT_RELEASED.notify_one();
+
+        let result = refreshing.await.unwrap();
+        assert!(result.is_err(), "the stale refresh unexpectedly succeeded");
+        let reopened = conn.open_materialized_view("doubled").await.unwrap();
+        assert_eq!(reopened.incarnation(), Some(replacement_token.as_str()));
+        assert_eq!(read(reopened.table(), "twice").await, Vec::<i32>::new());
+    }
+
+    /// Replacing the schema metadata wholesale drops the token. A refresh
+    /// bound to the old token is refused for that reason, not as a
+    /// recreation; an unbound refresh mints the view a fresh one.
+    #[tokio::test]
+    async fn test_a_view_whose_metadata_was_replaced_starts_a_new_incarnation() {
+        let (conn, _, view) = refreshed_doubled(vec![1]).await;
+        let token = view.incarnation().unwrap().to_string();
+        let mut metadata = HashMap::new();
+        metadata.insert(
+            crate::materialized_view::DEFINITION_META_KEY.to_string(),
+            crate::materialized_view::definition_to_metadata(view.definition()).unwrap(),
+        );
+        view.table()
+            .as_native()
+            .unwrap()
+            .replace_schema_metadata(metadata)
+            .await
+            .unwrap();
+        assert_eq!(
+            conn.open_materialized_view("doubled")
+                .await
+                .unwrap()
+                .incarnation(),
+            None
+        );
+
+        let err = view
+            .refresh()
+            .expect_incarnation(&token)
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("no incarnation token"), "{err}");
+
+        view.refresh().execute().await.unwrap();
+        let reopened = conn.open_materialized_view("doubled").await.unwrap();
+        assert!(reopened.incarnation().is_some());
+        assert_ne!(reopened.incarnation(), Some(token.as_str()));
+    }
+
    /// In-process refreshes of one view serialize: the loser of the race
    /// observes the winner's watermark instead of appending the same rows.
    #[tokio::test(flavor = "multi_thread")]
@@ -2528,7 +2767,7 @@ mod tests {
        let stale = view_native.dataset.get().await.unwrap().as_ref().clone();
        view.table().delete("x = 1").await.unwrap();

-        let err = stamp_watermark(view_native, stale, 99, 99).await;
+        let err = stamp_watermark(view_native, stale, 99, 99, None).await;
        assert!(err.is_err());

        let result = view.refresh().execute().await.unwrap();
@@ -344,6 +344,62 @@ impl<S: HttpSend> RemoteDatabase<S> {
        self.table_cache.remove(&cache_key).await;
        Ok((request_id, resp))
    }
+
+    /// Collect the tables of a namespace in name order, for `table_names`.
+    ///
+    /// `table_names` promises name order and resumes after a table name, but the namespace
+    /// route's `page_token` is opaque -- it belongs to the store the listing walks, and a
+    /// token this client invented would resume from the wrong place. So the whole namespace is
+    /// walked by handing each response's token straight back, and the name semantics are
+    /// applied here. Constructing no token is what makes this work against a server on either
+    /// side of the change: it only ever repeats what the server said.
+    ///
+    /// This is the cost `table_names` already paid -- the server used to enumerate and sort the
+    /// namespace on every request -- and it is why `list_tables` replaces it.
+    async fn table_names_in_namespace(
+        &self,
+        request: &TableNamesRequest,
+    ) -> Result<(Vec<String>, ServerVersion)> {
+        let namespace_id =
+            build_namespace_identifier(&request.namespace_path, &self.client.id_delimiter);
+        let path = format!("/v1/namespace/{}/table/list", namespace_id);
+
+        let mut names = Vec::new();
+        // Every page reports the same server, so keep the first page's version.
+        let mut version: Option<ServerVersion> = None;
+        let mut page_token: Option<String> = None;
+        loop {
+            let mut req = self.client.get(&path);
+            if let Some(ref token) = page_token {
+                req = req.query(&[("page_token", token)]);
+            }
+            let (request_id, rsp) = self.client.send_with_retry(req, None, true).await?;
+            let rsp = self.client.check_response(&request_id, rsp).await?;
+            if version.is_none() {
+                version = Some(parse_server_version(&request_id, &rsp)?);
+            }
+            let response: ListTablesResponse = rsp.json().await.err_to_http(request_id)?;
+            names.extend(response.tables);
+            // An empty token is the end of the listing, not a token to send back: a server
+            // that reads an empty token as "start from the beginning" would hand back the
+            // first page again.
+            match response.page_token.filter(|token| !token.is_empty()) {
+                // A server that repeated a token would never finish; treat that as the end
+                // rather than looping on it.
+                Some(token) if Some(&token) != page_token.as_ref() => page_token = Some(token),
+                _ => break,
+            }
+        }
+
+        names.sort();
+        if let Some(ref start_after) = request.start_after {
+            names.retain(|name| name > start_after);
+        }
+        if let Some(limit) = request.limit {
+            names.truncate(limit as usize);
+        }
+        Ok((names, version.unwrap_or_default()))
+    }
 }

 #[cfg(all(test, feature = "remote"))]
@@ -513,7 +569,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
    async fn get_function(&self, name: &str, version: &str) -> Result<FunctionVersion> {
        let req = self
            .client
-            .post("/v1/functions/get")
+            .post("/v1/functions/describe")
            .json(&serde_json::json!({
                "name": name,
                "version": version,
@@ -621,29 +677,29 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
    }

    async fn table_names(&self, request: TableNamesRequest) -> Result<Vec<String>> {
-        let mut req = if !request.namespace_path.is_empty() {
-            let namespace_id =
-                build_namespace_identifier(&request.namespace_path, &self.client.id_delimiter);
-            self.client
-                .get(&format!("/v1/namespace/{}/table/list", namespace_id))
+        let (tables, version) = if request.namespace_path.is_empty() {
+            // The flat route resumes after a table name and orders by name, which is exactly
+            // what `start_after` means, so the server does the paging.
+            let mut req = self.client.get("/v1/table/");
+            if let Some(limit) = request.limit {
+                req = req.query(&[("limit", limit)]);
+            }
+            if let Some(ref start_after) = request.start_after {
+                req = req.query(&[("page_token", start_after)]);
+            }
+            let (request_id, rsp) = self.client.send_with_retry(req, None, true).await?;
+            let rsp = self.client.check_response(&request_id, rsp).await?;
+            let version = parse_server_version(&request_id, &rsp)?;
+            let tables = rsp
+                .json::<ListTablesResponse>()
+                .await
+                .err_to_http(request_id)?
+                .tables;
+            (tables, version)
        } else {
-            self.client.get("/v1/table/")
+            self.table_names_in_namespace(&request).await?
        };

-        if let Some(limit) = request.limit {
-            req = req.query(&[("limit", limit)]);
-        }
-        if let Some(start_after) = request.start_after {
-            req = req.query(&[("page_token", start_after)]);
-        }
-        let (request_id, rsp) = self.client.send_with_retry(req, None, true).await?;
-        let rsp = self.client.check_response(&request_id, rsp).await?;
-        let version = parse_server_version(&request_id, &rsp)?;
-        let tables = rsp
-            .json::<ListTablesResponse>()
-            .await
-            .err_to_http(request_id)?
-            .tables;
        for table in &tables {
            let table_identifier =
                build_table_identifier(table, &request.namespace_path, &self.client.id_delimiter);
@@ -1227,6 +1283,101 @@ mod tests {
        assert_eq!(names, vec!["table1", "table2"]);
    }

+    #[tokio::test]
+    async fn test_table_names_in_a_namespace_never_invents_a_page_token() {
+        // The namespace route's token belongs to the store, so `table_names` cannot build one
+        // from `start_after`. It walks the namespace on the server's own tokens and applies the
+        // name semantics itself, which is what keeps it working either side of the change.
+        let page = Arc::new(AtomicUsize::new(0));
+        let conn = Connection::new_with_handler(move |request| {
+            assert_eq!(request.url().path(), "/v1/namespace/ns/table/list");
+            let query = request.url().query().unwrap_or("");
+            assert!(
+                !query.contains("page_token=users"),
+                "a table name must never be sent as a page token: {query}"
+            );
+            match page.fetch_add(1, Ordering::SeqCst) {
+                0 => {
+                    assert!(
+                        !query.contains("page_token"),
+                        "the walk starts with no token"
+                    );
+                    http::Response::builder()
+                        .status(200)
+                        .body(r#"{"tables": ["users", "orders"], "page_token": "opaque-1"}"#)
+                        .unwrap()
+                }
+                _ => {
+                    assert!(query.contains("page_token=opaque-1"));
+                    http::Response::builder()
+                        .status(200)
+                        .body(r#"{"tables": ["widgets"]}"#)
+                        .unwrap()
+                }
+            }
+        });
+
+        let names = conn
+            .table_names()
+            .namespace(vec!["ns".to_string()])
+            .start_after("users")
+            .execute()
+            .await
+            .unwrap();
+        // Name order, resumed after "users": "orders" sorts before it and is dropped.
+        assert_eq!(names, vec!["widgets"]);
+    }
+
+    #[tokio::test]
+    async fn test_table_names_in_a_namespace_stops_on_a_repeated_token() {
+        // A server that handed back the token it was given would never finish the walk.
+        let conn = Connection::new_with_handler(|_request| {
+            http::Response::builder()
+                .status(200)
+                .body(r#"{"tables": ["a"], "page_token": "same"}"#)
+                .unwrap()
+        });
+
+        let names = conn
+            .table_names()
+            .namespace(vec!["ns".to_string()])
+            .execute()
+            .await
+            .unwrap();
+        // The guard bounds the walk instead of letting it run forever. The repeat is the
+        // server breaking the token contract and is not papered over here.
+        assert_eq!(names, vec!["a", "a"]);
+    }
+
+    #[tokio::test]
+    async fn test_table_names_in_a_namespace_stops_on_an_empty_token() {
+        // An empty token ends the listing. Sending it back would ask a server that reads it
+        // as "start from the beginning" for the first page a second time, and every name on
+        // that page would be collected twice.
+        let requests = Arc::new(AtomicUsize::new(0));
+        let seen = requests.clone();
+        let conn = Connection::new_with_handler(move |request| {
+            seen.fetch_add(1, Ordering::SeqCst);
+            assert!(
+                !request.url().query().unwrap_or("").contains("page_token"),
+                "an empty token must never be sent back"
+            );
+            http::Response::builder()
+                .status(200)
+                .body(r#"{"tables": ["a"], "page_token": ""}"#)
+                .unwrap()
+        });
+
+        let names = conn
+            .table_names()
+            .namespace(vec!["ns".to_string()])
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(names, vec!["a"]);
+        assert_eq!(requests.load(Ordering::SeqCst), 1);
+    }
+
    #[tokio::test]
    async fn test_table_names_pagination() {
        let conn = Connection::new_with_handler(|request| {
@@ -2520,7 +2671,7 @@ mod tests {
        );
        let conn = Connection::new_with_handler(|request| {
            assert_eq!(request.method(), &reqwest::Method::POST);
-            assert_eq!(request.url().path(), "/v1/functions/get");
+            assert_eq!(request.url().path(), "/v1/functions/describe");
            let body: serde_json::Value =
                serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
            assert_eq!(
@@ -1775,6 +1775,18 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {

        Ok(())
    }
+    async fn snapshot_at_current_version(&self) -> Result<Option<Arc<dyn BaseTable>>> {
+        // A checked-out handle already names its snapshot. Otherwise resolve
+        // latest exactly once before creating the independent pinned handle.
+        let version = match self.current_version().await {
+            Some(version) => version,
+            None => self.describe().await?.version,
+        };
+
+        let snapshot = self.with_branch(self.branch.clone());
+        *snapshot.version.write().await = Some(version);
+        Ok(Some(Arc::new(snapshot)))
+    }
    async fn restore(&self) -> Result<()> {
        let mut request = self
            .client
@@ -2864,7 +2876,10 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
        })
    }

-    async fn refresh_column_async(&self, column: &str) -> Result<Job> {
+    async fn refresh_column_async(
+        &self,
+        column: &str,
+    ) -> Result<Job<crate::function::RefreshColumnResult>> {
        self.check_mutable().await?;
        let mut body = serde_json::json!({ "column": column });
        self.apply_branch_body(&mut body);
@@ -2885,7 +2900,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
            status_code: None,
        })?;

-        Ok(Job::new(Box::new(FreshnessJob {
+        Ok(Job::new_typed(Box::new(FreshnessJob {
            inner: RemoteJob::new(self.client.clone(), response.job_id),
            freshness: self.freshness.clone(),
            version: self.version.clone(),
@@ -3280,6 +3295,21 @@ mod tests {
        },
    };

+    fn refresh_done(job_id: &str) -> String {
+        json!({
+            "job_id": job_id,
+            "job_state": "DONE",
+            "result": {
+                "rows_assigned": 12,
+                "rows_failed": 0,
+                "rows_remaining": 0,
+                "source_version": 7,
+                "published_version": 8,
+            }
+        })
+        .to_string()
+    }
+
    #[tokio::test]
    async fn test_not_found() {
        let table = Table::new_with_handler("my_table", |_| {
@@ -4319,6 +4349,43 @@ mod tests {
        assert!(!table.base_table().scan_order_is_deterministic());
    }

+    #[tokio::test]
+    async fn test_checkout_branch_pins_without_touching_the_original() {
+        let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
+        let recorder = seen.clone();
+        let table = Table::new_with_handler_version(
+            "my_table",
+            semver::Version::new(0, 5, 0),
+            move |request| match request.url().path() {
+                "/v1/table/my_table/describe/" => http::Response::builder()
+                    .status(200)
+                    .body(br#"{"version": 42, "schema": {"fields": []}}"#.to_vec())
+                    .unwrap(),
+                "/v1/table/my_table/count_rows/" => {
+                    let body = request_body_json(&request);
+                    recorder.lock().unwrap().push(body["version"].clone());
+                    http::Response::builder()
+                        .status(200)
+                        .body(b"0".to_vec())
+                        .unwrap()
+                }
+                path => panic!("unexpected request path: {path}"),
+            },
+        );
+
+        let pinned = table.checkout_branch("main", Some(42)).await.unwrap();
+        pinned.count_rows(None).await.unwrap();
+        table.count_rows(None).await.unwrap();
+
+        let seen = seen.lock().unwrap();
+        assert_eq!(seen[0], 42, "the pinned handle must send its version");
+        assert!(
+            seen[1].is_null(),
+            "the original handle must still track latest, got {:?}",
+            seen[1]
+        );
+    }
+
    #[tokio::test]
    async fn test_fetch_blobs_sends_the_checked_out_version() {
        let ipc = one_row_blob_ipc_stream("image");
@@ -6735,6 +6802,53 @@ mod tests {
        assert_eq!(result.version, 8);
    }

+    #[tokio::test]
+    async fn test_add_fixed_size_list_function_column_declares_the_vector_type() {
+        let table = Table::new_with_handler("my_table", |request| {
+            match request.url().path() {
+            "/v1/table/my_table/describe/" => http::Response::builder()
+                .status(200)
+                .body(
+                    r#"{"version":1,"schema":{"fields":[{"name":"description","nullable":true,"type":{"type":"string"}}]}}"#,
+                )
+                .unwrap(),
+            "/v1/table/my_table/add_columns/" => {
+                let actual: serde_json::Value = serde_json::from_slice(
+                    request.body().unwrap().as_bytes().unwrap(),
+                )
+                .unwrap();
+                let expected: serde_json::Value = serde_json::from_str(include_str!(
+                    "../../tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json"
+                ))
+                .unwrap();
+                assert_eq!(actual, expected);
+                http::Response::builder()
+                    .status(200)
+                    .body(r#"{"version":8}"#)
+                    .unwrap()
+            }
+            path => panic!("Unexpected path: {path}"),
+        }
+        });
+        let application = crate::function::FunctionApplication::from_json(
+            r#"{
+                "function":{"name":"embed","version":"fv_01K3EXACT"},
+                "inputs":[{"parameter":"text","kind":"column","value":{"path":"description"}}],
+                "output":{"kind":"scalar","arrow_type":"fixed_size_list<float32, 3>","nullable":false},
+                "group_id":"fg_fixed"
+            }"#,
+        )
+        .unwrap();
+
+        let result = table
+            .add_columns()
+            .function_as("embedding", application)
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(result.version, 8);
+    }
+
    #[tokio::test]
    async fn test_add_named_struct_function_expands_one_atomic_sibling_group() {
        let table = Table::new_with_handler("my_table", |request| match request.url().path() {
@@ -6892,7 +7006,7 @@ mod tests {
                    .unwrap(),
                "/v1/jobs/describe" => http::Response::builder()
                    .status(200)
-                    .body(r#"{"job_id": "j-7", "job_state": "DONE"}"#.to_string())
+                    .body(refresh_done("j-7"))
                    .unwrap(),
                "/v1/table/my_table/count_rows/" => {
                    saw.store(
@@ -6908,7 +7022,9 @@ mod tests {
            });

        let job = table.refresh_column_async("doubled").await.unwrap();
-        job.wait().await.unwrap();
+        let result = job.wait().await.unwrap();
+        assert_eq!(result.rows_assigned, 12);
+        assert_eq!(result.published_version, Some(8));
        table.count_rows(None).await.unwrap();
        assert!(
            saw_min_timestamp.load(std::sync::atomic::Ordering::SeqCst),
@@ -6930,7 +7046,7 @@ mod tests {
                    .unwrap(),
                "/v1/jobs/describe" => http::Response::builder()
                    .status(200)
-                    .body(r#"{"job_id": "j-8", "job_state": "DONE"}"#.to_string())
+                    .body(refresh_done("j-8"))
                    .unwrap(),
                "/v1/table/my_table/describe/" => {
                    let schema = Schema::new(vec![Field::new("x", DataType::Int32, true)]);
@@ -6976,7 +7092,7 @@ mod tests {
                    .unwrap(),
                "/v1/jobs/describe" => http::Response::builder()
                    .status(200)
-                    .body(r#"{"job_id": "j-9", "job_state": "DONE"}"#.to_string())
+                    .body(refresh_done("j-9"))
                    .unwrap(),
                "/v1/table/my_table/tags/version/" => http::Response::builder()
                    .status(200)
@@ -7040,7 +7156,7 @@ mod tests {
                }
                "/v1/jobs/describe" => http::Response::builder()
                    .status(200)
-                    .body(r#"{"job_id": "j-10", "job_state": "DONE"}"#.to_string())
+                    .body(refresh_done("j-10"))
                    .unwrap(),
                "/v1/table/my_table/describe/" => {
                    let schema = Schema::new(vec![Field::new("x", DataType::Int32, true)]);
@@ -7113,7 +7229,7 @@ mod tests {
                }
                "/v1/jobs/describe" => http::Response::builder()
                    .status(200)
-                    .body(r#"{"job_id": "j-11", "job_state": "DONE"}"#.to_string())
+                    .body(refresh_done("j-11"))
                    .unwrap(),
                "/v1/table/my_table/count_rows/" => {
                    *saw.lock().unwrap() = request
@@ -771,7 +771,10 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
    }
    /// Fill a computed column's unfilled rows, returning a [`Job`] tracking
    /// the operation.
-    async fn refresh_column_async(&self, _column: &str) -> Result<Job> {
+    async fn refresh_column_async(
+        &self,
+        _column: &str,
+    ) -> Result<Job<crate::function::RefreshColumnResult>> {
        Err(Error::NotSupported {
            message: "computed columns are supported only on local tables".into(),
        })
@@ -789,6 +792,14 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
    async fn checkout_tag(&self, tag: &str) -> Result<()>;
    /// Checkout the latest version of the table.
    async fn checkout_latest(&self) -> Result<()>;
+    /// Return an independent handle pinned to the version currently selected.
+    ///
+    /// Backends that can advance between requests should override this for
+    /// multi-request operations that need snapshot consistency. Backends whose
+    /// existing handles already provide the desired behavior return `None`.
+    async fn snapshot_at_current_version(&self) -> Result<Option<Arc<dyn BaseTable>>> {
+        Ok(None)
+    }
    /// Whether repeated identical scans return rows in the same order.
    ///
    /// Callers that assign meaning to a row's position must order the results
@@ -1708,7 +1719,9 @@ impl Table {
    /// operation instead of blocking until it completes.
    ///
    /// The job may already be complete when returned, and callers must not
-    /// assume the column is filled until [`Job::wait`] returns. Invalid input
+    /// assume the column is filled until [`Job::wait`] returns. A successful
+    /// wait returns the durable [`crate::function::RefreshColumnResult`] for
+    /// both expression-backed and Function-backed columns. Invalid input
    /// -- an unknown column, or one that is not computed -- is reported by
    /// this call rather than by the job. On local tables the job runs as an
    /// in-process task; on LanceDB Cloud and Enterprise it is the server's
@@ -1719,11 +1732,15 @@ impl Table {
    /// # async fn refresh_in_background(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
    /// let job = table.refresh_column_async("doubled").await?;
    /// println!("refresh running: {:?}", job.status().await?);
-    /// job.wait().await?;
+    /// let result = job.wait().await?;
+    /// println!("assigned {} rows", result.rows_assigned);
    /// # Ok(())
    /// # }
    /// ```
-    pub async fn refresh_column_async(&self, column: impl AsRef<str>) -> Result<Job> {
+    pub async fn refresh_column_async(
+        &self,
+        column: impl AsRef<str>,
+    ) -> Result<Job<crate::function::RefreshColumnResult>> {
        self.inner.refresh_column_async(column.as_ref()).await
    }

@@ -3425,7 +3442,10 @@ impl BaseTable for NativeTable {
        Ok(result)
    }

-    async fn refresh_column_async(&self, column: &str) -> Result<Job> {
+    async fn refresh_column_async(
+        &self,
+        column: &str,
+    ) -> Result<Job<crate::function::RefreshColumnResult>> {
        refresh::execute_refresh_column_async(self, column).await
    }

@@ -586,6 +586,25 @@ fn canonical_input_arrow_type(field: &JsonArrowField) -> Result<String> {
    }
 }

+/// `fixed_size_list<item, size>` -> (`item`, `size`); the comma must sit outside
+/// any nested `<...>`.
+fn split_fixed_size_list(raw: &str) -> Option<(&str, i32)> {
+    let inner = raw.strip_prefix("fixed_size_list<")?.strip_suffix('>')?;
+    let mut depth = 0_u32;
+    let mut separator = None;
+    for (index, byte) in inner.bytes().enumerate() {
+        match byte {
+            b'<' => depth += 1,
+            b'>' => depth = depth.checked_sub(1)?,
+            b',' if depth == 0 => separator = Some(index),
+            _ => {}
+        }
+    }
+    let (item, size) = inner.split_at(separator?);
+    let size: i32 = size[1..].trim().parse().ok()?;
+    (size > 0).then_some((item.trim(), size))
+}
+
 fn parse_output_arrow_type(raw: &str) -> Result<JsonArrowDataType> {
    fn parse(raw: &str) -> Result<JsonArrowDataType> {
        let raw = raw.trim();
@@ -618,6 +637,16 @@ fn parse_output_arrow_type(raw: &str) -> Result<JsonArrowDataType> {
            )]);
            return Ok(data_type);
        }
+        if let Some((inner, size)) = split_fixed_size_list(raw) {
+            let mut data_type = JsonArrowDataType::new("fixed_size_list".to_string());
+            data_type.fields = Some(vec![JsonArrowField::new(
+                "item".to_string(),
+                false,
+                parse(inner)?,
+            )]);
+            data_type.length = Some(i64::from(size));
+            return Ok(data_type);
+        }
        let normalized = match raw {
            "boolean" => "bool",
            "string" => "utf8",
@@ -1322,6 +1351,32 @@ pub(super) async fn add_foreign_kind(table: &crate::Table, name: &str, kind: &st

 #[cfg(test)]
 mod tests {
+    #[test]
+    fn output_arrow_type_grammar_matches_the_shared_golden() {
+        let golden: serde_json::Value = serde_json::from_str(include_str!(
+            "../../tests/fixtures/first_class_functions/v1/arrow_types.json"
+        ))
+        .unwrap();
+        let valid = golden["valid"].as_array().unwrap().iter();
+        for case in valid.chain(golden["server_only"].as_array().unwrap()) {
+            let raw = case["arrow_type"].as_str().unwrap();
+            let parsed = super::parse_output_arrow_type(raw)
+                .unwrap_or_else(|error| panic!("{raw}: {error}"));
+            assert_eq!(
+                serde_json::to_value(&parsed).unwrap(),
+                case["json"],
+                "{raw}"
+            );
+        }
+        for raw in golden["invalid"].as_array().unwrap() {
+            let raw = raw.as_str().unwrap();
+            assert!(
+                super::parse_output_arrow_type(raw).is_err(),
+                "{raw:?} should be rejected"
+            );
+        }
+    }
+
    use arrow_array::record_batch;
    use arrow_schema::DataType;
    use futures::TryStreamExt;
@@ -70,6 +70,9 @@ async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> R
        .contains(&NamespaceClientPushdownOperation::QueryTable)
        && table.namespace_client.is_some()
        && table.dataset.current_branch().is_none()
+        // NsQueryTableRequest has no version field, so a pushed-down query would
+        // read latest and ignore the pin.
+        && table.dataset.time_travel_version().is_none()
        && !requires_local_namespace_execution(query))
    {
        return Ok(false);
@@ -49,11 +49,25 @@ pub struct RefreshColumnResult {
    pub version: u64,
 }

+struct RefreshExecution {
+    result: RefreshColumnResult,
+    source_version: u64,
+}
+
 /// Internal implementation of the refresh logic.
 pub(crate) async fn execute_refresh_column(
    table: &NativeTable,
    column: &str,
 ) -> Result<RefreshColumnResult> {
+    Ok(execute_refresh_column_with_source(table, column)
+        .await?
+        .result)
+}
+
+async fn execute_refresh_column_with_source(
+    table: &NativeTable,
+    column: &str,
+) -> Result<RefreshExecution> {
    table.dataset.ensure_mutable()?;
    ensure_no_lsm_write_spec(table).await?;
    let dataset = table.dataset.get().await?;
@@ -87,9 +101,13 @@ pub(crate) async fn execute_refresh_column(
    }

    if replacements.is_empty() {
-        return Ok(RefreshColumnResult {
-            rows_filled: 0,
-            version: dataset.version().version,
+        let source_version = dataset.version().version;
+        return Ok(RefreshExecution {
+            result: RefreshColumnResult {
+                rows_filled: 0,
+                version: source_version,
+            },
+            source_version,
        });
    }

@@ -110,14 +128,20 @@ pub(crate) async fn execute_refresh_column(

    let version = new_dataset.version().version;
    table.dataset.update(new_dataset);
-    Ok(RefreshColumnResult {
-        rows_filled,
-        version,
+    Ok(RefreshExecution {
+        result: RefreshColumnResult {
+            rows_filled,
+            version,
+        },
+        source_version: read_version,
    })
 }

 /// Run the refresh as a [`Job`] in this process.
-pub(crate) async fn execute_refresh_column_async(table: &NativeTable, column: &str) -> Result<Job> {
+pub(crate) async fn execute_refresh_column_async(
+    table: &NativeTable,
+    column: &str,
+) -> Result<Job<crate::function::RefreshColumnResult>> {
    // Validate before spawning so bad input is reported by this call rather
    // than only by the job.
    table.dataset.ensure_mutable()?;
@@ -129,9 +153,16 @@ pub(crate) async fn execute_refresh_column_async(table: &NativeTable, column: &s
    let table = table.clone();
    let column = column.to_string();
    Ok(Job::spawned(tokio::spawn(async move {
-        execute_refresh_column(&table, &column).await?;
+        let execution = execute_refresh_column_with_source(&table, &column).await?;
        table.bump_freshness();
-        Ok(())
+        Ok(crate::function::RefreshColumnResult {
+            rows_assigned: execution.result.rows_filled,
+            rows_failed: 0,
+            rows_remaining: 0,
+            source_version: execution.source_version,
+            published_version: (execution.result.rows_filled > 0)
+                .then_some(execution.result.version),
+        })
    })))
 }

@@ -396,6 +427,17 @@ mod tests {
            read(&table, "doubled").await,
            vec![Some(2), Some(4), Some(6)]
        );
+
+        let no_op = table
+            .refresh_column_async("doubled")
+            .await
+            .unwrap()
+            .wait()
+            .await
+            .unwrap();
+        assert_eq!(no_op.rows_assigned, 0);
+        assert_eq!(no_op.source_version, 3);
+        assert_eq!(no_op.published_version, None);
    }

    /// Values written after the last refresh must be reachable by another one.
@@ -646,7 +688,12 @@ mod tests {

        let job = table.refresh_column_async("doubled").await.unwrap();
        assert!(job.id().is_none(), "in-process jobs have no server id");
-        job.wait().await.unwrap();
+        let result = job.wait().await.unwrap();
+        assert_eq!(result.rows_assigned, 3);
+        assert_eq!(result.rows_failed, 0);
+        assert_eq!(result.rows_remaining, 0);
+        assert_eq!(result.source_version, 2);
+        assert_eq!(result.published_version, Some(3));
        assert_eq!(job.status().await.unwrap(), "finished");
        assert_eq!(
            read(&table, "doubled").await,
@@ -672,9 +719,9 @@ mod tests {
        declare_doubled(&table).await.unwrap();

        let job = table.refresh_column_async("doubled").await.unwrap();
-        job.wait().await.unwrap();
+        let first = job.wait().await.unwrap();
        // A second wait after completion observes the same outcome.
-        job.wait().await.unwrap();
+        assert_eq!(job.wait().await.unwrap(), first);
        assert_eq!(job.status().await.unwrap(), "finished");
    }

@@ -0,0 +1,333 @@
+{
+  "valid": [
+    {
+      "arrow_type": "bool",
+      "json": {
+        "type": "bool"
+      }
+    },
+    {
+      "arrow_type": "int8",
+      "json": {
+        "type": "int8"
+      }
+    },
+    {
+      "arrow_type": "int16",
+      "json": {
+        "type": "int16"
+      }
+    },
+    {
+      "arrow_type": "int32",
+      "json": {
+        "type": "int32"
+      }
+    },
+    {
+      "arrow_type": "int64",
+      "json": {
+        "type": "int64"
+      }
+    },
+    {
+      "arrow_type": "uint8",
+      "json": {
+        "type": "uint8"
+      }
+    },
+    {
+      "arrow_type": "uint16",
+      "json": {
+        "type": "uint16"
+      }
+    },
+    {
+      "arrow_type": "uint32",
+      "json": {
+        "type": "uint32"
+      }
+    },
+    {
+      "arrow_type": "uint64",
+      "json": {
+        "type": "uint64"
+      }
+    },
+    {
+      "arrow_type": "float16",
+      "json": {
+        "type": "float16"
+      }
+    },
+    {
+      "arrow_type": "float32",
+      "json": {
+        "type": "float32"
+      }
+    },
+    {
+      "arrow_type": "float64",
+      "json": {
+        "type": "float64"
+      }
+    },
+    {
+      "arrow_type": "utf8",
+      "json": {
+        "type": "utf8"
+      }
+    },
+    {
+      "arrow_type": "binary",
+      "json": {
+        "type": "binary"
+      }
+    },
+    {
+      "arrow_type": "date32",
+      "json": {
+        "type": "date32"
+      }
+    },
+    {
+      "arrow_type": "date64",
+      "json": {
+        "type": "date64"
+      }
+    },
+    {
+      "arrow_type": "list<float32>",
+      "json": {
+        "type": "list",
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "float32"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "large_list<float32>",
+      "json": {
+        "type": "large_list",
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "float32"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "list<int64>",
+      "json": {
+        "type": "list",
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "int64"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "large_list<int64>",
+      "json": {
+        "type": "large_list",
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "int64"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "list<utf8>",
+      "json": {
+        "type": "list",
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "utf8"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "large_list<utf8>",
+      "json": {
+        "type": "large_list",
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "utf8"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "fixed_size_list<float32, 384>",
+      "json": {
+        "type": "fixed_size_list",
+        "length": 384,
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "float32"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "fixed_size_list<float16, 8>",
+      "json": {
+        "type": "fixed_size_list",
+        "length": 8,
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "float16"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "fixed_size_list<uint8, 1>",
+      "json": {
+        "type": "fixed_size_list",
+        "length": 1,
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "uint8"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "list<list<float32>>",
+      "json": {
+        "type": "list",
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "list",
+              "fields": [
+                {
+                  "name": "item",
+                  "nullable": false,
+                  "type": {
+                    "type": "float32"
+                  }
+                }
+              ]
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "fixed_size_list<list<int32>, 2>",
+      "json": {
+        "type": "fixed_size_list",
+        "length": 2,
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "list",
+              "fields": [
+                {
+                  "name": "item",
+                  "nullable": false,
+                  "type": {
+                    "type": "int32"
+                  }
+                }
+              ]
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "list<fixed_size_list<float32, 3>>",
+      "json": {
+        "type": "list",
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "fixed_size_list",
+              "length": 3,
+              "fields": [
+                {
+                  "name": "item",
+                  "nullable": false,
+                  "type": {
+                    "type": "float32"
+                  }
+                }
+              ]
+            }
+          }
+        ]
+      }
+    }
+  ],
+  "server_only": [
+    {
+      "arrow_type": "null",
+      "json": {
+        "type": "null"
+      }
+    }
+  ],
+  "invalid": [
+    "",
+    "list<>",
+    "list<float32",
+    "fixed_size_list<float32>[3]",
+    "fixed_size_list<float32>",
+    "fixed_size_list<float32, 0>",
+    "fixed_size_list<float32, x>",
+    "map<utf8, int32>",
+    "decimal128(10, 2)",
+    "timestamp[us]",
+    "struct<a: int32>"
+  ]
+}
@@ -0,0 +1,79 @@
+{
+  "new_columns": [
+    {
+      "name": "embedding",
+      "all_null": true
+    }
+  ],
+  "function": {
+    "application": {
+      "function": {
+        "name": "embed",
+        "version": "fv_01K3EXACT"
+      },
+      "inputs": [
+        {
+          "parameter": "text",
+          "kind": "column",
+          "value": {
+            "path": "description"
+          }
+        }
+      ],
+      "output": {
+        "kind": "scalar",
+        "arrow_type": "fixed_size_list<float32, 3>",
+        "nullable": false
+      },
+      "group_id": "fg_fixed"
+    },
+    "binding_metadata_version": 1,
+    "input_bindings": [
+      {
+        "parameter": "text",
+        "field_path": "description",
+        "arrow_type": "utf8",
+        "nullable": true
+      }
+    ],
+    "input_schema": {
+      "fields": [
+        {
+          "name": "text",
+          "nullable": true,
+          "type": {
+            "type": "utf8"
+          }
+        }
+      ]
+    },
+    "output_schema": {
+      "fields": [
+        {
+          "name": "embedding",
+          "nullable": true,
+          "type": {
+            "type": "fixed_size_list",
+            "length": 3,
+            "fields": [
+              {
+                "name": "item",
+                "nullable": false,
+                "type": {
+                  "type": "float32"
+                }
+              }
+            ]
+          }
+        }
+      ]
+    },
+    "outputs": [
+      {
+        "result_field": "$value",
+        "output_name": "embedding",
+        "output_ordinal": 0
+      }
+    ]
+  }
+}