diff --git a/.agents/skills/README.md b/.agents/skills/README.md
index d4e3dc45d..296ae3f86 100644
--- a/.agents/skills/README.md
+++ b/.agents/skills/README.md
@@ -5,7 +5,3 @@ This directory contains repo-scoped code agent skills for the LanceDB project.
 Each skill is a folder that contains a required `SKILL.md` and optional bundled resources.
 
 Codex discovers skills from `.agents/skills` in the current working directory and parent directories.
-
-The `lancedb` skill lives in the `plugins/lancedb` plugin (see `plugins/lancedb/skills/lancedb`)
-so it can be installed via the plugin marketplaces (`.claude-plugin/marketplace.json` and
-`.agents/plugins/marketplace.json`); the `lancedb` entry here is a symlink into that plugin.
diff --git a/.agents/skills/lancedb b/.agents/skills/lancedb
deleted file mode 120000
index 1a303efd6..000000000
--- a/.agents/skills/lancedb
+++ /dev/null
@@ -1 +0,0 @@
-../../plugins/lancedb/skills/lancedb
\ No newline at end of file
diff --git a/.bumpversion.toml b/.bumpversion.toml
index 601b14d3f..1c4aea809 100644
--- a/.bumpversion.toml
+++ b/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.37.1-beta.0"
+current_version = "0.38.0-beta.10"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.
diff --git a/.cargo/config.toml b/.cargo/config.toml
index 0a4e3990e..95f9e7df4 100644
--- a/.cargo/config.toml
+++ b/.cargo/config.toml
@@ -9,6 +9,18 @@ debug = true
 codegen-units = 16
 lto = "thin"
 
+[profile.release-no-lto]
+inherits = "release"
+debug = true
+lto = false
+# Prioritize compile time when LTO is not relevant to the measurement.
+codegen-units = 16
+
+[profile.bench]
+inherits = "release"
+lto = "thin"
+codegen-units = 16
+
 [target.'cfg(all())']
 rustflags = [
     "-Wclippy::all",
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index d5d4cab08..eee966f76 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -17,6 +17,18 @@ updates:
     # newer minimum versions.
     versioning-strategy: lockfile-only
     groups:
+      # The arrow-rs and datafusion crates are released in lockstep and have to
+      # move together, so keep them in one PR instead of one per sub-crate.
+      # Listed first: a dependency joins the first group it matches.
+      arrow-datafusion:
+        patterns:
+          - arrow
+          - arrow-*
+          - parquet
+          - parquet-*
+          - datafusion
+          - datafusion-*
+          - object_store
       rust-minor-patch:
         update-types:
           - minor
diff --git a/.github/workflows/ci-scripts.yml b/.github/workflows/ci-scripts.yml
new file mode 100644
index 000000000..9b9124ab2
--- /dev/null
+++ b/.github/workflows/ci-scripts.yml
@@ -0,0 +1,30 @@
+name: CI scripts
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - ci/set_lance_version.py
+      - ci/tests/**
+      - .github/workflows/ci-scripts.yml
+  pull_request:
+    paths:
+      - ci/set_lance_version.py
+      - ci/tests/**
+      - .github/workflows/ci-scripts.yml
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Test CI scripts
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+      - name: Run tests
+        run: python -m unittest discover -s ci/tests -v
diff --git a/.github/workflows/codex-update-lance-dependency.yml b/.github/workflows/codex-update-lance-dependency.yml
index 420daf650..79ef84364 100644
--- a/.github/workflows/codex-update-lance-dependency.yml
+++ b/.github/workflows/codex-update-lance-dependency.yml
@@ -4,14 +4,14 @@ on:
   workflow_call:
     inputs:
       tag:
-        description: "Tag name from Lance. If omitted, the skill will use the latest Lance release that needs an update."
+        description: "Tag name from Lance (e.g. `v7.2.0-beta.1`). If omitted, the newest release is resolved automatically — stable releases are preferred over pre-releases — and the run is skipped if it is not newer than the version currently pinned in Cargo.toml."
         required: false
         default: ""
         type: string
   workflow_dispatch:
     inputs:
       tag:
-        description: "Tag name from Lance. Leave empty to use the latest Lance release that needs an update."
+        description: "Tag name from Lance (e.g. `v7.2.0-beta.1`). Leave empty to resolve the newest release automatically — stable releases are preferred over pre-releases — and skip the run if it is not newer than the version currently pinned in Cargo.toml."
         required: false
         default: ""
         type: string
diff --git a/.github/workflows/docs-link-check.yml b/.github/workflows/docs-link-check.yml
new file mode 100644
index 000000000..1286819bc
--- /dev/null
+++ b/.github/workflows/docs-link-check.yml
@@ -0,0 +1,243 @@
+name: Check doc links
+
+# Checking external links is inherently noisy: third-party sites rate-limit
+# automated clients, reject non-browser user agents, and go down temporarily.
+# Blocking pull requests on that trades a lot of false failures for very little
+# signal, so this runs on a schedule and reports findings in a single tracking
+# issue instead of failing anyone's build.
+on:
+  schedule:
+    - cron: "0 7 * * *"
+  workflow_dispatch:
+
+# The report lives in one repository-global issue, so runs must not overlap: a
+# lookup racing a create produces duplicate issues, and a healthy run closing
+# the issue while a failing run only rewrites its body would leave a broken
+# report closed. The group is deliberately ref-independent so that a manual
+# dispatch serializes against the scheduled run.
+concurrency:
+  group: docs-link-check
+  cancel-in-progress: false
+
+permissions: {}
+
+env:
+  REPORT_TITLE: "Docs link checker report"
+
+jobs:
+  scan:
+    name: Scan links
+    runs-on: ubuntu-24.04
+    # lychee-action is pinned by SHA, but its wrapper downloads the lychee
+    # release tarball at run time without verifying a digest, and hands the
+    # resulting binary a GitHub token. Release assets remain replaceable, so
+    # that binary is confined to a job whose token can only read public
+    # content; everything that writes runs in the report job below.
+    permissions:
+      contents: read
+    outputs:
+      checker_outcome: ${{ steps.lychee.outcome }}
+      exit_code: ${{ steps.lychee.outputs.exit_code }}
+      status: ${{ steps.validate.outputs.status }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          # workflow_dispatch can run from any ref, but the report is
+          # repository-global. Always measure the default branch so a manual
+          # run from a topic branch cannot close a report that main warrants,
+          # or overwrite it with branch-only findings.
+          ref: ${{ github.event.repository.default_branch }}
+          persist-credentials: false
+
+      - name: Check links
+        id: lychee
+        continue-on-error: true
+        uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0
+        with:
+          # Restricted to http(s) on purpose. Much of docs/src is generated
+          # API reference (the js/ tree comes from `npm run docs` in nodejs)
+          # and the hand-written pages use mkdocstrings cross-references and
+          # nav-relative paths that only resolve in the site mkdocs builds,
+          # not in this checkout, so relative links would be reported as
+          # broken on every run.
+          args: >-
+            --scheme https
+            --scheme http
+            --no-progress
+            --max-retries 3
+            --timeout 20
+            'docs/src/**/*.md'
+          format: json
+          output: ./lychee/out.json
+          jobSummary: false
+          # The report issue, not a red workflow run, is the signal for link
+          # findings and checker failures alike.
+          fail: false
+
+      - name: Validate report
+        id: validate
+        # lychee does not reserve exit code 2 for broken links: its CLI
+        # parser also exits 2 on an invalid option, before any link was
+        # checked or any report written. Only a parseable report whose
+        # counts agree with a completed exit code (0 or 2) counts as a link
+        # verdict. Everything else becomes a checker-error report instead of
+        # failing the workflow. Exit 2 covers timeouts as well as errors, and a
+        # timed-out host is exactly the transient unavailability this report
+        # exists to surface, so both count as findings. Requiring total > 0
+        # also catches a glob that silently stopped matching any file.
+        if: always()
+        env:
+          CHECKER_OUTCOME: ${{ steps.lychee.outcome }}
+          EXIT_CODE: ${{ steps.lychee.outputs.exit_code }}
+        run: |
+          status=checker-error
+          if [[ "$CHECKER_OUTCOME" == success ]] &&
+             [[ "$EXIT_CODE" == 0 || "$EXIT_CODE" == 2 ]] &&
+             jq -e --argjson code "$EXIT_CODE" '
+               (.total > 0) and
+               (if $code == 0
+                then .errors == 0 and .timeouts == 0
+                  and (.error_map | length == 0) and (.timeout_map | length == 0)
+                else (.errors + .timeouts) > 0
+                  and ((.error_map | length) + (.timeout_map | length)) > 0
+                end)
+             ' ./lychee/out.json
+          then
+            if [[ "$EXIT_CODE" == 0 ]]; then
+              status=healthy
+            else
+              status=findings
+            fi
+          fi
+          echo "status=$status" >> "$GITHUB_OUTPUT"
+          echo "Validated link check as $status"
+
+      - name: Upload report
+        if: steps.validate.outputs.status == 'findings'
+        uses: actions/upload-artifact@v7
+        with:
+          name: link-report
+          path: ./lychee/out.json
+          retention-days: 7
+
+  report:
+    name: Update report issue
+    needs: scan
+    runs-on: ubuntu-24.04
+    # Deliberately no checkout: this job needs the report artifact and the
+    # issues API, not the repository contents.
+    permissions:
+      issues: write
+    env:
+      CHECKER_OUTCOME: ${{ needs.scan.outputs.checker_outcome }}
+      EXIT_CODE: ${{ needs.scan.outputs.exit_code }}
+      STATUS: ${{ needs.scan.outputs.status }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Find existing report issue
+        id: report
+        # Matched on title alone, and through search rather than a listing:
+        # the issue action applies labels in a separate call after creating the
+        # issue, so a label filter misses a half-created report, and this
+        # repository has far more open issues than one listing page holds.
+        # Closed issues are included because a healthy run closes the report:
+        # an open-only lookup would forget that identity and the next failing
+        # run would open a duplicate. The oldest match stays the canonical
+        # report and is reopened below when a problem recurs.
+        run: |
+          match=$(gh issue list --repo "$GITHUB_REPOSITORY" --state all \
+            --search "in:title \"$REPORT_TITLE\" author:app/github-actions" \
+            --limit 50 --json number,title,state \
+            --jq "[.[] | select(.title == \"$REPORT_TITLE\")] | sort_by(.number) | first // empty")
+          echo "number=$(jq -r '.number // empty' <<<"$match")" >> "$GITHUB_OUTPUT"
+          echo "state=$(jq -r '.state // empty' <<<"$match")" >> "$GITHUB_OUTPUT"
+
+      - name: Download report
+        if: env.STATUS == 'findings'
+        uses: actions/download-artifact@v8
+        with:
+          name: link-report
+          path: ./lychee
+
+      - name: Compose report
+        if: env.STATUS == 'findings'
+        run: |
+          run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
+          {
+            echo "Broken documentation links found by [\`$GITHUB_WORKFLOW\`]($run_url)."
+            echo
+            echo "This issue is rewritten by every scheduled run and closed automatically once all links resolve."
+            echo
+            echo "Entries can be false positives: some sites rate-limit or block automated clients while working fine in a browser. Confirm before editing the docs, and add persistent offenders to \`--exclude\` in \`.github/workflows/docs-link-check.yml\`."
+            echo
+            # Timeouts are reported alongside errors: entries land in
+            # timeout_map with a status text instead of an HTTP code.
+            jq -r '
+              "\(.errors) of \(.total) links failed, \(.timeouts) timed out.",
+              "",
+              ([(.error_map | to_entries[]), (.timeout_map | to_entries[])]
+               | group_by(.key)[] |
+                "### Errors in \(.[0].key)",
+                "",
+                (map(.value[])[] | "* [\(.status.code // .status.text // "ERR")] <\(.url)> — \(.status.details // .status.text // "unknown error")"),
+                "")
+            ' ./lychee/out.json
+          } > ./lychee/issue.md
+
+      - name: Compose checker error report
+        if: env.STATUS == 'checker-error'
+        run: |
+          mkdir -p ./lychee
+          run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
+          {
+            echo "The documentation link check did not complete in [the latest run]($run_url)."
+            echo
+            echo "This issue is rewritten by every scheduled run and closed automatically once a trustworthy run finds that all links resolve."
+            echo
+            echo "The checker did not produce a trustworthy link verdict. Treat the previous result, if any, as stale until a later run completes."
+            echo
+            echo "* Action outcome: \`$CHECKER_OUTCOME\`"
+            echo "* Exit code: \`${EXIT_CODE:-not reported}\`"
+            echo "* Verdict validation: \`failed\`"
+          } > ./lychee/issue.md
+
+      - name: Reopen report issue
+        # A healthy run closes the report, and the issue action below only
+        # rewrites the body of whatever number it is given. Without an
+        # explicit reopen, a later finding or checker error would rewrite a
+        # closed issue. A CLOSED state implies the lookup found a canonical
+        # issue, so no separate emptiness check.
+        if: >-
+          env.STATUS != 'healthy' &&
+          steps.report.outputs.state == 'CLOSED'
+        env:
+          ISSUE_NUMBER: ${{ steps.report.outputs.number }}
+        run: |
+          run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
+          gh issue reopen "$ISSUE_NUMBER" --repo "$GITHUB_REPOSITORY" \
+            --comment "The documentation link checker reported a problem again in [the latest run]($run_url)."
+
+      - name: Report link-check problem
+        if: env.STATUS != 'healthy'
+        uses: peter-evans/create-issue-from-file@fca9117c27cdc29c6c4db3b86c48e4115a786710 # v6.0.0
+        with:
+          # Empty on the first failing run, which creates the issue; afterwards
+          # the same issue is updated in place.
+          issue-number: ${{ steps.report.outputs.number }}
+          title: ${{ env.REPORT_TITLE }}
+          content-filepath: ./lychee/issue.md
+          labels: documentation
+
+      - name: Close report issue once links are healthy
+        # An OPEN state implies the lookup found a canonical issue; a report
+        # that is already closed needs nothing.
+        if: >-
+          env.STATUS == 'healthy' &&
+          steps.report.outputs.state == 'OPEN'
+        env:
+          ISSUE_NUMBER: ${{ steps.report.outputs.number }}
+        run: |
+          run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
+          gh issue close "$ISSUE_NUMBER" --repo "$GITHUB_REPOSITORY" \
+            --comment "All documentation links resolved in [the latest run]($run_url)."
diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml
index 74b7d05e6..b80c1b019 100644
--- a/.github/workflows/pypi-publish.yml
+++ b/.github/workflows/pypi-publish.yml
@@ -69,6 +69,16 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: "3.10"
+      - name: Add swap for Arm fat LTO
+        if: matrix.config.platform == 'aarch64'
+        shell: bash
+        run: |
+          swap_file="$RUNNER_TEMP/lancedb-swap"
+          sudo fallocate --length 16G "$swap_file"
+          sudo chmod 600 "$swap_file"
+          sudo mkswap "$swap_file"
+          sudo swapon "$swap_file"
+          free -h
       - uses: ./.github/workflows/build_linux_wheel
         with:
           python-minor-version: 10
@@ -119,6 +129,12 @@ jobs:
       # link.exe is single-threaded and the long pole on Windows builds. Use
       # rustc's bundled lld-link instead.
       CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld
+      # Fat LTO of the cdylib is single-threaded and the peak-memory step of the
+      # build. ThinLTO parallelizes it across the runner's cores, at some cost
+      # to runtime performance on our least performance-sensitive platform.
+      # Matches what the nodejs Windows builds already do in npm-publish.yml.
+      CARGO_PROFILE_RELEASE_LTO: thin
+      CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16
     steps:
       - uses: actions/checkout@v6
         with:
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index 52582395f..db8919ddc 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -229,7 +229,8 @@ jobs:
       # Make sure wheels are not included in the Rust cache
       - name: Delete wheels
         run: rm -rf target/wheels
-  pydantic1x:
+  min-deps:
+    name: "Minimum dependencies"
     timeout-minutes: 60
     runs-on: "ubuntu-24.04"
     defaults:
@@ -259,8 +260,7 @@ jobs:
           save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Install lancedb
         run: |
-          pip install "pydantic<2"
-          pip install pyarrow==16
+          pip install "pydantic==2.7.4" "pyarrow==16"
           pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
       - name: Run tests
         run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 1a0b65c4a..cd872621b 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -121,7 +121,6 @@ jobs:
       # Need up-to-date compilers for kernels
       CC: clang-18
       CXX: clang++-18
-      GH_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
     steps:
       - uses: actions/checkout@v6
         with:
@@ -165,11 +164,40 @@ jobs:
       - name: Run feature tests
         run: CARGO_ARGS="--profile ci" make -C ./lancedb feature-tests
       - name: Run examples
-        run: cargo run  --profile ci --example simple --locked
+        run: cargo run --profile ci --all-features --example simple --locked
+
+  remote:
+    timeout-minutes: 30
+    # Running this requires access to secrets, so skip if this is a PR from a
+    # fork. Keep it separate from the all-features build so Cargo does not
+    # retain both dependency graphs in one target directory.
+    if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
+    runs-on: ubuntu-2404-4x-x64
+    defaults:
+      run:
+        shell: bash
+        working-directory: rust
+    env:
+      CC: clang-18
+      CXX: clang++-18
+      GH_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+          lfs: true
+      - uses: Swatinem/rust-cache@v2
+        with:
+          # Remote tests use a different feature graph from the main Linux
+          # job. Cache downloads, but build into a fresh target directory.
+          cache-targets: false
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - uses: rui314/setup-mold@v1
       - name: Run remote tests
-        # Running this requires access to secrets, so skip if this is
-        # a PR from a fork.
-        if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
         run: CARGO_ARGS="--profile ci" make -C ./lancedb remote-tests
 
   macos:
@@ -296,16 +324,18 @@ jobs:
           cargo update -p aws-types --precise 1.3.9
           cargo update -p aws-sigv4 --precise 1.3.5
           cargo update -p aws-credential-types --precise 1.2.8
-          cargo update -p aws-smithy-checksums --precise 0.63.9
+          # aws-smithy-checksums must stay at or above 0.63.13: OpenDAL's S3
+          # service needs crc-fast ~1.9, and older releases pin it to ~1.3.
+          cargo update -p aws-smithy-checksums --precise 0.63.13
           cargo update -p aws-smithy-runtime --precise 1.9.3
-          cargo update -p aws-smithy-http --precise 0.62.4
-          cargo update -p aws-smithy-eventstream --precise 0.60.12
+          cargo update -p aws-smithy-http --precise 0.62.6
+          cargo update -p aws-smithy-eventstream --precise 0.60.14
           cargo update -p aws-smithy-http-client --precise 1.1.3
           cargo update -p aws-smithy-observability --precise 0.1.4
           cargo update -p aws-smithy-query --precise 0.60.8
-          cargo update -p aws-smithy-runtime-api --precise 1.9.1
-          cargo update -p aws-smithy-async --precise 1.2.6
-          cargo update -p aws-smithy-types --precise 1.3.5
+          cargo update -p aws-smithy-runtime-api --precise 1.9.3
+          cargo update -p aws-smithy-async --precise 1.2.7
+          cargo update -p aws-smithy-types --precise 1.3.6
           cargo update -p aws-smithy-xml --precise 0.60.11
           cargo update -p home --precise 0.5.9
       - name: cargo +${{ matrix.msrv }} check
diff --git a/AGENTS.md b/AGENTS.md
index 21631a2cd..1e072446a 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -18,6 +18,9 @@ Common commands:
 * Run specific test: `cargo test --quiet --features remote -p <package_name> --test <test_name>`
 * Lint: `cargo clippy --quiet --features remote --tests --examples`
 * Format Rust: `cargo fmt --all`
+* Use repository-defined Cargo profiles instead of ad hoc LTO overrides.
+* Use `release-with-debug` for benchmarks and profiling so optimized builds keep debug symbols without a rebuild.
+* Use `release-no-lto` only for local debugging, IO-bound benchmarks, or compile-time-sensitive performance investigation where LTO would not affect the measured bottleneck.
 * Format Python: `ruff format .`
 * Lint Python: `ruff check .`
 * Bootstrap Python dev env: `cd python && uv run --extra tests --extra dev maturin develop --extras tests,dev`
diff --git a/Cargo.lock b/Cargo.lock
index 995424b20..33e43f9fa 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -775,7 +775,7 @@ dependencies = [
  "http 0.2.12",
  "http 1.5.0",
  "http-body 1.1.0",
- "lru",
+ "lru 0.16.4",
  "percent-encoding",
  "regex-lite",
  "sha2 0.11.0",
@@ -959,7 +959,7 @@ dependencies = [
  "aws-smithy-runtime-api",
  "aws-smithy-types",
  "h2 0.3.27",
- "h2 0.4.14",
+ "h2 0.4.16",
  "http 0.2.12",
  "http 1.5.0",
  "http-body 0.4.6",
@@ -1241,6 +1241,12 @@ version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
+[[package]]
+name = "base64"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
+
 [[package]]
 name = "base64-simd"
 version = "0.8.0"
@@ -1734,9 +1740,9 @@ dependencies = [
 
 [[package]]
 name = "cmov"
-version = "0.5.3"
+version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746"
+checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"
 
 [[package]]
 name = "colorchoice"
@@ -1964,15 +1970,6 @@ dependencies = [
  "spin 0.10.1",
 ]
 
-[[package]]
-name = "crc32c"
-version = "0.6.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47"
-dependencies = [
- "rustc_version",
-]
-
 [[package]]
 name = "crc32fast"
 version = "1.5.0"
@@ -2174,9 +2171,9 @@ dependencies = [
 
 [[package]]
 name = "ctor"
-version = "1.0.5"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "378f0974ae2468eaf63aa036dbe9c926b0dc7ea64c156f2ea618bc2f75b934f0"
+checksum = "2d83cb7e7a873830708d6b02a78cd36a592c6fa14bf267b68725103b85c0d77f"
 dependencies = [
  "link-section",
  "linktime-proc-macro",
@@ -2902,6 +2899,37 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "defmt"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1"
+dependencies = [
+ "bitflags 1.3.2",
+ "defmt-macros",
+]
+
+[[package]]
+name = "defmt-macros"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8"
+dependencies = [
+ "defmt-parser",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
+[[package]]
+name = "defmt-parser"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e"
+dependencies = [
+ "thiserror 2.0.18",
+]
+
 [[package]]
 name = "der"
 version = "0.6.1"
@@ -3413,6 +3441,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "frostem"
+version = "1.20260804.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "82eb03a32a1d50555353c85a7b9d3279a6f1e91af9890b789acdf544ed57c8d7"
+
 [[package]]
 name = "fs_extra"
 version = "1.3.0"
@@ -3421,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arrow-array",
  "rand 0.9.5",
@@ -3779,14 +3813,23 @@ dependencies = [
 
 [[package]]
 name = "goosefs-sdk"
-version = "0.1.5"
+version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f"
+checksum = "e1ea4eee6dcbc31b25ab4fd577adc55b677d2bed3aa3016c44c58fbe1b2298a5"
 dependencies = [
+ "arc-swap",
  "async-trait",
  "bytes",
  "dashmap",
+ "fastrand",
+ "futures",
  "hostname",
+ "io-uring",
+ "itoa",
+ "libc",
+ "lru 0.18.2",
+ "memmap2 0.9.10",
+ "moka",
  "prost",
  "prost-types",
  "rand 0.9.5",
@@ -3799,6 +3842,7 @@ dependencies = [
  "tonic-prost",
  "tracing",
  "uuid",
+ "xxhash-rust",
 ]
 
 [[package]]
@@ -3833,9 +3877,9 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.4.14"
+version = "0.4.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733"
+checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27"
 dependencies = [
  "atomic-waker",
  "bytes",
@@ -4144,7 +4188,7 @@ dependencies = [
  "bytes",
  "futures-channel",
  "futures-core",
- "h2 0.4.14",
+ "h2 0.4.16",
  "http 1.5.0",
  "http-body 1.1.0",
  "httparse",
@@ -4599,10 +4643,12 @@ dependencies = [
 
 [[package]]
 name = "jiff"
-version = "0.2.24"
+version = "0.2.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d"
+checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc"
 dependencies = [
+ "defmt",
+ "jiff-core",
  "jiff-static",
  "jiff-tzdb-platform",
  "js-sys",
@@ -4611,15 +4657,25 @@ dependencies = [
  "portable-atomic-util",
  "serde_core",
  "wasm-bindgen",
- "windows-sys 0.61.2",
+ "windows-link",
+]
+
+[[package]]
+name = "jiff-core"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09"
+dependencies = [
+ "defmt",
 ]
 
 [[package]]
 name = "jiff-static"
-version = "0.2.24"
+version = "0.2.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7"
+checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204"
 dependencies = [
+ "jiff-core",
  "proc-macro2",
  "quote",
  "syn 2.0.117",
@@ -4731,24 +4787,6 @@ dependencies = [
  "zmij",
 ]
 
-[[package]]
-name = "jsonwebtoken"
-version = "10.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc"
-dependencies = [
- "aws-lc-rs",
- "base64 0.22.1",
- "getrandom 0.2.17",
- "js-sys",
- "pem",
- "serde",
- "serde_json",
- "signature 2.2.0",
- "simple_asn1",
- "zeroize",
-]
-
 [[package]]
 name = "kanaria"
 version = "0.2.0"
@@ -4777,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 
 [[package]]
 name = "lance"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -4794,7 +4832,6 @@ dependencies = [
  "async-recursion",
  "async-trait",
  "async_cell",
- "aws-credential-types",
  "aws-sdk-dynamodb",
  "byteorder",
  "bytes",
@@ -4810,7 +4847,6 @@ dependencies = [
  "either",
  "fst",
  "futures",
- "half",
  "humantime",
  "itertools 0.14.0",
  "lance-arrow",
@@ -4852,8 +4888,8 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4875,7 +4911,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4889,7 +4925,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -4898,8 +4934,8 @@ dependencies = [
 
 [[package]]
 name = "lance-bitpacking"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arrayref",
  "crunchy",
@@ -4909,8 +4945,8 @@ dependencies = [
 
 [[package]]
 name = "lance-core"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4918,12 +4954,10 @@ dependencies = [
  "arrow-schema",
  "async-trait",
  "blake3",
- "byteorder",
  "bytes",
  "datafusion-common",
  "datafusion-sql",
  "futures",
- "itertools 0.14.0",
  "lance-arrow",
  "lance-derive",
  "libc",
@@ -4941,7 +4975,6 @@ dependencies = [
  "snafu 0.9.0",
  "tempfile",
  "tokio",
- "tokio-stream",
  "tokio-util",
  "tracing",
  "twox-hash",
@@ -4950,8 +4983,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datafusion"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4970,7 +5003,6 @@ dependencies = [
  "jsonb",
  "lance-arrow",
  "lance-core",
- "lance-datagen",
  "log",
  "pin-project",
  "prost",
@@ -4981,8 +5013,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4999,8 +5031,8 @@ dependencies = [
 
 [[package]]
 name = "lance-derive"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5009,8 +5041,8 @@ dependencies = [
 
 [[package]]
 name = "lance-encoding"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -5035,8 +5067,6 @@ dependencies = [
  "num-traits",
  "prost",
  "prost-build",
- "rand 0.9.5",
- "strum 0.26.3",
  "tokio",
  "tracing",
  "xxhash-rust",
@@ -5045,8 +5075,8 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -5077,8 +5107,8 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -5093,7 +5123,6 @@ dependencies = [
  "async-trait",
  "bitvec",
  "bytes",
- "chrono",
  "crossbeam-queue",
  "datafusion",
  "datafusion-common",
@@ -5111,7 +5140,6 @@ dependencies = [
  "lance-bitpacking",
  "lance-core",
  "lance-datafusion",
- "lance-datagen",
  "lance-encoding",
  "lance-file",
  "lance-index-core",
@@ -5140,13 +5168,12 @@ dependencies = [
  "tempfile",
  "tokio",
  "tracing",
- "uuid",
 ]
 
 [[package]]
 name = "lance-index-core"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5168,8 +5195,8 @@ dependencies = [
 
 [[package]]
 name = "lance-io"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5181,10 +5208,8 @@ dependencies = [
  "bytes",
  "chrono",
  "futures",
- "goosefs-sdk",
  "http 1.5.0",
  "io-uring",
- "lance-arrow",
  "lance-core",
  "lance-namespace",
  "log",
@@ -5197,34 +5222,37 @@ dependencies = [
  "pin-project",
  "prost",
  "rand 0.9.5",
+ "reqsign-core",
+ "reqsign-file-read-tokio",
+ "reqsign-google",
  "serde",
+ "serde_json",
  "tempfile",
  "tokio",
  "tracing",
  "url",
+ "uuid",
 ]
 
 [[package]]
 name = "lance-linalg"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arrow-array",
- "arrow-buffer",
  "arrow-schema",
  "cc",
  "half",
  "lance-arrow",
  "lance-core",
  "num-traits",
- "rand 0.9.5",
  "rayon",
 ]
 
 [[package]]
 name = "lance-namespace"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arrow",
  "async-trait",
@@ -5236,8 +5264,8 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-impls"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -5267,7 +5295,6 @@ dependencies = [
  "serde",
  "serde_json",
  "sha2 0.10.9",
- "time",
  "tokio",
  "tower",
  "tower-http 0.5.2",
@@ -5277,9 +5304,9 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-reqwest-client"
-version = "0.8.6"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba3f0a235e3ed5f8805205649ccc7d7d0f3df23ce1294242c9265ad488d7f19d"
+checksum = "0a030196da1c994b63a96a4f0bf5b0cfa459fe6dadc9e962320246ca328da22a"
 dependencies = [
  "reqwest 0.12.28",
  "serde",
@@ -5291,14 +5318,13 @@ dependencies = [
 
 [[package]]
 name = "lance-select"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
  "arrow-schema",
  "byteorder",
- "bytes",
  "itertools 0.14.0",
  "lance-core",
  "roaring",
@@ -5307,8 +5333,8 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5318,6 +5344,7 @@ dependencies = [
  "async-trait",
  "aws-credential-types",
  "aws-sdk-dynamodb",
+ "blake3",
  "byteorder",
  "bytes",
  "chrono",
@@ -5347,8 +5374,8 @@ dependencies = [
 
 [[package]]
 name = "lance-testing"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5361,13 +5388,13 @@ dependencies = [
 
 [[package]]
 name = "lance-tokenizer"
-version = "10.1.0-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
+version = "11.0.0-beta.22"
+source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
 dependencies = [
+ "frostem",
  "icu_segmenter",
  "jieba-rs",
  "lindera",
- "rust-stemmers",
  "serde",
  "stop-words",
  "unicode-normalization",
@@ -5375,7 +5402,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb"
-version = "0.37.1-beta.0"
+version = "0.38.0-beta.10"
 dependencies = [
  "ahash",
  "anyhow",
@@ -5411,7 +5438,6 @@ dependencies = [
  "datafusion-physical-plan",
  "datafusion-sql",
  "futures",
- "goosefs-sdk",
  "half",
  "hf-hub",
  "http 1.5.0",
@@ -5444,6 +5470,7 @@ dependencies = [
  "random_word",
  "regex",
  "reqwest 0.12.28",
+ "roaring",
  "rstest",
  "semver",
  "serde",
@@ -5463,7 +5490,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb-nodejs"
-version = "0.37.1-beta.0"
+version = "0.38.0-beta.10"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5488,7 +5515,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb-python"
-version = "0.37.1-beta.0"
+version = "0.38.0-beta.10"
 dependencies = [
  "arrow",
  "async-trait",
@@ -5645,7 +5672,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_yaml_ng",
- "strum 0.28.0",
+ "strum",
  "strum_macros 0.28.0",
  "unicode-blocks",
  "unicode-normalization",
@@ -5675,22 +5702,22 @@ dependencies = [
  "rkyv",
  "serde",
  "serde_json",
- "strum 0.28.0",
+ "strum",
  "strum_macros 0.28.0",
  "thiserror 2.0.18",
 ]
 
 [[package]]
 name = "link-section"
-version = "0.16.1"
+version = "0.19.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c8600ca3dbe044f07955b443ff606c50f45295b863289bbe7d0844d50cf11e4"
+checksum = "5ee1a0d6e252afe82e7bc2db42fba60e02ddf3b1accaf8cb21d96e34ba61f3d4"
 
 [[package]]
 name = "linktime-proc-macro"
-version = "0.1.0"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a44cd706ff0d503ee32b2071166510ca27e281228de10cd3aa8d35ff94560f81"
+checksum = "348d0075b1fc163b26d72a7f75fc5141daf2fd1bdf128d873cbaf6785d495bdf"
 
 [[package]]
 name = "linux-raw-sys"
@@ -5747,6 +5774,15 @@ dependencies = [
  "hashbrown 0.16.1",
 ]
 
+[[package]]
+name = "lru"
+version = "0.18.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a"
+dependencies = [
+ "hashbrown 0.17.1",
+]
+
 [[package]]
 name = "lru-slab"
 version = "0.1.2"
@@ -6067,7 +6103,7 @@ checksum = "de33522036981030a75c231829566bc63414e08101a6f5ff4ac6cef19c8e0941"
 dependencies = [
  "bitflags 2.11.1",
  "chrono",
- "ctor 1.0.5",
+ "ctor 1.0.12",
  "futures",
  "napi-build",
  "napi-sys",
@@ -6091,7 +6127,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4d5c9c02556ea6dc99dffd36c1ce60141411657438501a125b675776d011ce92"
 dependencies = [
  "convert_case",
- "ctor 1.0.5",
+ "ctor 1.0.12",
  "napi-derive-backend",
  "proc-macro2",
  "quote",
@@ -6384,9 +6420,9 @@ dependencies = [
 
 [[package]]
 name = "object_store_opendal"
-version = "0.57.0"
+version = "0.58.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8"
+checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac"
 dependencies = [
  "async-trait",
  "bytes",
@@ -6447,12 +6483,13 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
 
 [[package]]
 name = "opendal"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1"
+checksum = "4f20562cc7447fcc915fc5c23df305a412ea80a733c9f2fd9e2d267e2815be6d"
 dependencies = [
- "ctor 1.0.5",
+ "ctor 1.0.12",
  "opendal-core",
+ "opendal-http-transport-reqwest",
  "opendal-layer-concurrent-limit",
  "opendal-layer-logging",
  "opendal-layer-retry",
@@ -6469,24 +6506,22 @@ dependencies = [
 
 [[package]]
 name = "opendal-core"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309"
+checksum = "ec75551ff4cf3e57da98979f6a937aaa9ddb3915bf68cc17d03df733be6646ed"
 dependencies = [
  "anyhow",
- "base64 0.22.1",
+ "base64 0.23.1",
  "bytes",
  "futures",
  "http 1.5.0",
- "http-body 1.1.0",
  "jiff",
  "log",
  "md-5 0.11.0",
  "mea",
  "percent-encoding",
- "quick-xml 0.39.4",
+ "quick-xml 0.41.0",
  "reqsign-core",
- "reqwest 0.13.3",
  "serde",
  "serde_json",
  "tokio",
@@ -6496,10 +6531,24 @@ dependencies = [
 ]
 
 [[package]]
-name = "opendal-layer-concurrent-limit"
-version = "0.57.0"
+name = "opendal-http-transport-reqwest"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f"
+checksum = "ad4d4f19c3ce01126a30611f8e544eaa217104a278c889ac17c9374fe4f9e4ef"
+dependencies = [
+ "bytes",
+ "futures",
+ "http 1.5.0",
+ "http-body 1.1.0",
+ "opendal-core",
+ "reqwest 0.13.4",
+]
+
+[[package]]
+name = "opendal-layer-concurrent-limit"
+version = "0.58.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "249ac5b0aa5a7a6c3737342d10456067937f9c9a6f3f02544271f7908ab91081"
 dependencies = [
  "futures",
  "http 1.5.0",
@@ -6509,9 +6558,9 @@ dependencies = [
 
 [[package]]
 name = "opendal-layer-logging"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2"
+checksum = "5c75411ab00f77851ff086b686c1e9ca8175ac18c15afa2cb75b9036436cb06c"
 dependencies = [
  "log",
  "opendal-core",
@@ -6519,9 +6568,9 @@ dependencies = [
 
 [[package]]
 name = "opendal-layer-retry"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7"
+checksum = "80b7738bd5f233ad8da39af9b9316b9b7a4eaddd91e8e32a1e19b7030688121d"
 dependencies = [
  "backon",
  "log",
@@ -6530,9 +6579,9 @@ dependencies = [
 
 [[package]]
 name = "opendal-layer-timeout"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3"
+checksum = "a704141924500f3803c05ed871b53305d2a2f11cb5ef20160c3ee688a1857f66"
 dependencies = [
  "opendal-core",
  "tokio",
@@ -6540,17 +6589,17 @@ dependencies = [
 
 [[package]]
 name = "opendal-service-azblob"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a"
+checksum = "b3310fbbb48f111c6f590473c2cd15e1b7f8e384444b0d4e328f0464c864d767"
 dependencies = [
- "base64 0.22.1",
+ "base64 0.23.1",
  "bytes",
  "http 1.5.0",
  "log",
  "opendal-core",
  "opendal-service-azure-common",
- "quick-xml 0.39.4",
+ "quick-xml 0.41.0",
  "reqsign-azure-storage",
  "reqsign-core",
  "reqsign-file-read-tokio",
@@ -6561,17 +6610,18 @@ dependencies = [
 
 [[package]]
 name = "opendal-service-azdls"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361"
+checksum = "2e3c406729935fe214ce574d68681a1ff7e0b322548f14094912bdbfe50e5c53"
 dependencies = [
- "base64 0.22.1",
+ "base64 0.23.1",
  "bytes",
  "http 1.5.0",
  "log",
+ "mea",
  "opendal-core",
  "opendal-service-azure-common",
- "quick-xml 0.39.4",
+ "quick-xml 0.41.0",
  "reqsign-azure-storage",
  "reqsign-core",
  "reqsign-file-read-tokio",
@@ -6581,9 +6631,9 @@ dependencies = [
 
 [[package]]
 name = "opendal-service-azure-common"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051"
+checksum = "7348c88edf15af435b7be930077746b569fac5e738c1bf6a363b675e7317c9df"
 dependencies = [
  "http 1.5.0",
  "opendal-core",
@@ -6591,15 +6641,15 @@ dependencies = [
 
 [[package]]
 name = "opendal-service-cos"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901"
+checksum = "d533d4582105d269c8aebeee5f0e8bcf960f41b8aab6197df7012254d9f39bf0"
 dependencies = [
  "bytes",
  "http 1.5.0",
  "log",
  "opendal-core",
- "quick-xml 0.39.4",
+ "quick-xml 0.41.0",
  "reqsign-core",
  "reqsign-file-read-tokio",
  "reqsign-tencent-cos",
@@ -6608,9 +6658,9 @@ dependencies = [
 
 [[package]]
 name = "opendal-service-gcs"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47"
+checksum = "007f3fba63c21e516c956b891e96ff9892d8175662bfb781cdada9d3766a11e6"
 dependencies = [
  "async-trait",
  "bytes",
@@ -6618,7 +6668,7 @@ dependencies = [
  "log",
  "opendal-core",
  "percent-encoding",
- "quick-xml 0.39.4",
+ "quick-xml 0.41.0",
  "reqsign-core",
  "reqsign-file-read-tokio",
  "reqsign-google",
@@ -6629,9 +6679,9 @@ dependencies = [
 
 [[package]]
 name = "opendal-service-goosefs"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4"
+checksum = "60871e6386f04d831e6a5bdbc032af4a91aeba49963252d0ef456a2cf36a9b78"
 dependencies = [
  "bytes",
  "goosefs-sdk",
@@ -6643,9 +6693,9 @@ dependencies = [
 
 [[package]]
 name = "opendal-service-hf"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848"
+checksum = "b41fd41eb7ed03c5e66cefda61e8e117808ffd2908f2916737cb020a6beb02c7"
 dependencies = [
  "bytes",
  "hf-xet",
@@ -6653,22 +6703,21 @@ dependencies = [
  "log",
  "opendal-core",
  "percent-encoding",
- "reqwest 0.13.3",
  "serde",
  "serde_json",
 ]
 
 [[package]]
 name = "opendal-service-oss"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace"
+checksum = "cd528ec2d49c5ca69e674ffed7b3e0686fb9cfcfea0596870de381467fda4f1b"
 dependencies = [
  "bytes",
  "http 1.5.0",
  "log",
  "opendal-core",
- "quick-xml 0.39.4",
+ "quick-xml 0.41.0",
  "reqsign-aliyun-oss",
  "reqsign-core",
  "reqsign-file-read-tokio",
@@ -6677,18 +6726,18 @@ dependencies = [
 
 [[package]]
 name = "opendal-service-s3"
-version = "0.57.0"
+version = "0.58.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb"
+checksum = "58e80cdf192d7eff05feed747894d64f81905ac4eaf132edf7ea270abdd2d663"
 dependencies = [
- "base64 0.22.1",
+ "base64 0.23.1",
  "bytes",
- "crc32c",
+ "crc-fast",
  "http 1.5.0",
  "log",
  "md-5 0.11.0",
  "opendal-core",
- "quick-xml 0.39.4",
+ "quick-xml 0.41.0",
  "reqsign-aws-v4",
  "reqsign-core",
  "reqsign-file-read-tokio",
@@ -7583,7 +7632,7 @@ version = "0.14.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
 dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
  "itertools 0.14.0",
  "log",
  "multimap",
@@ -7799,6 +7848,16 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "quick-xml"
+version = "0.41.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
 [[package]]
 name = "quick_cache"
 version = "0.6.24"
@@ -8220,9 +8279,9 @@ dependencies = [
 
 [[package]]
 name = "reqsign-aliyun-oss"
-version = "3.0.0"
+version = "3.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57ac2757f3140aa2e213b554148ae0b52733e624fc6723f0cc6bb3d440176c95"
+checksum = "a5e6d659fcdbca6fe2d7ef109c2e28499b7be80501f1bb86c10caf5ec8ac1219"
 dependencies = [
  "anyhow",
  "form_urlencoded",
@@ -8236,38 +8295,52 @@ dependencies = [
 ]
 
 [[package]]
-name = "reqsign-aws-v4"
-version = "3.0.0"
+name = "reqsign-aws-core"
+version = "3.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44eaca382e94505a49f1a4849658d153aebf79d9c1a58e5dd3b10361511e9f43"
+checksum = "e4af084e1f3cbf3e67e0c972765399bce54ecec804cceba46b39a8331f3c1bff"
 dependencies = [
- "anyhow",
  "bytes",
  "form_urlencoded",
+ "hex",
  "http 1.5.0",
  "log",
  "percent-encoding",
- "quick-xml 0.39.4",
+ "quick-xml 0.41.0",
  "reqsign-core",
  "rust-ini",
  "serde",
  "serde_json",
  "serde_urlencoded",
- "sha1 0.10.6",
+ "sha1 0.11.0",
+]
+
+[[package]]
+name = "reqsign-aws-v4"
+version = "3.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ac5b3b7cefa28933792b439186459f77f19f9b6edbeab41b8b187150361a206"
+dependencies = [
+ "bytes",
+ "http 1.5.0",
+ "log",
+ "quick-xml 0.41.0",
+ "reqsign-aws-core",
+ "reqsign-core",
+ "serde",
 ]
 
 [[package]]
 name = "reqsign-azure-storage"
-version = "3.0.0"
+version = "3.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a321980405d596bd34aaf95c4722a3de4128a67fd19e74a81a83aa3fdf082e6"
+checksum = "2824e7da3c2cc42ac3406c674eb57c89127fdcd97f3a73c608cfc680505ea134"
 dependencies = [
  "anyhow",
- "base64 0.22.1",
+ "base64 0.23.1",
  "bytes",
  "form_urlencoded",
  "http 1.5.0",
- "jsonwebtoken",
  "log",
  "pem",
  "percent-encoding",
@@ -8275,36 +8348,38 @@ dependencies = [
  "rsa",
  "serde",
  "serde_json",
- "sha1 0.10.6",
+ "sha1 0.11.0",
 ]
 
 [[package]]
 name = "reqsign-core"
-version = "3.0.0"
+version = "3.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b10302cf0a7d7e7352ba211fc92c3c5bebf1286153e49cc5aa87348078a8e102"
+checksum = "c07dd510b1e1b9b241883e483358147fb2ed2d497a7b39b065ba61eb93deceb0"
 dependencies = [
  "anyhow",
- "base64 0.22.1",
+ "base64 0.23.1",
  "bytes",
- "form_urlencoded",
  "futures",
  "hex",
- "hmac 0.12.1",
+ "hmac 0.13.0",
  "http 1.5.0",
  "jiff",
  "log",
  "percent-encoding",
- "sha1 0.10.6",
- "sha2 0.10.9",
+ "rsa",
+ "serde",
+ "serde_json",
+ "sha1 0.11.0",
+ "sha2 0.11.0",
  "windows-sys 0.61.2",
 ]
 
 [[package]]
 name = "reqsign-file-read-tokio"
-version = "3.0.0"
+version = "3.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2d89295b3d17abea31851cc8de55d843d89c52132c864963c38d41920613dc5"
+checksum = "663d9d55abd0df0830ef0ae43708297cc1371cf4e8ca91f3ac813c309cca8c98"
 dependencies = [
  "anyhow",
  "reqsign-core",
@@ -8313,13 +8388,12 @@ dependencies = [
 
 [[package]]
 name = "reqsign-google"
-version = "3.0.0"
+version = "3.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35cc609b49c69e76ecaceb775a03f792d1ed3e7755ab3548d4534fd801e3242e"
+checksum = "4080a227f82a09f68540ecd028622065d7ac4c0bcb8727a25bdcfc0526235792"
 dependencies = [
  "form_urlencoded",
  "http 1.5.0",
- "jsonwebtoken",
  "log",
  "percent-encoding",
  "reqsign-aws-v4",
@@ -8327,15 +8401,14 @@ dependencies = [
  "rsa",
  "serde",
  "serde_json",
- "sha2 0.10.9",
  "tokio",
 ]
 
 [[package]]
 name = "reqsign-tencent-cos"
-version = "3.0.0"
+version = "3.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e128f19525861dbded59e1e7c17653a8ed63d573ca04aed708d552dbef5bb32a"
+checksum = "764629c90f7c3566a6d4e4641ebab9acd604ce02e16eda4d37c7d7e79e16ed90"
 dependencies = [
  "anyhow",
  "http 1.5.0",
@@ -8357,7 +8430,7 @@ dependencies = [
  "encoding_rs",
  "futures-core",
  "futures-util",
- "h2 0.4.14",
+ "h2 0.4.16",
  "http 1.5.0",
  "http-body 1.1.0",
  "http-body-util",
@@ -8394,9 +8467,9 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.13.3"
+version = "0.13.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0"
+checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
 dependencies = [
  "base64 0.22.1",
  "bytes",
@@ -8457,7 +8530,7 @@ dependencies = [
  "anyhow",
  "async-trait",
  "http 1.5.0",
- "reqwest 0.13.3",
+ "reqwest 0.13.4",
  "thiserror 2.0.18",
  "tower-service",
 ]
@@ -8597,16 +8670,6 @@ dependencies = [
  "ordered-multimap",
 ]
 
-[[package]]
-name = "rust-stemmers"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54"
-dependencies = [
- "serde",
- "serde_derive",
-]
-
 [[package]]
 name = "rustc-demangle"
 version = "0.1.27"
@@ -9206,18 +9269,6 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
 
-[[package]]
-name = "simple_asn1"
-version = "0.6.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d"
-dependencies = [
- "num-bigint",
- "num-traits",
- "thiserror 2.0.18",
- "time",
-]
-
 [[package]]
 name = "siphasher"
 version = "1.0.3"
@@ -9277,7 +9328,7 @@ version = "0.8.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
 dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
  "proc-macro2",
  "quote",
  "syn 2.0.117",
@@ -9289,7 +9340,7 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40"
 dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
  "proc-macro2",
  "quote",
  "syn 2.0.117",
@@ -9492,15 +9543,6 @@ version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
-[[package]]
-name = "strum"
-version = "0.26.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
-dependencies = [
- "strum_macros 0.26.4",
-]
-
 [[package]]
 name = "strum"
 version = "0.28.0"
@@ -9523,19 +9565,6 @@ dependencies = [
  "syn 2.0.117",
 ]
 
-[[package]]
-name = "strum_macros"
-version = "0.26.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
-dependencies = [
- "heck 0.5.0",
- "proc-macro2",
- "quote",
- "rustversion",
- "syn 2.0.117",
-]
-
 [[package]]
 name = "strum_macros"
 version = "0.28.0"
@@ -9730,7 +9759,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.3.4",
+ "getrandom 0.4.2",
  "once_cell",
  "rustix",
  "windows-sys 0.61.2",
@@ -10057,7 +10086,7 @@ dependencies = [
  "async-trait",
  "base64 0.22.1",
  "bytes",
- "h2 0.4.14",
+ "h2 0.4.16",
  "http 1.5.0",
  "http-body 1.1.0",
  "http-body-util",
@@ -11161,7 +11190,7 @@ dependencies = [
  "more-asserts",
  "rand 0.10.1",
  "redb",
- "reqwest 0.13.3",
+ "reqwest 0.13.4",
  "reqwest-middleware",
  "serde",
  "serde_json",
@@ -11274,7 +11303,7 @@ dependencies = [
  "oneshot",
  "pin-project",
  "rand 0.10.1",
- "reqwest 0.13.3",
+ "reqwest 0.13.4",
  "serde",
  "serde_json",
  "shellexpand",
@@ -11370,20 +11399,6 @@ name = "zeroize"
 version = "1.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
-dependencies = [
- "zeroize_derive",
-]
-
-[[package]]
-name = "zeroize_derive"
-version = "1.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.117",
-]
 
 [[package]]
 name = "zerotrie"
diff --git a/Cargo.toml b/Cargo.toml
index 35370e58f..716b14d7a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,20 +13,21 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"
 
 [workspace.dependencies]
-lance = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lancedb = { path = "rust/lancedb", default-features = false }
 ahash = "0.8"
 # Note that this one does not include pyarrow
 arrow = { version = "58.0.0", optional = false }
@@ -39,6 +40,7 @@ arrow-schema = "58.0.0"
 arrow-select = "58.0.0"
 arrow-cast = "58.0.0"
 async-trait = "0"
+bytes = "1"
 datafusion = { version = "54.0.0", default-features = false }
 datafusion-catalog = "54.0.0"
 datafusion-common = { version = "54.0.0", default-features = false }
@@ -52,7 +54,7 @@ env_logger = "0.11"
 half = { "version" = "2.7.1", default-features = false, features = [
     "num-traits",
 ] }
-futures = "0"
+futures = "0.3"
 log = "0.4"
 metrics = "0.24"
 metrics-util = "0.19"
@@ -65,7 +67,12 @@ url = "2"
 num-traits = "0.2"
 regex = "1.10"
 semver = "1.0.25"
-chrono = "0.4"
+serde = "1"
+serde_json = "1"
+tempfile = "3.5.0"
+tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
+uuid = { version = "1.7.0", features = ["v4"] }
+chrono = { version = "0.4", default-features = false, features = ["clock"] }
 
 [profile.ci]
 debug = "line-tables-only"
diff --git a/ci/check_breaking_changes.py b/ci/check_breaking_changes.py
index bc7a562b8..e31eedf0c 100644
--- a/ci/check_breaking_changes.py
+++ b/ci/check_breaking_changes.py
@@ -2,6 +2,7 @@
 Check whether there are any breaking changes in the PRs between the base and head commits.
 If there are, assert that we have incremented the minor version.
 """
+
 import argparse
 import os
 from packaging.version import parse
@@ -27,7 +28,7 @@ if __name__ == "__main__":
     else:
         print("No breaking changes found.")
         exit(0)
-    
+
     last_stable_version = parse(args.last_stable_version)
     current_version = parse(args.current_version)
     if current_version.minor <= last_stable_version.minor:
diff --git a/ci/check_lance_release.py b/ci/check_lance_release.py
index 47f1cdbde..9fff955ac 100755
--- a/ci/check_lance_release.py
+++ b/ci/check_lance_release.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 """Determine whether a newer Lance tag exists and expose results for CI."""
+
 from __future__ import annotations
 
 import argparse
@@ -36,8 +37,16 @@ class SemVer:
     prerelease: Tuple[Union[int, str], ...]
 
     def __lt__(self, other: "SemVer") -> bool:  # pragma: no cover - simple comparison
-        if (self.major, self.minor, self.patch) != (other.major, other.minor, other.patch):
-            return (self.major, self.minor, self.patch) < (other.major, other.minor, other.patch)
+        if (self.major, self.minor, self.patch) != (
+            other.major,
+            other.minor,
+            other.patch,
+        ):
+            return (self.major, self.minor, self.patch) < (
+                other.major,
+                other.minor,
+                other.patch,
+            )
         if self.prerelease == other.prerelease:
             return False
         if not self.prerelease:
@@ -142,7 +151,9 @@ def read_current_version(repo_root: Path) -> str:
         deps = data["workspace"]["dependencies"]
         entry = deps["lance"]
     except KeyError as exc:  # pragma: no cover - configuration guard
-        raise RuntimeError("Failed to locate workspace.dependencies.lance in Cargo.toml") from exc
+        raise RuntimeError(
+            "Failed to locate workspace.dependencies.lance in Cargo.toml"
+        ) from exc
 
     if isinstance(entry, str):
         raw_version = entry
diff --git a/ci/mock_openai.py b/ci/mock_openai.py
index da3cb6c46..4fcb62ad9 100644
--- a/ci/mock_openai.py
+++ b/ci/mock_openai.py
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 """A zero-dependency mock OpenAI embeddings API endpoint for testing purposes."""
+
 import argparse
 import json
 import http.server
@@ -22,11 +23,13 @@ class MockOpenAIRequestHandler(http.server.BaseHTTPRequestHandler):
 
         data = []
         for i in range(num_inputs):
-            data.append({
-                "object": "embedding",
-                "embedding": [0.1] * 1536,
-                "index": i,
-            })
+            data.append(
+                {
+                    "object": "embedding",
+                    "embedding": [0.1] * 1536,
+                    "index": i,
+                }
+            )
 
         response = {
             "object": "list",
@@ -35,7 +38,7 @@ class MockOpenAIRequestHandler(http.server.BaseHTTPRequestHandler):
             "usage": {
                 "prompt_tokens": 0,
                 "total_tokens": 0,
-            }
+            },
         }
 
         self.send_response(200)
diff --git a/ci/semver_sort.py b/ci/semver_sort.py
index b90ba3319..5f99c6c4f 100644
--- a/ci/semver_sort.py
+++ b/ci/semver_sort.py
@@ -7,6 +7,7 @@ from packaging.version import parse, InvalidVersion
 
 if __name__ == "__main__":
     import argparse
+
     parser = argparse.ArgumentParser()
     parser.add_argument("prefix", default="v")
     args = parser.parse_args()
diff --git a/ci/set_lance_version.py b/ci/set_lance_version.py
index e8c573ad4..c7bf41f55 100644
--- a/ci/set_lance_version.py
+++ b/ci/set_lance_version.py
@@ -22,7 +22,7 @@ def run_command(command: str) -> str:
 def get_latest_stable_version() -> str:
     version_line = run_command("cargo info lance | grep '^version:'")
     # Example output: "version: 0.35.0 (latest 0.37.0)"
-    match = re.search(r'\(latest ([0-9.]+)\)', version_line)
+    match = re.search(r"\(latest ([0-9.]+)\)", version_line)
     if match:
         return match.group(1)
     # Fallback: use the first version after 'version:'
@@ -69,7 +69,7 @@ def extract_default_features(line: str) -> bool:
     """
     import re
 
-    match = re.search(r'default-features\s*=\s*false', line)
+    match = re.search(r"default-features\s*=\s*false", line)
     return match is not None
 
 
@@ -104,7 +104,7 @@ def dict_to_toml_line(package_name: str, config: dict) -> str:
             # This shouldn't happen with our current usage
             parts.append(f'"{key}" = {json.dumps(value)}')
 
-    return f'{package_name} = {{ {", ".join(parts)} }}\n'
+    return f"{package_name} = {{ {', '.join(parts)} }}\n"
 
 
 def update_cargo_toml(line_updater):
@@ -119,7 +119,7 @@ def update_cargo_toml(line_updater):
     lance_line = ""
     is_parsing_lance_line = False
     for line in lines:
-        if line.startswith("lance"):
+        if re.match(r"^lance(?:\s|[-_])", line):
             # Check if this is a single-line or multi-line entry
             # Single-line entries either:
             # 1. End with } (complete inline table)
diff --git a/ci/tests/test_set_lance_version.py b/ci/tests/test_set_lance_version.py
new file mode 100644
index 000000000..1493fc59d
--- /dev/null
+++ b/ci/tests/test_set_lance_version.py
@@ -0,0 +1,185 @@
+import os
+import stat
+import subprocess
+import sys
+import tempfile
+import textwrap
+import unittest
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+SCRIPT = REPO_ROOT / "ci" / "set_lance_version.py"
+LANCE_GIT_URL = "https://github.com/lance-format/lance.git"
+
+CARGO_TOML = """\
+[workspace.dependencies]
+lance = { "version" = "=1.0.0", default-features = false, "features" = ["dynamodb"] }
+lance-core = "1.0.0"
+lance_datafusion = {
+    "version" = "=1.0.0",
+    "features" = ["substrait"]
+}
+lancedb = { path = "rust/lancedb", default-features = false }
+lancedb-common = { path = "rust/lancedb-common" }
+lancewood = "1.0.0"
+my-lance = "1.0.0"
+"""
+
+UNTOUCHED_DEPENDENCIES = """\
+lancedb = { path = "rust/lancedb", default-features = false }
+lancedb-common = { path = "rust/lancedb-common" }
+lancewood = "1.0.0"
+my-lance = "1.0.0"
+"""
+
+
+class SetLanceVersionTest(unittest.TestCase):
+    def test_supported_update_modes_only_rewrite_lance_dependencies(self):
+        cases = {
+            "stable": (
+                """\
+lance = { "version" = "=9.9.9", default-features = false, "features" = ["dynamodb"] }
+lance-core = "=9.9.9"
+lance_datafusion = { "version" = "=9.9.9", "features" = ["substrait"] }
+""",
+                ["cargo info lance", "cargo metadata"],
+            ),
+            "preview": (
+                f"""\
+lance = {{ "version" = "=10.0.0-beta.3", default-features = false, "features" = ["dynamodb"], "tag" = "v10.0.0-beta.3", "git" = "{LANCE_GIT_URL}" }}
+lance-core = {{ "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "{LANCE_GIT_URL}" }}
+lance_datafusion = {{ "version" = "=10.0.0-beta.3", "features" = ["substrait"], "tag" = "v10.0.0-beta.3", "git" = "{LANCE_GIT_URL}" }}
+""",
+                ["git ls-remote --tags", "cargo metadata"],
+            ),
+            "local": (
+                """\
+lance = { "path" = "../lance/rust/lance", default-features = false, "features" = ["dynamodb"] }
+lance-core = { "path" = "../lance/rust/lance-core" }
+lance_datafusion = { "path" = "../lance/rust/lance_datafusion", "features" = ["substrait"] }
+""",
+                ["cargo metadata"],
+            ),
+            "v8.1.2": (
+                """\
+lance = { "version" = "=8.1.2", default-features = false, "features" = ["dynamodb"] }
+lance-core = "=8.1.2"
+lance_datafusion = { "version" = "=8.1.2", "features" = ["substrait"] }
+""",
+                ["cargo metadata"],
+            ),
+            "v8.2.0-beta.4": (
+                f"""\
+lance = {{ "version" = "=8.2.0-beta.4", default-features = false, "features" = ["dynamodb"], "tag" = "v8.2.0-beta.4", "git" = "{LANCE_GIT_URL}" }}
+lance-core = {{ "version" = "=8.2.0-beta.4", "tag" = "v8.2.0-beta.4", "git" = "{LANCE_GIT_URL}" }}
+lance_datafusion = {{ "version" = "=8.2.0-beta.4", "features" = ["substrait"], "tag" = "v8.2.0-beta.4", "git" = "{LANCE_GIT_URL}" }}
+""",
+                ["cargo metadata"],
+            ),
+        }
+
+        for version, (updated_dependencies, expected_commands) in cases.items():
+            with self.subTest(version=version), tempfile.TemporaryDirectory() as tmp:
+                workdir = Path(tmp)
+                (workdir / "Cargo.toml").write_text(CARGO_TOML)
+                command_log = workdir / "commands.log"
+                fake_bin = workdir / "bin"
+                fake_bin.mkdir()
+                self._write_fake_executables(fake_bin)
+                self._write_fake_python_dependencies(workdir)
+
+                env = os.environ.copy()
+                env["PATH"] = os.pathsep.join([str(fake_bin), env["PATH"]])
+                env["FAKE_COMMAND_LOG"] = str(command_log)
+                env["PYTHONPATH"] = os.pathsep.join(
+                    filter(None, [str(workdir), env.get("PYTHONPATH")])
+                )
+                result = subprocess.run(
+                    [sys.executable, str(SCRIPT), version],
+                    cwd=workdir,
+                    env=env,
+                    capture_output=True,
+                    text=True,
+                    timeout=10,
+                )
+
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertEqual(
+                    (workdir / "Cargo.toml").read_text(),
+                    "[workspace.dependencies]\n"
+                    + updated_dependencies
+                    + UNTOUCHED_DEPENDENCIES,
+                )
+                commands = command_log.read_text().splitlines()
+                for command in expected_commands:
+                    self.assertTrue(
+                        any(line.startswith(command) for line in commands),
+                        f"{command!r} not found in {commands!r}",
+                    )
+
+    def _write_fake_executables(self, fake_bin: Path) -> None:
+        cargo = fake_bin / "cargo"
+        cargo.write_text(
+            textwrap.dedent(
+                """\
+                #!/bin/sh
+                printf 'cargo %s\\n' "$*" >> "$FAKE_COMMAND_LOG"
+                case "$1" in
+                  info)
+                    printf '%s\\n' 'version: 8.8.8 (latest 9.9.9)'
+                    ;;
+                  metadata)
+                    ;;
+                  *)
+                    exit 2
+                    ;;
+                esac
+                """
+            )
+        )
+        cargo.chmod(cargo.stat().st_mode | stat.S_IXUSR)
+
+        git = fake_bin / "git"
+        git.write_text(
+            textwrap.dedent(
+                """\
+                #!/bin/sh
+                printf 'git %s\\n' "$*" >> "$FAKE_COMMAND_LOG"
+                if [ "$1" != "ls-remote" ]; then
+                  exit 2
+                fi
+                printf '%s\\n' \\
+                  '111111 refs/tags/v9.9.9' \\
+                  '222222 refs/tags/v10.0.0-beta.1' \\
+                  '333333 refs/tags/v10.0.0-beta.3'
+                """
+            )
+        )
+        git.chmod(git.stat().st_mode | stat.S_IXUSR)
+
+    def _write_fake_python_dependencies(self, workdir: Path) -> None:
+        packaging = workdir / "packaging"
+        packaging.mkdir()
+        (packaging / "__init__.py").write_text("")
+        (packaging / "version.py").write_text(
+            textwrap.dedent(
+                """\
+                class Version:
+                    def __init__(self, value):
+                        release, _, prerelease = value.partition("-beta.")
+                        self._key = (
+                            tuple(int(part) for part in release.split(".")),
+                            not prerelease,
+                            int(prerelease or 0),
+                        )
+
+                    def __lt__(self, other):
+                        return self._key < other._key
+                """
+            )
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/ci/validate_stable_lance.py b/ci/validate_stable_lance.py
index 4edd4c522..240e64174 100644
--- a/ci/validate_stable_lance.py
+++ b/ci/validate_stable_lance.py
@@ -12,7 +12,7 @@ with open("Cargo.toml", "rb") as f:
             elif isinstance(dep, dict):
                 # Version doesn't have the beta tag in it, so we instead look
                 # at the git tag.
-                version = dep.get('tag', dep.get('version'))
+                version = dep.get("tag", dep.get("version"))
             else:
                 raise ValueError("Unexpected type for dependency: " + str(dep))
 
diff --git a/deny.toml b/deny.toml
index 034b48c25..3672321d0 100644
--- a/deny.toml
+++ b/deny.toml
@@ -101,6 +101,19 @@ ignore = [
     # https://rustsec.org/advisories/RUSTSEC-2026-0195
     { id = "RUSTSEC-2026-0194", reason = "transitive via inferno/lance/opendal; XML from trusted cloud endpoints, not attacker-controlled" },
     { id = "RUSTSEC-2026-0195", reason = "transitive via inferno/lance/opendal; XML from trusted cloud endpoints, not attacker-controlled" },
+    # smartstring: unmaintained — the repository was archived by its author on
+    # 2026-05-03. Not a vulnerability. Reached only transitively through polars
+    # (polars-core/-io/-ops/-time/-utils); nothing in LanceDB depends on it directly.
+    # The advisory states no safe upgrade is available: upstream recommends
+    # compact_str/smol_str, so clearing this requires polars to migrate.
+    # https://rustsec.org/advisories/RUSTSEC-2026-0249
+    { id = "RUSTSEC-2026-0249", reason = "smartstring unmaintained via polars; no fixed upstream release" },
+
+    # h2 0.3: empty DATA frames can be queued without limit. The patched
+    # h2 0.4 line is locked to 0.4.16, but no patched 0.3 release exists.
+    # The old copy is pulled in by aws-smithy's legacy hyper 0.14 client.
+    # https://rustsec.org/advisories/RUSTSEC-2026-0258
+    { id = "RUSTSEC-2026-0258", reason = "h2 0.3 via legacy aws-smithy/hyper 0.14; no patched 0.3 release" },
 ]
 
 # ---------------------------------------------------------------------------
@@ -164,6 +177,11 @@ multiple-versions = "warn"
 # Wildcard version requirements (`foo = "*"`) are a footgun — they let any
 # future release in without review. Ban them outright.
 wildcards = "deny"
+# Lint every dependency declared by a workspace member against the shared
+# `[workspace.dependencies]` table: any crate used by more than one member must
+# go through `workspace = true`, and entries nothing uses are an error. This
+# keeps versions from drifting between the core crate and the bindings.
+workspace-dependencies = { duplicates = "deny", unused = "deny" }
 # Internal workspace crates reference each other via `path = "..."`, which
 # cargo-deny sees as a wildcard version. That's fine for private workspace
 # members (not published to crates.io), so allow it specifically for paths.
diff --git a/docs/requirements.txt b/docs/requirements.txt
index e5f3867cb..89de1cb71 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -5,5 +5,5 @@ mkdocs-autorefs>=0.5,<=1.0
 mkdocstrings[python]>=0.24,<1.0
 griffe>=0.40,<1.0
 mkdocs-render-swagger-plugin>=0.1.0
-pydantic>=2.0,<3.0
-mkdocs-redirects>=1.2.0
\ No newline at end of file
+pydantic>=2.7.4,<3
+mkdocs-redirects>=1.2.0
diff --git a/docs/src/java/java.md b/docs/src/java/java.md
index 11e901ad0..06dc267f3 100644
--- a/docs/src/java/java.md
+++ b/docs/src/java/java.md
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-core</artifactId>
-    <version>0.37.1-beta.0</version>
+    <version>0.38.0-beta.10</version>
 </dependency>
 ```
 
@@ -55,6 +55,38 @@ LanceNamespace namespaceClient = LanceDbNamespaceClientBuilder.newBuilder()
 | `region(String)` | AWS region (default: "us-east-1") | No |
 | `config(String, String)` | Additional configuration parameters | No |
 
+### Opening a Table with Vended Credentials
+
+When the catalog vends temporary object store credentials, open the table through the
+namespace client. The Lance dataset builder fetches the table location and storage options
+from the catalog and refreshes the credentials when they expire.
+
+```java
+import com.lancedb.LanceDbNamespaceClientBuilder;
+import org.lance.Dataset;
+import org.lance.namespace.LanceNamespace;
+
+import java.util.Arrays;
+
+LanceNamespace namespaceClient = LanceDbNamespaceClientBuilder.newBuilder()
+    .apiKey(System.getenv("LANCEDB_API_KEY"))
+    .database(System.getenv("LANCEDB_DATABASE"))
+    // Set the endpoint for a LanceDB Enterprise deployment.
+    // .endpoint("https://your-enterprise-endpoint")
+    .build();
+
+try (Dataset dataset = Dataset.open()
+        .namespaceClient(namespaceClient)
+        .tableId(Arrays.asList("my_namespace", "my_table"))
+        .build()) {
+    System.out.println("Rows: " + dataset.countRows());
+}
+```
+
+Do not call `describeTable()` and then open the returned location with `Dataset.open(uri)`.
+Opening through `namespaceClient()` is what applies the vended storage options and enables
+automatic credential refresh. No object store credentials need to be passed by the application.
+
 ## Metadata Operations
 
 ### Creating a Namespace Path
diff --git a/docs/src/js/classes/AutoQuery.md b/docs/src/js/classes/AutoQuery.md
new file mode 100644
index 000000000..1d7ea6952
--- /dev/null
+++ b/docs/src/js/classes/AutoQuery.md
@@ -0,0 +1,518 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / AutoQuery
+
+# Class: AutoQuery
+
+A builder for automatic string searches.
+
+Automatic search determines whether to use full-text or vector search from
+the table revision selected for each execution. This builder exposes the
+common operations supported by both query families.
+
+## Extends
+
+- `StandardQueryBase`&lt;`NativeQuery` \| `NativeVectorQuery`&gt;
+
+## Properties
+
+### inner
+
+```ts
+protected inner: Query | VectorQuery | Promise<Query | VectorQuery>;
+```
+
+#### Inherited from
+
+`StandardQueryBase.inner`
+
+## Methods
+
+### analyzePlan()
+
+```ts
+analyzePlan(distributedMetrics?): Promise<string>
+```
+
+Executes the query and returns the physical query plan annotated with runtime metrics.
+
+This is useful for debugging and performance analysis, as it shows how the query was executed
+and includes metrics such as elapsed time, rows processed, and I/O statistics.
+
+#### Parameters
+
+* **distributedMetrics?**: [`AnalyzePlanDistributedMetrics`](../type-aliases/AnalyzePlanDistributedMetrics.md)
+    How distributed worker metrics are displayed for remote query plans.
+    Defaults to `"aggregate"`.
+
+#### Returns
+
+`Promise`&lt;`string`&gt;
+
+A query execution plan with runtime metrics for each step.
+
+#### Example
+
+```ts
+import * as lancedb from "@lancedb/lancedb"
+
+const db = await lancedb.connect("./.lancedb");
+const table = await db.createTable("my_table", [
+  { vector: [1.1, 0.9], id: "1" },
+]);
+
+const plan = await table.query().nearestTo([0.5, 0.2]).analyzePlan();
+
+Example output (with runtime metrics inlined):
+AnalyzeExec verbose=true, metrics=[]
+ ProjectionExec: expr=[id@3 as id, vector@0 as vector, _distance@2 as _distance], metrics=[output_rows=1, elapsed_compute=3.292µs]
+  Take: columns="vector, _rowid, _distance, (id)", metrics=[output_rows=1, elapsed_compute=66.001µs, batches_processed=1, bytes_read=8, iops=1, requests=1]
+   CoalesceBatchesExec: target_batch_size=1024, metrics=[output_rows=1, elapsed_compute=3.333µs]
+    GlobalLimitExec: skip=0, fetch=10, metrics=[output_rows=1, elapsed_compute=167ns]
+     FilterExec: _distance@2 IS NOT NULL, metrics=[output_rows=1, elapsed_compute=8.542µs]
+      SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=63.25µs, row_replacements=1]
+       KNNVectorDistance: metric=l2, metrics=[output_rows=1, elapsed_compute=114.333µs, output_batches=1]
+        LanceScan: uri=/path/to/data, projection=[vector], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1, elapsed_compute=103.626µs, bytes_read=549, iops=2, requests=2]
+```
+
+#### Inherited from
+
+`StandardQueryBase.analyzePlan`
+
+***
+
+### execute()
+
+```ts
+protected execute(options?): AsyncGenerator<RecordBatch<any>, void, unknown>
+```
+
+Execute the query and return the results as an
+
+#### Parameters
+
+* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
+
+#### Returns
+
+`AsyncGenerator`&lt;`RecordBatch`&lt;`any`&gt;, `void`, `unknown`&gt;
+
+#### See
+
+ - AsyncIterator
+of
+ - RecordBatch.
+
+By default, LanceDb will use many threads to calculate results and, when
+the result set is large, multiple batches will be processed at one time.
+This readahead is limited however and backpressure will be applied if this
+stream is consumed slowly (this constrains the maximum memory used by a
+single query)
+
+#### Inherited from
+
+`StandardQueryBase.execute`
+
+***
+
+### explainPlan()
+
+```ts
+explainPlan(verbose): Promise<string>
+```
+
+Generates an explanation of the query execution plan.
+
+#### Parameters
+
+* **verbose**: `boolean` = `false`
+    If true, provides a more detailed explanation. Defaults to false.
+
+#### Returns
+
+`Promise`&lt;`string`&gt;
+
+A Promise that resolves to a string containing the query execution plan explanation.
+
+#### Example
+
+```ts
+import * as lancedb from "@lancedb/lancedb"
+const db = await lancedb.connect("./.lancedb");
+const table = await db.createTable("my_table", [
+  { vector: [1.1, 0.9], id: "1" },
+]);
+const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
+```
+
+#### Inherited from
+
+`StandardQueryBase.explainPlan`
+
+***
+
+### fastSearch()
+
+```ts
+fastSearch(): this
+```
+
+Skip searching un-indexed data. This can make search faster, but will miss
+any data that is not yet indexed.
+
+Use [Table#optimize](Table.md#optimize) to index all un-indexed data.
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+`StandardQueryBase.fastSearch`
+
+***
+
+### ~~filter()~~
+
+```ts
+filter(predicate): this
+```
+
+A filter statement to be applied to this query.
+
+#### Parameters
+
+* **predicate**: `string`
+
+#### Returns
+
+`this`
+
+#### See
+
+where
+
+#### Deprecated
+
+Use `where` instead
+
+#### Inherited from
+
+`StandardQueryBase.filter`
+
+***
+
+### fullTextSearch()
+
+```ts
+fullTextSearch(query, options?): this
+```
+
+#### Parameters
+
+* **query**: `string` \| [`FullTextQuery`](../interfaces/FullTextQuery.md)
+
+* **options?**: `Partial`&lt;[`FullTextSearchOptions`](../interfaces/FullTextSearchOptions.md)&gt;
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+`StandardQueryBase.fullTextSearch`
+
+***
+
+### limit()
+
+```ts
+limit(limit): this
+```
+
+Set the maximum number of results to return.
+
+By default, a plain search has no limit.  If this method is not
+called then every valid row from the table will be returned.
+
+#### Parameters
+
+* **limit**: `number`
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+`StandardQueryBase.limit`
+
+***
+
+### offset()
+
+```ts
+offset(offset): this
+```
+
+Set the number of rows to skip before returning results.
+
+This is useful for pagination.
+
+#### Parameters
+
+* **offset**: `number`
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+`StandardQueryBase.offset`
+
+***
+
+### orderBy()
+
+```ts
+orderBy(ordering): this
+```
+
+Sort the results by the specified column(s).
+
+#### Parameters
+
+* **ordering**: [`ColumnOrdering`](../interfaces/ColumnOrdering.md) \| [`ColumnOrdering`](../interfaces/ColumnOrdering.md)[]
+
+#### Returns
+
+`this`
+
+This query builder.
+
+#### Inherited from
+
+`StandardQueryBase.orderBy`
+
+***
+
+### outputSchema()
+
+```ts
+outputSchema(): Promise<Schema<any>>
+```
+
+Returns the schema of the output that will be returned by this query.
+
+This can be used to inspect the types and names of the columns that will be
+returned by the query before executing it.
+
+#### Returns
+
+`Promise`&lt;`Schema`&lt;`any`&gt;&gt;
+
+An Arrow Schema describing the output columns.
+
+#### Inherited from
+
+`StandardQueryBase.outputSchema`
+
+***
+
+### select()
+
+```ts
+select(columns): this
+```
+
+Return only the specified columns.
+
+By default a query will return all columns from the table.  However, this can have
+a very significant impact on latency.  LanceDb stores data in a columnar fashion.  This
+means we can finely tune our I/O to select exactly the columns we need.
+
+As a best practice you should always limit queries to the columns that you need.  If you
+pass in an array of column names then only those columns will be returned.
+
+You can also use this method to create new "dynamic" columns based on your existing columns.
+For example, you may not care about "a" or "b" but instead simply want "a + b".  This is often
+seen in the SELECT clause of an SQL query (e.g. `SELECT a+b FROM my_table`).
+
+To create dynamic columns you can pass in a Map<string, string>.  A column will be returned
+for each entry in the map.  The key provides the name of the column.  The value is
+an SQL string used to specify how the column is calculated.
+
+For example, an SQL query might state `SELECT a + b AS combined, c`.  The equivalent
+input to this method would be:
+
+#### Parameters
+
+* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
+
+#### Returns
+
+`this`
+
+#### Example
+
+```ts
+new Map([["combined", "a + b"], ["c", "c"]])
+
+Columns will always be returned in the order given, even if that order is different than
+the order used when adding the data.
+
+Note that you can pass in a `Record<string, string>` (e.g. an object literal). This method
+uses `Object.entries` which should preserve the insertion order of the object.  However,
+object insertion order is easy to get wrong and `Map` is more foolproof.
+```
+
+#### Inherited from
+
+`StandardQueryBase.select`
+
+***
+
+### toArray()
+
+```ts
+toArray(options?): Promise<any[]>
+```
+
+Collect the results as an array of objects.
+
+#### Parameters
+
+* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
+
+#### Returns
+
+`Promise`&lt;`any`[]&gt;
+
+#### Inherited from
+
+`StandardQueryBase.toArray`
+
+***
+
+### toArrow()
+
+```ts
+toArrow(options?): Promise<Table<any>>
+```
+
+Collect the results as an Arrow
+
+#### Parameters
+
+* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
+
+#### Returns
+
+`Promise`&lt;`Table`&lt;`any`&gt;&gt;
+
+#### See
+
+ArrowTable.
+
+#### Inherited from
+
+`StandardQueryBase.toArrow`
+
+***
+
+### useLsm()
+
+```ts
+useLsm(enable): this
+```
+
+Control MemWAL read routing for this query.
+
+By default (unset), when the table carries a MemWAL write spec (see
+[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so
+they also return data written via the `mergeInsert` LSM path that has not yet
+been compacted into the base table (the active/frozen in-memory memtables and
+the flushed generations), deduplicated by primary key; a table without a spec
+reads the base table.
+
+#### Parameters
+
+* **enable**: `boolean`
+    `true` forces the LSM scanner and errors if the table has no
+    MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
+    even when a spec is present.
+    Note: the LSM scanner does not support every query shape (e.g. reranking,
+    hybrid search, `orderBy`). On a MemWAL table those shapes error unless
+    `useLsm(false)` is set, because a base-only read would silently exclude
+    un-compacted MemWAL data.
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+`StandardQueryBase.useLsm`
+
+***
+
+### where()
+
+```ts
+where(predicate): this
+```
+
+A filter statement to be applied to this query.
+
+The filter should be supplied as an SQL query string.  For example:
+
+#### Parameters
+
+* **predicate**: `string`
+
+#### Returns
+
+`this`
+
+#### Example
+
+```ts
+x > 10
+y > 0 AND y < 100
+x > 5 OR y = 'test'
+
+Filtering performance can often be improved by creating a scalar index
+on the filter column(s).
+
+Calling this multiple times combines the filters with a logical AND rather
+than replacing the previous filter.
+```
+
+#### Inherited from
+
+`StandardQueryBase.where`
+
+***
+
+### withRowId()
+
+```ts
+withRowId(): this
+```
+
+Whether to return the row id in the results.
+
+This column can be used to match results between different queries. For
+example, to match results from a full text search and a vector search in
+order to perform hybrid search.
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+`StandardQueryBase.withRowId`
diff --git a/docs/src/js/classes/Branches.md b/docs/src/js/classes/Branches.md
index 5296d0d08..6680fbfee 100644
--- a/docs/src/js/classes/Branches.md
+++ b/docs/src/js/classes/Branches.md
@@ -37,6 +37,31 @@ latest and stays writable.
 
 ***
 
+### cherryPick()
+
+```ts
+cherryPick(fromBranch, dryRun): Promise<CherryPickResult>
+```
+
+Cherry-pick a branch onto main.
+
+Set `dryRun` to `true` to preview. A failed cherry-pick resolves
+with `status: "failed"` instead of throwing.
+
+#### Parameters
+
+* **fromBranch**: `string`
+    Branch to cherry-pick from.
+
+* **dryRun**: `boolean` = `false`
+    When true, only preview. Defaults to false.
+
+#### Returns
+
+`Promise`&lt;[`CherryPickResult`](../interfaces/CherryPickResult.md)&gt;
+
+***
+
 ### create()
 
 ```ts
@@ -112,28 +137,3 @@ List all branches, mapping name to branch metadata.
 #### Returns
 
 `Promise`&lt;`Record`&lt;`string`, [`BranchContents`](BranchContents.md)&gt;&gt;
-
-***
-
-### merge()
-
-```ts
-merge(fromBranch, dryRun): Promise<MergeBranchResult>
-```
-
-Merge a branch into main.
-
-Set `dryRun` to `true` to preview the merge. A rejected merge resolves
-with `status: "rejected"` instead of throwing.
-
-#### Parameters
-
-* **fromBranch**: `string`
-    Branch to merge from.
-
-* **dryRun**: `boolean` = `false`
-    When true, only preview the merge. Defaults to false.
-
-#### Returns
-
-`Promise`&lt;[`MergeBranchResult`](../interfaces/MergeBranchResult.md)&gt;
diff --git a/docs/src/js/classes/Connection.md b/docs/src/js/classes/Connection.md
index fa4e0748a..1c5abd89f 100644
--- a/docs/src/js/classes/Connection.md
+++ b/docs/src/js/classes/Connection.md
@@ -169,6 +169,45 @@ Creates a new empty Table
 
 ***
 
+### createMaterializedView()
+
+```ts
+abstract createMaterializedView(
+   name,
+   source,
+   options?): Promise<MaterializedView>
+```
+
+Define a materialized view named `name` over the table `source`.
+
+The view is created empty, with the query recorded in its schema
+metadata; `view.refresh()` computes the rows. The view is a normal
+table: it can be queried, indexed and searched, and it appears in
+`tableNames`. The source table must have stable row ids (create it with
+the `newTableEnableStableRowIds` storage option); they keep the view's
+provenance valid across source compactions and cannot be enabled after
+a table exists. Local databases only.
+
+#### Parameters
+
+* **name**: `string`
+
+* **source**: `string`
+
+* **options?**
+
+* **options.limit?**: `number`
+
+* **options.select?**: [`MaterializedViewSelect`](../type-aliases/MaterializedViewSelect.md)
+
+* **options.where?**: `string`
+
+#### Returns
+
+`Promise`&lt;[`MaterializedView`](MaterializedView.md)&gt;
+
+***
+
 ### createNamespace()
 
 ```ts
@@ -386,6 +425,29 @@ Drop an existing table.
 
 ***
 
+### dropTableAsync()
+
+```ts
+abstract dropTableAsync(name, namespacePath?): Promise<Job>
+```
+
+Start dropping a table and return its cleanup job.
+
+The table may become unavailable before its data files are removed. Wait
+on the returned job to know when cleanup has finished.
+
+#### Parameters
+
+* **name**: `string`
+
+* **namespacePath?**: `string`[]
+
+#### Returns
+
+`Promise`&lt;[`Job`](Job.md)&gt;
+
+***
+
 ### getJob()
 
 ```ts
@@ -476,6 +538,22 @@ List server-side jobs across the database's tables.
 
 ***
 
+### listMaterializedViews()
+
+```ts
+abstract listMaterializedViews(): Promise<string[]>
+```
+
+The names of the materialized views in this database.
+
+Found by reading every table's schema, so this costs an open per table.
+
+#### Returns
+
+`Promise`&lt;`string`[]&gt;
+
+***
+
 ### listNamespaces()
 
 ```ts
@@ -506,6 +584,90 @@ Child namespace names and
 
 ***
 
+### listTables()
+
+#### listTables(options)
+
+```ts
+abstract listTables(options?): Promise<ListTablesResponse>
+```
+
+List a page of the tables in this database.
+
+To retrieve the tables after the page, pass the `pageToken` the response
+carries back in. A page can be shorter than `limit` without being the last
+one, so walk until a response carries no page token:
+
+```ts
+const names = [];
+let pageToken = undefined;
+do {
+  const page = await conn.listTables({ pageToken, limit: 100 });
+  names.push(...page.tables);
+  pageToken = page.pageToken;
+} while (pageToken);
+```
+
+##### Parameters
+
+* **options?**: `Partial`&lt;[`ListTablesOptions`](../interfaces/ListTablesOptions.md)&gt;
+    Pagination options
+    (`pageToken`, `limit`).
+
+##### Returns
+
+`Promise`&lt;[`ListTablesResponse`](../interfaces/ListTablesResponse.md)&gt;
+
+A page of table names and an
+  optional token for the tables after it.
+
+#### listTables(namespacePath, options)
+
+```ts
+abstract listTables(namespacePath?, options?): Promise<ListTablesResponse>
+```
+
+List a page of the tables in this database.
+
+##### Parameters
+
+* **namespacePath?**: `string`[]
+    The namespace path to list tables from
+    (defaults to root namespace)
+
+* **options?**: `Partial`&lt;[`ListTablesOptions`](../interfaces/ListTablesOptions.md)&gt;
+    Pagination options
+    (`pageToken`, `limit`).
+
+##### Returns
+
+`Promise`&lt;[`ListTablesResponse`](../interfaces/ListTablesResponse.md)&gt;
+
+A page of table names and an
+  optional token for the tables after it.
+
+***
+
+### openMaterializedView()
+
+```ts
+abstract openMaterializedView(name): Promise<MaterializedView>
+```
+
+Open the materialized view named `name`.
+
+Rejects a table that exists but is not a materialized view.
+
+#### Parameters
+
+* **name**: `string`
+
+#### Returns
+
+`Promise`&lt;[`MaterializedView`](MaterializedView.md)&gt;
+
+***
+
 ### openTable()
 
 ```ts
@@ -515,18 +677,13 @@ abstract openTable(
    options?): Promise<Table>
 ```
 
-Open a table in the database.
-
 #### Parameters
 
 * **name**: `string`
-    The name of the table
 
 * **namespacePath?**: `string`[]
-    The namespace path of the table (defaults to root namespace)
 
 * **options?**: `Partial`&lt;[`OpenTableOptions`](../interfaces/OpenTableOptions.md)&gt;
-    Additional options
 
 #### Returns
 
@@ -567,7 +724,7 @@ a "not supported" error.
 
 ***
 
-### tableNames()
+### ~~tableNames()~~
 
 #### tableNames(options)
 
@@ -589,6 +746,10 @@ Tables will be returned in lexicographical order.
 
 `Promise`&lt;`string`[]&gt;
 
+##### Deprecated
+
+Use [Connection.listTables](Connection.md#listtables) instead.
+
 #### tableNames(namespacePath, options)
 
 ```ts
@@ -611,3 +772,7 @@ Tables will be returned in lexicographical order.
 ##### Returns
 
 `Promise`&lt;`string`[]&gt;
+
+##### Deprecated
+
+Use [Connection.listTables](Connection.md#listtables) instead.
diff --git a/docs/src/js/classes/MaterializedView.md b/docs/src/js/classes/MaterializedView.md
new file mode 100644
index 000000000..e6ff66142
--- /dev/null
+++ b/docs/src/js/classes/MaterializedView.md
@@ -0,0 +1,101 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / MaterializedView
+
+# Class: MaterializedView
+
+A handle on a materialized view: its table plus its definition.
+
+Obtained from [Connection#createMaterializedView](Connection.md#creatematerializedview) or
+[Connection#openMaterializedView](Connection.md#openmaterializedview). The view is a normal table --
+queries, indexes and search all apply through [MaterializedView#table](MaterializedView.md#table)
+-- whose contents are maintained by [MaterializedView#refresh](MaterializedView.md#refresh).
+
+## Constructors
+
+### new MaterializedView()
+
+```ts
+new MaterializedView(table): MaterializedView
+```
+
+#### Parameters
+
+* **table**: [`Table`](Table.md)
+
+#### Returns
+
+[`MaterializedView`](MaterializedView.md)
+
+## Accessors
+
+### name
+
+```ts
+get name(): string
+```
+
+#### Returns
+
+`string`
+
+## Methods
+
+### definition()
+
+```ts
+definition(): Promise<MaterializedViewDefinition>
+```
+
+The query that defines the view, read from its stored schema.
+
+#### Returns
+
+`Promise`&lt;[`MaterializedViewDefinition`](../interfaces/MaterializedViewDefinition.md)&gt;
+
+***
+
+### refresh()
+
+```ts
+refresh(options?): Promise<RefreshMaterializedViewResult>
+```
+
+Recompute the view from its source.
+
+The refresh is incremental when the source's changes can be reconciled
+into the view -- rows added, changed or removed since the last one --
+and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
+refreshes to that source version instead of the latest.
+
+Concurrent refreshes of one view do not duplicate its rows. Two that
+plan the same source rows conflict on commit, and the loser throws
+rather than writing them a second time.
+
+#### Parameters
+
+* **options?**
+
+* **options.full?**: `boolean`
+
+* **options.sourceVersion?**: `number`
+
+#### Returns
+
+`Promise`&lt;[`RefreshMaterializedViewResult`](../interfaces/RefreshMaterializedViewResult.md)&gt;
+
+***
+
+### table()
+
+```ts
+table(): Table
+```
+
+The view, as the table it is.
+
+#### Returns
+
+[`Table`](Table.md)
diff --git a/docs/src/js/classes/Query.md b/docs/src/js/classes/Query.md
index 7fc7ee668..6ebaebd75 100644
--- a/docs/src/js/classes/Query.md
+++ b/docs/src/js/classes/Query.md
@@ -16,6 +16,18 @@ A builder for LanceDB queries.
 
 - `StandardQueryBase`&lt;`NativeQuery`&gt;
 
+## Properties
+
+### inner
+
+```ts
+protected inner: Query | Promise<Query>;
+```
+
+#### Inherited from
+
+`StandardQueryBase.inner`
+
 ## Methods
 
 ### analyzePlan()
diff --git a/docs/src/js/classes/QueryBase.md b/docs/src/js/classes/QueryBase.md
index 31b154525..35c071525 100644
--- a/docs/src/js/classes/QueryBase.md
+++ b/docs/src/js/classes/QueryBase.md
@@ -25,6 +25,14 @@ Common methods supported by all query types
 
 - `AsyncIterable`&lt;`RecordBatch`&gt;
 
+## Properties
+
+### inner
+
+```ts
+protected inner: NativeQueryType | Promise<NativeQueryType>;
+```
+
 ## Methods
 
 ### analyzePlan()
diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md
index 11fca32d0..9a85d0d96 100644
--- a/docs/src/js/classes/Table.md
+++ b/docs/src/js/classes/Table.md
@@ -69,14 +69,34 @@ abstract addColumns(newColumnTransforms): Promise<AddColumnsResult>
 
 Add new columns with defined values.
 
+The `{ computed }` form stores the expression rather than evaluating it
+now: the column is committed with no values, and rows get them from
+[Table#refreshColumn](Table.md#refreshcolumn). Declaring one therefore costs the same on a
+large table as on an empty one.
+
+A refresh does not revisit rows it has already filled, so mutating an
+input leaves the value computed at fill time; recomputing means dropping
+the column and declaring it again. While a declaration reads a column,
+that column cannot be renamed, retyped or dropped.
+
+On LanceDB Cloud and Enterprise the expression is planned by the
+server, and the refresh runs as a server job -- see
+[Table#refreshColumnAsync](Table.md#refreshcolumnasync).
+
 #### Parameters
 
-* **newColumnTransforms**: `Field`&lt;`any`&gt; \| `Field`&lt;`any`&gt;[] \| `Schema`&lt;`any`&gt; \| [`AddColumnsSql`](../interfaces/AddColumnsSql.md)[]
+* **newColumnTransforms**:
+    \| `Field`&lt;`any`&gt;
+    \| `Field`&lt;`any`&gt;[]
+    \| `Schema`&lt;`any`&gt;
+    \| [`AddColumnsSql`](../interfaces/AddColumnsSql.md)[]
+    \| `object`
     Either:
     - An array of objects with column names and SQL expressions to calculate values
     - A single Arrow Field defining one column with its data type (column will be initialized with null values)
     - An array of Arrow Fields defining columns with their data types (columns will be initialized with null values)
     - An Arrow Schema defining columns with their data types (columns will be initialized with null values)
+    - `{ computed }`, declaring columns defined by a SQL expression whose type and inputs are derived from it
 
 #### Returns
 
@@ -85,6 +105,13 @@ Add new columns with defined values.
 A promise that resolves to an object
 containing the new version number of the table after adding the columns.
 
+#### Example
+
+```ts
+await table.addColumns({ computed: [{ name: "doubled", valueSql: "x * 2" }] });
+const { rowsFilled } = await table.refreshColumn("doubled");
+```
+
 ***
 
 ### alterColumns()
@@ -186,6 +213,39 @@ version of the table.
 
 ***
 
+### checkpointLsm()
+
+```ts
+abstract checkpointLsm(): Promise<void>
+```
+
+Converge this table's LSM write path into its base table.
+
+Seals once, then triggers compaction and polls until the L0 that existed
+at the start is gone. The target set is fixed at the start, so
+generations created *during* the checkpoint are ignored — that is what
+lets it terminate under write load, and what makes it best-effort: it
+converges the fresh tier as of some instant. Idempotent, abandonable at
+any point, and safe to run on a cadence.
+
+There is no liveness bound — the compactor pool is shared across tables,
+so a checkpoint queued behind unrelated work looks exactly like one that
+is merging. The caller owns the deadline.
+
+#### Returns
+
+`Promise`&lt;`void`&gt;
+
+#### Example
+
+```ts
+const before = await table.getLsmStats();
+await table.checkpointLsm();
+const after = await table.getLsmStats();
+```
+
+***
+
 ### close()
 
 ```ts
@@ -223,6 +283,24 @@ It is a no-op when no writers are cached.
 
 ***
 
+### compactLsm()
+
+```ts
+abstract compactLsm(): Promise<void>
+```
+
+Trigger a background L0 → base compaction pass per bucket.
+
+Returns once the passes are *dispatched*, not once they finish — watch
+[Table#getLsmStats](Table.md#getlsmstats) for progress, or use
+[Table#checkpointLsm](Table.md#checkpointlsm) to wait for convergence.
+
+#### Returns
+
+`Promise`&lt;`void`&gt;
+
+***
+
 ### countRows()
 
 ```ts
@@ -421,6 +499,48 @@ Drop an index from the table.
 
 ***
 
+### flushLsm()
+
+```ts
+abstract flushLsm(): Promise<void>
+```
+
+Seal every bucket's active memtable into a new L0 generation.
+
+Returns once the seal is committed. Sealing an empty memtable is a no-op,
+so this is safe to call repeatedly.
+
+#### Returns
+
+`Promise`&lt;`void`&gt;
+
+***
+
+### getLsmStats()
+
+```ts
+abstract getLsmStats(includeGenerationRows?): Promise<undefined | LsmStats>
+```
+
+Read live per-bucket LSM state.
+
+Answers "how far behind is my fresh tier", "which bucket is hot", and
+"why is my fresh-tier vector search brute-force". Mutates no table state.
+
+Resolves to `undefined` only when the LSM write path is not enabled.
+
+#### Parameters
+
+* **includeGenerationRows?**: `boolean`
+    Also count rows per L0 generation.
+    Off by default because each count opens an uncached Lance dataset.
+
+#### Returns
+
+`Promise`&lt;`undefined` \| [`LsmStats`](../interfaces/LsmStats.md)&gt;
+
+***
+
 ### getLsmWriteSpec()
 
 ```ts
@@ -431,9 +551,10 @@ Read the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) currently installed on th
 
 Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
 spec has been set, or it was removed with [Table#unsetLsmWriteSpec](Table.md#unsetlsmwritespec)).
-The returned spec — including its `maintainedIndexes` and
-`writerConfigDefaults` — mirrors what was passed to
-[Table#setLsmWriteSpec](Table.md#setlsmwritespec).
+The returned spec mirrors what was passed to
+[Table#setLsmWriteSpec](Table.md#setlsmwritespec), except that `maintainedIndexes` always
+reports the concrete list resolved when the spec was set — `undefined`
+never round-trips.
 
 #### Returns
 
@@ -717,6 +838,67 @@ for await (const batch of table.query()) {
 
 ***
 
+### refreshColumn()
+
+```ts
+abstract refreshColumn(column): Promise<RefreshColumnResult>
+```
+
+Fill the rows of a computed column that hold no value yet.
+
+Rows appended since the last refresh are filled by the next one; rows
+already filled are left as they are, so the call is idempotent and does
+not observe a mutated input. Local tables only: a remote refresh runs
+as a server job, through [Table#refreshColumnAsync](Table.md#refreshcolumnasync).
+
+#### Parameters
+
+* **column**: `string`
+    The name of the computed column to fill.
+
+#### Returns
+
+`Promise`&lt;[`RefreshColumnResult`](../interfaces/RefreshColumnResult.md)&gt;
+
+A promise that resolves to the
+number of rows filled and the new version number of the table.
+
+***
+
+### refreshColumnAsync()
+
+```ts
+abstract refreshColumnAsync(column): Promise<Job>
+```
+
+Like [Table#refreshColumn](Table.md#refreshcolumn), but returns a handle to the refresh
+job instead of blocking until it completes.
+
+The job may already be complete when returned; callers must not assume
+the column is filled until [Job.wait](Job.md#wait) resolves. Invalid input --
+an unknown column, or one that is not computed -- rejects here rather
+than failing the job. On local tables the job runs in-process; on
+LanceDB Cloud and Enterprise it is the server's backfill job.
+
+#### Parameters
+
+* **column**: `string`
+    The name of the computed column to fill.
+
+#### Returns
+
+`Promise`&lt;[`Job`](Job.md)&gt;
+
+#### Example
+
+```ts
+const job = await table.refreshColumnAsync("doubled");
+await job.wait();
+console.log(await job.status()); // "finished"
+```
+
+***
+
 ### restore()
 
 ```ts
@@ -760,7 +942,7 @@ Get the schema of the table.
 abstract search(
    query,
    queryType?,
-   ftsColumns?): Query | VectorQuery
+   ftsColumns?): Query | VectorQuery | AutoQuery
 ```
 
 Create a search query to find the nearest neighbors
@@ -782,7 +964,7 @@ of the given query
 
 #### Returns
 
-[`Query`](Query.md) \| [`VectorQuery`](VectorQuery.md)
+[`Query`](Query.md) \| [`VectorQuery`](VectorQuery.md) \| [`AutoQuery`](AutoQuery.md)
 
 ***
 
@@ -806,6 +988,11 @@ All variants require the table to have an unenforced primary key
 ([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally
 requires it to be the single column being bucketed.
 
+Omitting `maintainedIndexes` maintains every index on the table, resolved
+here, failing if one cannot be maintained — name them to install anyway.
+Naming them pins an exact set, and a still-building index is rejected
+rather than quietly omitted.
+
 #### Parameters
 
 * **spec**: [`LsmWriteSpec`](../interfaces/LsmWriteSpec.md)
diff --git a/docs/src/js/classes/TakeQuery.md b/docs/src/js/classes/TakeQuery.md
index c00a68844..6ae2f9c8c 100644
--- a/docs/src/js/classes/TakeQuery.md
+++ b/docs/src/js/classes/TakeQuery.md
@@ -12,6 +12,18 @@ A query that returns a subset of the rows in the table.
 
 - [`QueryBase`](QueryBase.md)&lt;`NativeTakeQuery`&gt;
 
+## Properties
+
+### inner
+
+```ts
+protected inner: TakeQuery | Promise<TakeQuery>;
+```
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`inner`](QueryBase.md#inner)
+
 ## Methods
 
 ### analyzePlan()
diff --git a/docs/src/js/classes/VectorQuery.md b/docs/src/js/classes/VectorQuery.md
index 1e81c1716..f9412c76d 100644
--- a/docs/src/js/classes/VectorQuery.md
+++ b/docs/src/js/classes/VectorQuery.md
@@ -18,6 +18,18 @@ This builder can be reused to execute the query many times.
 
 - `StandardQueryBase`&lt;`NativeVectorQuery`&gt;
 
+## Properties
+
+### inner
+
+```ts
+protected inner: VectorQuery | Promise<VectorQuery>;
+```
+
+#### Inherited from
+
+`StandardQueryBase.inner`
+
 ## Methods
 
 ### addQueryVector()
diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md
index 7455a81ce..beb9cbeff 100644
--- a/docs/src/js/globals.md
+++ b/docs/src/js/globals.md
@@ -18,6 +18,7 @@
 
 ## Classes
 
+- [AutoQuery](classes/AutoQuery.md)
 - [BooleanQuery](classes/BooleanQuery.md)
 - [BoostQuery](classes/BoostQuery.md)
 - [BranchContents](classes/BranchContents.md)
@@ -28,6 +29,7 @@
 - [Job](classes/Job.md)
 - [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
 - [MatchQuery](classes/MatchQuery.md)
+- [MaterializedView](classes/MaterializedView.md)
 - [MergeInsertBuilder](classes/MergeInsertBuilder.md)
 - [MultiMatchQuery](classes/MultiMatchQuery.md)
 - [NativeJsHeaderProvider](classes/NativeJsHeaderProvider.md)
@@ -58,6 +60,10 @@
 - [BranchDiff](interfaces/BranchDiff.md)
 - [BranchIndexSummary](interfaces/BranchIndexSummary.md)
 - [BranchRowCountSummary](interfaces/BranchRowCountSummary.md)
+- [BucketStats](interfaces/BucketStats.md)
+- [CherryPickError](interfaces/CherryPickError.md)
+- [CherryPickPreview](interfaces/CherryPickPreview.md)
+- [CherryPickResult](interfaces/CherryPickResult.md)
 - [ClientConfig](interfaces/ClientConfig.md)
 - [ColumnAlteration](interfaces/ColumnAlteration.md)
 - [ColumnOrdering](interfaces/ColumnOrdering.md)
@@ -81,6 +87,7 @@
 - [FtsToken](interfaces/FtsToken.md)
 - [FullTextQuery](interfaces/FullTextQuery.md)
 - [FullTextSearchOptions](interfaces/FullTextSearchOptions.md)
+- [GenerationStats](interfaces/GenerationStats.md)
 - [HnswPqOptions](interfaces/HnswPqOptions.md)
 - [HnswSqOptions](interfaces/HnswSqOptions.md)
 - [IndexConfig](interfaces/IndexConfig.md)
@@ -94,10 +101,12 @@
 - [JobInfo](interfaces/JobInfo.md)
 - [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
 - [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
+- [ListTablesOptions](interfaces/ListTablesOptions.md)
+- [ListTablesResponse](interfaces/ListTablesResponse.md)
+- [LsmStats](interfaces/LsmStats.md)
 - [LsmWriteSpec](interfaces/LsmWriteSpec.md)
-- [MergeBlocker](interfaces/MergeBlocker.md)
-- [MergeBranchResult](interfaces/MergeBranchResult.md)
-- [MergePreview](interfaces/MergePreview.md)
+- [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md)
+- [MemtableStats](interfaces/MemtableStats.md)
 - [MergeResult](interfaces/MergeResult.md)
 - [NativeOAuthConfig](interfaces/NativeOAuthConfig.md)
 - [OAuthConfig](interfaces/OAuthConfig.md)
@@ -105,6 +114,8 @@
 - [OptimizeOptions](interfaces/OptimizeOptions.md)
 - [OptimizeStats](interfaces/OptimizeStats.md)
 - [QueryExecutionOptions](interfaces/QueryExecutionOptions.md)
+- [RefreshColumnResult](interfaces/RefreshColumnResult.md)
+- [RefreshMaterializedViewResult](interfaces/RefreshMaterializedViewResult.md)
 - [RemovalStats](interfaces/RemovalStats.md)
 - [RenameTableOptions](interfaces/RenameTableOptions.md)
 - [RestNamespaceConfig](interfaces/RestNamespaceConfig.md)
@@ -137,6 +148,7 @@
 - [FieldLike](type-aliases/FieldLike.md)
 - [IntoSql](type-aliases/IntoSql.md)
 - [IntoVector](type-aliases/IntoVector.md)
+- [MaterializedViewSelect](type-aliases/MaterializedViewSelect.md)
 - [MultiVector](type-aliases/MultiVector.md)
 - [RecordBatchLike](type-aliases/RecordBatchLike.md)
 - [SchemaLike](type-aliases/SchemaLike.md)
diff --git a/docs/src/js/interfaces/BranchDiff.md b/docs/src/js/interfaces/BranchDiff.md
index 224be1991..b408ebe11 100644
--- a/docs/src/js/interfaces/BranchDiff.md
+++ b/docs/src/js/interfaces/BranchDiff.md
@@ -50,6 +50,14 @@ changedColumns: BranchColumnChange[];
 
 ***
 
+### errors
+
+```ts
+errors: CherryPickError[];
+```
+
+***
+
 ### fromBranch
 
 ```ts
@@ -66,22 +74,6 @@ mainVersion: number;
 
 ***
 
-### mergeBlockers
-
-```ts
-mergeBlockers: MergeBlocker[];
-```
-
-***
-
-### mergeable
-
-```ts
-mergeable: boolean;
-```
-
-***
-
 ### parentVersion
 
 ```ts
diff --git a/docs/src/js/interfaces/BucketStats.md b/docs/src/js/interfaces/BucketStats.md
new file mode 100644
index 000000000..3f5095672
--- /dev/null
+++ b/docs/src/js/interfaces/BucketStats.md
@@ -0,0 +1,116 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / BucketStats
+
+# Interface: BucketStats
+
+Live state of one bucket. A table is N buckets on one node; flattening to a
+single number hides the one hot bucket that is usually why someone opened
+this endpoint.
+
+## Properties
+
+### compacting
+
+```ts
+compacting: boolean;
+```
+
+Whether a pass owns this bucket's compaction latch right now. Says *a*
+driver is running, not *whose*, and the latch is held from dispatch —
+including while the pass queues for a pod-wide compactor permit. Read it
+as "do not pile on", never as "mine is progressing".
+
+***
+
+### currentGeneration
+
+```ts
+currentGeneration: number;
+```
+
+The generation the active memtable will become.
+
+***
+
+### generations
+
+```ts
+generations: GenerationStats[];
+```
+
+Flushed L0 generations not yet merged into the base table.
+
+***
+
+### manifestVersion
+
+```ts
+manifestVersion: number;
+```
+
+Version of the shard manifest these numbers were read from.
+
+***
+
+### memtables?
+
+```ts
+optional memtables: MemtableStats[];
+```
+
+Oldest first, active last. Absent for a `"Sealed"` bucket, whose
+in-memory state is torn down.
+
+***
+
+### replayAfterWalEntryPosition
+
+```ts
+replayAfterWalEntryPosition: number;
+```
+
+WAL position replay resumes from.
+
+***
+
+### shardId
+
+```ts
+shardId: string;
+```
+
+The shard this bucket writes.
+
+***
+
+### status
+
+```ts
+status: string;
+```
+
+`"Active"` or `"Sealed"` (drop-table 2PC in flight).
+
+***
+
+### walEntryPositionLastSeen
+
+```ts
+walEntryPositionLastSeen: number;
+```
+
+Highest WAL position the writer has seen. The difference against
+`replayAfterWalEntryPosition` is the WAL lag.
+
+***
+
+### writerEpoch
+
+```ts
+writerEpoch: number;
+```
+
+Epoch of the writer that currently owns the shard.
diff --git a/docs/src/js/interfaces/MergeBlocker.md b/docs/src/js/interfaces/CherryPickError.md
similarity index 54%
rename from docs/src/js/interfaces/MergeBlocker.md
rename to docs/src/js/interfaces/CherryPickError.md
index 6c8f84b37..84f8fe012 100644
--- a/docs/src/js/interfaces/MergeBlocker.md
+++ b/docs/src/js/interfaces/CherryPickError.md
@@ -2,11 +2,11 @@
 
 ***
 
-[@lancedb/lancedb](../globals.md) / MergeBlocker
+[@lancedb/lancedb](../globals.md) / CherryPickError
 
-# Interface: MergeBlocker
+# Interface: CherryPickError
 
-A reason why a branch cannot currently be merged.
+A reason why a cherry-pick cannot currently land.
 
 ## Properties
 
diff --git a/docs/src/js/interfaces/CherryPickPreview.md b/docs/src/js/interfaces/CherryPickPreview.md
new file mode 100644
index 000000000..9620068a2
--- /dev/null
+++ b/docs/src/js/interfaces/CherryPickPreview.md
@@ -0,0 +1,17 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / CherryPickPreview
+
+# Interface: CherryPickPreview
+
+Changes that would be, or were, promoted by a cherry-pick.
+
+## Properties
+
+### promotedColumns
+
+```ts
+promotedColumns: string[];
+```
diff --git a/docs/src/js/interfaces/MergeBranchResult.md b/docs/src/js/interfaces/CherryPickResult.md
similarity index 60%
rename from docs/src/js/interfaces/MergeBranchResult.md
rename to docs/src/js/interfaces/CherryPickResult.md
index 61c2d0d33..c837ab143 100644
--- a/docs/src/js/interfaces/MergeBranchResult.md
+++ b/docs/src/js/interfaces/CherryPickResult.md
@@ -2,11 +2,11 @@
 
 ***
 
-[@lancedb/lancedb](../globals.md) / MergeBranchResult
+[@lancedb/lancedb](../globals.md) / CherryPickResult
 
-# Interface: MergeBranchResult
+# Interface: CherryPickResult
 
-Result of previewing or attempting a branch merge.
+Result of previewing or attempting a cherry-pick.
 
 ## Properties
 
@@ -29,7 +29,7 @@ optional mainVersionAfter: number;
 ### preview
 
 ```ts
-preview: MergePreview;
+preview: CherryPickPreview;
 ```
 
 ***
@@ -38,9 +38,9 @@ preview: MergePreview;
 
 ```ts
 status:
+  | "failed"
   | "unknown"
-  | "rejected"
   | "ready"
   | "notImplemented"
-  | "merged";
+  | "cherryPicked";
 ```
diff --git a/docs/src/js/interfaces/GenerationStats.md b/docs/src/js/interfaces/GenerationStats.md
new file mode 100644
index 000000000..19dd2afda
--- /dev/null
+++ b/docs/src/js/interfaces/GenerationStats.md
@@ -0,0 +1,40 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / GenerationStats
+
+# Interface: GenerationStats
+
+One flushed L0 generation.
+
+## Properties
+
+### bytes
+
+```ts
+bytes: number;
+```
+
+On-disk size of the generation.
+
+***
+
+### generation
+
+```ts
+generation: number;
+```
+
+The generation number. Increases as memtables are sealed into L0.
+
+***
+
+### rows?
+
+```ts
+optional rows: number;
+```
+
+Present only when `includeGenerationRows` was requested. Off by default
+because each count opens an uncached Lance dataset.
diff --git a/docs/src/js/interfaces/ListTablesOptions.md b/docs/src/js/interfaces/ListTablesOptions.md
new file mode 100644
index 000000000..52ace47e5
--- /dev/null
+++ b/docs/src/js/interfaces/ListTablesOptions.md
@@ -0,0 +1,34 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ListTablesOptions
+
+# Interface: ListTablesOptions
+
+## Properties
+
+### limit?
+
+```ts
+optional limit: number;
+```
+
+An upper bound on how many tables to return.
+
+A page may hold fewer than this and still not be the last one, so keep
+going while the response carries a page token rather than while pages are
+full.
+
+***
+
+### pageToken?
+
+```ts
+optional pageToken: string;
+```
+
+Token from a previous response, to resume listing where it left off.
+
+The token is opaque: it carries whatever the database needs to resume, and
+callers should not construct or interpret one.
diff --git a/docs/src/js/interfaces/ListTablesResponse.md b/docs/src/js/interfaces/ListTablesResponse.md
new file mode 100644
index 000000000..76cac2b23
--- /dev/null
+++ b/docs/src/js/interfaces/ListTablesResponse.md
@@ -0,0 +1,23 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ListTablesResponse
+
+# Interface: ListTablesResponse
+
+## Properties
+
+### pageToken?
+
+```ts
+optional pageToken: string;
+```
+
+***
+
+### tables
+
+```ts
+tables: string[];
+```
diff --git a/docs/src/js/interfaces/LsmStats.md b/docs/src/js/interfaces/LsmStats.md
new file mode 100644
index 000000000..76a2f50db
--- /dev/null
+++ b/docs/src/js/interfaces/LsmStats.md
@@ -0,0 +1,22 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / LsmStats
+
+# Interface: LsmStats
+
+Live per-bucket LSM state, as returned by `Table#getLsmStats`.
+
+Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are
+the caller's to compute.
+
+## Properties
+
+### buckets
+
+```ts
+buckets: BucketStats[];
+```
+
+One entry per bucket backing this table.
diff --git a/docs/src/js/interfaces/LsmWriteSpec.md b/docs/src/js/interfaces/LsmWriteSpec.md
index 8a588df6a..f2ae91186 100644
--- a/docs/src/js/interfaces/LsmWriteSpec.md
+++ b/docs/src/js/interfaces/LsmWriteSpec.md
@@ -34,7 +34,9 @@ Bucket and identity variants: the sharding column.
 optional maintainedIndexes: string[];
 ```
 
-Names of indexes the MemWAL should keep up to date during writes.
+Indexes the MemWAL keeps up to date. Omit to maintain every supported
+index, resolved on install — a snapshot, so indexes created later are not
+maintained. Pass `[]` for none.
 
 ***
 
diff --git a/docs/src/js/interfaces/MaterializedViewDefinition.md b/docs/src/js/interfaces/MaterializedViewDefinition.md
new file mode 100644
index 000000000..741bbba31
--- /dev/null
+++ b/docs/src/js/interfaces/MaterializedViewDefinition.md
@@ -0,0 +1,59 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / MaterializedViewDefinition
+
+# Interface: MaterializedViewDefinition
+
+The query that defines a materialized view.
+
+## Properties
+
+### filter?
+
+```ts
+optional filter: string;
+```
+
+SQL predicate selecting the source rows the view holds.
+
+***
+
+### inputs
+
+```ts
+inputs: string[];
+```
+
+Source columns the projections and filter read.
+
+***
+
+### limit?
+
+```ts
+optional limit: number;
+```
+
+Cap on the number of rows the view holds.
+
+***
+
+### projections
+
+```ts
+projections: [string, string][];
+```
+
+`[output column, SQL expression]` pairs, in view schema order.
+
+***
+
+### sourceTable
+
+```ts
+sourceTable: string;
+```
+
+Name of the source table, in the same database as the view.
diff --git a/docs/src/js/interfaces/MemtableStats.md b/docs/src/js/interfaces/MemtableStats.md
new file mode 100644
index 000000000..fdc1e4467
--- /dev/null
+++ b/docs/src/js/interfaces/MemtableStats.md
@@ -0,0 +1,60 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / MemtableStats
+
+# Interface: MemtableStats
+
+One in-memory memtable.
+
+## Properties
+
+### batches
+
+```ts
+batches: number;
+```
+
+Record batches currently buffered.
+
+***
+
+### bytes
+
+```ts
+bytes: number;
+```
+
+Estimated in-memory size.
+
+***
+
+### generation
+
+```ts
+generation: number;
+```
+
+The generation this memtable will become once sealed.
+
+***
+
+### indexes
+
+```ts
+indexes: string[];
+```
+
+Names of the indexes this memtable carries. An absent name is the whole
+answer to "why is my fresh-tier search on that column brute-force".
+
+***
+
+### rows
+
+```ts
+rows: number;
+```
+
+Rows currently buffered.
diff --git a/docs/src/js/interfaces/MergePreview.md b/docs/src/js/interfaces/MergePreview.md
deleted file mode 100644
index 0d9717289..000000000
--- a/docs/src/js/interfaces/MergePreview.md
+++ /dev/null
@@ -1,17 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / MergePreview
-
-# Interface: MergePreview
-
-Changes that would be, or were, promoted by a branch merge.
-
-## Properties
-
-### promotedColumns
-
-```ts
-promotedColumns: string[];
-```
diff --git a/docs/src/js/interfaces/RefreshColumnResult.md b/docs/src/js/interfaces/RefreshColumnResult.md
new file mode 100644
index 000000000..d2854fda6
--- /dev/null
+++ b/docs/src/js/interfaces/RefreshColumnResult.md
@@ -0,0 +1,23 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / RefreshColumnResult
+
+# Interface: RefreshColumnResult
+
+## Properties
+
+### rowsFilled
+
+```ts
+rowsFilled: number;
+```
+
+***
+
+### version
+
+```ts
+version: number;
+```
diff --git a/docs/src/js/interfaces/RefreshMaterializedViewResult.md b/docs/src/js/interfaces/RefreshMaterializedViewResult.md
new file mode 100644
index 000000000..cb7100cd8
--- /dev/null
+++ b/docs/src/js/interfaces/RefreshMaterializedViewResult.md
@@ -0,0 +1,41 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / RefreshMaterializedViewResult
+
+# Interface: RefreshMaterializedViewResult
+
+## Properties
+
+### mode
+
+```ts
+mode: string;
+```
+
+How the view was brought up to date: "rebuild", "incremental" or "no_op".
+
+***
+
+### rowsWritten
+
+```ts
+rowsWritten: number;
+```
+
+***
+
+### sourceVersion
+
+```ts
+sourceVersion: number;
+```
+
+***
+
+### version
+
+```ts
+version: number;
+```
diff --git a/docs/src/js/interfaces/TableNamesOptions.md b/docs/src/js/interfaces/TableNamesOptions.md
index 45fa3d1b0..9254e9fa7 100644
--- a/docs/src/js/interfaces/TableNamesOptions.md
+++ b/docs/src/js/interfaces/TableNamesOptions.md
@@ -4,11 +4,16 @@
 
 [@lancedb/lancedb](../globals.md) / TableNamesOptions
 
-# Interface: TableNamesOptions
+# Interface: ~~TableNamesOptions~~
+
+## Deprecated
+
+Use [ListTablesOptions](ListTablesOptions.md) with [Connection.listTables](../classes/Connection.md#listtables)
+instead.
 
 ## Properties
 
-### limit?
+### ~~limit?~~
 
 ```ts
 optional limit: number;
@@ -18,7 +23,7 @@ An optional limit to the number of results to return.
 
 ***
 
-### startAfter?
+### ~~startAfter?~~
 
 ```ts
 optional startAfter: string;
diff --git a/docs/src/js/interfaces/TableStatistics.md b/docs/src/js/interfaces/TableStatistics.md
index e19cba119..e2e8ef34d 100644
--- a/docs/src/js/interfaces/TableStatistics.md
+++ b/docs/src/js/interfaces/TableStatistics.md
@@ -44,4 +44,7 @@ The number of rows in the table
 totalBytes: number;
 ```
 
-The total number of bytes in the table
+The total size, in bytes, of the table's data files, index files, and
+overlay files
+
+Read from the manifest, so this excludes deletion files and manifests.
diff --git a/docs/src/js/namespaces/embedding/README.md b/docs/src/js/namespaces/embedding/README.md
index 157018e16..a736674c0 100644
--- a/docs/src/js/namespaces/embedding/README.md
+++ b/docs/src/js/namespaces/embedding/README.md
@@ -25,9 +25,12 @@
 ### Type Aliases
 
 - [CreateReturnType](type-aliases/CreateReturnType.md)
+- [EmbeddingMetadataEntry](type-aliases/EmbeddingMetadataEntry.md)
+- [ResolvedEmbeddingFunctionConfig](type-aliases/ResolvedEmbeddingFunctionConfig.md)
 
 ### Functions
 
 - [LanceSchema](functions/LanceSchema.md)
 - [getRegistry](functions/getRegistry.md)
+- [parseEmbeddingMetadata](functions/parseEmbeddingMetadata.md)
 - [register](functions/register.md)
diff --git a/docs/src/js/namespaces/embedding/functions/getRegistry.md b/docs/src/js/namespaces/embedding/functions/getRegistry.md
index 331dbd60b..b149a4a88 100644
--- a/docs/src/js/namespaces/embedding/functions/getRegistry.md
+++ b/docs/src/js/namespaces/embedding/functions/getRegistry.md
@@ -10,16 +10,12 @@
 function getRegistry(): EmbeddingFunctionRegistry
 ```
 
-Utility function to get the global instance of the registry
+Get the global embedding function registry.
+
+LanceDB built-in providers are initialized when this public API is first
+used, so importing the root package does not change automatic search
+selection for tables without embedding metadata.
 
 ## Returns
 
 [`EmbeddingFunctionRegistry`](../classes/EmbeddingFunctionRegistry.md)
-
-`EmbeddingFunctionRegistry` The global instance of the registry
-
-## Example
-
-```ts
-const registry = getRegistry();
-const openai = registry.get("openai").create();
diff --git a/docs/src/js/namespaces/embedding/functions/parseEmbeddingMetadata.md b/docs/src/js/namespaces/embedding/functions/parseEmbeddingMetadata.md
new file mode 100644
index 000000000..d6c381bb3
--- /dev/null
+++ b/docs/src/js/namespaces/embedding/functions/parseEmbeddingMetadata.md
@@ -0,0 +1,22 @@
+[**@lancedb/lancedb**](../../../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / parseEmbeddingMetadata
+
+# Function: parseEmbeddingMetadata()
+
+```ts
+function parseEmbeddingMetadata(json): EmbeddingMetadataEntry[]
+```
+
+The single parser for `embedding_functions` schema metadata: every reader
+goes through here, so the wire contract cannot fork between them.
+
+## Parameters
+
+* **json**: `string`
+
+## Returns
+
+[`EmbeddingMetadataEntry`](../type-aliases/EmbeddingMetadataEntry.md)[]
diff --git a/docs/src/js/namespaces/embedding/type-aliases/EmbeddingMetadataEntry.md b/docs/src/js/namespaces/embedding/type-aliases/EmbeddingMetadataEntry.md
new file mode 100644
index 000000000..a1bd247f6
--- /dev/null
+++ b/docs/src/js/namespaces/embedding/type-aliases/EmbeddingMetadataEntry.md
@@ -0,0 +1,40 @@
+[**@lancedb/lancedb**](../../../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / EmbeddingMetadataEntry
+
+# Type Alias: EmbeddingMetadataEntry
+
+```ts
+type EmbeddingMetadataEntry: object;
+```
+
+One entry of the `embedding_functions` schema metadata, with the column
+keys normalized across the bindings' spellings.
+
+## Type declaration
+
+### model
+
+```ts
+model: EmbeddingFunction["TOptions"];
+```
+
+### name
+
+```ts
+name: string;
+```
+
+### sourceColumn
+
+```ts
+sourceColumn: string;
+```
+
+### vectorColumn
+
+```ts
+vectorColumn: string;
+```
diff --git a/docs/src/js/namespaces/embedding/type-aliases/ResolvedEmbeddingFunctionConfig.md b/docs/src/js/namespaces/embedding/type-aliases/ResolvedEmbeddingFunctionConfig.md
new file mode 100644
index 000000000..864dfedfc
--- /dev/null
+++ b/docs/src/js/namespaces/embedding/type-aliases/ResolvedEmbeddingFunctionConfig.md
@@ -0,0 +1,22 @@
+[**@lancedb/lancedb**](../../../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / ResolvedEmbeddingFunctionConfig
+
+# Type Alias: ResolvedEmbeddingFunctionConfig
+
+```ts
+type ResolvedEmbeddingFunctionConfig: EmbeddingFunctionConfig & object;
+```
+
+An [EmbeddingFunctionConfig] read back from table metadata, where the
+vector column is always recorded.
+
+## Type declaration
+
+### vectorColumn
+
+```ts
+vectorColumn: string;
+```
diff --git a/docs/src/js/type-aliases/MaterializedViewSelect.md b/docs/src/js/type-aliases/MaterializedViewSelect.md
new file mode 100644
index 000000000..7b246e945
--- /dev/null
+++ b/docs/src/js/type-aliases/MaterializedViewSelect.md
@@ -0,0 +1,14 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / MaterializedViewSelect
+
+# Type Alias: MaterializedViewSelect
+
+```ts
+type MaterializedViewSelect: (string | [string, string])[] | Record<string, string>;
+```
+
+The view's columns: column names, `[alias, SQL expression]` pairs, or a
+record of the same. A bare name projects itself.
diff --git a/docs/src/python/python.md b/docs/src/python/python.md
index 36044d35d..3cbeee6f0 100644
--- a/docs/src/python/python.md
+++ b/docs/src/python/python.md
@@ -31,7 +31,7 @@ is also an [asynchronous API client](#connections-asynchronous).
 ## Namespaces (Synchronous)
 
 A namespace-backed connection resolves tables through a
-[Lance namespace](https://lancedb.github.io/lance-namespace/) service instead of
+[Lance namespace](https://lance-format.github.io/lance-namespace/) service instead of
 listing a storage directory.
 
 ::: lancedb.connect_namespace
@@ -52,6 +52,62 @@ listing a storage directory.
 
 ::: lancedb.table.Branches
 
+::: lancedb.LsmWriteSpec
+
+## Functions and Jobs
+
+::: lancedb.functions.FunctionArtifact
+
+::: lancedb.functions.FunctionParameter
+
+::: lancedb.functions.FunctionResultField
+
+::: lancedb.functions.FunctionOutput
+
+::: lancedb.functions.FunctionSignature
+
+::: lancedb.functions.PythonEnvironmentSpec
+
+::: lancedb.functions.udf
+
+::: lancedb.functions.UdfDefinition
+
+::: lancedb.functions.FunctionRegistrationRequest
+
+::: lancedb.functions.FunctionArtifactRequest
+
+::: lancedb.functions.FunctionArtifactContent
+
+::: lancedb.functions.PythonAdapterSpec
+
+::: lancedb.functions.FunctionVersion
+
+::: lancedb.functions.PythonRuntimeSpec
+
+::: lancedb.functions.FunctionVersionRef
+
+::: lancedb.functions.ApplicationInput
+
+::: lancedb.functions.FunctionApplication
+
+::: lancedb.functions.InputBinding
+
+::: lancedb.functions.OutputMapping
+
+::: lancedb.functions.FunctionBinding
+
+::: lancedb.functions.RefreshColumnResult
+
+::: lancedb.job.Job
+
+::: lancedb.job.AsyncJob
+
+## Materialized Views (Synchronous)
+
+::: lancedb.materialized_view.MaterializedView
+
+::: lancedb.materialized_view.MaterializedViewDefinition
+
 ## Expressions
 
 Type-safe expression builder for filters and projections. Use these instead
@@ -151,8 +207,9 @@ The same option is available on `lancedb.tokenize(...)` and the deprecated
 ```python
 import lancedb
 
-tokens = list(lancedb.tokenize("acme makes searchable data",
-                               custom_stop_words=["acme"]))
+tokens = list(
+    lancedb.tokenize("acme makes searchable data", custom_stop_words=["acme"])
+)
 ```
 
 ::: lancedb.tokenize
@@ -204,6 +261,8 @@ instead of being materialized with the rest of the row.
 
 ::: lancedb.streaming.StreamingDataset
 
+::: lancedb.streaming.StreamingDataLoader
+
 ::: lancedb.permutation.permutation_builder
 
 ::: lancedb.permutation.PermutationBuilder
@@ -244,6 +303,10 @@ Table hold your actual data as a collection of records / rows.
 
 ::: lancedb.table.AsyncBranches
 
+## Materialized Views (Asynchronous)
+
+::: lancedb.materialized_view.AsyncMaterializedView
+
 ## Indices (Asynchronous)
 
 Indices can be created on a table to speed up queries. This section
diff --git a/java/README.md b/java/README.md
index d3560ba4d..c46c8174b 100644
--- a/java/README.md
+++ b/java/README.md
@@ -29,6 +29,48 @@ LanceNamespace namespaceClient = LanceDbNamespaceClientBuilder.newBuilder()
     .build();
 ```
 
+## MemWAL LSM write path
+
+Most table operations reach LanceDB through the `LanceNamespace` above, which is
+generated from the Lance Namespace specification. The MemWAL LSM routes are not part
+of that specification, so they are issued through a separate client:
+
+```java
+import com.lancedb.LanceDbRestClient;
+import com.lancedb.LanceDbTableLsm;
+import com.lancedb.LsmWriteSpec;
+
+LanceDbRestClient client = LanceDbNamespaceClientBuilder.newBuilder()
+    .apiKey("your_lancedb_cloud_api_key")
+    .database("your_database_name")
+    .buildRestClient();
+
+LanceDbTableLsm lsm = new LanceDbTableLsm(client, "my_table");
+
+// Route future merge_insert upserts through the MemWAL, hash-bucketed by `id`.
+lsm.setLsmWriteSpec(LsmWriteSpec.bucket("id", 16));
+
+// ... merge_insert traffic ...
+
+// Converge the fresh tier into the base table.
+lsm.checkpointLsm();
+
+// Inspect live per-bucket state.
+lsm.getLsmStats().ifPresent(stats -> stats.buckets().forEach(bucket ->
+    System.out.println(bucket.shardId() + ": " + bucket.generations().size() + " L0 generations")));
+
+client.close();
+```
+
+`maintainedIndexes` is tri-state, and the null default is the opposite of what a Java
+reader usually expects:
+
+| Value | Meaning |
+| --- | --- |
+| unset (null) | Maintain **every** index the MemWAL can, resolved on install |
+| `Collections.emptyList()` | Maintain **none** |
+| `Arrays.asList("id_idx")` | Maintain exactly those |
+
 ## Development
 
 Build:
diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml
index 3df2ac178..25e3b10e3 100644
--- a/java/lancedb-core/pom.xml
+++ b/java/lancedb-core/pom.xml
@@ -8,7 +8,7 @@
     <parent>
       <groupId>com.lancedb</groupId>
       <artifactId>lancedb-parent</artifactId>
-      <version>0.37.1-beta.0</version>
+      <version>0.38.0-beta.10</version>
       <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -33,6 +33,20 @@
             <artifactId>arrow-memory-netty</artifactId>
         </dependency>
 
+        <!-- Transport for the LanceDB routes outside the Lance Namespace spec.
+             Versions match what lance-namespace-apache-client resolves to. -->
+        <dependency>
+            <groupId>org.apache.httpcomponents.client5</groupId>
+            <artifactId>httpclient5</artifactId>
+            <version>5.2.1</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.17.1</version>
+        </dependency>
+
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter</artifactId>
diff --git a/java/lancedb-core/src/main/java/com/lancedb/BucketStats.java b/java/lancedb-core/src/main/java/com/lancedb/BucketStats.java
new file mode 100644
index 000000000..2a8060c5d
--- /dev/null
+++ b/java/lancedb-core/src/main/java/com/lancedb/BucketStats.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.lancedb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.OptionalLong;
+
+/**
+ * Live state of one bucket. A table is N buckets on one node; flattening to a single number hides
+ * the one hot bucket that is usually why someone opened this endpoint.
+ */
+public class BucketStats {
+  private static final String CONTEXT = "bucket stats";
+
+  private final String shardId;
+  private final String status;
+  private final long writerEpoch;
+  private final long manifestVersion;
+  private final long currentGeneration;
+  private final long replayAfterWalEntryPosition;
+  private final long walEntryPositionLastSeen;
+  private final List<GenerationStats> generations;
+  private final boolean compacting;
+  private final List<MemtableStats> memtables;
+
+  BucketStats(
+      String shardId,
+      String status,
+      long writerEpoch,
+      long manifestVersion,
+      long currentGeneration,
+      long replayAfterWalEntryPosition,
+      long walEntryPositionLastSeen,
+      List<GenerationStats> generations,
+      boolean compacting,
+      List<MemtableStats> memtables) {
+    this.shardId = shardId;
+    this.status = status;
+    this.writerEpoch = writerEpoch;
+    this.manifestVersion = manifestVersion;
+    this.currentGeneration = currentGeneration;
+    this.replayAfterWalEntryPosition = replayAfterWalEntryPosition;
+    this.walEntryPositionLastSeen = walEntryPositionLastSeen;
+    this.generations = Collections.unmodifiableList(generations);
+    this.compacting = compacting;
+    this.memtables = memtables == null ? null : Collections.unmodifiableList(memtables);
+  }
+
+  /** The shard this bucket writes. */
+  public String shardId() {
+    return shardId;
+  }
+
+  /** {@code "Active"} or {@code "Sealed"} (drop-table 2PC in flight). */
+  public String status() {
+    return status;
+  }
+
+  /** Epoch of the writer that currently owns the shard. */
+  public long writerEpoch() {
+    return writerEpoch;
+  }
+
+  /** Version of the shard manifest these numbers were read from. */
+  public long manifestVersion() {
+    return manifestVersion;
+  }
+
+  /** The generation the active memtable will become. */
+  public long currentGeneration() {
+    return currentGeneration;
+  }
+
+  /** WAL position replay resumes from. */
+  public long replayAfterWalEntryPosition() {
+    return replayAfterWalEntryPosition;
+  }
+
+  /**
+   * Highest WAL position the writer has seen. The difference against {@link
+   * #replayAfterWalEntryPosition()} is the WAL lag.
+   */
+  public long walEntryPositionLastSeen() {
+    return walEntryPositionLastSeen;
+  }
+
+  /** Flushed L0 generations not yet merged into the base table. */
+  public List<GenerationStats> generations() {
+    return generations;
+  }
+
+  /**
+   * Whether a pass owns this bucket's compaction latch right now. Says <em>a</em> driver is
+   * running, not <em>whose</em>, and the latch is held from dispatch — including while the pass
+   * queues for a pod-wide compactor permit. Read it as "do not pile on", never as "mine is
+   * progressing".
+   */
+  public boolean compacting() {
+    return compacting;
+  }
+
+  /** Oldest first, active last. Empty for a {@code "Sealed"} bucket, whose state is torn down. */
+  public Optional<List<MemtableStats>> memtables() {
+    return Optional.ofNullable(memtables);
+  }
+
+  /** The newest flushed generation, or empty when L0 is empty. */
+  OptionalLong newestGeneration() {
+    OptionalLong newest = OptionalLong.empty();
+    for (GenerationStats generation : generations) {
+      if (!newest.isPresent() || generation.generation() > newest.getAsLong()) {
+        newest = OptionalLong.of(generation.generation());
+      }
+    }
+    return newest;
+  }
+
+  /**
+   * How many generations at or below {@code target} are still in L0.
+   *
+   * <p>A count, not a boolean: one pass drains a bounded prefix rather than the whole target set,
+   * so a boolean would read as "no progress" for every pass but the last. Compaction drains
+   * oldest-first, so this decreases monotonically.
+   */
+  long outstandingGenerations(long target) {
+    long count = 0;
+    for (GenerationStats generation : generations) {
+      if (generation.generation() <= target) {
+        count++;
+      }
+    }
+    return count;
+  }
+
+  static BucketStats fromJson(JsonNode node) {
+    JsonFields.requiredObject(node, CONTEXT);
+    List<GenerationStats> generations = new ArrayList<GenerationStats>();
+    for (JsonNode generation : JsonFields.requiredArray(node, "generations", CONTEXT)) {
+      generations.add(GenerationStats.fromJson(generation));
+    }
+
+    JsonNode memtablesNode = JsonFields.optionalArray(node, "memtables", CONTEXT);
+    List<MemtableStats> memtables = null;
+    if (memtablesNode != null) {
+      memtables = new ArrayList<MemtableStats>();
+      for (JsonNode memtable : memtablesNode) {
+        memtables.add(MemtableStats.fromJson(memtable));
+      }
+    }
+
+    return new BucketStats(
+        JsonFields.requiredText(node, "shard_id", CONTEXT),
+        JsonFields.requiredText(node, "status", CONTEXT),
+        JsonFields.requiredLong(node, "writer_epoch", CONTEXT),
+        JsonFields.requiredLong(node, "manifest_version", CONTEXT),
+        JsonFields.requiredLong(node, "current_generation", CONTEXT),
+        JsonFields.requiredLong(node, "replay_after_wal_entry_position", CONTEXT),
+        JsonFields.requiredLong(node, "wal_entry_position_last_seen", CONTEXT),
+        generations,
+        JsonFields.requiredBoolean(node, "compacting", CONTEXT),
+        memtables);
+  }
+
+  @Override
+  public String toString() {
+    return "BucketStats{shardId="
+        + shardId
+        + ", status="
+        + status
+        + ", currentGeneration="
+        + currentGeneration
+        + ", generations="
+        + generations
+        + ", compacting="
+        + compacting
+        + "}";
+  }
+}
diff --git a/java/lancedb-core/src/main/java/com/lancedb/GenerationStats.java b/java/lancedb-core/src/main/java/com/lancedb/GenerationStats.java
new file mode 100644
index 000000000..12222407c
--- /dev/null
+++ b/java/lancedb-core/src/main/java/com/lancedb/GenerationStats.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.lancedb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import java.util.OptionalLong;
+
+/** One flushed L0 generation. */
+public class GenerationStats {
+  private static final String CONTEXT = "generation stats";
+
+  private final long generation;
+  private final long bytes;
+  private final Long rows;
+
+  GenerationStats(long generation, long bytes, Long rows) {
+    this.generation = generation;
+    this.bytes = bytes;
+    this.rows = rows;
+  }
+
+  /** The generation number. Increases as memtables are sealed into L0. */
+  public long generation() {
+    return generation;
+  }
+
+  /** On-disk size of the generation. */
+  public long bytes() {
+    return bytes;
+  }
+
+  /**
+   * Rows in this generation, present only when {@code includeGenerationRows} was requested. Off by
+   * default because each count opens an uncached Lance dataset.
+   */
+  public OptionalLong rows() {
+    return rows == null ? OptionalLong.empty() : OptionalLong.of(rows);
+  }
+
+  static GenerationStats fromJson(JsonNode node) {
+    JsonFields.requiredObject(node, CONTEXT);
+    return new GenerationStats(
+        JsonFields.requiredLong(node, "generation", CONTEXT),
+        JsonFields.requiredLong(node, "bytes", CONTEXT),
+        JsonFields.optionalLong(node, "rows", CONTEXT));
+  }
+
+  @Override
+  public String toString() {
+    return "GenerationStats{generation=" + generation + ", bytes=" + bytes + ", rows=" + rows + "}";
+  }
+}
diff --git a/java/lancedb-core/src/main/java/com/lancedb/JsonFields.java b/java/lancedb-core/src/main/java/com/lancedb/JsonFields.java
new file mode 100644
index 000000000..b78e2411a
--- /dev/null
+++ b/java/lancedb-core/src/main/java/com/lancedb/JsonFields.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.lancedb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+/**
+ * Strict readers for decoding LanceDB JSON responses.
+ *
+ * <p>Every reader fails closed: a missing, null, or wrong-typed field throws rather than
+ * defaulting. That mirrors the serde decoding the Rust client applies to the same payloads in
+ * {@code rust/lancedb/src/table/lsm_stats.rs}, where a required field has no default and a
+ * malformed response is an error rather than a zero.
+ *
+ * <p>The alternative — Jackson's {@code path()}, which yields a missing node that reads as an empty
+ * array or a zero — is unsafe here because {@link LanceDbTableLsm#checkpointLsm()} decides
+ * convergence from these numbers. A defaulted {@code generations} array is indistinguishable from a
+ * drained one, so a malformed response would report a checkpoint that never happened.
+ */
+final class JsonFields {
+  private JsonFields() {}
+
+  /** The node itself, once confirmed to be a JSON object. */
+  static JsonNode requiredObject(JsonNode node, String context) {
+    if (node == null || !node.isObject()) {
+      throw new IllegalStateException(context + " is not a JSON object: " + node);
+    }
+    return node;
+  }
+
+  static String requiredText(JsonNode owner, String field, String context) {
+    JsonNode value = required(owner, field, context);
+    if (!value.isTextual()) {
+      throw new IllegalStateException(fieldIs(context, field, "a string", value));
+    }
+    return value.asText();
+  }
+
+  static long requiredLong(JsonNode owner, String field, String context) {
+    JsonNode value = required(owner, field, context);
+    if (!value.isIntegralNumber()) {
+      throw new IllegalStateException(fieldIs(context, field, "an integer", value));
+    }
+    return value.asLong();
+  }
+
+  static boolean requiredBoolean(JsonNode owner, String field, String context) {
+    JsonNode value = required(owner, field, context);
+    if (!value.isBoolean()) {
+      throw new IllegalStateException(fieldIs(context, field, "a boolean", value));
+    }
+    return value.asBoolean();
+  }
+
+  static JsonNode requiredArray(JsonNode owner, String field, String context) {
+    JsonNode value = required(owner, field, context);
+    if (!value.isArray()) {
+      throw new IllegalStateException(fieldIs(context, field, "an array", value));
+    }
+    return value;
+  }
+
+  /** Null when the field is absent or JSON null, mirroring a serde {@code Option}. */
+  static Long optionalLong(JsonNode owner, String field, String context) {
+    JsonNode value = owner.get(field);
+    if (value == null || value.isNull()) {
+      return null;
+    }
+    if (!value.isIntegralNumber()) {
+      throw new IllegalStateException(fieldIs(context, field, "an integer", value));
+    }
+    return value.asLong();
+  }
+
+  /** Null when the field is absent or JSON null, mirroring a serde {@code Option}. */
+  static JsonNode optionalArray(JsonNode owner, String field, String context) {
+    JsonNode value = owner.get(field);
+    if (value == null || value.isNull()) {
+      return null;
+    }
+    if (!value.isArray()) {
+      throw new IllegalStateException(fieldIs(context, field, "an array", value));
+    }
+    return value;
+  }
+
+  private static JsonNode required(JsonNode owner, String field, String context) {
+    JsonNode value = owner.get(field);
+    if (value == null || value.isNull()) {
+      throw new IllegalStateException(context + " is missing required field '" + field + "'");
+    }
+    return value;
+  }
+
+  private static String fieldIs(String context, String field, String expected, JsonNode value) {
+    return context + " field '" + field + "' is not " + expected + ": " + value;
+  }
+}
diff --git a/java/lancedb-core/src/main/java/com/lancedb/LanceDbNamespaceClientBuilder.java b/java/lancedb-core/src/main/java/com/lancedb/LanceDbNamespaceClientBuilder.java
index 5e31aaaa1..da241dfd5 100644
--- a/java/lancedb-core/src/main/java/com/lancedb/LanceDbNamespaceClientBuilder.java
+++ b/java/lancedb-core/src/main/java/com/lancedb/LanceDbNamespaceClientBuilder.java
@@ -136,29 +136,48 @@ public class LanceDbNamespaceClientBuilder {
    * @throws IllegalStateException if required parameters are missing
    */
   public LanceNamespace build() {
-    // Validate required fields
+    validate();
+
+    // Build configuration map
+    Map<String, String> config = new HashMap<>(additionalConfig);
+    config.put("header.x-lancedb-database", database);
+    config.put("header.x-api-key", apiKey);
+    config.put("uri", resolveUri());
+
+    return LanceNamespace.connect("rest", config, null);
+  }
+
+  /**
+   * Build a {@link LanceDbRestClient} for the same endpoint.
+   *
+   * <p>Needed only for LanceDB routes that the Lance Namespace specification does not cover — the
+   * MemWAL LSM write path, reached through {@link LanceDbTableLsm}. Every other table operation
+   * belongs on the {@link LanceNamespace} from {@link #build()}.
+   *
+   * <p>The returned client owns an HTTP connection pool; close it when you are done with it.
+   *
+   * @return A configured LanceDbRestClient
+   * @throws IllegalStateException if required parameters are missing
+   */
+  public LanceDbRestClient buildRestClient() {
+    validate();
+    return new LanceDbRestClient(resolveUri(), apiKey, database);
+  }
+
+  private void validate() {
     if (apiKey == null) {
       throw new IllegalStateException("API key is required");
     }
     if (database == null) {
       throw new IllegalStateException("Database is required");
     }
+  }
 
-    // Build configuration map
-    Map<String, String> config = new HashMap<>(additionalConfig);
-    config.put("header.x-lancedb-database", database);
-    config.put("header.x-api-key", apiKey);
-
-    // Determine base URL
-    String uri;
+  /** The custom endpoint when set, else the LanceDB Cloud URL for this database and region. */
+  private String resolveUri() {
     if (endpoint.isPresent()) {
-      uri = endpoint.get();
-    } else {
-      String effectiveRegion = region.orElse(DEFAULT_REGION);
-      uri = String.format(CLOUD_URL_PATTERN, database, effectiveRegion);
+      return endpoint.get();
     }
-    config.put("uri", uri);
-
-    return LanceNamespace.connect("rest", config, null);
+    return String.format(CLOUD_URL_PATTERN, database, region.orElse(DEFAULT_REGION));
   }
 }
diff --git a/java/lancedb-core/src/main/java/com/lancedb/LanceDbRestClient.java b/java/lancedb-core/src/main/java/com/lancedb/LanceDbRestClient.java
new file mode 100644
index 000000000..baafbb9df
--- /dev/null
+++ b/java/lancedb-core/src/main/java/com/lancedb/LanceDbRestClient.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.lancedb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hc.client5.http.classic.methods.HttpPost;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.impl.classic.HttpClients;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
+import org.apache.hc.core5.http.io.entity.StringEntity;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+
+/**
+ * Minimal HTTP client for LanceDB Cloud and Enterprise routes that the Lance Namespace
+ * specification does not cover.
+ *
+ * <p>Most table operations reach LanceDB through {@link org.lance.namespace.LanceNamespace}, which
+ * is generated from the namespace spec. A handful of routes — the MemWAL LSM write path in
+ * particular — are served by the same endpoint but are not part of that spec, so they are issued
+ * directly here. See {@link LanceDbTableLsm}.
+ *
+ * <p>Obtain one from {@link LanceDbNamespaceClientBuilder#buildRestClient()}.
+ */
+public class LanceDbRestClient implements Closeable {
+  private static final ObjectMapper MAPPER = new ObjectMapper();
+
+  private final String baseUri;
+  private final String apiKey;
+  private final String database;
+  private final CloseableHttpClient http;
+
+  LanceDbRestClient(String baseUri, String apiKey, String database) {
+    this.baseUri = baseUri.endsWith("/") ? baseUri.substring(0, baseUri.length() - 1) : baseUri;
+    this.apiKey = apiKey;
+    this.database = database;
+    // Automatic retries off, deliberately. The default strategy retries 429 and 503 —
+    // exactly the two statuses LanceDbTableLsm.checkpointLsm() acts on — which would
+    // silently double its explicit retry budget and would also retry compact_lsm in
+    // place, where the loop is designed to fall through to a fresh stats poll instead.
+    // The checkpoint loop owns the 421/429/503 transitions; the transport must not.
+    this.http = HttpClients.custom().disableAutomaticRetries().build();
+  }
+
+  /**
+   * POST {@code path}, sending {@code body} as JSON when it is non-null.
+   *
+   * @param path Absolute request path, beginning with {@code /}.
+   * @param body Object to serialize as the request body, or null to send no body.
+   * @return The parsed response body, or null when the response carried no content.
+   * @throws HttpException if the server returned a non-2xx status.
+   */
+  public JsonNode post(String path, Object body) {
+    HttpPost request = new HttpPost(baseUri + path);
+    request.setHeader("x-api-key", apiKey);
+    request.setHeader("x-lancedb-database", database);
+    try {
+      if (body != null) {
+        request.setEntity(
+            new StringEntity(MAPPER.writeValueAsString(body), ContentType.APPLICATION_JSON));
+      }
+      return http.execute(
+          request,
+          response -> {
+            String text =
+                response.getEntity() == null ? "" : EntityUtils.toString(response.getEntity());
+            int status = response.getCode();
+            if (status < 200 || status >= 300) {
+              throw new HttpException(status, "LanceDB request to " + path + " failed: " + text);
+            }
+            return text.isEmpty() ? null : MAPPER.readTree(text);
+          });
+    } catch (IOException e) {
+      throw new UncheckedIOException("LanceDB request to " + path + " failed", e);
+    }
+  }
+
+  @Override
+  public void close() throws IOException {
+    http.close();
+  }
+
+  /**
+   * A non-2xx response.
+   *
+   * <p>The status is exposed because callers act on it: {@link LanceDbTableLsm#checkpointLsm()}
+   * treats 429 and 503 as retryable and 421 as a lost node claim.
+   */
+  public static class HttpException extends RuntimeException {
+    private static final long serialVersionUID = 1L;
+
+    private final int statusCode;
+
+    public HttpException(int statusCode, String message) {
+      super(message);
+      this.statusCode = statusCode;
+    }
+
+    /** The HTTP status the failed response carried. */
+    public int statusCode() {
+      return statusCode;
+    }
+  }
+}
diff --git a/java/lancedb-core/src/main/java/com/lancedb/LanceDbTableLsm.java b/java/lancedb-core/src/main/java/com/lancedb/LanceDbTableLsm.java
new file mode 100644
index 000000000..23b18199e
--- /dev/null
+++ b/java/lancedb-core/src/main/java/com/lancedb/LanceDbTableLsm.java
@@ -0,0 +1,394 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.lancedb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalLong;
+
+/**
+ * The MemWAL LSM write path for one LanceDB Cloud or Enterprise table.
+ *
+ * <p>Installing an {@link LsmWriteSpec} routes {@code mergeInsert} upserts through Lance's MemWAL —
+ * an LSM-style append — instead of the standard merge path. Rows land in an in-memory memtable,
+ * seal into L0 generations, and are merged into the base table by compaction.
+ *
+ * <p>These routes are not part of the Lance Namespace specification, so they are issued directly
+ * rather than through {@link org.lance.namespace.LanceNamespace}.
+ *
+ * <pre>{@code
+ * LanceDbRestClient client = LanceDbNamespaceClientBuilder.newBuilder()
+ *     .apiKey("your_lancedb_cloud_api_key")
+ *     .database("your_database_name")
+ *     .buildRestClient();
+ *
+ * LanceDbTableLsm lsm = new LanceDbTableLsm(client, "my_table");
+ * lsm.setLsmWriteSpec(LsmWriteSpec.bucket("id", 16));
+ * // ... merge_insert traffic ...
+ * lsm.checkpointLsm();
+ * }</pre>
+ */
+public class LanceDbTableLsm {
+
+  /**
+   * Interval between {@code get_lsm_stats} polls during a checkpoint. One interval is roughly one
+   * compaction pass, the granularity at which the answer can change.
+   */
+  private static final long POLL_INTERVAL_MS = 5_000L;
+
+  /**
+   * Cap on re-issues from {@code flushLsm} after a 421, so a crash-looping node cannot turn flush →
+   * compact → 421 → flush into a spin.
+   *
+   * <p>Deliberately not shared with {@link #MAX_RETRIES}: a claim that keeps evaporating is a
+   * broken node, while contention is routine and wants a real budget.
+   */
+  private static final int MAX_REISSUES = 3;
+
+  /**
+   * Retryable faults tolerated on a <em>single</em> request, reset on every success — scattered
+   * contention across a long checkpoint must not accumulate toward a cap.
+   */
+  private static final int MAX_RETRIES = 8;
+
+  private static final long RETRY_BACKOFF_BASE_MS = 100L;
+  private static final long RETRY_BACKOFF_MAX_MS = 5_000L;
+
+  private final LanceDbRestClient client;
+  private final String tableIdentifier;
+
+  /**
+   * Bind the LSM routes for one table.
+   *
+   * @param client Transport for the LanceDB endpoint.
+   * @param tableIdentifier The table's full identifier, {@code $}-delimited when it sits inside a
+   *     namespace, such as {@code analytics$events}.
+   */
+  public LanceDbTableLsm(LanceDbRestClient client, String tableIdentifier) {
+    if (client == null) {
+      throw new IllegalArgumentException("Client cannot be null");
+    }
+    if (tableIdentifier == null || tableIdentifier.trim().isEmpty()) {
+      throw new IllegalArgumentException("Table identifier cannot be null or empty");
+    }
+    this.client = client;
+    this.tableIdentifier = tableIdentifier;
+  }
+
+  /**
+   * Install an {@link LsmWriteSpec} on this table, selecting the MemWAL LSM write path for future
+   * {@code mergeInsert} calls.
+   *
+   * <p>All variants require the table to have an unenforced primary key; bucket sharding
+   * additionally requires it to be the single column being bucketed.
+   */
+  public void setLsmWriteSpec(LsmWriteSpec spec) {
+    if (spec == null) {
+      throw new IllegalArgumentException("Spec cannot be null");
+    }
+    client.post(route("set_lsm_write_spec"), spec.toRequestBody());
+  }
+
+  /**
+   * Remove the {@link LsmWriteSpec} from this table, reverting to the standard {@code mergeInsert}
+   * write path.
+   *
+   * <p>Errors if no spec is currently set.
+   */
+  public void unsetLsmWriteSpec() {
+    client.post(route("unset_lsm_write_spec"), null);
+  }
+
+  /**
+   * Read the {@link LsmWriteSpec} currently installed on this table.
+   *
+   * <p>Empty when the LSM write path is not enabled. The returned spec mirrors what was installed,
+   * except that {@link LsmWriteSpec#maintainedIndexes()} always reports the concrete list resolved
+   * when the spec was set — a null selection never round-trips.
+   */
+  public Optional<LsmWriteSpec> getLsmWriteSpec() {
+    JsonNode response = client.post(route("get_lsm_write_spec"), null);
+    if (response == null || !response.hasNonNull("lsm_write_spec")) {
+      return Optional.empty();
+    }
+    return Optional.of(LsmWriteSpec.fromJson(response.get("lsm_write_spec")));
+  }
+
+  /**
+   * Seal every bucket's active memtable into a new L0 generation.
+   *
+   * <p>Returns once the seal is committed. Sealing an empty memtable is a no-op, so this is safe to
+   * call repeatedly.
+   */
+  public void flushLsm() {
+    client.post(route("flush_lsm"), null);
+  }
+
+  /**
+   * Trigger a background L0 → base compaction pass per bucket.
+   *
+   * <p>Returns once the passes are <em>dispatched</em>, not once they finish — watch {@link
+   * #getLsmStats}, or use {@link #checkpointLsm} to wait for convergence.
+   */
+  public void compactLsm() {
+    client.post(route("compact_lsm"), null);
+  }
+
+  /**
+   * Read live per-bucket LSM state.
+   *
+   * <p>Answers "how far behind is my fresh tier", "which bucket is hot", and "why is my fresh-tier
+   * vector search brute-force". Mutates no table state.
+   *
+   * <p>Empty only when the LSM write path is not enabled — that is, when the server sends an absent
+   * or null {@code lsm_stats}. A stats object that is present is decoded strictly, and a malformed
+   * one throws rather than decoding to something empty, because {@link #checkpointLsm} reads
+   * convergence out of these numbers and cannot tell a defaulted array from a drained one.
+   *
+   * @param includeGenerationRows Also count rows per L0 generation. Off by default because each
+   *     count opens an uncached Lance dataset.
+   * @throws IllegalStateException if the response is absent or does not decode.
+   */
+  public Optional<LsmStats> getLsmStats(boolean includeGenerationRows) {
+    Map<String, Object> body = new LinkedHashMap<String, Object>();
+    body.put("include_generation_rows", includeGenerationRows);
+    JsonNode response = client.post(route("get_lsm_stats"), body);
+    if (response == null) {
+      throw new IllegalStateException("get_lsm_stats returned an empty response body");
+    }
+    JsonNode stats = response.get("lsm_stats");
+    if (stats == null || stats.isNull()) {
+      return Optional.empty();
+    }
+    return Optional.of(LsmStats.fromJson(stats));
+  }
+
+  /** Equivalent to {@code getLsmStats(false)}. */
+  public Optional<LsmStats> getLsmStats() {
+    return getLsmStats(false);
+  }
+
+  /**
+   * Converge this table's LSM write path into its base table.
+   *
+   * <p>Seals once, fixes a target watermark from the resulting L0, then triggers compaction and
+   * polls until that L0 is gone. The target set is fixed at the start, so generations created
+   * <em>during</em> the checkpoint are ignored — that is what lets it terminate under write load,
+   * and what makes it best-effort: it converges the fresh tier as of some instant. Idempotent,
+   * abandonable at any point, safe on a cadence.
+   *
+   * <p>The loop runs here, not on the server: {@link #compactLsm} dispatches a pass and returns, so
+   * nothing holds a socket and a client can vanish mid-operation with nothing to reconcile.
+   * Completion is read from generation numbers in the shard manifest — durable state, unlike a
+   * count in a compact response, which a concurrent write invalidates.
+   *
+   * <p>No liveness bound — the caller owns the deadline. The compactor pool is shared across
+   * tables, so a checkpoint queued behind unrelated work looks exactly like one that is merging.
+   */
+  public void checkpointLsm() {
+    for (int reissue = 0; reissue <= MAX_REISSUES; reissue++) {
+      // The seal turns everything written before this call into a generation, so the
+      // watermark has to be read after it. Idempotent: sealing an empty memtable is a
+      // no-op, so a re-issue does not churn empty generations.
+      if (issueVoid(this::flushLsm)) {
+        backoff(reissue);
+        continue;
+      }
+
+      Attempt<Optional<LsmStats>> stats = issue(() -> getLsmStats(false));
+      if (stats.lostClaim) {
+        backoff(reissue);
+        continue;
+      }
+      if (!stats.value.isPresent()) {
+        // Not WAL-backed; flushLsm would have errored first but for a race.
+        return;
+      }
+
+      Map<String, Long> targets = newestGenerations(stats.value.get());
+      if (targets.isEmpty()) {
+        return;
+      }
+
+      if (drainToTargets(targets)) {
+        return;
+      }
+      backoff(reissue);
+    }
+    throw new IllegalStateException(
+        "checkpointLsm: the owning node kept losing its claim; re-issued from flush the maximum "
+            + "number of times");
+  }
+
+  /**
+   * Trigger and poll until no bucket holds a generation at or below its target.
+   *
+   * @return true when the drain finished, false when the table needs re-claiming from flush.
+   */
+  private boolean drainToTargets(Map<String, Long> targets) {
+    while (true) {
+      Attempt<Optional<LsmStats>> stats = issue(() -> getLsmStats(false));
+      if (stats.lostClaim) {
+        return false;
+      }
+      if (!stats.value.isPresent()) {
+        return true;
+      }
+
+      // `compacting` is the bucket's compaction latch, held from dispatch until the pass
+      // ends — including while it waits on a pod-wide permit. So it answers one question
+      // only: do not pile on. Buckets with nothing outstanding are skipped, not counted
+      // as idle.
+      long outstanding = 0;
+      boolean allCompacting = true;
+      for (BucketStats bucket : stats.value.get().buckets()) {
+        Long target = targets.get(bucket.shardId());
+        if (target == null) {
+          continue;
+        }
+        long remaining = bucket.outstandingGenerations(target);
+        if (remaining > 0) {
+          outstanding += remaining;
+          allCompacting &= bucket.compacting();
+        }
+      }
+      if (outstanding == 0) {
+        return true;
+      }
+
+      if (!allCompacting) {
+        try {
+          compactLsm();
+        } catch (LanceDbRestClient.HttpException e) {
+          if (isLostClaim(e)) {
+            return false;
+          }
+          if (!isRetryable(e)) {
+            throw e;
+          }
+          // A 429 here means the server could latch no bucket at all, which the poll
+          // above already handles. Not retried in place: the latch it would contend for
+          // is the one doing the work, so fall through and re-read — POLL_INTERVAL_MS is
+          // the backoff.
+        }
+      }
+      sleep(POLL_INTERVAL_MS);
+    }
+  }
+
+  /** The newest generation held by each bucket, skipping buckets holding none. */
+  private static Map<String, Long> newestGenerations(LsmStats stats) {
+    Map<String, Long> targets = new HashMap<String, Long>();
+    for (BucketStats bucket : stats.buckets()) {
+      OptionalLong newest = bucket.newestGeneration();
+      if (newest.isPresent()) {
+        targets.put(bucket.shardId(), newest.getAsLong());
+      }
+    }
+    return targets;
+  }
+
+  /**
+   * 429 (latch held, pool saturated, or the pod replaying its WAL) and 503 (a draining node, or a
+   * proxy between here and it).
+   */
+  private static boolean isRetryable(LanceDbRestClient.HttpException e) {
+    return e.statusCode() == 429 || e.statusCode() == 503;
+  }
+
+  /**
+   * 421: the owning node holds no claim. Only {@code flush} re-claims and replays, so this cannot
+   * be retried in place — the caller has to start over.
+   */
+  private static boolean isLostClaim(LanceDbRestClient.HttpException e) {
+    return e.statusCode() == 421;
+  }
+
+  /**
+   * Issue one LSM request, retrying in place while the fault is retryable.
+   *
+   * <p>The two recoverable faults have separate budgets: contention clears on its own and retries
+   * here against {@link #MAX_RETRIES}, while a 421 needs {@code flush} to re-claim, which only the
+   * caller can drive.
+   *
+   * <p>An exhausted budget propagates the last error as itself rather than a synthesized one — "429
+   * after nine tries" beats "checkpoint failed".
+   */
+  private static <T> Attempt<T> issue(Call<T> call) {
+    int retries = 0;
+    while (true) {
+      try {
+        return new Attempt<T>(call.run(), false);
+      } catch (LanceDbRestClient.HttpException e) {
+        if (isLostClaim(e)) {
+          return new Attempt<T>(null, true);
+        }
+        if (!isRetryable(e) || retries >= MAX_RETRIES) {
+          throw e;
+        }
+        backoff(retries);
+        retries++;
+      }
+    }
+  }
+
+  /** {@link #issue} for a call with no return value. Returns true when the claim was lost. */
+  private static boolean issueVoid(Runnable call) {
+    return issue(
+            () -> {
+              call.run();
+              return Boolean.TRUE;
+            })
+        .lostClaim;
+  }
+
+  /** Sleep before re-issuing a retryable request. Doubles up to {@link #RETRY_BACKOFF_MAX_MS}. */
+  private static void backoff(int attempt) {
+    long delay = RETRY_BACKOFF_BASE_MS << Math.min(attempt, 8);
+    sleep(Math.min(delay, RETRY_BACKOFF_MAX_MS));
+  }
+
+  private static void sleep(long millis) {
+    try {
+      Thread.sleep(millis);
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      throw new IllegalStateException("Interrupted while waiting on the LSM checkpoint", e);
+    }
+  }
+
+  private String route(String operation) {
+    return "/v1/table/" + tableIdentifier + "/" + operation + "/";
+  }
+
+  /** What one LSM request produced: its value, or word that the owning node holds no claim. */
+  private static final class Attempt<T> {
+    private final T value;
+    private final boolean lostClaim;
+
+    private Attempt(T value, boolean lostClaim) {
+      this.value = value;
+      this.lostClaim = lostClaim;
+    }
+  }
+
+  @FunctionalInterface
+  private interface Call<T> {
+    T run();
+  }
+}
diff --git a/java/lancedb-core/src/main/java/com/lancedb/LsmStats.java b/java/lancedb-core/src/main/java/com/lancedb/LsmStats.java
new file mode 100644
index 000000000..3496ebc96
--- /dev/null
+++ b/java/lancedb-core/src/main/java/com/lancedb/LsmStats.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.lancedb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Live per-bucket LSM state, as returned by {@link LanceDbTableLsm#getLsmStats()}.
+ *
+ * <p>Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are the caller's to
+ * compute. There is no "LSM is off" shape — that case is an empty {@link java.util.Optional},
+ * because a stats object of zeros would read as measurements.
+ */
+public class LsmStats {
+  private static final String CONTEXT = "lsm stats";
+
+  private final List<BucketStats> buckets;
+
+  LsmStats(List<BucketStats> buckets) {
+    this.buckets = Collections.unmodifiableList(buckets);
+  }
+
+  /** One entry per bucket. */
+  public List<BucketStats> buckets() {
+    return buckets;
+  }
+
+  static LsmStats fromJson(JsonNode node) {
+    JsonFields.requiredObject(node, CONTEXT);
+    List<BucketStats> buckets = new ArrayList<BucketStats>();
+    for (JsonNode bucket : JsonFields.requiredArray(node, "buckets", CONTEXT)) {
+      buckets.add(BucketStats.fromJson(bucket));
+    }
+    return new LsmStats(buckets);
+  }
+
+  @Override
+  public String toString() {
+    return "LsmStats{buckets=" + buckets + "}";
+  }
+}
diff --git a/java/lancedb-core/src/main/java/com/lancedb/LsmWriteSpec.java b/java/lancedb-core/src/main/java/com/lancedb/LsmWriteSpec.java
new file mode 100644
index 000000000..da0966910
--- /dev/null
+++ b/java/lancedb-core/src/main/java/com/lancedb/LsmWriteSpec.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.lancedb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Specification selecting Lance's MemWAL LSM-style write path for {@code mergeInsert}.
+ *
+ * <p>Construct via {@link #bucket}, {@link #identity}, or {@link #unsharded}, then optionally chain
+ * {@link #withMaintainedIndexes} and {@link #withWriterConfigDefaults}. Install it with {@link
+ * LanceDbTableLsm#setLsmWriteSpec} and remove it with {@link LanceDbTableLsm#unsetLsmWriteSpec}.
+ *
+ * <p>This is deliberately not {@code org.lance.memwal.InitializeMemWalParams}. That type is Lance's
+ * own, and its maintained-index default is the opposite of this one: it defaults to maintaining
+ * <em>nothing</em>, while a fresh spec here maintains <em>every</em> index. It also cannot express
+ * the null that asks the server to resolve the set.
+ */
+public class LsmWriteSpec {
+
+  /** How writes are routed to MemWAL shards. */
+  public enum Sharding {
+    /** Hash-bucket writes by a scalar column. */
+    BUCKET("bucket"),
+    /** Shard by the raw value of a scalar column. */
+    IDENTITY("identity"),
+    /** Route every write to a single shard. */
+    UNSHARDED("unsharded");
+
+    private final String wireName;
+
+    Sharding(String wireName) {
+      this.wireName = wireName;
+    }
+
+    String wireName() {
+      return wireName;
+    }
+
+    static Sharding fromWireName(String name) {
+      for (Sharding s : values()) {
+        if (s.wireName.equals(name)) {
+          return s;
+        }
+      }
+      throw new IllegalArgumentException("Unknown sharding mode: " + name);
+    }
+  }
+
+  private final Sharding sharding;
+  private final String column;
+  private final Integer numBuckets;
+  private final List<String> maintainedIndexes;
+  private final Map<String, String> writerConfigDefaults;
+
+  private LsmWriteSpec(
+      Sharding sharding,
+      String column,
+      Integer numBuckets,
+      List<String> maintainedIndexes,
+      Map<String, String> writerConfigDefaults) {
+    this.sharding = sharding;
+    this.column = column;
+    this.numBuckets = numBuckets;
+    this.maintainedIndexes = maintainedIndexes;
+    this.writerConfigDefaults = writerConfigDefaults;
+  }
+
+  /**
+   * Hash-bucket sharding by a scalar column, maintaining every index on the table.
+   *
+   * <p>Iceberg-compatible Murmur3-x86-32 (seed 0) is used, so each row's {@code bucket(column,
+   * numBuckets)} value is stable across processes.
+   *
+   * @param column A non-nested column with a supported scalar type.
+   * @param numBuckets The number of buckets, in {@code [1, 1024]}.
+   */
+  public static LsmWriteSpec bucket(String column, int numBuckets) {
+    if (column == null || column.trim().isEmpty()) {
+      throw new IllegalArgumentException("Column cannot be null or empty");
+    }
+    return new LsmWriteSpec(
+        Sharding.BUCKET, column, numBuckets, null, new HashMap<String, String>());
+  }
+
+  /**
+   * Identity sharding — shard by the raw value of {@code column} — maintaining every index on the
+   * table.
+   *
+   * <p>{@code column} must be a deterministic function of the unenforced primary key: every row
+   * with a given primary key must always produce the same {@code column} value, or upserts of that
+   * key can land in different shards and a stale version can win.
+   */
+  public static LsmWriteSpec identity(String column) {
+    if (column == null || column.trim().isEmpty()) {
+      throw new IllegalArgumentException("Column cannot be null or empty");
+    }
+    return new LsmWriteSpec(Sharding.IDENTITY, column, null, null, new HashMap<String, String>());
+  }
+
+  /** No sharding — every write goes to a single MemWAL shard — maintaining every index. */
+  public static LsmWriteSpec unsharded() {
+    return new LsmWriteSpec(Sharding.UNSHARDED, null, null, null, new HashMap<String, String>());
+  }
+
+  /**
+   * Set the indexes the MemWAL keeps up to date as rows are appended.
+   *
+   * <p>Pass {@code null} — the default for a fresh spec — to maintain every index the MemWAL can,
+   * resolved when the spec is installed. That is a snapshot: indexes created later are not
+   * maintained until the spec is unset and set again. Pass an empty list to maintain none.
+   *
+   * <p>Note that {@code null} and the empty list mean opposite things here.
+   */
+  public LsmWriteSpec withMaintainedIndexes(List<String> maintainedIndexes) {
+    return new LsmWriteSpec(
+        sharding,
+        column,
+        numBuckets,
+        maintainedIndexes == null ? null : new ArrayList<String>(maintainedIndexes),
+        writerConfigDefaults);
+  }
+
+  /**
+   * Set default {@code ShardWriter} configuration recorded in the MemWAL index.
+   *
+   * <p>A sparse override map — only the keys you set are recorded. Recognized keys include {@code
+   * durable_write}, {@code max_wal_buffer_size}, {@code max_memtable_size}, {@code
+   * max_memtable_rows}, {@code max_memtable_batches}, {@code manifest_scan_batch_size}, {@code
+   * max_unflushed_memtable_bytes}, and {@code enable_memtable}. Duration knobs carry an {@code _ms}
+   * suffix, such as {@code max_wal_flush_interval_ms}.
+   */
+  public LsmWriteSpec withWriterConfigDefaults(Map<String, String> writerConfigDefaults) {
+    if (writerConfigDefaults == null) {
+      throw new IllegalArgumentException("writerConfigDefaults cannot be null");
+    }
+    return new LsmWriteSpec(
+        sharding,
+        column,
+        numBuckets,
+        maintainedIndexes,
+        new HashMap<String, String>(writerConfigDefaults));
+  }
+
+  /** How writes are routed to shards. */
+  public Sharding sharding() {
+    return sharding;
+  }
+
+  /** The sharding column for {@link Sharding#BUCKET} and {@link Sharding#IDENTITY}, else null. */
+  public String column() {
+    return column;
+  }
+
+  /** The bucket count for {@link Sharding#BUCKET}, else null. */
+  public Integer numBuckets() {
+    return numBuckets;
+  }
+
+  /**
+   * The indexes the MemWAL maintains, or null to have the server resolve every maintainable index
+   * on install. An empty list means none.
+   */
+  public List<String> maintainedIndexes() {
+    return maintainedIndexes == null ? null : Collections.unmodifiableList(maintainedIndexes);
+  }
+
+  /** Default {@code ShardWriter} configuration recorded in the MemWAL index. */
+  public Map<String, String> writerConfigDefaults() {
+    return Collections.unmodifiableMap(writerConfigDefaults);
+  }
+
+  /** Render this spec as the {@code set_lsm_write_spec} request body. */
+  Map<String, Object> toRequestBody() {
+    Map<String, Object> shardingBody = new LinkedHashMap<String, Object>();
+    shardingBody.put("mode", sharding.wireName());
+    if (column != null) {
+      shardingBody.put("column", column);
+    }
+    if (numBuckets != null) {
+      shardingBody.put("num_buckets", numBuckets);
+    }
+
+    Map<String, Object> body = new LinkedHashMap<String, Object>();
+    body.put("sharding", shardingBody);
+    // Null is meaningful: it asks the server to resolve every maintainable index.
+    body.put("maintained_indexes", maintainedIndexes);
+    body.put("writer_config_defaults", writerConfigDefaults);
+    return body;
+  }
+
+  /**
+   * Rebuild a spec from a {@code get_lsm_write_spec} response body.
+   *
+   * <p>The server always reports a concrete maintained-index list, so a null selection never
+   * round-trips.
+   */
+  static LsmWriteSpec fromJson(JsonNode node) {
+    JsonNode shardingNode = node.get("sharding");
+    if (shardingNode == null || shardingNode.get("mode") == null) {
+      throw new IllegalStateException("get_lsm_write_spec response has no sharding mode");
+    }
+    Sharding sharding = Sharding.fromWireName(shardingNode.get("mode").asText());
+
+    String column = shardingNode.hasNonNull("column") ? shardingNode.get("column").asText() : null;
+    Integer numBuckets =
+        shardingNode.hasNonNull("num_buckets") ? shardingNode.get("num_buckets").asInt() : null;
+
+    List<String> maintainedIndexes = new ArrayList<String>();
+    JsonNode indexesNode = node.get("maintained_indexes");
+    if (indexesNode != null && indexesNode.isArray()) {
+      for (JsonNode index : indexesNode) {
+        maintainedIndexes.add(index.asText());
+      }
+    }
+
+    Map<String, String> defaults = new HashMap<String, String>();
+    JsonNode defaultsNode = node.get("writer_config_defaults");
+    if (defaultsNode != null && defaultsNode.isObject()) {
+      defaultsNode
+          .fieldNames()
+          .forEachRemaining(name -> defaults.put(name, defaultsNode.get(name).asText()));
+    }
+
+    return new LsmWriteSpec(sharding, column, numBuckets, maintainedIndexes, defaults);
+  }
+
+  @Override
+  public String toString() {
+    return "LsmWriteSpec{sharding="
+        + sharding
+        + ", column="
+        + column
+        + ", numBuckets="
+        + numBuckets
+        + ", maintainedIndexes="
+        + maintainedIndexes
+        + ", writerConfigDefaults="
+        + writerConfigDefaults
+        + "}";
+  }
+}
diff --git a/java/lancedb-core/src/main/java/com/lancedb/MemtableStats.java b/java/lancedb-core/src/main/java/com/lancedb/MemtableStats.java
new file mode 100644
index 000000000..777e915aa
--- /dev/null
+++ b/java/lancedb-core/src/main/java/com/lancedb/MemtableStats.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.lancedb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/** One in-memory memtable. */
+public class MemtableStats {
+  private static final String CONTEXT = "memtable stats";
+
+  private final long generation;
+  private final long rows;
+  private final long bytes;
+  private final long batches;
+  private final List<String> indexes;
+
+  MemtableStats(long generation, long rows, long bytes, long batches, List<String> indexes) {
+    this.generation = generation;
+    this.rows = rows;
+    this.bytes = bytes;
+    this.batches = batches;
+    this.indexes = Collections.unmodifiableList(indexes);
+  }
+
+  /** The generation this memtable will become once sealed. */
+  public long generation() {
+    return generation;
+  }
+
+  /** Rows currently buffered. */
+  public long rows() {
+    return rows;
+  }
+
+  /** Estimated in-memory size. */
+  public long bytes() {
+    return bytes;
+  }
+
+  /** Record batches currently buffered. */
+  public long batches() {
+    return batches;
+  }
+
+  /**
+   * Names of the indexes this memtable carries. An absent name is the whole answer to "why is my
+   * fresh-tier search on that column brute-force".
+   */
+  public List<String> indexes() {
+    return indexes;
+  }
+
+  static MemtableStats fromJson(JsonNode node) {
+    JsonFields.requiredObject(node, CONTEXT);
+    List<String> indexes = new ArrayList<String>();
+    for (JsonNode index : JsonFields.requiredArray(node, "indexes", CONTEXT)) {
+      if (!index.isTextual()) {
+        throw new IllegalStateException(CONTEXT + " has a non-string index name: " + index);
+      }
+      indexes.add(index.asText());
+    }
+    return new MemtableStats(
+        JsonFields.requiredLong(node, "generation", CONTEXT),
+        JsonFields.requiredLong(node, "rows", CONTEXT),
+        JsonFields.requiredLong(node, "bytes", CONTEXT),
+        JsonFields.requiredLong(node, "batches", CONTEXT),
+        indexes);
+  }
+
+  @Override
+  public String toString() {
+    return "MemtableStats{generation="
+        + generation
+        + ", rows="
+        + rows
+        + ", bytes="
+        + bytes
+        + ", batches="
+        + batches
+        + ", indexes="
+        + indexes
+        + "}";
+  }
+}
diff --git a/java/lancedb-core/src/test/java/com/lancedb/LanceDbTableLsmTest.java b/java/lancedb-core/src/test/java/com/lancedb/LanceDbTableLsmTest.java
new file mode 100644
index 000000000..e84fa5421
--- /dev/null
+++ b/java/lancedb-core/src/test/java/com/lancedb/LanceDbTableLsmTest.java
@@ -0,0 +1,570 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.lancedb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.sun.net.httpserver.HttpServer;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * Unit tests for the MemWAL LSM routes, run against a scripted local HTTP server.
+ *
+ * <p>The wire assertions mirror the Rust mocked-endpoint tests in {@code
+ * rust/lancedb/src/remote/table.rs}, which are the contract these routes have to match.
+ */
+public class LanceDbTableLsmTest {
+  private static final ObjectMapper MAPPER = new ObjectMapper();
+
+  private HttpServer server;
+  private LanceDbRestClient client;
+  private LanceDbTableLsm lsm;
+
+  private final List<String> requestPaths = Collections.synchronizedList(new ArrayList<String>());
+  private final List<String> requestBodies = Collections.synchronizedList(new ArrayList<String>());
+  private final Map<String, Deque<Reply>> replies = new ConcurrentHashMap<String, Deque<Reply>>();
+
+  @BeforeEach
+  public void setUp() throws IOException {
+    start();
+  }
+
+  /** Tear down and restart the scripted server, for a test that scripts several exchanges. */
+  private void setUpFresh() {
+    try {
+      client.close();
+      server.stop(0);
+      requestPaths.clear();
+      requestBodies.clear();
+      replies.clear();
+      start();
+    } catch (IOException e) {
+      throw new UncheckedIOException(e);
+    }
+  }
+
+  private void start() throws IOException {
+    server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+    server.createContext(
+        "/",
+        exchange -> {
+          String path = exchange.getRequestURI().getPath();
+          requestPaths.add(path);
+          requestBodies.add(readAll(exchange.getRequestBody()));
+
+          Reply reply = nextReply(path);
+          byte[] out = reply.body.getBytes(StandardCharsets.UTF_8);
+          exchange.sendResponseHeaders(reply.status, out.length == 0 ? -1 : out.length);
+          if (out.length > 0) {
+            exchange.getResponseBody().write(out);
+          }
+          exchange.close();
+        });
+    server.start();
+
+    client =
+        LanceDbNamespaceClientBuilder.newBuilder()
+            .apiKey("test-key")
+            .database("test-db")
+            .endpoint("http://127.0.0.1:" + server.getAddress().getPort())
+            .buildRestClient();
+    lsm = new LanceDbTableLsm(client, "my_table");
+  }
+
+  @AfterEach
+  public void tearDown() throws IOException {
+    client.close();
+    server.stop(0);
+  }
+
+  // ===========================================================================
+  // set / unset / get spec
+  // ===========================================================================
+
+  @Test
+  public void testSetLsmWriteSpecUnsharded() throws Exception {
+    enqueue("set_lsm_write_spec", 200, "");
+
+    lsm.setLsmWriteSpec(LsmWriteSpec.unsharded());
+
+    assertEquals("/v1/table/my_table/set_lsm_write_spec/", requestPaths.get(0));
+    JsonNode body = MAPPER.readTree(requestBodies.get(0));
+    assertEquals("unsharded", body.get("sharding").get("mode").asText());
+    assertFalse(body.get("sharding").has("column"));
+    assertFalse(body.get("sharding").has("num_buckets"));
+  }
+
+  @Test
+  public void testSetLsmWriteSpecBucket() throws Exception {
+    enqueue("set_lsm_write_spec", 200, "");
+
+    lsm.setLsmWriteSpec(
+        LsmWriteSpec.bucket("id", 16).withMaintainedIndexes(Arrays.asList("id_idx")));
+
+    JsonNode body = MAPPER.readTree(requestBodies.get(0));
+    assertEquals("bucket", body.get("sharding").get("mode").asText());
+    assertEquals("id", body.get("sharding").get("column").asText());
+    assertEquals(16, body.get("sharding").get("num_buckets").asInt());
+    assertEquals(1, body.get("maintained_indexes").size());
+    assertEquals("id_idx", body.get("maintained_indexes").get(0).asText());
+  }
+
+  @Test
+  public void testSetLsmWriteSpecIdentity() throws Exception {
+    enqueue("set_lsm_write_spec", 200, "");
+
+    lsm.setLsmWriteSpec(LsmWriteSpec.identity("tenant"));
+
+    JsonNode body = MAPPER.readTree(requestBodies.get(0));
+    assertEquals("identity", body.get("sharding").get("mode").asText());
+    assertEquals("tenant", body.get("sharding").get("column").asText());
+    assertFalse(body.get("sharding").has("num_buckets"));
+  }
+
+  /**
+   * The tri-state that motivated a LanceDB-owned spec type: a null selection asks the server to
+   * resolve every maintainable index, while an empty list asks for none. They must not collapse.
+   */
+  @Test
+  public void testMaintainedIndexesNullAndEmptyAreDistinctOnTheWire() throws Exception {
+    enqueue("set_lsm_write_spec", 200, "");
+
+    lsm.setLsmWriteSpec(LsmWriteSpec.unsharded());
+    JsonNode fresh = MAPPER.readTree(requestBodies.get(0));
+    assertTrue(fresh.has("maintained_indexes"), "the key must be present");
+    assertTrue(fresh.get("maintained_indexes").isNull(), "a fresh spec sends null, not []");
+
+    lsm.setLsmWriteSpec(
+        LsmWriteSpec.unsharded().withMaintainedIndexes(Collections.<String>emptyList()));
+    JsonNode none = MAPPER.readTree(requestBodies.get(1));
+    assertTrue(none.get("maintained_indexes").isArray());
+    assertEquals(0, none.get("maintained_indexes").size());
+  }
+
+  @Test
+  public void testSetLsmWriteSpecWriterConfigDefaults() throws Exception {
+    enqueue("set_lsm_write_spec", 200, "");
+
+    Map<String, String> defaults = new HashMap<String, String>();
+    defaults.put("max_memtable_rows", "50000");
+    lsm.setLsmWriteSpec(LsmWriteSpec.unsharded().withWriterConfigDefaults(defaults));
+
+    JsonNode body = MAPPER.readTree(requestBodies.get(0));
+    assertEquals("50000", body.get("writer_config_defaults").get("max_memtable_rows").asText());
+  }
+
+  @Test
+  public void testUnsetLsmWriteSpec() {
+    enqueue("unset_lsm_write_spec", 200, "");
+
+    lsm.unsetLsmWriteSpec();
+
+    assertEquals("/v1/table/my_table/unset_lsm_write_spec/", requestPaths.get(0));
+    assertEquals("", requestBodies.get(0));
+  }
+
+  @Test
+  public void testGetLsmWriteSpec() {
+    enqueue(
+        "get_lsm_write_spec",
+        200,
+        "{\"lsm_write_spec\":{\"sharding\":{\"mode\":\"bucket\",\"column\":\"id\","
+            + "\"num_buckets\":16},\"maintained_indexes\":[\"id_idx\"],"
+            + "\"writer_config_defaults\":{\"durable_write\":\"true\"}}}");
+
+    Optional<LsmWriteSpec> spec = lsm.getLsmWriteSpec();
+
+    assertTrue(spec.isPresent());
+    assertEquals(LsmWriteSpec.Sharding.BUCKET, spec.get().sharding());
+    assertEquals("id", spec.get().column());
+    assertEquals(Integer.valueOf(16), spec.get().numBuckets());
+    assertEquals(Arrays.asList("id_idx"), spec.get().maintainedIndexes());
+    assertEquals("true", spec.get().writerConfigDefaults().get("durable_write"));
+  }
+
+  @Test
+  public void testGetLsmWriteSpecAbsent() {
+    enqueue("get_lsm_write_spec", 200, "{\"lsm_write_spec\":null}");
+
+    assertFalse(lsm.getLsmWriteSpec().isPresent());
+  }
+
+  // ===========================================================================
+  // stats
+  // ===========================================================================
+
+  @Test
+  public void testGetLsmStats() throws Exception {
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 7L, 8L)));
+
+    Optional<LsmStats> got = lsm.getLsmStats(true);
+
+    assertEquals("/v1/table/my_table/get_lsm_stats/", requestPaths.get(0));
+    assertTrue(MAPPER.readTree(requestBodies.get(0)).get("include_generation_rows").asBoolean());
+    assertTrue(got.isPresent());
+    BucketStats decoded = got.get().buckets().get(0);
+    assertEquals("shard-0", decoded.shardId());
+    assertEquals("Active", decoded.status());
+    assertEquals(1, decoded.writerEpoch());
+    assertEquals(2, decoded.manifestVersion());
+    assertEquals(9, decoded.currentGeneration());
+    assertFalse(decoded.compacting());
+    assertEquals(Arrays.asList(7L, 8L), generationNumbers(decoded));
+    assertEquals(1024, decoded.generations().get(0).bytes());
+    assertFalse(decoded.generations().get(0).rows().isPresent(), "rows absent unless requested");
+    assertFalse(decoded.memtables().isPresent(), "absent memtables stay absent");
+  }
+
+  /** The optional fields decode when the server does send them. */
+  @Test
+  public void testGetLsmStatsDecodesOptionalFields() {
+    enqueue(
+        "get_lsm_stats",
+        200,
+        "{\"lsm_stats\":{\"buckets\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
+            + "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9,"
+            + "\"replay_after_wal_entry_position\":3,\"wal_entry_position_last_seen\":11,"
+            + "\"generations\":[{\"generation\":7,\"bytes\":1024,\"rows\":42}],"
+            + "\"compacting\":true,\"memtables\":[{\"generation\":8,\"rows\":5,"
+            + "\"bytes\":64,\"batches\":2,\"indexes\":[\"id_idx\"]}]}]}}");
+
+    BucketStats decoded = lsm.getLsmStats(true).get().buckets().get(0);
+
+    assertEquals(3, decoded.replayAfterWalEntryPosition());
+    assertEquals(11, decoded.walEntryPositionLastSeen());
+    assertTrue(decoded.compacting());
+    assertEquals(42, decoded.generations().get(0).rows().getAsLong());
+    assertTrue(decoded.memtables().isPresent());
+    MemtableStats memtable = decoded.memtables().get().get(0);
+    assertEquals(8, memtable.generation());
+    assertEquals(5, memtable.rows());
+    assertEquals(64, memtable.bytes());
+    assertEquals(2, memtable.batches());
+    assertEquals(Arrays.asList("id_idx"), memtable.indexes());
+  }
+
+  @Test
+  public void testGetLsmStatsAbsentWhenLsmDisabled() {
+    enqueue("get_lsm_stats", 200, "{\"lsm_stats\":null}");
+
+    assertFalse(lsm.getLsmStats().isPresent());
+  }
+
+  @Test
+  public void testGetLsmStatsDefaultsToExcludingGenerationRows() throws Exception {
+    enqueue("get_lsm_stats", 200, stats());
+
+    lsm.getLsmStats();
+
+    assertFalse(MAPPER.readTree(requestBodies.get(0)).get("include_generation_rows").asBoolean());
+  }
+
+  // ===========================================================================
+  // flush / compact
+  // ===========================================================================
+
+  @Test
+  public void testFlushAndCompactRoutes() {
+    enqueue("flush_lsm", 200, "");
+    enqueue("compact_lsm", 200, "");
+
+    lsm.flushLsm();
+    lsm.compactLsm();
+
+    assertEquals("/v1/table/my_table/flush_lsm/", requestPaths.get(0));
+    assertEquals("/v1/table/my_table/compact_lsm/", requestPaths.get(1));
+  }
+
+  @Test
+  public void testHttpErrorCarriesStatus() {
+    enqueue("flush_lsm", 404, "no such table");
+
+    LanceDbRestClient.HttpException e =
+        assertThrows(LanceDbRestClient.HttpException.class, () -> lsm.flushLsm());
+    assertEquals(404, e.statusCode());
+  }
+
+  // ===========================================================================
+  // checkpoint
+  // ===========================================================================
+
+  @Test
+  public void testCheckpointReturnsWhenLsmDisabled() {
+    enqueue("flush_lsm", 200, "");
+    enqueue("get_lsm_stats", 200, "{\"lsm_stats\":null}");
+
+    lsm.checkpointLsm();
+
+    assertEquals(0, countCalls("compact_lsm"), "nothing to compact when the LSM path is off");
+  }
+
+  @Test
+  public void testCheckpointReturnsWhenNoGenerationsOutstanding() {
+    enqueue("flush_lsm", 200, "");
+    // A bucket with no L0 generations yields no target, so the drain never starts.
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false)));
+
+    lsm.checkpointLsm();
+
+    assertEquals(0, countCalls("compact_lsm"));
+  }
+
+  @Test
+  public void testCheckpointConvergesOnceTargetGenerationsAreGone() {
+    enqueue("flush_lsm", 200, "");
+    // Watermark read: shard-0 holds generations 7 and 8, so target = 8.
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 7L, 8L)));
+    // First drain poll: both still outstanding, nothing compacting -> dispatch a pass.
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 7L, 8L)));
+    // Second drain poll: drained past the target -> done.
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 9L)));
+    enqueue("compact_lsm", 200, "");
+
+    lsm.checkpointLsm();
+
+    assertEquals(1, countCalls("compact_lsm"), "one pass dispatched");
+    assertEquals(3, countCalls("get_lsm_stats"), "watermark read plus two drain polls");
+  }
+
+  @Test
+  public void testCheckpointDoesNotPileOnWhileEveryTargetBucketIsCompacting() {
+    enqueue("flush_lsm", 200, "");
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", true, 4L)));
+    // Still compacting on the first poll, so no pass is dispatched; then it drains.
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", true, 4L)));
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 5L)));
+
+    lsm.checkpointLsm();
+
+    assertEquals(0, countCalls("compact_lsm"), "a latched bucket is left alone");
+  }
+
+  @Test
+  public void testCheckpointRetriesFromFlushAfterLostClaim() {
+    // 421 on the watermark read: the node lost its claim, so the whole thing restarts
+    // from flush rather than retrying the read in place.
+    enqueue("flush_lsm", 200, "");
+    enqueue("get_lsm_stats", 421, "no claim");
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false)));
+
+    lsm.checkpointLsm();
+
+    assertEquals(2, countCalls("flush_lsm"), "re-issued from flush");
+  }
+
+  @Test
+  public void testCheckpointRetriesRetryableStatusInPlace() {
+    enqueue("flush_lsm", 429, "latch held");
+    enqueue("flush_lsm", 200, "");
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false)));
+
+    lsm.checkpointLsm();
+
+    assertEquals(2, countCalls("flush_lsm"), "429 retried in place, not re-issued");
+  }
+
+  @Test
+  public void testCheckpointPropagatesTerminalStatus() {
+    enqueue("flush_lsm", 400, "bad request");
+
+    LanceDbRestClient.HttpException e =
+        assertThrows(LanceDbRestClient.HttpException.class, () -> lsm.checkpointLsm());
+    assertEquals(400, e.statusCode());
+    assertEquals(1, countCalls("flush_lsm"), "a terminal status is not retried");
+  }
+
+  @Test
+  public void testCheckpointGivesUpAfterRepeatedLostClaims() {
+    enqueue("flush_lsm", 421, "no claim");
+
+    IllegalStateException e = assertThrows(IllegalStateException.class, () -> lsm.checkpointLsm());
+    assertTrue(e.getMessage().contains("kept losing its claim"), e.getMessage());
+    assertEquals(4, countCalls("flush_lsm"), "the initial attempt plus MAX_REISSUES");
+  }
+
+  // ===========================================================================
+  // strict decoding
+  // ===========================================================================
+
+  /**
+   * A stats payload that does not decode must fail closed. Every one of these bodies used to be
+   * read as "no buckets", which is indistinguishable from a drained table, so {@code checkpointLsm}
+   * reported convergence for a checkpoint that never ran.
+   */
+  @Test
+  public void testCheckpointRejectsMalformedStats() {
+    Map<String, String> malformed = new LinkedHashMap<String, String>();
+    malformed.put("no response body at all", "");
+    malformed.put("stats object with no buckets", "{\"lsm_stats\":{}}");
+    malformed.put("bucket missing its required fields", "{\"lsm_stats\":{\"buckets\":[{}]}}");
+    malformed.put(
+        "bucket missing generations",
+        "{\"lsm_stats\":{\"buckets\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
+            + "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9,"
+            + "\"replay_after_wal_entry_position\":0,\"wal_entry_position_last_seen\":0,"
+            + "\"compacting\":false}]}}");
+    malformed.put(
+        "generation with a non-numeric generation number",
+        "{\"lsm_stats\":{\"buckets\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
+            + "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9,"
+            + "\"replay_after_wal_entry_position\":0,\"wal_entry_position_last_seen\":0,"
+            + "\"generations\":[{\"generation\":\"7\",\"bytes\":1024}],"
+            + "\"compacting\":false}]}}");
+
+    for (Map.Entry<String, String> each : malformed.entrySet()) {
+      setUpFresh();
+      enqueue("flush_lsm", 200, "");
+      enqueue("get_lsm_stats", 200, each.getValue());
+
+      assertThrows(
+          IllegalStateException.class,
+          () -> lsm.checkpointLsm(),
+          each.getKey() + " must not report convergence");
+    }
+  }
+
+  /** The one shape that legitimately means "this table has no LSM write path". */
+  @Test
+  public void testCheckpointTreatsNullStatsAsNotWalBacked() {
+    enqueue("flush_lsm", 200, "");
+    enqueue("get_lsm_stats", 200, "{\"lsm_stats\":null}");
+
+    lsm.checkpointLsm();
+
+    assertEquals(1, countCalls("get_lsm_stats"));
+  }
+
+  // ===========================================================================
+  // retry budget
+  // ===========================================================================
+
+  /**
+   * The transport must not retry on the checkpoint loop's behalf. Apache HttpClient's default
+   * strategy retries exactly 429 and 503 — the two statuses {@code isRetryable} owns — which
+   * doubled every budget here and also retried {@code compact_lsm} in place, where the loop is
+   * built to fall through to a fresh stats poll instead.
+   */
+  @Test
+  public void testCheckpointRetryBudgetIsNotDoubledByTheTransport() {
+    enqueue("flush_lsm", 429, "latch held");
+
+    LanceDbRestClient.HttpException e =
+        assertThrows(LanceDbRestClient.HttpException.class, () -> lsm.checkpointLsm());
+
+    assertEquals(429, e.statusCode(), "the exhausted budget propagates the last error as itself");
+    assertEquals(9, countCalls("flush_lsm"), "the initial request plus MAX_RETRIES, and no more");
+  }
+
+  // ===========================================================================
+  // harness
+  // ===========================================================================
+
+  private static List<Long> generationNumbers(BucketStats bucket) {
+    List<Long> numbers = new ArrayList<Long>();
+    for (GenerationStats generation : bucket.generations()) {
+      numbers.add(generation.generation());
+    }
+    return numbers;
+  }
+
+  /** Build an {@code lsm_stats} response body from bucket fragments. */
+  private static String stats(String... buckets) {
+    return "{\"lsm_stats\":{\"buckets\":[" + String.join(",", buckets) + "]}}";
+  }
+
+  private static String bucket(String shardId, boolean compacting, Long... generations) {
+    StringBuilder gens = new StringBuilder();
+    for (Long generation : generations) {
+      if (gens.length() > 0) {
+        gens.append(",");
+      }
+      gens.append("{\"generation\":").append(generation).append(",\"bytes\":1024}");
+    }
+    return "{\"shard_id\":\""
+        + shardId
+        + "\",\"status\":\"Active\",\"writer_epoch\":1,\"manifest_version\":2,"
+        + "\"current_generation\":9,\"replay_after_wal_entry_position\":0,"
+        + "\"wal_entry_position_last_seen\":0,\"generations\":["
+        + gens
+        + "],\"compacting\":"
+        + compacting
+        + "}";
+  }
+
+  /** Queue a reply for an operation. The last queued reply repeats once the queue drains. */
+  private void enqueue(String operation, int status, String body) {
+    replies.computeIfAbsent(operation, key -> new ArrayDeque<Reply>()).add(new Reply(status, body));
+  }
+
+  private Reply nextReply(String path) {
+    String operation = operationOf(path);
+    Deque<Reply> queued = replies.get(operation);
+    if (queued == null || queued.isEmpty()) {
+      return new Reply(200, "");
+    }
+    return queued.size() > 1 ? queued.poll() : queued.peek();
+  }
+
+  private long countCalls(String operation) {
+    return requestPaths.stream().filter(path -> operationOf(path).equals(operation)).count();
+  }
+
+  /** {@code /v1/table/my_table/flush_lsm/} -> {@code flush_lsm}. */
+  private static String operationOf(String path) {
+    String[] segments = path.split("/");
+    return segments.length == 0 ? "" : segments[segments.length - 1];
+  }
+
+  private static String readAll(InputStream in) throws IOException {
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    byte[] buffer = new byte[4096];
+    int read;
+    while ((read = in.read(buffer)) != -1) {
+      out.write(buffer, 0, read);
+    }
+    return new String(out.toByteArray(), StandardCharsets.UTF_8);
+  }
+
+  private static final class Reply {
+    private final int status;
+    private final String body;
+
+    private Reply(int status, String body) {
+      this.status = status;
+      this.body = body;
+    }
+  }
+}
diff --git a/java/pom.xml b/java/pom.xml
index 1b96cbcab..4d83153ab 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.37.1-beta.0</version>
+    <version>0.38.0-beta.10</version>
     <packaging>pom</packaging>
     <name>${project.artifactId}</name>
     <description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>10.1.0-beta.1</lance-core.version>
+        <lance-core.version>11.0.0-beta.22</lance-core.version>
         <spotless.skip>false</spotless.skip>
         <spotless.version>2.30.0</spotless.version>
         <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml
index 10e12edc8..fd08e7a5e 100644
--- a/nodejs/Cargo.toml
+++ b/nodejs/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.37.1-beta.0"
+version = "0.38.0-beta.10"
 publish = false
 license.workspace = true
 description.workspace = true
@@ -16,12 +16,12 @@ crate-type = ["cdylib"]
 async-trait.workspace = true
 arrow-ipc.workspace = true
 arrow-array.workspace = true
-arrow-buffer = "58.0.0"
+arrow-buffer.workspace = true
 half.workspace = true
 arrow-schema.workspace = true
 env_logger.workspace = true
 futures.workspace = true
-lancedb = { path = "../rust/lancedb", default-features = false }
+lancedb.workspace = true
 lance-namespace.workspace = true
 napi = { version = "3.8.3", default-features = false, features = [
     "napi9",
@@ -29,8 +29,8 @@ napi = { version = "3.8.3", default-features = false, features = [
     "chrono_date",
     "serde-json",
 ] }
-chrono = { version = "0.4", default-features = false, features = ["clock"] }
-serde_json = "1"
+chrono.workspace = true
+serde_json.workspace = true
 napi-derive = "3.5.2"
 # Prevent dynamic linking of lzma, which comes from datafusion
 lzma-sys = { version = "0.1", features = ["static"] }
diff --git a/nodejs/__test__/arrow.test.ts b/nodejs/__test__/arrow.test.ts
index 9e20e3c04..c5bbbf169 100644
--- a/nodejs/__test__/arrow.test.ts
+++ b/nodejs/__test__/arrow.test.ts
@@ -6,7 +6,9 @@ import * as arrow17 from "apache-arrow-17";
 import * as arrow18 from "apache-arrow-18";
 
 import {
+  Vector as CurrentVector,
   convertToTable,
+  tableFromIPC as currentTableFromIPC,
   fromBufferToRecordBatch,
   fromDataToBuffer,
   fromRecordBatchToBuffer,
@@ -19,6 +21,7 @@ import {
   FunctionOptions,
 } from "../lancedb/embedding/embedding_function";
 import { EmbeddingFunctionConfig } from "../lancedb/embedding/registry";
+import { sanitizeTable } from "../lancedb/sanitize";
 
 // biome-ignore lint/suspicious/noExplicitAny: skip
 function sampleRecords(): Array<Record<string, any>> {
@@ -64,7 +67,11 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       tableFromIPC,
       DataType,
       Dictionary,
+      RecordBatch: ArrowRecordBatch,
+      Table: ArrowTable,
       Uint8: ArrowUint8,
+      makeData: arrowMakeData,
+      vectorFromArray,
       // biome-ignore lint/suspicious/noExplicitAny: <explanation>
     } = <any>arrow;
     type Schema = ApacheArrow["Schema"];
@@ -166,6 +173,36 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
     }
 
     describe("The function makeArrowTable", function () {
+      it("accepts snake_case embedding metadata like camelCase", function () {
+        const spellings = [
+          // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+          { source_column: "text", vector_column: "vector" },
+          { sourceColumn: "text", vectorColumn: "vector" },
+        ];
+        for (const columns of spellings) {
+          const schema = new Schema(
+            [
+              new Field("text", new Utf8(), false),
+              new Field(
+                "vector",
+                new FixedSizeList(3, new Field("item", new Float32(), true)),
+                false,
+              ),
+            ],
+            new Map([
+              [
+                "embedding_functions",
+                JSON.stringify([{ name: "mock", model: {}, ...columns }]),
+              ],
+            ]),
+          );
+          // The vector field is non-nullable and absent from the data; only a
+          // recognized embedding config makes that acceptable.
+          const table = makeArrowTable([{ text: "hello" }], { schema });
+          expect(table.numRows).toBe(1);
+        }
+      });
+
       it("will use data types from a provided schema instead of inference", async function () {
         const schema = new Schema([
           new Field("a", new Int32(), false),
@@ -197,6 +234,35 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
         expect(table.getChild("d")?.toJSON()).toEqual([9n, 10n, null]);
       });
 
+      it("will use a provided FixedSizeList schema with typed array values", function () {
+        const schema = new Schema([
+          new Field("text", new Utf8(), false),
+          new Field(
+            "vector",
+            new FixedSizeList(3, new Field("item", new Float32(), false)),
+            false,
+          ),
+        ]);
+
+        const table = makeArrowTable(
+          [
+            {
+              text: "foo",
+              vector: new Float32Array([1, 2, 3]),
+            },
+          ],
+          { schema },
+        );
+
+        expect(table.getChild("text")?.toJSON()).toEqual(["foo"]);
+        expect(
+          table
+            .getChild("vector")
+            ?.toJSON()
+            .map((value) => value.toJSON()),
+        ).toEqual([[1, 2, 3]]);
+      });
+
       it("will assume the column `vector` is FixedSizeList<Float32> by default", async function () {
         const schema = new Schema([
           new Field("a", new Float(Precision.DOUBLE), true),
@@ -449,6 +515,137 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
         );
       });
 
+      it("will allow matching inferred types across records", function () {
+        expect(() =>
+          makeArrowTable([{ value: 1 }, { value: 2 }]),
+        ).not.toThrow();
+      });
+
+      it("will reject mismatched inferred types across records", function () {
+        expect(() => makeArrowTable([{ value: 1 }, { value: "two" }])).toThrow(
+          "Failed to infer schema for data. Previously inferred type Float64 but found Utf8 for field value at row 1. Consider providing an explicit schema.",
+        );
+      });
+
+      it("will ignore generated dictionary IDs when comparing inferred types", function () {
+        const table = makeArrowTable([{ str: "a" }, { str: "b" }], {
+          dictionaryEncodeStrings: true,
+        });
+
+        expect(table.getChild("str")?.toJSON()).toEqual(["a", "b"]);
+      });
+
+      it("will preserve null values without treating them as type mismatches", function () {
+        for (const records of [
+          [{ vector: [1, 2, 3] }, { vector: null }],
+          [{ vector: null }, { vector: [1, 2, 3] }],
+        ]) {
+          const table = makeArrowTable(records);
+
+          expect(table.numRows).toBe(2);
+          expect(table.getChild("vector")?.nullCount).toBe(1);
+        }
+      });
+
+      it("will preserve empty variable-size lists", function () {
+        for (const records of [
+          [{ items: [1] }, { items: [] }],
+          [{ items: [] }, { items: [1] }],
+        ]) {
+          const table = makeArrowTable(records);
+          expect(
+            table
+              .getChild("items")
+              ?.toJSON()
+              .map((value) => value.toJSON()),
+          ).toEqual(records.map((record) => record.items));
+        }
+      });
+
+      it("will propagate deferred evidence through nested lists", function () {
+        for (const records of [
+          [{ items: [1] }, { items: [null] }],
+          [{ items: [null] }, { items: [1] }],
+          [{ items: [null, 1] }, { items: [2, null] }],
+        ]) {
+          const table = makeArrowTable(records);
+          expect(
+            table
+              .getChild("items")
+              ?.toJSON()
+              .map((value) => value.toJSON()),
+          ).toEqual(records.map((record) => record.items));
+        }
+
+        const nestedRecords = [{ items: [[1]] }, { items: [[null]] }];
+        const nestedTable = makeArrowTable(nestedRecords);
+        expect(
+          nestedTable
+            .getChild("items")
+            ?.toJSON()
+            .map((value) =>
+              value
+                .toJSON()
+                .map((nestedValue: { toJSON: () => unknown[] }) =>
+                  nestedValue.toJSON(),
+                ),
+            ),
+        ).toEqual(nestedRecords.map((record) => record.items));
+      });
+
+      it("will reject incompatible deferred evidence within a list", function () {
+        for (const items of [
+          [[], 1],
+          [1, []],
+          [[null], 1],
+          [1, [null]],
+        ]) {
+          expect(() => makeArrowTable([{ items }])).toThrow(
+            "Failed to infer data type for field items at row 0.",
+          );
+        }
+      });
+
+      it("will reject empty fixed-size lists", function () {
+        expect(() =>
+          makeArrowTable([{ vector: [1, 2, 3] }, { vector: [] }]),
+        ).toThrow(
+          "Failed to infer schema for data. Previously inferred type FixedSizeList[3]<Float32> but found List[0] for field vector at row 1.",
+        );
+      });
+
+      it("will reject inferred leaf and branch shape changes", function () {
+        expect(() =>
+          makeArrowTable([{ value: 1 }, { value: { nested: 2 } }]),
+        ).toThrow(
+          "Failed to infer schema for data. Previously inferred type Float64 but found Struct for field value at row 1.",
+        );
+        expect(() =>
+          makeArrowTable([{ value: { nested: 1 } }, { value: 2 }]),
+        ).toThrow(
+          "Failed to infer schema for data. Previously inferred type Struct but found Float64 for field value at row 1.",
+        );
+      });
+
+      it("will allow null values around inferred struct values", function () {
+        for (const { records, nullIndex } of [
+          {
+            records: [{ value: null }, { value: { nested: 2 } }],
+            nullIndex: 0,
+          },
+          {
+            records: [{ value: { nested: 1 } }, { value: null }],
+            nullIndex: 1,
+          },
+        ]) {
+          const table = makeArrowTable(records);
+          const values = table.getChild("value");
+
+          expect(values?.nullCount).toBe(1);
+          expect(values?.get(nullIndex)).toBeNull();
+        }
+      });
+
       it("will allow a schema to be provided", async function () {
         await checkTableCreation(
           async (records, _, schema) =>
@@ -1025,6 +1222,114 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
     });
 
     describe("when using two versions of arrow", function () {
+      it("preserves a dictionary shared by multiple fields", async function () {
+        const values = ["alpha", "beta", "alpha"];
+        const dictionaryVector = vectorFromArray(values);
+        const batch = new ArrowRecordBatch({
+          first: dictionaryVector.data[0],
+          second: dictionaryVector.data[0],
+        });
+        const table = new ArrowTable([batch]);
+
+        const sanitized = sanitizeTable(table);
+        expect([...sanitized.getChild("first")!]).toEqual(values);
+        expect([...sanitized.getChild("second")!]).toEqual(values);
+        const firstType = sanitized.schema.fields[0].type as {
+          dictionary: unknown;
+        };
+        const secondType = sanitized.schema.fields[1].type as {
+          dictionary: unknown;
+        };
+        expect(secondType.dictionary).toBe(firstType.dictionary);
+        expect(sanitized.batches[0].data.children[1].dictionary).toBe(
+          sanitized.batches[0].data.children[0].dictionary,
+        );
+
+        const buf = await fromDataToBuffer(table);
+        const actual = currentTableFromIPC(buf);
+        expect([...actual.getChild("first")!]).toEqual(values);
+        expect([...actual.getChild("second")!]).toEqual(values);
+      });
+
+      it("preserves shared dictionary data from another Arrow version", async function () {
+        const values = ["alpha", "beta", "alpha"];
+        const dictionaryVector = vectorFromArray(values);
+        const firstBatch = new ArrowRecordBatch({
+          label: dictionaryVector.slice(0, 2).data[0],
+        });
+        const secondBatch = new ArrowRecordBatch({
+          label: dictionaryVector.slice(2).data[0],
+        });
+        const table = new ArrowTable([firstBatch, secondBatch]);
+
+        const sanitized = sanitizeTable(table);
+        expect([...sanitized.getChild("label")!]).toEqual(values);
+
+        const dictionaries = sanitized.batches.map(
+          (batch) => batch.data.children[0].dictionary,
+        );
+        expect(dictionaries[0]).toBeInstanceOf(CurrentVector);
+        expect(dictionaries[1]).toBe(dictionaries[0]);
+
+        const buf = await fromDataToBuffer(table);
+        const actual = currentTableFromIPC(buf);
+        expect([...actual.getChild("label")!]).toEqual(values);
+      });
+
+      it("preserves shared chunks in growing dictionaries", async function () {
+        const type = new Dictionary(new Utf8(), new Int32(), 42, false);
+        const firstDictionary = vectorFromArray(["alpha", "beta"], new Utf8());
+        const secondDictionary = firstDictionary.concat(
+          vectorFromArray(["gamma"], new Utf8()),
+        );
+        const firstData = arrowMakeData({
+          type,
+          data: Int32Array.from([0, 1]),
+          dictionary: firstDictionary,
+        });
+        const secondData = arrowMakeData({
+          type,
+          data: Int32Array.from([2]),
+          dictionary: secondDictionary,
+        });
+        const table = new ArrowTable([
+          new ArrowRecordBatch({ label: firstData }),
+          new ArrowRecordBatch({ label: secondData }),
+        ]);
+
+        const sanitized = sanitizeTable(table);
+        const expected = ["alpha", "beta", "gamma"];
+        expect([...sanitized.getChild("label")!]).toEqual(expected);
+        const firstLocalDictionary =
+          sanitized.batches[0].data.children[0].dictionary!;
+        const secondLocalDictionary =
+          sanitized.batches[1].data.children[0].dictionary!;
+        expect(secondLocalDictionary.data[0]).toBe(
+          firstLocalDictionary.data[0],
+        );
+
+        const buf = await fromTableToBuffer(sanitized);
+        const actual = currentTableFromIPC(buf);
+        expect([...actual.getChild("label")!]).toEqual(expected);
+      });
+
+      it("can serialize list data from another Arrow version", async function () {
+        const values = [["anime", "action"], [], null];
+        const vector = vectorFromArray(
+          values,
+          new List(new Field("item", new Utf8(), true)),
+        );
+        const table = new ArrowTable({ tags: vector });
+
+        const buf = await fromDataToBuffer(table);
+        const actual = currentTableFromIPC(buf);
+        const actualTags = actual.getChild("tags");
+
+        expect(actualTags?.get(0)?.toJSON()).toEqual(values[0]);
+        expect(actualTags?.get(1)?.toJSON()).toEqual(values[1]);
+        expect(actualTags?.get(2)).toBeNull();
+      });
+
       it("can still import data", async function () {
         const schema = new arrow15.Schema([
           new arrow15.Field("id", new arrow15.Int32()),
diff --git a/nodejs/__test__/connection.test.ts b/nodejs/__test__/connection.test.ts
index 68180471a..816f8c3de 100644
--- a/nodejs/__test__/connection.test.ts
+++ b/nodejs/__test__/connection.test.ts
@@ -4,7 +4,13 @@
 import { readdirSync } from "fs";
 import { Field, Float64, Schema } from "apache-arrow";
 import * as tmp from "tmp";
-import { Connection, Table, connect, connectNamespace } from "../lancedb";
+import {
+  Connection,
+  ListTablesResponse,
+  Table,
+  connect,
+  connectNamespace,
+} from "../lancedb";
 import { LocalTable } from "../lancedb/table";
 
 describe("when connecting", () => {
@@ -47,6 +53,7 @@ describe("given a connection", () => {
     await db.close();
     expect(db.isOpen()).toBe(false);
     await expect(db.tableNames()).rejects.toThrow("Connection is closed");
+    await expect(db.listTables()).rejects.toThrow("Connection is closed");
     await expect(db.renameTable("a", "b")).rejects.toThrow(
       "Connection is closed",
     );
@@ -89,6 +96,16 @@ describe("given a connection", () => {
     await db.createTable("test4", [{ id: 1 }, { id: 2 }]);
   });
 
+  it("should return a completed job when dropping a local table", async () => {
+    await db.createTable("async-drop", [{ id: 1 }]);
+
+    const job = await db.dropTableAsync("async-drop");
+    expect(job.id).toBeNull();
+    await expect(job.status()).resolves.toBe("finished");
+    await job.wait();
+    await expect(db.tableNames()).resolves.toEqual([]);
+  });
+
   it("should fail if creating table twice, unless overwrite is true", async () => {
     let tbl = await db.createTable("test", [{ id: 1 }, { id: 2 }]);
     await expect(tbl.countRows()).resolves.toBe(2);
@@ -119,6 +136,66 @@ describe("given a connection", () => {
     expect(tables).toEqual(["b", "c"]);
   });
 
+  it("should respect limit and page token when listing tables", async () => {
+    const db = await connect(tmpDir.name);
+
+    await db.createTable("b", [{ id: 1 }]);
+    await db.createTable("a", [{ id: 1 }]);
+    await db.createTable("c", [{ id: 1 }]);
+
+    const all = await db.listTables();
+    expect(all.tables).toEqual(["a", "b", "c"]);
+    expect(all.pageToken).toBeUndefined();
+
+    const first = await db.listTables({ limit: 1 });
+    expect(first.tables).toEqual(["a"]);
+    expect(first.pageToken).toBeDefined();
+
+    const second = await db.listTables({
+      limit: 1,
+      pageToken: first.pageToken,
+    });
+    expect(second.tables).toEqual(["b"]);
+  });
+
+  it("should visit every table exactly once when walking pages", async () => {
+    const db = await connect(tmpDir.name);
+
+    const created = ["a", "b", "c", "d", "e"];
+    for (const name of created) {
+      await db.createTable(name, [{ id: 1 }]);
+    }
+
+    const seen: string[] = [];
+    let pageToken: string | undefined = undefined;
+    do {
+      const page: ListTablesResponse = await db.listTables({
+        limit: 2,
+        pageToken,
+      });
+      seen.push(...page.tables);
+      pageToken = page.pageToken;
+    } while (pageToken);
+
+    expect(seen).toEqual(created);
+  });
+
+  it("should list tables in a namespace", async () => {
+    const db = await connect(tmpDir.name, {
+      // biome-ignore lint/style/useNamingConvention: opaque backend property key, must match Rust
+      namespaceClientProperties: { manifest_enabled: "true" },
+    });
+    await db.createNamespace(["child"]);
+    await db.createTable("nested", [{ id: 1 }], ["child"]);
+
+    await expect(db.listTables(["child"])).resolves.toEqual(
+      expect.objectContaining({ tables: ["nested"] }),
+    );
+    await expect(db.listTables()).resolves.toEqual(
+      expect.objectContaining({ tables: [] }),
+    );
+  });
+
   it("should create tables in v2 mode", async () => {
     const db = await connect(tmpDir.name);
     const data = [...Array(10000).keys()].map((i) => ({ id: i }));
diff --git a/nodejs/__test__/embedding.test.ts b/nodejs/__test__/embedding.test.ts
index e56e80631..2a8494e0f 100644
--- a/nodejs/__test__/embedding.test.ts
+++ b/nodejs/__test__/embedding.test.ts
@@ -11,8 +11,11 @@ import {
   Float16,
   Float32,
   Float64,
+  Int32,
   Schema,
   Utf8,
+  fromDataToBuffer,
+  tableFromIPC,
 } from "../lancedb/arrow";
 import { EmbeddingFunction, LanceSchema } from "../lancedb/embedding";
 import { getRegistry, register } from "../lancedb/embedding/registry";
@@ -184,6 +187,63 @@ describe("embedding functions", () => {
     const vector0 = JSON.parse(JSON.stringify(arr[0].vector));
     expect(vector0).toEqual([1, 2, 3]);
   });
+
+  it("should append generated vectors to a non-nullable schema", async () => {
+    @register("non_nullable_schema_test")
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      ndims() {
+        return 3;
+      }
+      embeddingDataType(): Float {
+        return new Float64();
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        return data.map(() => [1, 2, 3]);
+      }
+    }
+
+    const schema = new Schema([
+      new Field("id", new Int32()),
+      new Field("text", new Utf8()),
+      new Field("type", new Utf8()),
+      new Field(
+        "vector",
+        new FixedSizeList(3, new Field("item", new Float64())),
+      ),
+    ]);
+    const func = new MockEmbeddingFunction();
+    const db = await connect(tmpDir.name);
+    const table = await db.createEmptyTable("test_non_nullable", schema, {
+      embeddingFunction: {
+        function: func,
+        sourceColumn: "text",
+      },
+    });
+
+    const data = [
+      { id: 1, text: "Carrot", type: "vegetable" },
+      { id: 2, text: "Apple", type: "fruit" },
+    ];
+    const buffer = await fromDataToBuffer(
+      data,
+      undefined,
+      await table.schema(),
+    );
+    const generatedTable = tableFromIPC(buffer);
+    const vectorField = generatedTable.schema.fields.find(
+      (field) => field.name === "vector",
+    );
+    expect(vectorField?.nullable).toBe(false);
+
+    await table.add(data);
+
+    const rows = await table.query().toArray();
+    expect(rows).toHaveLength(2);
+    for (const row of rows) {
+      expect([...row.vector]).toEqual([1, 2, 3]);
+    }
+  });
+
   it("should error when appending to a table with an unregistered embedding function", async () => {
     @register("mock")
     class MockEmbeddingFunction extends EmbeddingFunction<string> {
@@ -427,4 +487,52 @@ describe("embedding functions", () => {
       expect(stringSchema3).toEqual(stringExpectedSchema);
     },
   );
+  test("parses one function writing several vector columns", async () => {
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      ndims() {
+        return 3;
+      }
+      embeddingDataType(): Float {
+        return new Float32();
+      }
+      async computeQueryEmbeddings(_data: string) {
+        return [1, 2, 3];
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        return Array.from({ length: data.length }).fill([
+          1, 2, 3,
+        ]) as number[][];
+      }
+    }
+    const registry = getRegistry();
+    registry.register("multi_output_mock")(MockEmbeddingFunction);
+
+    // A materialized view can project one source vector column under two
+    // names, so a table's configuration names the same function twice.
+    const parsed = await registry.parseFunctions(
+      new Map([
+        [
+          "embedding_functions",
+          JSON.stringify([
+            {
+              name: "multi_output_mock",
+              sourceColumn: "text",
+              vectorColumn: "vector_a",
+              model: {},
+            },
+            {
+              name: "multi_output_mock",
+              sourceColumn: "text",
+              vectorColumn: "vector_b",
+              model: {},
+            },
+          ]),
+        ],
+      ]),
+    );
+
+    expect(
+      [...parsed.values()].map(({ vectorColumn }) => vectorColumn).sort(),
+    ).toEqual(["vector_a", "vector_b"]);
+  });
 });
diff --git a/nodejs/__test__/embedding_registry.test.ts b/nodejs/__test__/embedding_registry.test.ts
new file mode 100644
index 000000000..83933399a
--- /dev/null
+++ b/nodejs/__test__/embedding_registry.test.ts
@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import { execFileSync } from "node:child_process";
+import { resolve } from "node:path";
+
+import type { OpenAIEmbeddingFunction } from "../lancedb/embedding/openai";
+import type { EmbeddingFunctionRegistry } from "../lancedb/embedding/registry";
+
+type EmbeddingModule = typeof import("../lancedb/embedding");
+type OpenAIModule = typeof import("../lancedb/embedding/openai");
+type RegistryModule = typeof import("../lancedb/embedding/registry");
+
+describe("embedding function registry", () => {
+  const registries: EmbeddingFunctionRegistry[] = [];
+
+  afterEach(() => {
+    for (const registry of registries) {
+      registry.reset();
+    }
+    registries.length = 0;
+  });
+
+  it("defers built-in providers until the public registry API is used", () => {
+    jest.isolateModules(() => {
+      const embedding = require("../lancedb/embedding") as EmbeddingModule;
+      const { getRegistry: getInternalRegistry } =
+        require("../lancedb/embedding/registry") as RegistryModule;
+      const registry = getInternalRegistry();
+      registries.push(registry);
+
+      expect(registry.length()).toBe(0);
+      expect(embedding.getRegistry()).toBe(registry);
+      expect(registry.get("openai")).toBeDefined();
+      expect(registry.get("huggingface")).toBeDefined();
+    });
+  });
+
+  it("preserves automatic FTS search in a fresh process", () => {
+    execFileSync(
+      process.execPath,
+      [resolve(__dirname, "fixtures", "auto_fts_search.cjs")],
+      { stdio: "pipe" },
+    );
+  });
+
+  it("shares registrations across duplicated provider module graphs", () => {
+    let registeringRegistry: EmbeddingFunctionRegistry | undefined;
+    let latestOpenAIConstructor: typeof OpenAIEmbeddingFunction | undefined;
+
+    jest.isolateModules(() => {
+      require("../lancedb/embedding/openai");
+      const { getRegistry } =
+        require("../lancedb/embedding/registry") as RegistryModule;
+      registeringRegistry = getRegistry();
+      registries.push(registeringRegistry);
+      expect(registeringRegistry.get("openai")).toBeDefined();
+    });
+
+    expect(() => {
+      jest.isolateModules(() => {
+        const { OpenAIEmbeddingFunction } =
+          require("../lancedb/embedding/openai") as OpenAIModule;
+        latestOpenAIConstructor = OpenAIEmbeddingFunction;
+        const { getRegistry } =
+          require("../lancedb/embedding/registry") as RegistryModule;
+        registries.push(getRegistry());
+      });
+    }).not.toThrow();
+
+    const previousApiKey = process.env.OPENAI_API_KEY;
+    process.env.OPENAI_API_KEY = "test";
+    try {
+      const latestOpenAI = registeringRegistry!
+        .get<OpenAIEmbeddingFunction>("openai")!
+        .create();
+      expect(latestOpenAI).toBeInstanceOf(latestOpenAIConstructor!);
+    } finally {
+      if (previousApiKey === undefined) {
+        delete process.env.OPENAI_API_KEY;
+      } else {
+        process.env.OPENAI_API_KEY = previousApiKey;
+      }
+    }
+
+    jest.isolateModules(() => {
+      const { getRegistry } =
+        require("../lancedb/embedding") as EmbeddingModule;
+      const publicRegistry = getRegistry();
+      registries.push(publicRegistry);
+      expect(publicRegistry).toBe(registeringRegistry);
+      expect(publicRegistry.get("openai")).toBeDefined();
+    });
+  });
+});
diff --git a/nodejs/__test__/fixtures/auto_fts_search.cjs b/nodejs/__test__/fixtures/auto_fts_search.cjs
new file mode 100644
index 000000000..b5ab060b3
--- /dev/null
+++ b/nodejs/__test__/fixtures/auto_fts_search.cjs
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+const assert = require("node:assert/strict");
+const tmp = require("tmp");
+const { connect, embedding, Index } = require("../../dist");
+const { getRegistry } = require("../../dist/embedding/registry");
+
+async function main() {
+  assert.equal(typeof embedding.getRegistry, "function");
+  assert.equal(getRegistry().length(), 0);
+  assert.equal(embedding.getRegistry(), getRegistry());
+  assert.equal(getRegistry().length(), 2);
+
+  const dir = tmp.dirSync({ unsafeCleanup: true });
+  let db;
+  try {
+    db = await connect(dir.name);
+    const table = await db.createTable("docs", [{ text: "hello world" }]);
+    await table.createIndex("text", { config: Index.fts() });
+
+    const rows = await table.search("hello").toArray();
+    assert.equal(rows[0].text, "hello world");
+  } finally {
+    db?.close();
+    dir.removeCallback();
+  }
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});
diff --git a/nodejs/__test__/materialized_view.test.ts b/nodejs/__test__/materialized_view.test.ts
new file mode 100644
index 000000000..2e7b2ec4d
--- /dev/null
+++ b/nodejs/__test__/materialized_view.test.ts
@@ -0,0 +1,147 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import * as tmp from "tmp";
+
+import { Connection, connect } from "../lancedb";
+import {
+  DEFINITION_META_KEY,
+  definitionFromMetadata,
+} from "../lancedb/materialized_view";
+
+describe("materialized views", () => {
+  let tmpDir: tmp.DirResult;
+  let db: Connection;
+
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    db = await connect(tmpDir.name);
+    await db.createTable(
+      "people",
+      [
+        { name: "ada", age: 36 },
+        { name: "kid", age: 7 },
+        { name: "grace", age: 85 },
+      ],
+      { storageOptions: { newTableEnableStableRowIds: "true" } },
+    );
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  it("rejects a stored limit a number cannot carry", () => {
+    const big = new Map([
+      [
+        DEFINITION_META_KEY,
+        '{"kind":"select","source_table":"people","limit":9007199254740993}',
+      ],
+    ]);
+    expect(() => definitionFromMetadata(big, "v")).toThrow(
+      /too large to represent exactly/,
+    );
+
+    const safe = new Map([
+      [
+        DEFINITION_META_KEY,
+        '{"kind":"select","source_table":"people","limit":42}',
+      ],
+    ]);
+    expect(definitionFromMetadata(safe, "v").limit).toBe(42);
+  });
+
+  it("creates, refreshes and queries a view", async () => {
+    const view = await db.createMaterializedView("adults", "people", {
+      select: ["name", ["shout", "upper(name)"]],
+      where: "age >= 18",
+    });
+    expect(view.name).toBe("adults");
+    expect(await view.table().countRows()).toBe(0);
+
+    const result = await view.refresh();
+    expect(result.mode).toBe("rebuild");
+    expect(Number(result.rowsWritten)).toBe(2);
+
+    const rows = await view.table().query().toArray();
+    expect(rows.map((r) => r.shout).sort()).toEqual(["ADA", "GRACE"]);
+  });
+
+  it("round-trips the definition", async () => {
+    await db.createMaterializedView("adults", "people", {
+      where: "age >= 18",
+    });
+    const view = await db.openMaterializedView("adults");
+    const definition = await view.definition();
+    expect(definition.sourceTable).toBe("people");
+    expect(definition.filter).toBe("age >= 18");
+    expect(definition.projections).toEqual([
+      ["name", "`name`"],
+      ["age", "`age`"],
+    ]);
+    expect(definition.inputs).toEqual(["age", "name"]);
+  });
+
+  it("refreshes incrementally after an append", async () => {
+    const view = await db.createMaterializedView("copy", "people");
+    await view.refresh();
+
+    const people = await db.openTable("people");
+    await people.add([{ name: "alan", age: 41 }]);
+    const result = await view.refresh();
+    expect(result.mode).toBe("incremental");
+    expect(Number(result.rowsWritten)).toBe(1);
+    expect(await view.table().countRows()).toBe(4);
+
+    expect((await view.refresh()).mode).toBe("no_op");
+  });
+
+  it("lists views and rejects non-views", async () => {
+    await db.createMaterializedView("adults", "people", {
+      where: "age >= 18",
+    });
+    expect(await db.listMaterializedViews()).toEqual(["adults"]);
+    await expect(db.openMaterializedView("people")).rejects.toThrow(
+      "not a materialized view",
+    );
+  });
+
+  it("rejects an invalid expression at create time", async () => {
+    await expect(
+      db.createMaterializedView("bad", "people", {
+        select: [["x", "missing + 1"]],
+      }),
+    ).rejects.toThrow("missing");
+  });
+
+  it("rejects invalid numeric options before creating anything", async () => {
+    for (const limit of [-5, 1.5, Infinity, NaN]) {
+      await expect(
+        db.createMaterializedView("bad", "people", { limit }),
+      ).rejects.toThrow("non-negative integer");
+    }
+    expect(await db.listMaterializedViews()).toEqual([]);
+
+    const view = await db.createMaterializedView("copy", "people");
+    for (const sourceVersion of [-1, 1.5, Infinity, NaN]) {
+      await expect(view.refresh({ sourceVersion })).rejects.toThrow(
+        "non-negative integer",
+      );
+    }
+  });
+
+  it("quotes bare select names", async () => {
+    await db.createTable("odd_names", [{ "order item": "widget" }], {
+      storageOptions: { newTableEnableStableRowIds: "true" },
+    });
+    const view = await db.createMaterializedView("quoted", "odd_names", {
+      select: ["order item"],
+    });
+    const result = await view.refresh();
+    expect(Number(result.rowsWritten)).toBe(1);
+  });
+
+  it("requires stable row ids on the source", async () => {
+    await db.createTable("plain", [{ x: 1 }]);
+    await expect(db.createMaterializedView("v", "plain")).rejects.toThrow(
+      "stable row ids",
+    );
+  });
+});
diff --git a/nodejs/__test__/package.test.ts b/nodejs/__test__/package.test.ts
new file mode 100644
index 000000000..7743d73d6
--- /dev/null
+++ b/nodejs/__test__/package.test.ts
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import packageJson = require("../package.json");
+
+describe("package metadata", () => {
+  it("requires Node.js type declarations compatible with the runtime", () => {
+    expect(packageJson.engines.node).toBe(">= 18");
+    expect(packageJson.peerDependencies["@types/node"]).toBe(">=18");
+    expect(packageJson.peerDependenciesMeta["@types/node"]).toEqual({
+      optional: true,
+    });
+  });
+});
diff --git a/nodejs/__test__/query.test.ts b/nodejs/__test__/query.test.ts
index da001b1eb..5f3e68b16 100644
--- a/nodejs/__test__/query.test.ts
+++ b/nodejs/__test__/query.test.ts
@@ -110,6 +110,81 @@ describe("Query outputSchema", () => {
   });
 });
 
+describe("Search pagination", () => {
+  let tmpDir: tmp.DirResult;
+  let table: Table;
+
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    const db = await connect(tmpDir.name);
+    const schema = new Schema([
+      new Field("id", new Int64(), false),
+      new Field("text", new Utf8(), false),
+      new Field(
+        "vector",
+        new FixedSizeList(2, new Field("item", new Float32())),
+        false,
+      ),
+    ]);
+    const data = makeArrowTable(
+      [
+        { id: 1n, text: "common", vector: [0, 0] },
+        { id: 2n, text: "common common", vector: [1, 1] },
+        { id: 3n, text: "common common common", vector: [2, 2] },
+        { id: 4n, text: "common common common common", vector: [3, 3] },
+      ],
+      { schema },
+    );
+    table = await db.createTable("test", data);
+  });
+
+  afterEach(() => {
+    tmpDir.removeCallback();
+  });
+
+  it("applies offset after the vector search limit", async () => {
+    const allResults = await table
+      .vectorSearch([0, 0])
+      .select(["id"])
+      .limit(4)
+      .toArray();
+    const secondPage = await table
+      .vectorSearch([0, 0])
+      .select(["id"])
+      .limit(2)
+      .offset(2)
+      .toArray();
+
+    expect(allResults).toHaveLength(4);
+    expect(secondPage).toHaveLength(2);
+    expect(secondPage.map((row) => row.id)).toEqual(
+      allResults.slice(2, 4).map((row) => row.id),
+    );
+  });
+
+  it("applies offset after the full-text search limit", async () => {
+    await table.createIndex("text", { config: Index.fts() });
+
+    const allResults = await table
+      .search("common", "fts")
+      .select(["id"])
+      .limit(4)
+      .toArray();
+    const secondPage = await table
+      .search("common", "fts")
+      .select(["id"])
+      .limit(2)
+      .offset(2)
+      .toArray();
+
+    expect(allResults).toHaveLength(4);
+    expect(secondPage).toHaveLength(2);
+    expect(secondPage.map((row) => row.id)).toEqual(
+      allResults.slice(2, 4).map((row) => row.id),
+    );
+  });
+});
+
 describe("Query orderBy", () => {
   let tmpDir: tmp.DirResult;
   let table: Table;
diff --git a/nodejs/__test__/registry.test.ts b/nodejs/__test__/registry.test.ts
index a5cf73e74..973ad8a25 100644
--- a/nodejs/__test__/registry.test.ts
+++ b/nodejs/__test__/registry.test.ts
@@ -106,6 +106,77 @@ describe.each([arrow15, arrow16, arrow17, arrow18])("Registry", (arrow) => {
       'Embedding function with alias "mock-embedding" already exists',
     );
   });
+  test("parseFunctions keeps entries sharing a function name", async () => {
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      ndims() {
+        return 3;
+      }
+      embeddingDataType() {
+        return new arrow.Float32() as apiArrow.Float;
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        return data.map(() => [1, 2, 3]);
+      }
+    }
+    register("mock-embedding")(MockEmbeddingFunction);
+    const parsed = await getRegistry().parseFunctions(
+      new Map([
+        [
+          "embedding_functions",
+          JSON.stringify([
+            {
+              name: "mock-embedding",
+              sourceColumn: "text",
+              vectorColumn: "vector_a",
+              model: {},
+            },
+            {
+              name: "mock-embedding",
+              sourceColumn: "text",
+              vectorColumn: "vector_b",
+              model: {},
+            },
+          ]),
+        ],
+      ]),
+    );
+    expect([...parsed.values()].map((f) => f.vectorColumn)).toEqual([
+      "vector_a",
+      "vector_b",
+    ]);
+
+    // The Python bindings write snake_case keys.
+    const snake = await getRegistry().parseFunctions(
+      new Map([
+        [
+          "embedding_functions",
+          JSON.stringify([
+            {
+              name: "mock-embedding",
+              // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+              source_column: "text",
+              // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+              vector_column: "vector_a",
+              model: {},
+            },
+            {
+              name: "mock-embedding",
+              // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+              source_column: "text",
+              // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+              vector_column: "vector_b",
+              model: {},
+            },
+          ]),
+        ],
+      ]),
+    );
+    expect([...snake.keys()]).toEqual(["vector_a", "vector_b"]);
+    expect([...snake.values()].map((f) => f.sourceColumn)).toEqual([
+      "text",
+      "text",
+    ]);
+  });
   test("schema should contain correct metadata", async () => {
     class MockEmbeddingFunction extends EmbeddingFunction<string> {
       constructor(args: FunctionOptions = {}) {
diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts
index 89a9e992c..c51cbbbb7 100644
--- a/nodejs/__test__/remote.test.ts
+++ b/nodejs/__test__/remote.test.ts
@@ -75,6 +75,25 @@ async function withMockDatabase(
 }
 
 describe("remote connection", () => {
+  it("refuses materialized views before issuing any request", async () => {
+    const paths: string[] = [];
+    await withMockDatabase(
+      (req, res) => {
+        paths.push(req.url ?? "");
+        res.writeHead(404).end();
+      },
+      async (db) => {
+        await expect(db.openMaterializedView("secret_table")).rejects.toThrow(
+          /only on local databases/,
+        );
+        await expect(db.listMaterializedViews()).rejects.toThrow(
+          /only on local databases/,
+        );
+        expect(paths).toEqual([]);
+      },
+    );
+  });
+
   it("should accept partial connection options", async () => {
     await connect("db://test", {
       apiKey: "fake",
@@ -170,6 +189,38 @@ describe("remote connection", () => {
     );
   });
 
+  it("surfaces JSON server errors from remote table operations", async () => {
+    await withMockDatabase(
+      (req, res) => {
+        const path = req.url ?? "";
+        if (path.endsWith("/describe/")) {
+          res.writeHead(200, { "Content-Type": "application/json" }).end(
+            JSON.stringify({
+              name: "broken_table",
+              version: 1,
+              schema: { fields: [] },
+            }),
+          );
+          return;
+        }
+
+        if (path.endsWith("/count_rows/")) {
+          res
+            .writeHead(400, { "Content-Type": "application/json" })
+            .end(JSON.stringify({ error: "count rows failed" }));
+          return;
+        }
+
+        res.writeHead(404).end();
+      },
+      async (db) => {
+        const table = await db.openTable("broken_table");
+
+        await expect(table.countRows()).rejects.toThrow("count rows failed");
+      },
+    );
+  });
+
   it("should pass on requested extra headers", async () => {
     await withMockDatabase(
       (req, res) => {
@@ -279,7 +330,7 @@ describe("remote connection", () => {
     expect(createIndexBody?.["custom_stop_words"]).toEqual(["the"]);
   });
 
-  it("diffs and merges remote branches", async () => {
+  it("diffs and cherry-picks remote branches", async () => {
     const sampleDiff = {
       fromBranch: "exp",
       parentVersion: 1,
@@ -301,10 +352,9 @@ describe("remote connection", () => {
       changedColumns: [],
       addedIndexes: [],
       removedIndexes: [],
-      mergeable: true,
-      mergeBlockers: [],
+      errors: [],
     };
-    const mergeBodies: Record<string, unknown>[] = [];
+    const cherryPickBodies: Record<string, unknown>[] = [];
 
     await withMockDatabase(
       (req, res) => {
@@ -334,17 +384,16 @@ describe("remote connection", () => {
               .end(JSON.stringify(sampleDiff));
             return;
           }
-          if (path.endsWith("/branches/merge/")) {
-            mergeBodies.push(body);
+          if (path.endsWith("/branches/cherry_pick/")) {
+            cherryPickBodies.push(body);
             const dryRun = body["dry_run"] === true;
             const response = {
-              status: dryRun ? "ready" : "rejected",
+              status: dryRun ? "ready" : "failed",
               diff: dryRun
                 ? sampleDiff
                 : {
                     ...sampleDiff,
-                    mergeable: false,
-                    mergeBlockers: [
+                    errors: [
                       { code: "baseMoved", message: "main has advanced" },
                     ],
                   },
@@ -366,19 +415,19 @@ describe("remote connection", () => {
 
         await expect(branches.diff("exp")).resolves.toEqual(sampleDiff);
 
-        const rejected = await branches.merge("exp");
-        expect(rejected.status).toBe("rejected");
-        expect(rejected.diff.mergeBlockers).toEqual([
+        const failed = await branches.cherryPick("exp");
+        expect(failed.status).toBe("failed");
+        expect(failed.diff.errors).toEqual([
           { code: "baseMoved", message: "main has advanced" },
         ]);
 
-        const preview = await branches.merge("exp", true);
+        const preview = await branches.cherryPick("exp", true);
         expect(preview.status).toBe("ready");
         expect(preview.preview.promotedColumns).toEqual(["tag"]);
       },
     );
 
-    expect(mergeBodies).toEqual([
+    expect(cherryPickBodies).toEqual([
       // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
       { from_branch: "exp", dry_run: false },
       // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts
index 47f80c3d9..345121ab1 100644
--- a/nodejs/__test__/table.test.ts
+++ b/nodejs/__test__/table.test.ts
@@ -11,10 +11,13 @@ import * as arrow17 from "apache-arrow-17";
 import * as arrow18 from "apache-arrow-18";
 
 import {
+  AutoQuery,
   Connection,
   MatchQuery,
   PhraseQuery,
+  Query,
   Table,
+  VectorQuery,
   connect,
   tokenize,
 } from "../lancedb";
@@ -47,7 +50,6 @@ import {
   BooleanQuery,
   Occur,
   Operator,
-  VectorQuery,
   instanceOfFullTextQuery,
 } from "../lancedb/query";
 
@@ -87,6 +89,44 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       await expect(table.countRows()).resolves.toBe(3);
     });
 
+    it("should support a foreign Float64 vector schema end to end", async () => {
+      const conn = await connect(tmpDir.name);
+      const schema = new arrow.Schema([
+        new arrow.Field("resource_id", new arrow.Int32(), false),
+        new arrow.Field(
+          "vector",
+          new arrow.FixedSizeList(
+            3,
+            new arrow.Field("value", new arrow.Float64(), true),
+          ),
+          false,
+        ),
+      ]);
+      const data = [
+        {
+          // biome-ignore lint/style/useNamingConvention: matches the reported schema
+          resource_id: 0,
+          vector: [0.1, 0.1, 0.1],
+        },
+      ];
+
+      const resources = await conn.createTable("resources", data, { schema });
+
+      const existing = await resources
+        .query()
+        .where("resource_id = 0")
+        .limit(1)
+        .toArray();
+      expect(existing).toHaveLength(1);
+
+      const matched = await resources
+        .search(Float64Array.from(data[0].vector))
+        .limit(1)
+        .toArray();
+      expect(matched).toHaveLength(1);
+      expect(matched[0]["resource_id"]).toBe(0);
+    });
+
     it("should support branches", async () => {
       await table.add([{ id: 1 }]);
       expect(await table.countRows()).toBe(1);
@@ -240,8 +280,16 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
         },
         numIndices: 0,
         numRows: 3,
-        totalBytes: 44,
+        // Full on-disk size of the two data files, footers and metadata included.
+        totalBytes: 684,
       });
+
+      // Index files count toward totalBytes too (only deletion files and
+      // manifests are excluded).
+      await table.createIndex("id", { config: Index.btree() });
+      const statsWithIndex = await table.stats();
+      expect(statsWithIndex.numIndices).toBe(1);
+      expect(statsWithIndex.totalBytes).toBeGreaterThan(684);
     });
 
     it("should overwrite data if asked", async () => {
@@ -1732,6 +1780,194 @@ describe("Read consistency interval", () => {
   });
 });
 
+describe("automatic search schema consistency", () => {
+  let tmpDir: tmp.DirResult;
+
+  class SchemaRefreshEmbedding extends EmbeddingFunction<string> {
+    ndims() {
+      return 2;
+    }
+
+    embeddingDataType() {
+      return new Float32();
+    }
+
+    async computeSourceEmbeddings(data: string[]) {
+      return data.map((value) => [value.length, 1]);
+    }
+
+    async computeQueryEmbeddings(value: string) {
+      return [value.length, 1];
+    }
+  }
+
+  function embeddingSchema() {
+    const func = new SchemaRefreshEmbedding();
+    return LanceSchema({
+      text: func.sourceField(new Utf8()),
+      vector: func.vectorField(),
+    });
+  }
+
+  beforeEach(() => {
+    getRegistry().reset();
+    register("schema-refresh")(SchemaRefreshEmbedding);
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+
+  afterEach(() => {
+    getRegistry().reset();
+    tmpDir.removeCallback();
+  });
+
+  it("uses the schema refreshed from another connection", async () => {
+    const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
+    const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
+
+    try {
+      const stale = await first.createTable("docs", [{ text: "before" }], {
+        schema: embeddingSchema(),
+      });
+      const replacement = await second.createTable(
+        "docs",
+        [{ text: "after hello" }],
+        { mode: "overwrite" },
+      );
+      await replacement.createIndex("text", { config: Index.fts() });
+
+      const search = stale.search("hello");
+      expect(search).toBeInstanceOf(AutoQuery);
+      expect(search).not.toBeInstanceOf(Query);
+      expect(search).not.toBeInstanceOf(VectorQuery);
+      expect("nprobes" in search).toBe(false);
+
+      const rows = await search.toArray();
+      expect(rows[0].text).toBe("after hello");
+      expect((await stale.schema()).metadata.has("embedding_functions")).toBe(
+        false,
+      );
+    } finally {
+      first.close();
+      second.close();
+    }
+  });
+
+  it("tracks embedding metadata across checkout and restore", async () => {
+    const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
+    const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
+
+    try {
+      await first.createTable("docs", [{ text: "before" }], {
+        schema: embeddingSchema(),
+      });
+      const table = await second.createTable(
+        "docs",
+        [{ text: "after hello" }],
+        { mode: "overwrite" },
+      );
+      await table.createIndex("text", { config: Index.fts() });
+
+      await table.checkout(1);
+      expect((await table.search("before").toArray())[0].text).toBe("before");
+
+      await table.checkoutLatest();
+      expect((await table.search("hello").toArray())[0].text).toBe(
+        "after hello",
+      );
+
+      await table.checkout(1);
+      await table.restore();
+      expect((await table.search("before").toArray())[0].text).toBe("before");
+    } finally {
+      first.close();
+      second.close();
+    }
+  });
+
+  it("pins automatic search while computing an embedding", async () => {
+    let markStarted!: () => void;
+    let releaseEmbedding!: () => void;
+    const started = new Promise<void>((resolve) => {
+      markStarted = resolve;
+    });
+    const released = new Promise<void>((resolve) => {
+      releaseEmbedding = resolve;
+    });
+
+    class BlockingEmbedding extends SchemaRefreshEmbedding {
+      async computeQueryEmbeddings(value: string) {
+        markStarted();
+        await released;
+        return [value.length, 1];
+      }
+    }
+
+    register("schema-refresh-blocking")(BlockingEmbedding);
+    const func = new BlockingEmbedding();
+    const schema = LanceSchema({
+      text: func.sourceField(new Utf8()),
+      vector: func.vectorField(),
+    });
+    const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
+    const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
+
+    try {
+      const table = await first.createTable(
+        "docs",
+        [{ text: "hello before" }],
+        { schema },
+      );
+      const pending = table.search("hello").toArray();
+      await started;
+
+      const replacement = await second.createTable(
+        "docs",
+        [{ text: "hello after" }],
+        { mode: "overwrite" },
+      );
+      await replacement.createIndex("text", { config: Index.fts() });
+      releaseEmbedding();
+
+      expect((await pending)[0].text).toBe("hello before");
+    } finally {
+      releaseEmbedding();
+      first.close();
+      second.close();
+    }
+  });
+
+  it("refreshes a reused automatic search for every execution", async () => {
+    const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
+    const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
+
+    try {
+      const table = await first.createTable("docs", [
+        { text: "hello before", marker: "before" },
+      ]);
+      await table.createIndex("text", { config: Index.fts() });
+      const search = table.search("hello").select(["text"]);
+
+      const before = (await search.toArray())[0];
+      expect(before.text).toBe("hello before");
+      expect(before.marker).toBeUndefined();
+
+      const replacement = await second.createTable(
+        "docs",
+        [{ text: "hello after", marker: "after" }],
+        { mode: "overwrite" },
+      );
+      await replacement.createIndex("text", { config: Index.fts() });
+
+      const after = (await search.toArray())[0];
+      expect(after.text).toBe("hello after");
+      expect(after.marker).toBeUndefined();
+    } finally {
+      first.close();
+      second.close();
+    }
+  });
+});
+
 describe("schema evolution", function () {
   let tmpDir: tmp.DirResult;
   beforeEach(() => {
@@ -2393,10 +2629,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
         snapshotCalls += 1;
         return await querySnapshot();
       };
-      const autoQuery = (tracked.search("greetings") as VectorQuery)
-        .nprobes(1)
-        .select(["text"])
-        .limit(1);
+      const autoQuery = tracked.search("greetings").select(["text"]).limit(1);
 
       const func = new TestEmbedding();
       const schema = LanceSchema({
@@ -2414,31 +2647,13 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       expect(results[0].text).toBe(data[0].text);
       expect(initCalls).toBe(baselineInitCalls + 1);
       expect(queryCalls).toBe(1);
-      expect(snapshotCalls).toBe(2);
+      expect(snapshotCalls).toBe(1);
 
       const repeatedResults = await autoQuery.toArray();
       expect(repeatedResults[0].text).toBe(data[0].text);
       expect(initCalls).toBe(baselineInitCalls + 1);
       expect(queryCalls).toBe(1);
-      expect(snapshotCalls).toBe(3);
-
-      await expect(
-        (tracked.search("greetings") as VectorQuery)
-          .addQueryVector(Promise.reject(new Error("extra vector failed")))
-          .toArray(),
-      ).rejects.toThrow("extra vector failed");
-
-      const multiVectorResults = await (
-        tracked.search("greetings") as VectorQuery
-      )
-        .addQueryVector(Promise.resolve([0.2]))
-        .select(["text"])
-        .limit(1)
-        .toArray();
-      expect(multiVectorResults).toHaveLength(2);
-      expect(multiVectorResults.map((row) => row.text).sort()).toEqual(
-        data.map((row) => row.text).sort(),
-      );
+      expect(snapshotCalls).toBe(2);
 
       const pending = tracked
         .search("blocked")
@@ -2458,20 +2673,13 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       releaseEmbedding();
 
       const pendingResults = await pending;
-      expect(pendingResults[0].text).toBe(ftsData[1].text);
+      expect(pendingResults[0].text).toBe(data[1].text);
 
       expect(
         (await tracked.schema()).metadata.get("embedding_functions"),
       ).toBeUndefined();
       const ftsResults = await autoQuery.toArray();
       expect(ftsResults[0].text).toBe(ftsData[0].text);
-
-      const rejectedVector = Promise.reject(new Error("unused vector failed"));
-      const ftsWithRejectedVector = (
-        tracked.search("greetings") as VectorQuery
-      ).addQueryVector(rejectedVector);
-      await expect(ftsWithRejectedVector.toArray()).resolves.toBeDefined();
-      await new Promise<void>((resolve) => setImmediate(resolve));
     });
 
     test("auto search keeps newer preparation during a revision race", async () => {
@@ -3253,7 +3461,7 @@ describe("column name options", () => {
       .limit(10)
       .toArray();
     expect(results2.length).toBe(10);
-  });
+  }, 30_000);
 });
 
 describe("when creating an empty table", () => {
@@ -3640,3 +3848,120 @@ describe("LSM merge insert", () => {
     await expect(table.query().useLsm(true).toArray()).rejects.toThrow();
   });
 });
+
+describe("LSM convergence and stats", () => {
+  let tmpDir: tmp.DirResult;
+
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  async function lsmTable(conn: Connection): Promise<Table> {
+    const table = await conn.createEmptyTable(
+      "t",
+      new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]),
+    );
+    await table.setUnenforcedPrimaryKey("id");
+    await table.setLsmWriteSpec({ specType: "unsharded" });
+    return table;
+  }
+
+  // These four route through the server that owns the MemWAL, so a local table
+  // rejects them rather than answering. What is asserted here is that the
+  // bindings reach the core at all; the behavior against a real endpoint is
+  // covered by the mocked endpoint tests in rust/lancedb/src/remote/table.rs.
+  it("rejects flushLsm on a local table", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await lsmTable(conn);
+
+    await expect(table.flushLsm()).rejects.toThrow(/not supported/i);
+  });
+
+  it("rejects compactLsm on a local table", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await lsmTable(conn);
+
+    await expect(table.compactLsm()).rejects.toThrow(/not supported/i);
+  });
+
+  it("rejects getLsmStats on a local table", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await lsmTable(conn);
+
+    await expect(table.getLsmStats()).rejects.toThrow(/not supported/i);
+    await expect(table.getLsmStats(true)).rejects.toThrow(/not supported/i);
+  });
+
+  it("rejects checkpointLsm on a local table", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await lsmTable(conn);
+
+    // checkpointLsm seals first, so it surfaces flushLsm's rejection.
+    await expect(table.checkpointLsm()).rejects.toThrow(/not supported/i);
+  });
+});
+
+describe("computed columns", () => {
+  let tmpDir: tmp.DirResult;
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  it("declares a column and fills it on refresh", async () => {
+    const db = await connect(tmpDir.name);
+    const table = await db.createTable("computed", [{ x: 1 }, { x: 2 }]);
+
+    await table.addColumns({
+      computed: [{ name: "doubled", valueSql: "x * 2" }],
+    });
+    let rows = await table.query().toArray();
+    expect(rows.map((r) => r.doubled)).toEqual([null, null]);
+
+    const result = await table.refreshColumn("doubled");
+    expect(result.rowsFilled).toBe(2);
+
+    rows = await table.query().toArray();
+    expect(rows.map((r) => r.doubled).sort()).toEqual([2, 4]);
+  });
+
+  it("returns a job handle from refreshColumnAsync", async () => {
+    const db = await connect(tmpDir.name);
+    const table = await db.createTable("computed_job", [{ x: 1 }, { x: 2 }]);
+
+    await table.addColumns({
+      computed: [{ name: "doubled", valueSql: "x * 2" }],
+    });
+
+    const job = await table.refreshColumnAsync("doubled");
+    expect(job.id).toBeNull();
+    await job.wait();
+    expect(await job.status()).toBe("finished");
+
+    const rows = await table.query().toArray();
+    expect(rows.map((r) => r.doubled).sort()).toEqual([2, 4]);
+
+    // Bad input rejects at the call, not through the job.
+    await expect(table.refreshColumnAsync("x")).rejects.toThrow(
+      "not a computed column",
+    );
+  });
+
+  it("fills rows added since the last refresh", async () => {
+    const db = await connect(tmpDir.name);
+    const table = await db.createTable("computed_append", [{ x: 1 }]);
+
+    await table.addColumns({
+      computed: [{ name: "doubled", valueSql: "x * 2" }],
+    });
+    await table.refreshColumn("doubled");
+    await table.add([{ x: 5 }]);
+
+    const result = await table.refreshColumn("doubled");
+    expect(result.rowsFilled).toBe(1);
+
+    const rows = await table.query().toArray();
+    expect(rows.map((r) => r.doubled).sort()).toEqual([10, 2]);
+  });
+});
diff --git a/nodejs/lancedb/arrow.ts b/nodejs/lancedb/arrow.ts
index 587d30b19..b52ab50ef 100644
--- a/nodejs/lancedb/arrow.ts
+++ b/nodejs/lancedb/arrow.ts
@@ -5,7 +5,6 @@ import {
   Data as ArrowData,
   Table as ArrowTable,
   Binary,
-  Bool,
   BufferType,
   DataType,
   DateUnit,
@@ -18,12 +17,7 @@ import {
   FixedSizeList,
   Float,
   Float32,
-  Float64,
   Int,
-  Int8,
-  Int16,
-  Int32,
-  Int64,
   LargeBinary,
   List,
   Null,
@@ -36,33 +30,29 @@ import {
   Struct,
   Timestamp,
   Type,
-  Uint8,
-  Uint16,
-  Uint32,
   Utf8,
   Vector,
   makeVector as arrowMakeVector,
+  util as arrowUtil,
   vectorFromArray as badVectorFromArray,
   makeBuilder,
   makeData,
 } from "apache-arrow";
 import { Buffers } from "apache-arrow/data";
+import { typedArrayToArrowType } from "./arrow_type";
 import { type EmbeddingFunction } from "./embedding/embedding_function";
-import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
+import {
+  EmbeddingFunctionConfig,
+  getRegistry,
+  parseEmbeddingMetadata,
+} from "./embedding/registry";
 import {
   sanitizeField,
   sanitizeSchema,
   sanitizeTable,
   sanitizeType,
 } from "./sanitize";
-
-/**
- * Check if a field name indicates a vector column.
- */
-function nameSuggestsVectorColumn(fieldName: string): boolean {
-  const nameLower = fieldName.toLowerCase();
-  return nameLower.includes("vector") || nameLower.includes("embedding");
-}
+import { inferSchema } from "./schema";
 
 export * from "apache-arrow";
 export type SchemaLike =
@@ -455,110 +445,6 @@ export function makeArrowTable(
   return new ArrowTable(inferredSchema, finalColumns);
 }
 
-function inferSchema(
-  data: Array<Record<string, unknown>>,
-  schema: Schema | undefined,
-  opts: MakeArrowTableOptions,
-): Schema {
-  // We will collect all fields we see in the data.
-  const pathTree = new PathTree<DataType>();
-
-  for (const [rowI, row] of data.entries()) {
-    for (const [path, value] of rowPathsAndValues(row)) {
-      if (!pathTree.has(path)) {
-        // First time seeing this field.
-        if (schema !== undefined) {
-          const field = getFieldForPath(schema, path);
-          if (field === undefined) {
-            throw new Error(
-              `Found field not in schema: ${path.join(".")} at row ${rowI}`,
-            );
-          } else {
-            pathTree.set(path, field.type);
-          }
-        } else {
-          const inferredType = inferType(value, path, opts);
-          if (inferredType === undefined) {
-            throw new Error(`Failed to infer data type for field ${path.join(
-              ".",
-            )} at row ${rowI}. \
-                             Consider providing an explicit schema.`);
-          }
-          pathTree.set(path, inferredType);
-        }
-      } else if (schema === undefined) {
-        const currentType = pathTree.get(path);
-        const newType = inferType(value, path, opts);
-        if (currentType !== newType) {
-          new Error(`Failed to infer schema for data. Previously inferred type \
-                     ${currentType} but found ${newType} at row ${rowI}. Consider \
-                     providing an explicit schema.`);
-        }
-      }
-    }
-  }
-
-  if (schema === undefined) {
-    function fieldsFromPathTree(pathTree: PathTree<DataType>): Field[] {
-      const fields = [];
-      for (const [name, value] of pathTree.map.entries()) {
-        if (value instanceof PathTree) {
-          const children = fieldsFromPathTree(value);
-          fields.push(new Field(name, new Struct(children), true));
-        } else {
-          fields.push(new Field(name, value, true));
-        }
-      }
-      return fields;
-    }
-    const fields = fieldsFromPathTree(pathTree);
-    return new Schema(fields);
-  } else {
-    function takeMatchingFields(
-      fields: Field[],
-      pathTree: PathTree<DataType>,
-    ): Field[] {
-      const outFields = [];
-      for (const field of fields) {
-        if (pathTree.map.has(field.name)) {
-          const value = pathTree.get([field.name]);
-          if (value instanceof PathTree) {
-            const struct = field.type as Struct;
-            const children = takeMatchingFields(struct.children, value);
-            outFields.push(
-              new Field(field.name, new Struct(children), field.nullable),
-            );
-          } else {
-            outFields.push(
-              new Field(field.name, value as DataType, field.nullable),
-            );
-          }
-        }
-      }
-      return outFields;
-    }
-    const fields = takeMatchingFields(schema.fields, pathTree);
-    return new Schema(fields);
-  }
-}
-
-function* rowPathsAndValues(
-  row: Record<string, unknown>,
-  basePath: string[] = [],
-): Generator<[string[], unknown]> {
-  for (const [key, value] of Object.entries(row)) {
-    if (isObject(value)) {
-      yield* rowPathsAndValues(value, [...basePath, key]);
-    } else {
-      // Skip undefined values - they should be treated the same as missing fields
-      // for embedding function purposes
-      if (value !== undefined) {
-        yield [[...basePath, key], value];
-      }
-    }
-  }
-}
-
 function isObject(value: unknown): value is Record<string, unknown> {
   return (
     typeof value === "object" &&
@@ -573,146 +459,19 @@ function isObject(value: unknown): value is Record<string, unknown> {
   );
 }
 
-function getFieldForPath(schema: Schema, path: string[]): Field | undefined {
-  let current: Field | Schema = schema;
+function valueAtPath(datum: Record<string, unknown>, path: string[]): unknown {
+  let current: unknown = datum;
   for (const key of path) {
-    if (current instanceof Schema) {
-      const field: Field | undefined = current.fields.find(
-        (f) => f.name === key,
-      );
-      if (field === undefined) {
-        return undefined;
-      }
-      current = field;
-    } else if (current instanceof Field && DataType.isStruct(current.type)) {
-      const struct: Struct = current.type;
-      const field = struct.children.find((f) => f.name === key);
-      if (field === undefined) {
-        return undefined;
-      }
-      current = field;
+    if (current == null) {
+      return null;
+    }
+    if (isObject(current) && (Object.hasOwn(current, key) || key in current)) {
+      current = current[key];
     } else {
       return undefined;
     }
   }
-  if (current instanceof Field) {
-    return current;
-  } else {
-    return undefined;
-  }
-}
-
-/**
- * Try to infer which Arrow type to use for a given value.
- *
- * May return undefined if the type cannot be inferred.
- */
-function inferType(
-  value: unknown,
-  path: string[],
-  opts: MakeArrowTableOptions,
-): DataType | undefined {
-  if (typeof value === "bigint") {
-    return new Int64();
-  } else if (typeof value === "number") {
-    // Even if it's an integer, it's safer to assume Float64. Users can
-    // always provide an explicit schema or use BigInt if they mean integer.
-    return new Float64();
-  } else if (typeof value === "string") {
-    if (opts.dictionaryEncodeStrings) {
-      return new Dictionary(new Utf8(), new Int32());
-    } else {
-      return new Utf8();
-    }
-  } else if (typeof value === "boolean") {
-    return new Bool();
-  } else if (value instanceof Buffer) {
-    return new Binary();
-  } else if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
-    const info = typedArrayToArrowType(value);
-    if (info !== undefined) {
-      const child = new Field("item", info.elementType, true);
-      return new FixedSizeList(info.length, child);
-    }
-    return undefined;
-  } else if (Array.isArray(value)) {
-    if (value.length === 0) {
-      return undefined; // Without any values we can't infer the type
-    }
-    if (path.length === 1 && Object.hasOwn(opts.vectorColumns, path[0])) {
-      const floatType = sanitizeType(opts.vectorColumns[path[0]].type);
-      return new FixedSizeList(
-        value.length,
-        new Field("item", floatType, true),
-      );
-    }
-    const valueType = inferType(value[0], path, opts);
-    if (valueType === undefined) {
-      return undefined;
-    }
-    // Try to automatically detect embedding columns.
-    if (nameSuggestsVectorColumn(path[path.length - 1])) {
-      // Check if value is a Uint8Array for integer vector type determination
-      if (value instanceof Uint8Array) {
-        // For integer vectors, we default to Uint8 (matching Python implementation)
-        const child = new Field("item", new Uint8(), true);
-        return new FixedSizeList(value.length, child);
-      } else {
-        // For float vectors, we default to Float32
-        const child = new Field("item", new Float32(), true);
-        return new FixedSizeList(value.length, child);
-      }
-    } else {
-      const child = new Field("item", valueType, true);
-      return new List(child);
-    }
-  } else {
-    // TODO: timestamp
-    return undefined;
-  }
-}
-
-class PathTree<V> {
-  map: Map<string, V | PathTree<V>>;
-
-  constructor(entries?: [string[], V][]) {
-    this.map = new Map();
-    if (entries !== undefined) {
-      for (const [path, value] of entries) {
-        this.set(path, value);
-      }
-    }
-  }
-  has(path: string[]): boolean {
-    let ref: PathTree<V> = this;
-    for (const part of path) {
-      if (!(ref instanceof PathTree) || !ref.map.has(part)) {
-        return false;
-      }
-      ref = ref.map.get(part) as PathTree<V>;
-    }
-    return true;
-  }
-  get(path: string[]): V | undefined {
-    let ref: PathTree<V> = this;
-    for (const part of path) {
-      if (!(ref instanceof PathTree) || !ref.map.has(part)) {
-        return undefined;
-      }
-      ref = ref.map.get(part) as PathTree<V>;
-    }
-    return ref as V;
-  }
-  set(path: string[], value: V): void {
-    let ref: PathTree<V> = this;
-    for (const part of path.slice(0, path.length - 1)) {
-      if (!ref.map.has(part)) {
-        ref.map.set(part, new PathTree<V>());
-      }
-      ref = ref.map.get(part) as PathTree<V>;
-    }
-    ref.map.set(path[path.length - 1], value);
-  }
+  return current;
 }
 
 function transposeData(
@@ -720,37 +479,26 @@ function transposeData(
   field: Field,
   path: string[] = [],
 ): Vector {
+  const valuesPath = [...path, field.name];
+  const values = data.map((datum) => valueAtPath(datum, valuesPath));
   if (field.type instanceof Struct) {
     const childFields = field.type.children;
-    const fullPath = [...path, field.name];
     const childVectors = childFields.map((child) => {
-      return transposeData(data, child, fullPath);
+      return transposeData(data, child, valuesPath);
     });
+    const nullCount = values.filter((value) => value === null).length;
     const structData = makeData({
       type: field.type,
+      length: values.length,
+      nullCount,
+      nullBitmap:
+        nullCount > 0
+          ? arrowUtil.packBools(values.map((value) => value !== null))
+          : undefined,
       children: childVectors as unknown as ArrowData<DataType>[],
     });
     return arrowMakeVector(structData);
   } else {
-    const valuesPath = [...path, field.name];
-    const values = data.map((datum) => {
-      let current: unknown = datum;
-      for (const key of valuesPath) {
-        if (current == null) {
-          return null;
-        }
-
-        if (
-          isObject(current) &&
-          (Object.hasOwn(current, key) || key in current)
-        ) {
-          current = current[key];
-        } else {
-          return null;
-        }
-      }
-      return current;
-    });
     return makeVector(values, field.type, undefined, field.nullable);
   }
 }
@@ -793,32 +541,6 @@ function makeListVector(lists: unknown[][]): Vector<unknown> {
   return listBuilder.finish().toVector();
 }
 
-/**
- * Map a JS TypedArray instance to the corresponding Arrow element DataType
- * and its length. Returns undefined if the value is not a recognized TypedArray.
- */
-function typedArrayToArrowType(
-  value: ArrayBufferView,
-): { elementType: DataType; length: number } | undefined {
-  if (value instanceof Float32Array)
-    return { elementType: new Float32(), length: value.length };
-  if (value instanceof Float64Array)
-    return { elementType: new Float64(), length: value.length };
-  if (value instanceof Uint8Array)
-    return { elementType: new Uint8(), length: value.length };
-  if (value instanceof Uint16Array)
-    return { elementType: new Uint16(), length: value.length };
-  if (value instanceof Uint32Array)
-    return { elementType: new Uint32(), length: value.length };
-  if (value instanceof Int8Array)
-    return { elementType: new Int8(), length: value.length };
-  if (value instanceof Int16Array)
-    return { elementType: new Int16(), length: value.length };
-  if (value instanceof Int32Array)
-    return { elementType: new Int32(), length: value.length };
-  return undefined;
-}
-
 /** Helper function to convert an Array of JS values to an Arrow Vector */
 function makeVector(
   values: unknown[],
@@ -933,7 +655,7 @@ async function applyEmbeddingsFromMetadata(
 
   for (const functionEntry of functions.values()) {
     const sourceColumn = columns[functionEntry.sourceColumn];
-    const destColumn = functionEntry.vectorColumn ?? "vector";
+    const destColumn = functionEntry.vectorColumn;
     if (sourceColumn === undefined) {
       throw new Error(
         `Cannot apply embedding function because the source column '${functionEntry.sourceColumn}' was not present in the data`,
@@ -1385,11 +1107,10 @@ function validateSchemaEmbeddings(
 
         // Check schema metadata for embedding functions
         if (schema.metadata.has("embedding_functions")) {
-          const embeddings = JSON.parse(
+          const entries = parseEmbeddingMetadata(
             schema.metadata.get("embedding_functions")!,
           );
-          // biome-ignore lint/suspicious/noExplicitAny: we don't know the type of `f`
-          if (embeddings.find((f: any) => f["vectorColumn"] === field.name)) {
+          if (entries.some((f) => f.vectorColumn === field.name)) {
             hasEmbeddingFunction = true;
           }
         }
@@ -1459,8 +1180,12 @@ export function ensureNestedFieldsExist(
           completeRow[field.name] = row[field.name];
         }
       } else {
-        // Field is missing from the data - set to null
-        completeRow[field.name] = null;
+        // Keep a missing struct valid while filling each of its children with
+        // null. This is distinct from an explicitly null struct value.
+        completeRow[field.name] =
+          field.type.constructor.name === "Struct"
+            ? ensureStructFieldsExist({}, field.type as Struct)
+            : null;
       }
     }
 
@@ -1495,8 +1220,12 @@ function ensureStructFieldsExist(
         completeStruct[childField.name] = data[childField.name];
       }
     } else {
-      // Field is missing - set to null
-      completeStruct[childField.name] = null;
+      // Keep a missing struct valid while filling each of its children with
+      // null. This is distinct from an explicitly null struct value.
+      completeStruct[childField.name] =
+        childField.type.constructor.name === "Struct"
+          ? ensureStructFieldsExist({}, childField.type as Struct)
+          : null;
     }
   }
 
diff --git a/nodejs/lancedb/arrow_type.ts b/nodejs/lancedb/arrow_type.ts
new file mode 100644
index 000000000..35da346ef
--- /dev/null
+++ b/nodejs/lancedb/arrow_type.ts
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import {
+  type DataType,
+  Float32,
+  Float64,
+  Int8,
+  Int16,
+  Int32,
+  Uint8,
+  Uint16,
+  Uint32,
+} from "apache-arrow";
+
+/**
+ * Map a JS TypedArray instance to the corresponding Arrow element type and
+ * length. Returns undefined when the view is not a supported TypedArray.
+ */
+export function typedArrayToArrowType(
+  value: ArrayBufferView,
+): { elementType: DataType; length: number } | undefined {
+  if (value instanceof Float32Array)
+    return { elementType: new Float32(), length: value.length };
+  if (value instanceof Float64Array)
+    return { elementType: new Float64(), length: value.length };
+  if (value instanceof Uint8Array)
+    return { elementType: new Uint8(), length: value.length };
+  if (value instanceof Uint16Array)
+    return { elementType: new Uint16(), length: value.length };
+  if (value instanceof Uint32Array)
+    return { elementType: new Uint32(), length: value.length };
+  if (value instanceof Int8Array)
+    return { elementType: new Int8(), length: value.length };
+  if (value instanceof Int16Array)
+    return { elementType: new Int16(), length: value.length };
+  if (value instanceof Int32Array)
+    return { elementType: new Int32(), length: value.length };
+  return undefined;
+}
diff --git a/nodejs/lancedb/connection.ts b/nodejs/lancedb/connection.ts
index e63a7ae65..263a338ab 100644
--- a/nodejs/lancedb/connection.ts
+++ b/nodejs/lancedb/connection.ts
@@ -16,6 +16,12 @@ import {
   makeEmptyTable,
 } from "./arrow";
 import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
+import {
+  MaterializedView,
+  MaterializedViewSelect,
+  normalizeSelect,
+  validateNonNegativeInteger,
+} from "./materialized_view";
 import { Connection as LanceDbConnection } from "./native";
 import type {
   CreateNamespaceResponse,
@@ -25,12 +31,14 @@ import type {
   JobDescription,
   JobInfo,
   ListNamespacesResponse,
+  ListTablesResponse,
 } from "./native";
 export type {
   CreateNamespaceResponse,
   DescribeNamespaceResponse,
   DropNamespaceResponse,
   ListNamespacesResponse,
+  ListTablesResponse,
 };
 import { sanitizeTable } from "./sanitize";
 import { LocalTable, Table } from "./table";
@@ -128,6 +136,10 @@ export interface OpenTableOptions {
   indexCacheSize?: number;
 }
 
+/**
+ * @deprecated Use {@link ListTablesOptions} with {@link Connection.listTables}
+ * instead.
+ */
 export interface TableNamesOptions {
   /**
    * If present, only return names that come lexicographically after the
@@ -141,6 +153,24 @@ export interface TableNamesOptions {
   limit?: number;
 }
 
+export interface ListTablesOptions {
+  /**
+   * Token from a previous response, to resume listing where it left off.
+   *
+   * The token is opaque: it carries whatever the database needs to resume, and
+   * callers should not construct or interpret one.
+   */
+  pageToken?: string;
+  /**
+   * An upper bound on how many tables to return.
+   *
+   * A page may hold fewer than this and still not be the last one, so keep
+   * going while the response carries a page token rather than while pages are
+   * full.
+   */
+  limit?: number;
+}
+
 export interface ListNamespacesOptions {
   /** Token from a previous response for pagination. */
   pageToken?: string;
@@ -225,6 +255,7 @@ export abstract class Connection {
    * @param {Partial<TableNamesOptions>} options - options to control the
    * paging / start point (backwards compatibility)
    *
+   * @deprecated Use {@link Connection.listTables} instead.
    */
   abstract tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
   /**
@@ -235,18 +266,94 @@ export abstract class Connection {
    * @param {Partial<TableNamesOptions>} options - options to control the
    * paging / start point
    *
+   * @deprecated Use {@link Connection.listTables} instead.
    */
   abstract tableNames(
     namespacePath?: string[],
     options?: Partial<TableNamesOptions>,
   ): Promise<string[]>;
 
+  /**
+   * List a page of the tables in this database.
+   *
+   * To retrieve the tables after the page, pass the `pageToken` the response
+   * carries back in. A page can be shorter than `limit` without being the last
+   * one, so walk until a response carries no page token:
+   *
+   * ```ts
+   * const names = [];
+   * let pageToken = undefined;
+   * do {
+   *   const page = await conn.listTables({ pageToken, limit: 100 });
+   *   names.push(...page.tables);
+   *   pageToken = page.pageToken;
+   * } while (pageToken);
+   * ```
+   *
+   * @param {Partial<ListTablesOptions>} options - Pagination options
+   *   (`pageToken`, `limit`).
+   * @returns {Promise<ListTablesResponse>} A page of table names and an
+   *   optional token for the tables after it.
+   */
+  abstract listTables(
+    options?: Partial<ListTablesOptions>,
+  ): Promise<ListTablesResponse>;
+  /**
+   * List a page of the tables in this database.
+   *
+   * @param {string[]} namespacePath - The namespace path to list tables from
+   *   (defaults to root namespace)
+   * @param {Partial<ListTablesOptions>} options - Pagination options
+   *   (`pageToken`, `limit`).
+   * @returns {Promise<ListTablesResponse>} A page of table names and an
+   *   optional token for the tables after it.
+   */
+  abstract listTables(
+    namespacePath?: string[],
+    options?: Partial<ListTablesOptions>,
+  ): Promise<ListTablesResponse>;
+
   /**
    * Open a table in the database.
    * @param {string} name - The name of the table
    * @param {string[]} namespacePath - The namespace path of the table (defaults to root namespace)
    * @param {Partial<OpenTableOptions>} options - Additional options
    */
+  /**
+   * Define a materialized view named `name` over the table `source`.
+   *
+   * The view is created empty, with the query recorded in its schema
+   * metadata; `view.refresh()` computes the rows. The view is a normal
+   * table: it can be queried, indexed and searched, and it appears in
+   * `tableNames`. The source table must have stable row ids (create it with
+   * the `newTableEnableStableRowIds` storage option); they keep the view's
+   * provenance valid across source compactions and cannot be enabled after
+   * a table exists. Local databases only.
+   */
+  abstract createMaterializedView(
+    name: string,
+    source: string,
+    options?: {
+      select?: MaterializedViewSelect;
+      where?: string;
+      limit?: number;
+    },
+  ): Promise<MaterializedView>;
+
+  /**
+   * Open the materialized view named `name`.
+   *
+   * Rejects a table that exists but is not a materialized view.
+   */
+  abstract openMaterializedView(name: string): Promise<MaterializedView>;
+
+  /**
+   * The names of the materialized views in this database.
+   *
+   * Found by reading every table's schema, so this costs an open per table.
+   */
+  abstract listMaterializedViews(): Promise<string[]>;
+
   abstract openTable(
     name: string,
     namespacePath?: string[],
@@ -327,6 +434,14 @@ export abstract class Connection {
    */
   abstract dropTable(name: string, namespacePath?: string[]): Promise<void>;
 
+  /**
+   * Start dropping a table and return its cleanup job.
+   *
+   * The table may become unavailable before its data files are removed. Wait
+   * on the returned job to know when cleanup has finished.
+   */
+  abstract dropTableAsync(name: string, namespacePath?: string[]): Promise<Job>;
+
   /**
    * Drop all tables in the database.
    * @param {string[]} namespacePath The namespace path to drop tables from (defaults to root namespace).
@@ -523,6 +638,54 @@ export class LocalConnection extends Connection {
     );
   }
 
+  async createMaterializedView(
+    name: string,
+    source: string,
+    options?: {
+      select?: MaterializedViewSelect;
+      where?: string;
+      limit?: number;
+    },
+  ): Promise<MaterializedView> {
+    validateNonNegativeInteger(options?.limit, "limit");
+    const innerTable = await this.inner.createMaterializedView(
+      name,
+      source,
+      normalizeSelect(options?.select),
+      options?.where,
+      options?.limit,
+    );
+    return new MaterializedView(new LocalTable(innerTable));
+  }
+
+  async openMaterializedView(name: string): Promise<MaterializedView> {
+    const innerTable = await this.inner.openMaterializedView(name);
+    return new MaterializedView(new LocalTable(innerTable));
+  }
+
+  async listMaterializedViews(): Promise<string[]> {
+    return await this.inner.listMaterializedViews();
+  }
+
+  async listTables(
+    namespacePathOrOptions?: string[] | Partial<ListTablesOptions>,
+    options?: Partial<ListTablesOptions>,
+  ): Promise<ListTablesResponse> {
+    // Detect if first argument is namespacePath array or options object
+    const namespacePath = Array.isArray(namespacePathOrOptions)
+      ? namespacePathOrOptions
+      : undefined;
+    const listTablesOptions = Array.isArray(namespacePathOrOptions)
+      ? options
+      : namespacePathOrOptions;
+
+    return this.inner.listTables(
+      namespacePath ?? [],
+      listTablesOptions?.pageToken,
+      listTablesOptions?.limit,
+    );
+  }
+
   async openTable(
     name: string,
     namespacePath?: string[],
@@ -705,6 +868,10 @@ export class LocalConnection extends Connection {
     return this.inner.dropTable(name, namespacePath ?? []);
   }
 
+  async dropTableAsync(name: string, namespacePath?: string[]): Promise<Job> {
+    return this.inner.dropTableAsync(name, namespacePath ?? []);
+  }
+
   async dropAllTables(namespacePath?: string[]): Promise<void> {
     return this.inner.dropAllTables(namespacePath ?? []);
   }
diff --git a/nodejs/lancedb/embedding/index.ts b/nodejs/lancedb/embedding/index.ts
index d0ffaec0d..d748e245a 100644
--- a/nodejs/lancedb/embedding/index.ts
+++ b/nodejs/lancedb/embedding/index.ts
@@ -4,7 +4,15 @@
 import { Field, Schema } from "../arrow";
 import { sanitizeType } from "../sanitize";
 import { EmbeddingFunction } from "./embedding_function";
-import { EmbeddingFunctionConfig, getRegistry } from "./registry";
+import {
+  EmbeddingFunctionConfig,
+  EmbeddingFunctionRegistry,
+  getRegistry as getGlobalRegistry,
+  registerBuiltIn,
+} from "./registry";
+
+type OpenAIModule = typeof import("./openai");
+type TransformersModule = typeof import("./transformers");
 
 export {
   FieldOptions,
@@ -14,7 +22,39 @@ export {
   EmbeddingFunctionConstructor,
 } from "./embedding_function";
 
-export * from "./registry";
+export {
+  EmbeddingFunctionRegistry,
+  parseEmbeddingMetadata,
+  register,
+} from "./registry";
+export type {
+  CreateReturnType,
+  EmbeddingFunctionConfig,
+  EmbeddingFunctionCreate,
+  EmbeddingMetadataEntry,
+  ResolvedEmbeddingFunctionConfig,
+} from "./registry";
+
+function initializeBuiltInProviders() {
+  const { OpenAIEmbeddingFunction } = require("./openai") as OpenAIModule;
+  const { TransformersEmbeddingFunction } =
+    require("./transformers") as TransformersModule;
+
+  registerBuiltIn("openai", OpenAIEmbeddingFunction);
+  registerBuiltIn("huggingface", TransformersEmbeddingFunction);
+}
+
+/**
+ * Get the global embedding function registry.
+ *
+ * LanceDB built-in providers are initialized when this public API is first
+ * used, so importing the root package does not change automatic search
+ * selection for tables without embedding metadata.
+ */
+export function getRegistry(): EmbeddingFunctionRegistry {
+  initializeBuiltInProviders();
+  return getGlobalRegistry();
+}
 
 /**
  * Create a schema with embedding functions.
diff --git a/nodejs/lancedb/embedding/openai.ts b/nodejs/lancedb/embedding/openai.ts
index 5771cfeb5..2218d44bc 100644
--- a/nodejs/lancedb/embedding/openai.ts
+++ b/nodejs/lancedb/embedding/openai.ts
@@ -5,14 +5,13 @@ import type OpenAI from "openai";
 import type { EmbeddingCreateParams } from "openai/resources/index";
 import { Float, Float32 } from "../arrow";
 import { EmbeddingFunction } from "./embedding_function";
-import { register } from "./registry";
+import { registerBuiltIn } from "./registry";
 
 export type OpenAIOptions = {
   apiKey: string;
   model: EmbeddingCreateParams["model"];
 };
 
-@register("openai")
 export class OpenAIEmbeddingFunction extends EmbeddingFunction<
   string,
   Partial<OpenAIOptions>
@@ -100,3 +99,5 @@ export class OpenAIEmbeddingFunction extends EmbeddingFunction<
     return response.data[0].embedding;
   }
 }
+
+registerBuiltIn("openai", OpenAIEmbeddingFunction);
diff --git a/nodejs/lancedb/embedding/registry.ts b/nodejs/lancedb/embedding/registry.ts
index 2eae90ed3..c9ee9135e 100644
--- a/nodejs/lancedb/embedding/registry.ts
+++ b/nodejs/lancedb/embedding/registry.ts
@@ -7,6 +7,10 @@ import {
 } from "./embedding_function";
 import "reflect-metadata";
 
+const builtInFunctionsKey = Symbol.for(
+  "@lancedb/lancedb::embedding-built-in-functions::v1",
+);
+
 export type CreateReturnType<T> = T extends { init: () => Promise<void> }
   ? Promise<T>
   : T;
@@ -59,6 +63,15 @@ export class EmbeddingFunctionRegistry {
     };
   }
 
+  /** @ignore */
+  setBuiltIn<
+    T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
+  >(name: string, ctor: T): T {
+    this.#functions.set(name, ctor);
+    Reflect.defineMetadata("lancedb::embedding::name", name, ctor);
+    return ctor;
+  }
+
   get<T extends EmbeddingFunction<unknown>>(
     name: string,
   ): EmbeddingFunctionCreate<T> | undefined;
@@ -96,6 +109,7 @@ export class EmbeddingFunctionRegistry {
    */
   reset(this: EmbeddingFunctionRegistry) {
     this.#functions.clear();
+    getBuiltInFunctions(this).clear();
   }
 
   /**
@@ -104,41 +118,29 @@ export class EmbeddingFunctionRegistry {
   async parseFunctions(
     this: EmbeddingFunctionRegistry,
     metadata: Map<string, string>,
-  ): Promise<Map<string, EmbeddingFunctionConfig>> {
+  ): Promise<Map<string, ResolvedEmbeddingFunctionConfig>> {
     if (!metadata.has("embedding_functions")) {
       return new Map();
-    } else {
-      type FunctionConfig = {
-        name: string;
-        sourceColumn: string;
-        vectorColumn: string;
-        model: EmbeddingFunction["TOptions"];
-      };
-
-      const functions = <FunctionConfig[]>(
-        JSON.parse(metadata.get("embedding_functions")!)
-      );
-
-      const items: [string, EmbeddingFunctionConfig][] = await Promise.all(
-        functions.map(async (f) => {
-          const fn = this.get(f.name);
-          if (!fn) {
-            throw new Error(`Function "${f.name}" not found in registry`);
-          }
-          const func = await this.get(f.name)!.create(f.model);
-          return [
-            f.name,
-            {
-              sourceColumn: f.sourceColumn,
-              vectorColumn: f.vectorColumn,
-              function: func,
-            },
-          ];
-        }),
-      );
-
-      return new Map(items);
     }
+    const entries = parseEmbeddingMetadata(
+      metadata.get("embedding_functions")!,
+    );
+    const items = await Promise.all(
+      entries.map(async (f): Promise<ResolvedEmbeddingFunctionConfig> => {
+        const fn = this.get(f.name);
+        if (!fn) {
+          throw new Error(`Function "${f.name}" not found in registry`);
+        }
+        const func = await fn.create(f.model);
+        return {
+          sourceColumn: f.sourceColumn,
+          vectorColumn: f.vectorColumn,
+          function: func,
+        };
+      }),
+    );
+    // Keyed by output column: one function may serve several columns.
+    return new Map(items.map((config) => [config.vectorColumn, config]));
   }
   // biome-ignore lint/suspicious/noExplicitAny: <explanation>
   functionToMetadata(conf: EmbeddingFunctionConfig): Record<string, any> {
@@ -195,12 +197,56 @@ export class EmbeddingFunctionRegistry {
   }
 }
 
-const _REGISTRY = new EmbeddingFunctionRegistry();
+function getBuiltInFunctions(registry: EmbeddingFunctionRegistry): Set<string> {
+  const registryWithBuiltIns = registry as EmbeddingFunctionRegistry & {
+    [key: symbol]: Set<string> | undefined;
+  };
+  let builtInFunctions = registryWithBuiltIns[builtInFunctionsKey];
+  if (builtInFunctions === undefined) {
+    builtInFunctions = new Set<string>();
+    registryWithBuiltIns[builtInFunctionsKey] = builtInFunctions;
+  }
+  return builtInFunctions;
+}
+
+// Server bundlers can load the side-effect embedding entry points and the public
+// embedding API from separate module graphs. Keep their registry shared.
+const registryKey = Symbol.for(
+  "@lancedb/lancedb::embedding-function-registry::v1",
+);
+const registryGlobal = globalThis as typeof globalThis & {
+  [key: symbol]: EmbeddingFunctionRegistry | undefined;
+};
+
+function getGlobalRegistry(): EmbeddingFunctionRegistry {
+  const existingRegistry = registryGlobal[registryKey];
+  if (existingRegistry !== undefined) {
+    return existingRegistry;
+  }
+  const registry = new EmbeddingFunctionRegistry();
+  registryGlobal[registryKey] = registry;
+  return registry;
+}
+
+const _REGISTRY = getGlobalRegistry();
 
 export function register(name?: string) {
   return _REGISTRY.register(name);
 }
 
+/** @ignore */
+export function registerBuiltIn<
+  T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
+>(name: string, ctor: T): T {
+  const builtInFunctions = getBuiltInFunctions(_REGISTRY);
+  if (builtInFunctions.has(name)) {
+    return _REGISTRY.setBuiltIn(name, ctor);
+  }
+  _REGISTRY.register(name)(ctor);
+  builtInFunctions.add(name);
+  return ctor;
+}
+
 /**
  * Utility function to get the global instance of the registry
  * @returns `EmbeddingFunctionRegistry` The global instance of the registry
@@ -218,3 +264,52 @@ export interface EmbeddingFunctionConfig {
   vectorColumn?: string;
   function: EmbeddingFunction;
 }
+
+/** An [EmbeddingFunctionConfig] read back from table metadata, where the
+ * vector column is always recorded. */
+export type ResolvedEmbeddingFunctionConfig = EmbeddingFunctionConfig & {
+  vectorColumn: string;
+};
+
+/** One entry of the `embedding_functions` schema metadata, with the column
+ * keys normalized across the bindings' spellings. */
+export type EmbeddingMetadataEntry = {
+  name: string;
+  sourceColumn: string;
+  vectorColumn: string;
+  model: EmbeddingFunction["TOptions"];
+};
+
+/** The single parser for `embedding_functions` schema metadata: every reader
+ * goes through here, so the wire contract cannot fork between them. */
+export function parseEmbeddingMetadata(json: string): EmbeddingMetadataEntry[] {
+  // The wire format, honestly: the Python bindings write snake_case keys.
+  type Raw = {
+    name: string;
+    sourceColumn?: string;
+    // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+    source_column?: string;
+    vectorColumn?: string;
+    // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+    vector_column?: string;
+    model: EmbeddingFunction["TOptions"];
+  };
+  const entries = <Raw[]>JSON.parse(json);
+  const seen = new Set<string>();
+  return entries.map((f) => {
+    const sourceColumn = f.sourceColumn ?? f.source_column;
+    const vectorColumn = f.vectorColumn ?? f.vector_column;
+    if (sourceColumn === undefined || vectorColumn === undefined) {
+      throw new Error(
+        `Embedding function "${f.name}" metadata names no source or vector column`,
+      );
+    }
+    if (seen.has(vectorColumn)) {
+      throw new Error(
+        `Multiple embedding configs claim vector column "${vectorColumn}"`,
+      );
+    }
+    seen.add(vectorColumn);
+    return { name: f.name, sourceColumn, vectorColumn, model: f.model };
+  });
+}
diff --git a/nodejs/lancedb/embedding/transformers.ts b/nodejs/lancedb/embedding/transformers.ts
index 161575285..06043ea7c 100644
--- a/nodejs/lancedb/embedding/transformers.ts
+++ b/nodejs/lancedb/embedding/transformers.ts
@@ -3,7 +3,7 @@
 
 import { Float, Float32 } from "../arrow";
 import { EmbeddingFunction } from "./embedding_function";
-import { register } from "./registry";
+import { registerBuiltIn } from "./registry";
 
 export type XenovaTransformerOptions = {
   /** The wasm compatible model to use */
@@ -31,7 +31,6 @@ export type XenovaTransformerOptions = {
   };
 };
 
-@register("huggingface")
 export class TransformersEmbeddingFunction extends EmbeddingFunction<
   string,
   Partial<XenovaTransformerOptions>
@@ -158,6 +157,8 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
   }
 }
 
+registerBuiltIn("huggingface", TransformersEmbeddingFunction);
+
 const tensorDiv = (
   src: import("@huggingface/transformers").Tensor,
   divBy: number,
diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts
index 319222421..34d7ce4d9 100644
--- a/nodejs/lancedb/index.ts
+++ b/nodejs/lancedb/index.ts
@@ -21,6 +21,11 @@ import type { BaseTokenizer } from "./indices";
 import type { FtsToken } from "./table";
 
 // Re-export native header provider for use with connectWithHeaderProvider
+export {
+  MaterializedView,
+  MaterializedViewDefinition,
+  MaterializedViewSelect,
+} from "./materialized_view";
 export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
 
 // OpenTelemetry metrics bridge. Only the high-level entry point is public; the
@@ -50,6 +55,8 @@ export {
   MergeResult,
   AddResult,
   AddColumnsResult,
+  RefreshColumnResult,
+  RefreshMaterializedViewResult,
   AlterColumnsResult,
   UpdateFieldMetadataResult,
   DeleteResult,
@@ -74,11 +81,13 @@ export {
   Connection,
   CreateTableOptions,
   TableNamesOptions,
+  ListTablesOptions,
   OpenTableOptions,
   ListNamespacesOptions,
   CreateNamespaceOptions,
   DropNamespaceOptions,
   ListNamespacesResponse,
+  ListTablesResponse,
   CreateNamespaceResponse,
   DropNamespaceResponse,
   DescribeNamespaceResponse,
@@ -94,6 +103,7 @@ export {
 } from "./native.js";
 
 export {
+  AutoQuery,
   ExecutableQuery,
   Query,
   QueryBase,
@@ -134,10 +144,10 @@ export {
   BranchColumnChange,
   BranchIndexSummary,
   BranchRowCountSummary,
-  MergeBlocker,
+  CherryPickError,
   BranchDiff,
-  MergePreview,
-  MergeBranchResult,
+  CherryPickPreview,
+  CherryPickResult,
   AddDataOptions,
   UpdateOptions,
   OptimizeOptions,
@@ -146,6 +156,10 @@ export {
   FtsToken,
   TokenizeTableOptions,
   LsmWriteSpec,
+  LsmStats,
+  BucketStats,
+  GenerationStats,
+  MemtableStats,
   ColumnAlteration,
   FieldMetadataUpdate,
 } from "./table";
diff --git a/nodejs/lancedb/materialized_view.ts b/nodejs/lancedb/materialized_view.ts
new file mode 100644
index 000000000..b26dee59e
--- /dev/null
+++ b/nodejs/lancedb/materialized_view.ts
@@ -0,0 +1,161 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import { RefreshMaterializedViewResult } from "./native";
+import { Table } from "./table";
+
+/** Schema metadata key holding a materialized view's definition. */
+export const DEFINITION_META_KEY = "mv.definition";
+
+/** The query that defines a materialized view. */
+export interface MaterializedViewDefinition {
+  /** Name of the source table, in the same database as the view. */
+  sourceTable: string;
+  /** `[output column, SQL expression]` pairs, in view schema order. */
+  projections: [string, string][];
+  /** SQL predicate selecting the source rows the view holds. */
+  filter?: string;
+  /** Cap on the number of rows the view holds. */
+  limit?: number;
+  /** Source columns the projections and filter read. */
+  inputs: string[];
+}
+
+/**
+ * The view's columns: column names, `[alias, SQL expression]` pairs, or a
+ * record of the same. A bare name projects itself.
+ */
+export type MaterializedViewSelect =
+  | (string | [string, string])[]
+  | Record<string, string>;
+
+/**
+ * @internal Reject a numeric option N-API would otherwise silently coerce:
+ * `Infinity` reaches Rust as 0, `1.5` as 1.
+ */
+export function validateNonNegativeInteger(
+  value: number | undefined,
+  name: string,
+): void {
+  if (value !== undefined && !(Number.isSafeInteger(value) && value >= 0)) {
+    throw new Error(`${name} must be a non-negative integer`);
+  }
+}
+
+/** @internal Quote a column name as a Lance SQL identifier (backticks). */
+function quoteIdentifier(name: string): string {
+  return "`" + name.replace(/`/g, "``") + "`";
+}
+
+/**
+ * @internal Normalize a select argument into `[alias, expression]` pairs.
+ * A bare name projects itself and is quoted, so any valid column name works;
+ * pair and record entries are kept verbatim because their right side is an
+ * expression.
+ */
+export function normalizeSelect(
+  select?: MaterializedViewSelect,
+): [string, string][] | undefined {
+  if (select === undefined) {
+    return undefined;
+  }
+  if (Array.isArray(select)) {
+    return select.map((item) =>
+      typeof item === "string" ? [item, quoteIdentifier(item)] : item,
+    );
+  }
+  return Object.entries(select);
+}
+
+/** @internal Parse a definition off a table's stored schema metadata. */
+export function definitionFromMetadata(
+  metadata: Map<string, string>,
+  name: string,
+): MaterializedViewDefinition {
+  const raw = metadata.get(DEFINITION_META_KEY);
+  if (raw === undefined) {
+    throw new Error(`Table '${name}' is not a materialized view`);
+  }
+  // biome-ignore lint/suspicious/noExplicitAny: raw JSON
+  const value: any = JSON.parse(raw);
+  if (value.kind !== "select") {
+    throw new Error(
+      `materialized view '${name}' is defined by '${value.kind}', which this ` +
+        "version of lancedb cannot refresh",
+    );
+  }
+  const limit = value.limit ?? undefined;
+  // JSON.parse rounds integers past 2^53; every exact u64 parses to a safe
+  // integer and every rounded one does not, so this rejects precisely the
+  // values a number cannot carry.
+  if (limit !== undefined && !Number.isSafeInteger(limit)) {
+    throw new Error(
+      `materialized view '${name}' has a stored limit too large to represent exactly`,
+    );
+  }
+  return {
+    sourceTable: value.source_table,
+    // biome-ignore lint/suspicious/noExplicitAny: raw JSON
+    projections: (value.projections ?? []).map((p: any) => [
+      p.output,
+      p.expression,
+    ]),
+    filter: value.filter ?? undefined,
+    limit,
+    inputs: value.inputs ?? [],
+  };
+}
+
+/**
+ * A handle on a materialized view: its table plus its definition.
+ *
+ * Obtained from {@link Connection#createMaterializedView} or
+ * {@link Connection#openMaterializedView}. The view is a normal table --
+ * queries, indexes and search all apply through {@link MaterializedView#table}
+ * -- whose contents are maintained by {@link MaterializedView#refresh}.
+ */
+export class MaterializedView {
+  private readonly inner: Table;
+
+  constructor(table: Table) {
+    this.inner = table;
+  }
+
+  get name(): string {
+    return this.inner.name;
+  }
+
+  /** The view, as the table it is. */
+  table(): Table {
+    return this.inner;
+  }
+
+  /** The query that defines the view, read from its stored schema. */
+  async definition(): Promise<MaterializedViewDefinition> {
+    const schema = await this.inner.schema();
+    return definitionFromMetadata(schema.metadata, this.name);
+  }
+
+  /**
+   * Recompute the view from its source.
+   *
+   * The refresh is incremental when the source's changes can be reconciled
+   * into the view -- rows added, changed or removed since the last one --
+   * and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
+   * refreshes to that source version instead of the latest.
+   *
+   * Concurrent refreshes of one view do not duplicate its rows. Two that
+   * plan the same source rows conflict on commit, and the loser throws
+   * rather than writing them a second time.
+   */
+  async refresh(options?: {
+    full?: boolean;
+    sourceVersion?: number;
+  }): Promise<RefreshMaterializedViewResult> {
+    validateNonNegativeInteger(options?.sourceVersion, "sourceVersion");
+    return await this.inner.refreshMaterializedView(
+      options?.full,
+      options?.sourceVersion,
+    );
+  }
+}
diff --git a/nodejs/lancedb/query.ts b/nodejs/lancedb/query.ts
index eff712d9a..8532222f9 100644
--- a/nodejs/lancedb/query.ts
+++ b/nodejs/lancedb/query.ts
@@ -100,26 +100,6 @@ export interface FullTextSearchOptions {
   columns?: string | string[];
 }
 
-type NativeQueryLike = NativeQuery | NativeVectorQuery | NativeTakeQuery;
-
-class DeferredNativeQuery<NativeQueryType extends NativeQueryLike> {
-  protected readonly calls: Array<(inner: NativeQueryType) => void> = [];
-
-  constructor(private readonly factory: () => Promise<NativeQueryType>) {}
-
-  doCall(fn: (inner: NativeQueryType) => void) {
-    this.calls.push(fn);
-  }
-
-  async resolve(): Promise<NativeQueryType> {
-    const inner = await this.factory();
-    for (const call of this.calls) {
-      call(inner);
-    }
-    return inner;
-  }
-}
-
 function nearestToNative(
   inner: NativeQuery,
   vector: Awaited<IntoVector>,
@@ -154,24 +134,15 @@ export class QueryBase<
   NativeQueryType extends NativeQuery | NativeVectorQuery | NativeTakeQuery,
 > implements AsyncIterable<RecordBatch>
 {
-  /**
-   * @hidden
-   */
-  protected inner:
-    | NativeQueryType
-    | Promise<NativeQueryType>
-    | DeferredNativeQuery<NativeQueryType>;
+  protected inner!: NativeQueryType | Promise<NativeQueryType>;
 
   /**
    * @hidden
    */
-  protected constructor(
-    inner:
-      | NativeQueryType
-      | Promise<NativeQueryType>
-      | DeferredNativeQuery<NativeQueryType>,
-  ) {
-    this.inner = inner;
+  protected constructor(inner?: NativeQueryType | Promise<NativeQueryType>) {
+    if (inner !== undefined) {
+      this.inner = inner;
+    }
   }
 
   // call a function on the inner (either a promise or the actual object)
@@ -179,9 +150,7 @@ export class QueryBase<
    * @hidden
    */
   protected doCall(fn: (inner: NativeQueryType) => void) {
-    if (this.inner instanceof DeferredNativeQuery) {
-      this.inner.doCall(fn);
-    } else if (this.inner instanceof Promise) {
+    if (this.inner instanceof Promise) {
       this.inner = this.inner.then((inner) => {
         fn(inner);
         return inner;
@@ -192,12 +161,11 @@ export class QueryBase<
   }
 
   /**
+   * Return the native query used by the next terminal operation.
+   *
    * @hidden
    */
-  protected resolveInner(): NativeQueryType | Promise<NativeQueryType> {
-    if (this.inner instanceof DeferredNativeQuery) {
-      return this.inner.resolve();
-    }
+  protected async getInner(): Promise<NativeQueryType> {
     return this.inner;
   }
 
@@ -273,17 +241,11 @@ export class QueryBase<
   /**
    * @hidden
    */
-  protected nativeExecute(
+  protected async nativeExecute(
     options?: Partial<QueryExecutionOptions>,
   ): Promise<NativeBatchIterator> {
-    const inner = this.resolveInner();
-    if (inner instanceof Promise) {
-      return inner.then((inner) =>
-        inner.execute(options?.maxBatchLength, options?.timeoutMs),
-      );
-    } else {
-      return inner.execute(options?.maxBatchLength, options?.timeoutMs);
-    }
+    const inner = await this.getInner();
+    return inner.execute(options?.maxBatchLength, options?.timeoutMs);
   }
 
   /**
@@ -312,7 +274,7 @@ export class QueryBase<
   /** Collect the results as an Arrow @see {@link ArrowTable}. */
   async toArrow(options?: Partial<QueryExecutionOptions>): Promise<ArrowTable> {
     const batches = [];
-    const inner = await this.resolveInner();
+    const inner = await this.getInner();
     for await (const batch of new RecordBatchIterable(inner, options)) {
       batches.push(batch);
     }
@@ -341,12 +303,8 @@ export class QueryBase<
    * @returns A Promise that resolves to a string containing the query execution plan explanation.
    */
   async explainPlan(verbose = false): Promise<string> {
-    const inner = this.resolveInner();
-    if (inner instanceof Promise) {
-      return inner.then((inner) => inner.explainPlan(verbose));
-    } else {
-      return inner.explainPlan(verbose);
-    }
+    const inner = await this.getInner();
+    return inner.explainPlan(verbose);
   }
 
   /**
@@ -384,12 +342,8 @@ export class QueryBase<
     distributedMetrics?: AnalyzePlanDistributedMetrics,
   ): Promise<string> {
     const distributedMetricsMode = distributedMetrics ?? "aggregate";
-    const inner = this.resolveInner();
-    if (inner instanceof Promise) {
-      return inner.then((inner) => inner.analyzePlan(distributedMetricsMode));
-    } else {
-      return inner.analyzePlan(distributedMetricsMode);
-    }
+    const inner = await this.getInner();
+    return inner.analyzePlan(distributedMetricsMode);
   }
 
   /**
@@ -401,13 +355,8 @@ export class QueryBase<
    * @returns An Arrow Schema describing the output columns.
    */
   async outputSchema(): Promise<import("./arrow").Schema> {
-    let schemaBuffer: Buffer;
-    const inner = this.resolveInner();
-    if (inner instanceof Promise) {
-      schemaBuffer = await inner.then((inner) => inner.outputSchema());
-    } else {
-      schemaBuffer = await inner.outputSchema();
-    }
+    const inner = await this.getInner();
+    const schemaBuffer = await inner.outputSchema();
     const schema = tableFromIPC(schemaBuffer).schema;
     return schema;
   }
@@ -419,12 +368,7 @@ export class StandardQueryBase<
   extends QueryBase<NativeQueryType>
   implements ExecutableQuery
 {
-  constructor(
-    inner:
-      | NativeQueryType
-      | Promise<NativeQueryType>
-      | DeferredNativeQuery<NativeQueryType>,
-  ) {
+  constructor(inner?: NativeQueryType | Promise<NativeQueryType>) {
     super(inner);
   }
 
@@ -574,12 +518,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
   /**
    * @hidden
    */
-  constructor(
-    inner:
-      | NativeVectorQuery
-      | Promise<NativeVectorQuery>
-      | DeferredNativeQuery<NativeVectorQuery>,
-  ) {
+  constructor(inner: NativeVectorQuery | Promise<NativeVectorQuery>) {
     super(inner);
   }
 
@@ -797,7 +736,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
   addQueryVector(vector: IntoVector): VectorQuery {
     if (vector instanceof Promise) {
       const res = (async () => {
-        const inner = (await this.resolveInner()) as NativeVectorQuery;
+        const inner = await this.getInner();
         addQueryVectorToNative(inner, await vector);
         return inner;
       })();
@@ -828,70 +767,6 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
   }
 }
 
-type AutoQueryResolution = {
-  inner: NativeQuery | NativeVectorQuery;
-  route: "fts" | "vector";
-};
-
-class DeferredAutoNativeQuery extends DeferredNativeQuery<NativeVectorQuery> {
-  private readonly vectorCalls: Array<
-    (inner: NativeVectorQuery) => void | Promise<void>
-  > = [];
-
-  constructor(
-    private readonly autoFactory: () => Promise<AutoQueryResolution>,
-  ) {
-    super(async () => (await autoFactory()).inner as NativeVectorQuery);
-  }
-
-  doVectorCall(fn: (inner: NativeVectorQuery) => void | Promise<void>) {
-    this.vectorCalls.push(fn);
-  }
-
-  async resolve(): Promise<NativeVectorQuery> {
-    const resolution = await this.autoFactory();
-    for (const call of this.calls) {
-      call(resolution.inner as NativeVectorQuery);
-    }
-    if (resolution.route === "vector") {
-      for (const call of this.vectorCalls) {
-        await call(resolution.inner as NativeVectorQuery);
-      }
-    }
-    return resolution.inner as NativeVectorQuery;
-  }
-}
-
-class DeferredAutoQuery extends VectorQuery {
-  constructor(private readonly deferred: DeferredAutoNativeQuery) {
-    super(deferred);
-  }
-
-  protected doVectorCall(fn: (inner: NativeVectorQuery) => void) {
-    this.deferred.doVectorCall(fn);
-  }
-
-  addQueryVector(vector: IntoVector): VectorQuery {
-    // Observe promised vectors immediately so a rejection cannot become an
-    // unhandled rejection while auto routing is still resolving (or when the
-    // eventual route is FTS and vector-only calls are intentionally skipped).
-    // The settled outcome remains fulfilled and is rethrown only if a vector
-    // execution actually consumes it.
-    const settledVector = Promise.resolve(vector).then(
-      (value) => ({ status: "fulfilled" as const, value }),
-      (reason) => ({ status: "rejected" as const, reason }),
-    );
-    this.deferred.doVectorCall(async (inner) => {
-      const outcome = await settledVector;
-      if (outcome.status === "rejected") {
-        throw outcome.reason;
-      }
-      addQueryVectorToNative(inner, outcome.value);
-    });
-    return this;
-  }
-}
-
 /**
  * Create a string query whose vector/FTS routing is resolved against the active
  * table schema when the query executes.
@@ -903,7 +778,7 @@ export function createAutoQuery(
   query: string,
   columns: string[] | null,
   getVector: (metadata: string) => Promise<Awaited<IntoVector>>,
-): VectorQuery {
+): AutoQuery {
   type RouteSnapshot = {
     table: NativeTable;
     embeddingMetadata: string | undefined;
@@ -911,7 +786,6 @@ export function createAutoQuery(
   type CachedPreparation = {
     metadata: string;
     vector: Promise<Awaited<IntoVector>>;
-    settled: boolean;
   };
 
   let cachedPreparation: CachedPreparation | undefined;
@@ -925,75 +799,37 @@ export function createAutoQuery(
     };
   };
 
-  const deferred = new DeferredAutoNativeQuery(async () => {
-    while (true) {
-      const initial = await snapshotRoute();
-      if (initial.embeddingMetadata === undefined) {
-        const inner = initial.table.query();
-        inner.fullTextSearch({ query, columns });
-        return { inner, route: "fts" };
-      }
+  const createInner = async (): Promise<NativeQuery | NativeVectorQuery> => {
+    const route = await snapshotRoute();
+    if (route.embeddingMetadata === undefined) {
+      const inner = route.table.query();
+      inner.fullTextSearch({ query, columns });
+      return inner;
+    }
 
-      const metadata = initial.embeddingMetadata;
-      if (cachedPreparation?.metadata !== metadata) {
-        const vector = Promise.resolve().then(() => getVector(metadata));
-        const preparation: CachedPreparation = {
-          metadata,
-          settled: false,
-          vector,
-        };
-        void vector.then(
-          () => {
-            preparation.settled = true;
-          },
-          () => {
-            preparation.settled = true;
-          },
-        );
-        cachedPreparation = preparation;
-      }
-
-      const preparation = cachedPreparation;
-      const preparationWasWarm = preparation.settled;
-      let vector: Awaited<IntoVector>;
-      try {
-        vector = await preparation.vector;
-      } catch (error) {
-        if (cachedPreparation === preparation) {
-          cachedPreparation = undefined;
-        }
-        throw error;
-      }
-
-      // Newly started (or still in-flight) provider preparation can perform
-      // arbitrary asynchronous work. Revalidate after that work, but reuse the
-      // initial pinned snapshot once preparation was already warm. This avoids
-      // a second remote describe request on every repeated execution.
-      if (!preparationWasWarm) {
-        const current = await snapshotRoute();
-        if (current.embeddingMetadata !== metadata) {
-          // A stale execution must not erase another revision's newer in-flight
-          // preparation.
-          if (cachedPreparation === preparation) {
-            cachedPreparation = undefined;
-          }
-          continue;
-        }
-
-        return {
-          inner: nearestToNative(current.table.query(), vector),
-          route: "vector",
-        };
-      }
-
-      return {
-        inner: nearestToNative(initial.table.query(), vector),
-        route: "vector",
+    const metadata = route.embeddingMetadata;
+    if (cachedPreparation?.metadata !== metadata) {
+      cachedPreparation = {
+        metadata,
+        vector: Promise.resolve().then(() => getVector(metadata)),
       };
     }
-  });
 
-  return new DeferredAutoQuery(deferred);
+    const preparation = cachedPreparation;
+    let vector: Awaited<IntoVector>;
+    try {
+      vector = await preparation.vector;
+    } catch (error) {
+      if (cachedPreparation === preparation) {
+        cachedPreparation = undefined;
+      }
+      throw error;
+    }
+
+    return nearestToNative(route.table.query(), vector);
+  };
+
+  return new AutoQuery(createInner);
 }
 
 /**
@@ -1021,6 +857,51 @@ export class TakeQuery extends QueryBase<NativeTakeQuery> {
   }
 }
 
+/**
+ * A builder for automatic string searches.
+ *
+ * Automatic search determines whether to use full-text or vector search from
+ * the table revision selected for each execution. This builder exposes the
+ * common operations supported by both query families.
+ *
+ * @hideconstructor
+ */
+export class AutoQuery extends StandardQueryBase<
+  NativeQuery | NativeVectorQuery
+> {
+  private readonly calls: Array<
+    (inner: NativeQuery | NativeVectorQuery) => void
+  > = [];
+
+  /** @hidden */
+  constructor(
+    private readonly createInner: () => Promise<
+      NativeQuery | NativeVectorQuery
+    >,
+  ) {
+    super();
+  }
+
+  /** @hidden */
+  protected override doCall(
+    fn: (inner: NativeQuery | NativeVectorQuery) => void,
+  ) {
+    this.calls.push(fn);
+  }
+
+  /** @hidden */
+  protected override async getInner(): Promise<
+    NativeQuery | NativeVectorQuery
+  > {
+    const calls = [...this.calls];
+    const inner = await this.createInner();
+    for (const call of calls) {
+      call(inner);
+    }
+    return inner;
+  }
+}
+
 /** A builder for LanceDB queries.
  *
  * @see {@link Table#query}, {@link Table#search}
@@ -1073,33 +954,19 @@ export class Query extends StandardQueryBase<NativeQuery> {
    * a default `limit` of 10 will be used.  @see {@link Query#limit}
    */
   nearestTo(vector: IntoVector): VectorQuery {
-    const inner = this.resolveInner();
+    const inner = this.inner;
     if (inner instanceof Promise) {
-      const nativeQuery = inner.then(async (inner) => {
-        const resolved = vector instanceof Promise ? await vector : vector;
-        return nearestToNative(inner, resolved);
-      });
+      const nativeQuery = inner.then(async (resolvedInner) =>
+        nearestToNative(resolvedInner, await vector),
+      );
       return new VectorQuery(nativeQuery);
     }
     if (vector instanceof Promise) {
-      const res = (async () => {
-        try {
-          const v = await vector;
-          // biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
-          const value: any = this.nearestTo(v);
-          const inner = value.inner as
-            | NativeVectorQuery
-            | Promise<NativeVectorQuery>;
-          return inner;
-        } catch (e) {
-          return Promise.reject(e);
-        }
-      })();
-      return new VectorQuery(res);
-    } else {
-      const vectorQuery = nearestToNative(inner, vector);
-      return new VectorQuery(vectorQuery);
+      return new VectorQuery(
+        vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)),
+      );
     }
+    return new VectorQuery(nearestToNative(inner, vector));
   }
 
   nearestToText(query: string | FullTextQuery, columns?: string[]): Query {
diff --git a/nodejs/lancedb/sanitize.ts b/nodejs/lancedb/sanitize.ts
index ae0bc0179..8fb2f1a0a 100644
--- a/nodejs/lancedb/sanitize.ts
+++ b/nodejs/lancedb/sanitize.ts
@@ -9,7 +9,7 @@
 // comes from the exact same library instance.  This is not always the case
 // and so we must sanitize the input to ensure that it is compatible.
 
-import { BufferType, Data } from "apache-arrow";
+import { BufferType, Data, Vector } from "apache-arrow";
 import type { IntBitWidth, TKeys, TimeBitWidth } from "apache-arrow/type";
 import {
   Binary,
@@ -74,6 +74,20 @@ import {
   Utf8,
 } from "./arrow";
 
+type SanitizationContext = {
+  types: WeakMap<object, DataType>;
+  vectors: WeakMap<object, Vector>;
+  data: WeakMap<object, Data<DataType>>;
+};
+
+function createSanitizationContext(): SanitizationContext {
+  return {
+    types: new WeakMap(),
+    vectors: new WeakMap(),
+    data: new WeakMap(),
+  };
+}
+
 export function sanitizeMetadata(
   metadataLike?: unknown,
 ): Map<string, string> | undefined {
@@ -186,6 +200,13 @@ export function sanitizeInterval(typeLike: object) {
 }
 
 export function sanitizeList(typeLike: object) {
+  return sanitizeListWithContext(typeLike, createSanitizationContext());
+}
+
+function sanitizeListWithContext(
+  typeLike: object,
+  context: SanitizationContext,
+) {
   if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
     throw Error(
       "Expected a List type to have an array-like `children` property",
@@ -194,19 +215,35 @@ export function sanitizeList(typeLike: object) {
   if (typeLike.children.length !== 1) {
     throw Error("Expected a List type to have exactly one child");
   }
-  return new List(sanitizeField(typeLike.children[0]));
+  return new List(sanitizeFieldWithContext(typeLike.children[0], context));
 }
 
 export function sanitizeStruct(typeLike: object) {
+  return sanitizeStructWithContext(typeLike, createSanitizationContext());
+}
+
+function sanitizeStructWithContext(
+  typeLike: object,
+  context: SanitizationContext,
+) {
   if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
     throw Error(
       "Expected a Struct type to have an array-like `children` property",
     );
   }
-  return new Struct(typeLike.children.map((child) => sanitizeField(child)));
+  return new Struct(
+    typeLike.children.map((child) => sanitizeFieldWithContext(child, context)),
+  );
 }
 
 export function sanitizeUnion(typeLike: object) {
+  return sanitizeUnionWithContext(typeLike, createSanitizationContext());
+}
+
+function sanitizeUnionWithContext(
+  typeLike: object,
+  context: SanitizationContext,
+) {
   if (
     !("typeIds" in typeLike) ||
     !("mode" in typeLike) ||
@@ -226,7 +263,7 @@ export function sanitizeUnion(typeLike: object) {
     typeLike.mode,
     // biome-ignore lint/suspicious/noExplicitAny: skip
     typeLike.typeIds as any,
-    typeLike.children.map((child) => sanitizeField(child)),
+    typeLike.children.map((child) => sanitizeFieldWithContext(child, context)),
   );
 }
 
@@ -234,6 +271,19 @@ export function sanitizeTypedUnion(
   typeLike: object,
   // eslint-disable-next-line @typescript-eslint/naming-convention
   UnionType: typeof DenseUnion | typeof SparseUnion,
+) {
+  return sanitizeTypedUnionWithContext(
+    typeLike,
+    UnionType,
+    createSanitizationContext(),
+  );
+}
+
+function sanitizeTypedUnionWithContext(
+  typeLike: object,
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  UnionType: typeof DenseUnion | typeof SparseUnion,
+  context: SanitizationContext,
 ) {
   if (!("typeIds" in typeLike)) {
     throw Error(
@@ -248,7 +298,7 @@ export function sanitizeTypedUnion(
 
   return new UnionType(
     typeLike.typeIds as Int32Array | number[],
-    typeLike.children.map((child) => sanitizeField(child)),
+    typeLike.children.map((child) => sanitizeFieldWithContext(child, context)),
   );
 }
 
@@ -262,6 +312,16 @@ export function sanitizeFixedSizeBinary(typeLike: object) {
 }
 
 export function sanitizeFixedSizeList(typeLike: object) {
+  return sanitizeFixedSizeListWithContext(
+    typeLike,
+    createSanitizationContext(),
+  );
+}
+
+function sanitizeFixedSizeListWithContext(
+  typeLike: object,
+  context: SanitizationContext,
+) {
   if (!("listSize" in typeLike) || typeof typeLike.listSize !== "number") {
     throw Error("Expected a FixedSizeList type to have a `listSize` property");
   }
@@ -275,11 +335,18 @@ export function sanitizeFixedSizeList(typeLike: object) {
   }
   return new FixedSizeList(
     typeLike.listSize,
-    sanitizeField(typeLike.children[0]),
+    sanitizeFieldWithContext(typeLike.children[0], context),
   );
 }
 
 export function sanitizeMap(typeLike: object) {
+  return sanitizeMapWithContext(typeLike, createSanitizationContext());
+}
+
+function sanitizeMapWithContext(
+  typeLike: object,
+  context: SanitizationContext,
+) {
   if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
     throw Error(
       "Expected a Map type to have an array-like `children` property",
@@ -292,7 +359,10 @@ export function sanitizeMap(typeLike: object) {
     throw Error("Expected a Map type to have exactly one child");
   }
 
-  return new Map_(sanitizeField(typeLike.children[0]), typeLike.keysSorted);
+  return new Map_(
+    sanitizeFieldWithContext(typeLike.children[0], context),
+    typeLike.keysSorted,
+  );
 }
 
 export function sanitizeDuration(typeLike: object) {
@@ -303,6 +373,13 @@ export function sanitizeDuration(typeLike: object) {
 }
 
 export function sanitizeDictionary(typeLike: object) {
+  return sanitizeDictionaryWithContext(typeLike, createSanitizationContext());
+}
+
+function sanitizeDictionaryWithContext(
+  typeLike: object,
+  context: SanitizationContext,
+) {
   if (!("id" in typeLike) || typeof typeLike.id !== "number") {
     throw Error("Expected a Dictionary type to have an `id` property");
   }
@@ -316,8 +393,8 @@ export function sanitizeDictionary(typeLike: object) {
     throw Error("Expected a Dictionary type to have an `isOrdered` property");
   }
   return new Dictionary(
-    sanitizeType(typeLike.dictionary),
-    sanitizeType(typeLike.indices) as TKeys,
+    sanitizeTypeWithContext(typeLike.dictionary, context),
+    sanitizeTypeWithContext(typeLike.indices, context) as TKeys,
     typeLike.id,
     typeLike.isOrdered,
   );
@@ -325,12 +402,23 @@ export function sanitizeDictionary(typeLike: object) {
 
 // biome-ignore lint/suspicious/noExplicitAny: skip
 export function sanitizeType(typeLike: unknown): DataType<any> {
+  return sanitizeTypeWithContext(typeLike, createSanitizationContext());
+}
+
+function sanitizeTypeWithContext(
+  typeLike: unknown,
+  context: SanitizationContext,
+): DataType {
   if (typeof typeLike === "string") {
     return dataTypeFromName(typeLike);
   }
   if (typeof typeLike !== "object" || typeLike === null) {
     throw Error("Expected a Type but object was null/undefined");
   }
+  const cached = context.types.get(typeLike);
+  if (cached !== undefined) {
+    return cached;
+  }
   if (
     !("typeId" in typeLike) ||
     !(
@@ -349,6 +437,16 @@ export function sanitizeType(typeLike: unknown): DataType<any> {
     throw Error("Type's typeId property was not a function or number");
   }
 
+  const type = sanitizeTypeById(typeLike, typeId, context);
+  context.types.set(typeLike, type);
+  return type;
+}
+
+function sanitizeTypeById(
+  typeLike: object,
+  typeId: Type,
+  context: SanitizationContext,
+): DataType {
   switch (typeId) {
     case Type.NONE:
       throw Error("Received a Type with a typeId of NONE");
@@ -375,21 +473,21 @@ export function sanitizeType(typeLike: unknown): DataType<any> {
     case Type.Interval:
       return sanitizeInterval(typeLike);
     case Type.List:
-      return sanitizeList(typeLike);
+      return sanitizeListWithContext(typeLike, context);
     case Type.Struct:
-      return sanitizeStruct(typeLike);
+      return sanitizeStructWithContext(typeLike, context);
     case Type.Union:
-      return sanitizeUnion(typeLike);
+      return sanitizeUnionWithContext(typeLike, context);
     case Type.FixedSizeBinary:
       return sanitizeFixedSizeBinary(typeLike);
     case Type.FixedSizeList:
-      return sanitizeFixedSizeList(typeLike);
+      return sanitizeFixedSizeListWithContext(typeLike, context);
     case Type.Map:
-      return sanitizeMap(typeLike);
+      return sanitizeMapWithContext(typeLike, context);
     case Type.Duration:
       return sanitizeDuration(typeLike);
     case Type.Dictionary:
-      return sanitizeDictionary(typeLike);
+      return sanitizeDictionaryWithContext(typeLike, context);
     case Type.Int8:
       return new Int8();
     case Type.Int16:
@@ -433,9 +531,9 @@ export function sanitizeType(typeLike: unknown): DataType<any> {
     case Type.TimestampSecond:
       return sanitizeTypedTimestamp(typeLike, TimestampSecond);
     case Type.DenseUnion:
-      return sanitizeTypedUnion(typeLike, DenseUnion);
+      return sanitizeTypedUnionWithContext(typeLike, DenseUnion, context);
     case Type.SparseUnion:
-      return sanitizeTypedUnion(typeLike, SparseUnion);
+      return sanitizeTypedUnionWithContext(typeLike, SparseUnion, context);
     case Type.IntervalDayTime:
       return new IntervalDayTime();
     case Type.IntervalYearMonth:
@@ -454,6 +552,13 @@ export function sanitizeType(typeLike: unknown): DataType<any> {
 }
 
 export function sanitizeField(fieldLike: unknown): Field {
+  return sanitizeFieldWithContext(fieldLike, createSanitizationContext());
+}
+
+function sanitizeFieldWithContext(
+  fieldLike: unknown,
+  context: SanitizationContext,
+): Field {
   if (fieldLike instanceof Field) {
     return fieldLike;
   }
@@ -471,7 +576,7 @@ export function sanitizeField(fieldLike: unknown): Field {
   }
   let type: DataType;
   try {
-    type = sanitizeType(fieldLike.type);
+    type = sanitizeTypeWithContext(fieldLike.type, context);
   } catch (error: unknown) {
     throw Error(
       `Unable to sanitize type for field: ${fieldLike.name} due to error: ${error}`,
@@ -501,6 +606,13 @@ export function sanitizeField(fieldLike: unknown): Field {
  * than lancedb is using.
  */
 export function sanitizeSchema(schemaLike: SchemaLike): Schema {
+  return sanitizeSchemaWithContext(schemaLike, createSanitizationContext());
+}
+
+function sanitizeSchemaWithContext(
+  schemaLike: SchemaLike,
+  context: SanitizationContext,
+): Schema {
   if (schemaLike instanceof Schema) {
     return schemaLike;
   }
@@ -522,7 +634,7 @@ export function sanitizeSchema(schemaLike: SchemaLike): Schema {
     );
   }
   const sanitizedFields = schemaLike.fields.map((field) =>
-    sanitizeField(field),
+    sanitizeFieldWithContext(field, context),
   );
   return new Schema(sanitizedFields, metadata);
 }
@@ -544,13 +656,18 @@ export function sanitizeTable(tableLike: TableLike): Table {
       "The table passed in does not appear to be a table (no 'columns' property)",
     );
   }
-  const schema = sanitizeSchema(tableLike.schema);
-
-  const batches = tableLike.batches.map(sanitizeRecordBatch);
+  const context = createSanitizationContext();
+  const schema = sanitizeSchemaWithContext(tableLike.schema, context);
+  const batches = tableLike.batches.map((batch) =>
+    sanitizeRecordBatch(batch, context),
+  );
   return new Table(schema, batches);
 }
 
-function sanitizeRecordBatch(batchLike: RecordBatchLike): RecordBatch {
+function sanitizeRecordBatch(
+  batchLike: RecordBatchLike,
+  context: SanitizationContext,
+): RecordBatch {
   if (batchLike instanceof RecordBatch) {
     return batchLike;
   }
@@ -567,19 +684,43 @@ function sanitizeRecordBatch(batchLike: RecordBatchLike): RecordBatch {
       "The record batch passed in does not appear to be a record batch (no 'data' property)",
     );
   }
-  const schema = sanitizeSchema(batchLike.schema);
-  const data = sanitizeData(batchLike.data);
+  const schema = sanitizeSchemaWithContext(batchLike.schema, context);
+  const data = sanitizeData(batchLike.data, context) as Data<Struct>;
   return new RecordBatch(schema, data);
 }
+
+type DictionaryVectorLike = {
+  data: readonly DataLike[];
+};
+
+type DictionaryDataLike = DataLike & {
+  dictionary?: DictionaryVectorLike;
+};
+
 function sanitizeData(
   dataLike: DataLike,
-  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-): import("apache-arrow").Data<Struct<any>> {
+  context: SanitizationContext,
+): Data<DataType> {
   if (dataLike instanceof Data) {
     return dataLike;
   }
-  return new Data(
-    dataLike.type,
+  const cachedData = context.data.get(dataLike);
+  if (cachedData !== undefined) {
+    return cachedData;
+  }
+  const dictionaryLike = (dataLike as DictionaryDataLike).dictionary;
+  let dictionary: Vector | undefined;
+  if (dictionaryLike !== undefined) {
+    dictionary = context.vectors.get(dictionaryLike);
+    if (dictionary === undefined) {
+      dictionary = new Vector(
+        dictionaryLike.data.map((data) => sanitizeData(data, context)),
+      );
+      context.vectors.set(dictionaryLike, dictionary);
+    }
+  }
+  const data = new Data(
+    sanitizeTypeWithContext(dataLike.type, context),
     dataLike.offset,
     dataLike.length,
     dataLike.nullCount,
@@ -589,7 +730,11 @@ function sanitizeData(
       [BufferType.VALIDITY]: dataLike.nullBitmap,
       [BufferType.TYPE]: dataLike.typeIds,
     },
+    dataLike.children.map((child) => sanitizeData(child, context)),
+    dictionary,
   );
+  context.data.set(dataLike, data);
+  return data;
 }
 
 const constructorsByTypeName = {
diff --git a/nodejs/lancedb/schema.ts b/nodejs/lancedb/schema.ts
new file mode 100644
index 000000000..e4749ef37
--- /dev/null
+++ b/nodejs/lancedb/schema.ts
@@ -0,0 +1,566 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import {
+  Binary,
+  Bool,
+  DataType,
+  Dictionary,
+  Field,
+  FixedSizeList,
+  Float32,
+  Float64,
+  Int32,
+  Int64,
+  List,
+  Schema,
+  Struct,
+  Utf8,
+  util as arrowUtil,
+} from "apache-arrow";
+import { typedArrayToArrowType } from "./arrow_type";
+import { sanitizeType } from "./sanitize";
+
+type InferenceOptions = {
+  dictionaryEncodeStrings: boolean;
+  vectorColumns: Record<string, { type: unknown }>;
+};
+
+/**
+ * Infer the Arrow schema represented by a set of records.
+ *
+ * This is the intentionally small interface to schema inference. The stateful
+ * details of combining partial type evidence are encapsulated below so callers
+ * only need to provide records, an optional schema, and inference options.
+ */
+export function inferSchema(
+  data: Array<Record<string, unknown>>,
+  schema: Schema | undefined,
+  options: InferenceOptions,
+): Schema {
+  return new SchemaInferrer(schema, options).infer(data);
+}
+
+class SchemaInferrer {
+  private readonly fields = new FieldTree();
+
+  constructor(
+    private readonly providedSchema: Schema | undefined,
+    private readonly options: InferenceOptions,
+  ) {}
+
+  infer(data: Array<Record<string, unknown>>): Schema {
+    for (const [row, record] of data.entries()) {
+      for (const [path, value] of recordPathsAndValues(record)) {
+        this.observe(path, value, row);
+      }
+    }
+
+    return this.providedSchema === undefined
+      ? new Schema(fieldsFromTree(this.fields))
+      : new Schema(matchingFields(this.providedSchema.fields, this.fields));
+  }
+
+  private observe(path: string[], value: unknown, row: number): void {
+    const current = this.fields.get(path);
+    if (current === undefined) {
+      this.addField(path, value, row);
+    } else if (this.providedSchema === undefined) {
+      this.updateInferredField(path, value, row, current);
+    }
+  }
+
+  private addField(path: string[], value: unknown, row: number): void {
+    if (this.providedSchema !== undefined) {
+      this.addSchemaField(this.providedSchema, path, row);
+      return;
+    }
+
+    const evidence =
+      this.inferType(value, path) ?? DeferredTypeEvidence.from(value, row);
+    if (evidence === undefined) {
+      throw typeInferenceError(path, row);
+    }
+
+    const conflict = this.fields.set(
+      path,
+      evidence,
+      (existing) =>
+        existing instanceof DeferredTypeEvidence && existing.isOnlyNulls(),
+    );
+    if (conflict !== undefined) {
+      throw branchConflictError(conflict, row, "Struct");
+    }
+  }
+
+  private addSchemaField(schema: Schema, path: string[], row: number): void {
+    const field = fieldAtPath(schema, path);
+    if (field === undefined) {
+      throw new Error(
+        `Found field not in schema: ${path.join(".")} at row ${row}`,
+      );
+    }
+
+    const conflict = this.fields.set(path, field.type);
+    if (conflict !== undefined) {
+      throw branchConflictError(conflict, row, "Struct");
+    }
+  }
+
+  private updateInferredField(
+    path: string[],
+    value: unknown,
+    row: number,
+    current: FieldNode,
+  ): void {
+    const newType = this.inferType(value, path);
+    const deferred = DeferredTypeEvidence.from(value, row);
+
+    if (current instanceof FieldTree) {
+      if (deferred?.isOnlyNulls()) {
+        return;
+      }
+      throw schemaInferenceError(
+        path,
+        row,
+        "Struct",
+        describeEvidence(newType ?? deferred),
+      );
+    }
+
+    if (current instanceof DeferredTypeEvidence) {
+      this.resolveDeferredField(path, row, current, newType, deferred);
+      return;
+    }
+
+    if (newType !== undefined) {
+      if (!inferredTypesEqual(current, newType)) {
+        throw schemaInferenceError(
+          path,
+          row,
+          describeEvidence(current),
+          describeEvidence(newType),
+        );
+      }
+      return;
+    }
+
+    if (deferred === undefined || !deferred.matches(current)) {
+      throw schemaInferenceError(
+        path,
+        row,
+        describeEvidence(current),
+        describeEvidence(deferred),
+      );
+    }
+  }
+
+  private resolveDeferredField(
+    path: string[],
+    row: number,
+    current: DeferredTypeEvidence,
+    newType: DataType | undefined,
+    deferred: DeferredTypeEvidence | undefined,
+  ): void {
+    if (newType !== undefined) {
+      if (!current.matches(newType)) {
+        throw schemaInferenceError(
+          path,
+          row,
+          current.describe(),
+          describeEvidence(newType),
+        );
+      }
+      this.fields.set(path, newType);
+      return;
+    }
+
+    if (deferred !== undefined) {
+      this.fields.set(path, current.merge(deferred));
+      return;
+    }
+
+    throw schemaInferenceError(
+      path,
+      row,
+      current.describe(),
+      describeEvidence(newType),
+    );
+  }
+
+  private inferType(value: unknown, path: string[]): DataType | undefined {
+    if (typeof value === "bigint") {
+      return new Int64();
+    }
+    if (typeof value === "number") {
+      return new Float64();
+    }
+    if (typeof value === "string") {
+      return this.options.dictionaryEncodeStrings
+        ? new Dictionary(new Utf8(), new Int32())
+        : new Utf8();
+    }
+    if (typeof value === "boolean") {
+      return new Bool();
+    }
+    if (value instanceof Buffer) {
+      return new Binary();
+    }
+    if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
+      const typedArray = typedArrayToArrowType(value);
+      return typedArray === undefined
+        ? undefined
+        : new FixedSizeList(
+            typedArray.length,
+            new Field("item", typedArray.elementType, true),
+          );
+    }
+    if (!Array.isArray(value) || value.length === 0) {
+      return undefined;
+    }
+
+    const configuredVector =
+      path.length === 1 ? this.options.vectorColumns[path[0]] : undefined;
+    if (configuredVector !== undefined) {
+      return new FixedSizeList(
+        value.length,
+        new Field("item", sanitizeType(configuredVector.type), true),
+      );
+    }
+
+    const itemType = this.inferArrayItemType(value, path);
+    if (itemType === undefined) {
+      return undefined;
+    }
+
+    return nameSuggestsVectorColumn(path[path.length - 1])
+      ? new FixedSizeList(value.length, new Field("item", new Float32(), true))
+      : new List(new Field("item", itemType, true));
+  }
+
+  private inferArrayItemType(
+    values: unknown[],
+    path: string[],
+  ): DataType | undefined {
+    let itemType: DataType | undefined;
+    const deferredItems: unknown[] = [];
+
+    for (const value of values) {
+      const candidate = this.inferType(value, path);
+      if (candidate === undefined) {
+        if (!isDeferredValue(value)) {
+          return undefined;
+        }
+        deferredItems.push(value);
+      } else if (itemType === undefined) {
+        itemType = candidate;
+      } else if (!inferredTypesEqual(itemType, candidate)) {
+        return undefined;
+      }
+    }
+
+    if (itemType === undefined) {
+      return undefined;
+    }
+    return deferredItems.every((value) =>
+      deferredValueMatchesType(value, itemType),
+    )
+      ? itemType
+      : undefined;
+  }
+}
+
+/** Nulls and empty/all-null lists that do not determine a type by themselves. */
+class DeferredTypeEvidence {
+  private constructor(
+    private readonly values: Array<{ value: unknown; row: number }>,
+  ) {}
+
+  static from(value: unknown, row: number): DeferredTypeEvidence | undefined {
+    return isDeferredValue(value)
+      ? new DeferredTypeEvidence([{ value, row }])
+      : undefined;
+  }
+
+  isOnlyNulls(): boolean {
+    return this.values.every(({ value }) => value == null);
+  }
+
+  matches(type: DataType): boolean {
+    return this.values.every(({ value }) =>
+      deferredValueMatchesType(value, type),
+    );
+  }
+
+  merge(other: DeferredTypeEvidence): DeferredTypeEvidence {
+    return new DeferredTypeEvidence([...this.values, ...other.values]);
+  }
+
+  describe(): string {
+    const list = this.values.find(({ value }) => Array.isArray(value));
+    return list === undefined
+      ? "null"
+      : `List[${(list.value as unknown[]).length}]`;
+  }
+
+  firstRow(): number {
+    return this.values[0].row;
+  }
+}
+
+type FieldNode = DataType | DeferredTypeEvidence | FieldTree;
+type LeafNode = Exclude<FieldNode, FieldTree>;
+type FieldConflict = { path: string[]; value: FieldNode };
+
+/** Nested field state, kept separate from Arrow's eventual Struct types. */
+class FieldTree {
+  private readonly children = new Map<string, FieldNode>();
+
+  get(path: string[]): FieldNode | undefined {
+    let current: FieldNode = this;
+    for (const part of path) {
+      if (!(current instanceof FieldTree)) {
+        return undefined;
+      }
+      const child = current.children.get(part);
+      if (child === undefined) {
+        return undefined;
+      }
+      current = child;
+    }
+    return current;
+  }
+
+  set(
+    path: string[],
+    value: LeafNode,
+    canReplaceLeaf: (value: LeafNode) => boolean = () => false,
+  ): FieldConflict | undefined {
+    let branch: FieldTree = this;
+    for (const [index, part] of path.slice(0, -1).entries()) {
+      const child = branch.children.get(part);
+      if (child === undefined || (isLeaf(child) && canReplaceLeaf(child))) {
+        const nextBranch = new FieldTree();
+        branch.children.set(part, nextBranch);
+        branch = nextBranch;
+      } else if (child instanceof FieldTree) {
+        branch = child;
+      } else {
+        return { path: path.slice(0, index + 1), value: child };
+      }
+    }
+
+    const name = path[path.length - 1];
+    const current = branch.children.get(name);
+    if (current instanceof FieldTree) {
+      return { path, value: current };
+    }
+    branch.children.set(name, value);
+    return undefined;
+  }
+
+  entries(): IterableIterator<[string, FieldNode]> {
+    return this.children.entries();
+  }
+
+  has(name: string): boolean {
+    return this.children.has(name);
+  }
+}
+
+function isLeaf(value: FieldNode): value is LeafNode {
+  return !(value instanceof FieldTree);
+}
+
+function fieldsFromTree(tree: FieldTree, path: string[] = []): Field[] {
+  const fields: Field[] = [];
+  for (const [name, value] of tree.entries()) {
+    if (value instanceof FieldTree) {
+      fields.push(
+        new Field(
+          name,
+          new Struct(fieldsFromTree(value, [...path, name])),
+          true,
+        ),
+      );
+    } else if (value instanceof DeferredTypeEvidence) {
+      throw typeInferenceError([...path, name], value.firstRow());
+    } else {
+      fields.push(new Field(name, value, true));
+    }
+  }
+  return fields;
+}
+
+function matchingFields(fields: Field[], tree: FieldTree): Field[] {
+  const matches: Field[] = [];
+  for (const field of fields) {
+    if (!tree.has(field.name)) {
+      continue;
+    }
+    const value = tree.get([field.name]);
+    if (value instanceof FieldTree) {
+      const struct = field.type as Struct;
+      matches.push(
+        new Field(
+          field.name,
+          new Struct(matchingFields(struct.children, value)),
+          field.nullable,
+        ),
+      );
+    } else {
+      matches.push(new Field(field.name, value as DataType, field.nullable));
+    }
+  }
+  return matches;
+}
+
+function* recordPathsAndValues(
+  record: Record<string, unknown>,
+  path: string[] = [],
+): Generator<[string[], unknown]> {
+  for (const [name, value] of Object.entries(record)) {
+    if (isRecord(value)) {
+      yield* recordPathsAndValues(value, [...path, name]);
+    } else if (value !== undefined) {
+      yield [[...path, name], value];
+    }
+  }
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    !Array.isArray(value) &&
+    !(value instanceof RegExp) &&
+    !(value instanceof Date) &&
+    !(value instanceof Set) &&
+    !(value instanceof Map) &&
+    !(value instanceof Buffer) &&
+    !ArrayBuffer.isView(value)
+  );
+}
+
+function fieldAtPath(schema: Schema, path: string[]): Field | undefined {
+  let fields = schema.fields;
+  let field: Field | undefined;
+  for (const [index, name] of path.entries()) {
+    field = fields.find((candidate) => candidate.name === name);
+    if (field === undefined || index === path.length - 1) {
+      return field;
+    }
+    if (!DataType.isStruct(field.type)) {
+      return undefined;
+    }
+    fields = field.type.children;
+  }
+  return field;
+}
+
+function isDeferredValue(value: unknown): boolean {
+  return (
+    value == null || (Array.isArray(value) && value.every(isDeferredValue))
+  );
+}
+
+function deferredValueMatchesType(value: unknown, type: DataType): boolean {
+  if (value == null) {
+    return true;
+  }
+  if (!Array.isArray(value)) {
+    return false;
+  }
+  if (DataType.isList(type)) {
+    return value.every((item) =>
+      deferredValueMatchesType(item, type.valueType),
+    );
+  }
+  if (DataType.isFixedSizeList(type)) {
+    return (
+      value.length === type.listSize &&
+      value.every((item) => deferredValueMatchesType(item, type.valueType))
+    );
+  }
+  return false;
+}
+
+function inferredTypesEqual(current: DataType, candidate: DataType): boolean {
+  if (DataType.isDictionary(current)) {
+    return (
+      DataType.isDictionary(candidate) &&
+      current.isOrdered === candidate.isOrdered &&
+      inferredTypesEqual(current.indices, candidate.indices) &&
+      inferredTypesEqual(current.dictionary, candidate.dictionary)
+    );
+  }
+  if (DataType.isList(current)) {
+    return (
+      DataType.isList(candidate) &&
+      current.valueField.name === candidate.valueField.name &&
+      current.valueField.nullable === candidate.valueField.nullable &&
+      inferredTypesEqual(current.valueType, candidate.valueType)
+    );
+  }
+  if (DataType.isFixedSizeList(current)) {
+    return (
+      DataType.isFixedSizeList(candidate) &&
+      current.listSize === candidate.listSize &&
+      current.valueField.name === candidate.valueField.name &&
+      current.valueField.nullable === candidate.valueField.nullable &&
+      inferredTypesEqual(current.valueType, candidate.valueType)
+    );
+  }
+  return arrowUtil.compareTypes(current, candidate);
+}
+
+function describeEvidence(
+  evidence: DataType | DeferredTypeEvidence | undefined,
+): string {
+  if (evidence === undefined) {
+    return "an unsupported value";
+  }
+  return evidence instanceof DeferredTypeEvidence
+    ? evidence.describe()
+    : evidence.toString();
+}
+
+function branchConflictError(
+  conflict: FieldConflict,
+  row: number,
+  candidate: string,
+): Error {
+  return schemaInferenceError(
+    conflict.path,
+    row,
+    conflict.value instanceof FieldTree
+      ? "Struct"
+      : describeEvidence(conflict.value),
+    candidate,
+  );
+}
+
+function schemaInferenceError(
+  path: string[],
+  row: number,
+  currentType: string,
+  newType: string,
+): Error {
+  return new Error(
+    `Failed to infer schema for data. Previously inferred type ${currentType} ` +
+      `but found ${newType} for field ${path.join(".")} at row ${row}. ` +
+      "Consider providing an explicit schema.",
+  );
+}
+
+function typeInferenceError(path: string[], row: number): Error {
+  return new Error(
+    `Failed to infer data type for field ${path.join(".")} at row ${row}. ` +
+      "Consider providing an explicit schema.",
+  );
+}
+
+function nameSuggestsVectorColumn(name: string): boolean {
+  const normalized = name.toLowerCase();
+  return normalized.includes("vector") || normalized.includes("embedding");
+}
diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts
index c148c63d4..82f23e2f2 100644
--- a/nodejs/lancedb/table.ts
+++ b/nodejs/lancedb/table.ts
@@ -31,8 +31,11 @@ import {
   IndexConfig,
   IndexStatistics,
   Job,
+  LsmStats,
   Branches as NativeBranches,
   OptimizeStats,
+  RefreshColumnResult,
+  RefreshMaterializedViewResult,
   TableStatistics,
   Tags,
   UpdateFieldMetadataResult,
@@ -40,6 +43,7 @@ import {
   Table as _NativeTable,
 } from "./native";
 import {
+  AutoQuery,
   FullTextQuery,
   Query,
   TakeQuery,
@@ -50,6 +54,12 @@ import {
 import { sanitizeType } from "./sanitize";
 import { IntoSql, toSQL } from "./util";
 export { IndexConfig } from "./native";
+export {
+  BucketStats,
+  GenerationStats,
+  LsmStats,
+  MemtableStats,
+} from "./native";
 
 /**
  * Progress snapshot for a write operation, delivered to the `progress`
@@ -198,7 +208,11 @@ export interface LsmWriteSpec {
   column?: string;
   /** Bucket variant: the number of buckets, in `[1, 1024]`. */
   numBuckets?: number;
-  /** Names of indexes the MemWAL should keep up to date during writes. */
+  /**
+   * Indexes the MemWAL keeps up to date. Omit to maintain every supported
+   * index, resolved on install — a snapshot, so indexes created later are not
+   * maintained. Pass `[]` for none.
+   */
   maintainedIndexes?: string[];
   /** Default `ShardWriter` configuration recorded in the MemWAL index. */
   writerConfigDefaults?: Record<string, string>;
@@ -511,7 +525,7 @@ export abstract class Table {
     query: string | IntoVector | MultiVector | FullTextQuery,
     queryType?: string,
     ftsColumns?: string | string[],
-  ): VectorQuery | Query;
+  ): VectorQuery | Query | AutoQuery;
   /**
    * Search the table with a given query vector.
    *
@@ -522,18 +536,87 @@ export abstract class Table {
   abstract vectorSearch(vector: IntoVector | MultiVector): VectorQuery;
   /**
    * Add new columns with defined values.
+   *
+   * The `{ computed }` form stores the expression rather than evaluating it
+   * now: the column is committed with no values, and rows get them from
+   * {@link Table#refreshColumn}. Declaring one therefore costs the same on a
+   * large table as on an empty one.
+   *
+   * A refresh does not revisit rows it has already filled, so mutating an
+   * input leaves the value computed at fill time; recomputing means dropping
+   * the column and declaring it again. While a declaration reads a column,
+   * that column cannot be renamed, retyped or dropped.
+   *
+   * On LanceDB Cloud and Enterprise the expression is planned by the
+   * server, and the refresh runs as a server job -- see
+   * {@link Table#refreshColumnAsync}.
    * @param {AddColumnsSql[] | Field | Field[] | Schema} newColumnTransforms Either:
    *   - An array of objects with column names and SQL expressions to calculate values
    *   - A single Arrow Field defining one column with its data type (column will be initialized with null values)
    *   - An array of Arrow Fields defining columns with their data types (columns will be initialized with null values)
    *   - An Arrow Schema defining columns with their data types (columns will be initialized with null values)
+   *   - `{ computed }`, declaring columns defined by a SQL expression whose type and inputs are derived from it
    * @returns {Promise<AddColumnsResult>} A promise that resolves to an object
    * containing the new version number of the table after adding the columns.
+   * @example
+   * ```ts
+   * await table.addColumns({ computed: [{ name: "doubled", valueSql: "x * 2" }] });
+   * const { rowsFilled } = await table.refreshColumn("doubled");
+   * ```
    */
   abstract addColumns(
-    newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema,
+    newColumnTransforms:
+      | AddColumnsSql[]
+      | Field
+      | Field[]
+      | Schema
+      | { computed: AddColumnsSql[] },
   ): Promise<AddColumnsResult>;
 
+  /**
+   * Fill the rows of a computed column that hold no value yet.
+   *
+   * Rows appended since the last refresh are filled by the next one; rows
+   * already filled are left as they are, so the call is idempotent and does
+   * not observe a mutated input. Local tables only: a remote refresh runs
+   * as a server job, through {@link Table#refreshColumnAsync}.
+   * @param {string} column The name of the computed column to fill.
+   * @returns {Promise<RefreshColumnResult>} A promise that resolves to the
+   * number of rows filled and the new version number of the table.
+   */
+  abstract refreshColumn(column: string): Promise<RefreshColumnResult>;
+
+  /**
+   * Like {@link Table#refreshColumn}, but returns a handle to the refresh
+   * job instead of blocking until it completes.
+   *
+   * The job may already be complete when returned; callers must not assume
+   * the column is filled until {@link Job.wait} resolves. Invalid input --
+   * an unknown column, or one that is not computed -- rejects here rather
+   * than failing the job. On local tables the job runs in-process; on
+   * LanceDB Cloud and Enterprise it is the server's backfill job.
+   * @param {string} column The name of the computed column to fill.
+   * @example
+   * ```ts
+   * const job = await table.refreshColumnAsync("doubled");
+   * await job.wait();
+   * console.log(await job.status()); // "finished"
+   * ```
+   */
+  abstract refreshColumnAsync(column: string): Promise<Job>;
+
+  /**
+   * Recompute this table's contents from its materialized-view definition.
+   *
+   * Plumbing for {@link MaterializedView.refresh}, which is the way to call
+   * it: rejects tables that carry no view definition. Local tables only.
+   * @ignore
+   */
+  abstract refreshMaterializedView(
+    full?: boolean,
+    sourceVersion?: number,
+  ): Promise<RefreshMaterializedViewResult>;
+
   /**
    * Alter the name or nullability of columns.
    * @param {ColumnAlteration[]} columnAlterations One or more alterations to
@@ -596,6 +679,11 @@ export abstract class Table {
    * All variants require the table to have an unenforced primary key
    * ({@link Table#setUnenforcedPrimaryKey}); bucket sharding additionally
    * requires it to be the single column being bucketed.
+   *
+   * Omitting `maintainedIndexes` maintains every index on the table, resolved
+   * here, failing if one cannot be maintained — name them to install anyway.
+   * Naming them pins an exact set, and a still-building index is rejected
+   * rather than quietly omitted.
    * @param {LsmWriteSpec} spec The sharding spec to install.
    * @returns {Promise<void>}
    * @example
@@ -623,9 +711,10 @@ export abstract class Table {
    *
    * Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
    * spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}).
-   * The returned spec — including its `maintainedIndexes` and
-   * `writerConfigDefaults` — mirrors what was passed to
-   * {@link Table#setLsmWriteSpec}.
+   * The returned spec mirrors what was passed to
+   * {@link Table#setLsmWriteSpec}, except that `maintainedIndexes` always
+   * reports the concrete list resolved when the spec was set — `undefined`
+   * never round-trips.
    * @returns {Promise<LsmWriteSpec | undefined>}
    */
   abstract getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
@@ -639,6 +728,59 @@ export abstract class Table {
    * @returns {Promise<void>}
    */
   abstract closeLsmWriters(): Promise<void>;
+  /**
+   * Seal every bucket's active memtable into a new L0 generation.
+   *
+   * Returns once the seal is committed. Sealing an empty memtable is a no-op,
+   * so this is safe to call repeatedly.
+   * @returns {Promise<void>}
+   */
+  abstract flushLsm(): Promise<void>;
+  /**
+   * Trigger a background L0 → base compaction pass per bucket.
+   *
+   * Returns once the passes are *dispatched*, not once they finish — watch
+   * {@link Table#getLsmStats} for progress, or use
+   * {@link Table#checkpointLsm} to wait for convergence.
+   * @returns {Promise<void>}
+   */
+  abstract compactLsm(): Promise<void>;
+  /**
+   * Converge this table's LSM write path into its base table.
+   *
+   * Seals once, then triggers compaction and polls until the L0 that existed
+   * at the start is gone. The target set is fixed at the start, so
+   * generations created *during* the checkpoint are ignored — that is what
+   * lets it terminate under write load, and what makes it best-effort: it
+   * converges the fresh tier as of some instant. Idempotent, abandonable at
+   * any point, and safe to run on a cadence.
+   *
+   * There is no liveness bound — the compactor pool is shared across tables,
+   * so a checkpoint queued behind unrelated work looks exactly like one that
+   * is merging. The caller owns the deadline.
+   * @returns {Promise<void>}
+   * @example
+   * ```ts
+   * const before = await table.getLsmStats();
+   * await table.checkpointLsm();
+   * const after = await table.getLsmStats();
+   * ```
+   */
+  abstract checkpointLsm(): Promise<void>;
+  /**
+   * Read live per-bucket LSM state.
+   *
+   * Answers "how far behind is my fresh tier", "which bucket is hot", and
+   * "why is my fresh-tier vector search brute-force". Mutates no table state.
+   *
+   * Resolves to `undefined` only when the LSM write path is not enabled.
+   * @param {boolean} includeGenerationRows Also count rows per L0 generation.
+   *   Off by default because each count opens an uncached Lance dataset.
+   * @returns {Promise<LsmStats | undefined>}
+   */
+  abstract getLsmStats(
+    includeGenerationRows?: boolean,
+  ): Promise<LsmStats | undefined>;
   /** Retrieve the version of the table */
 
   abstract version(): Promise<number>;
@@ -835,10 +977,11 @@ export class LocalTable extends Table {
     return this.inner.display();
   }
 
-  private async getEmbeddingFunctions(): Promise<
-    Map<string, EmbeddingFunctionConfig>
-  > {
-    const schema = await this.schema();
+  private async getEmbeddingFunctions(
+    inner: _NativeTable = this.inner,
+  ): Promise<Map<string, EmbeddingFunctionConfig>> {
+    const schemaBuf = await inner.schema();
+    const schema = tableFromIPC(schemaBuf).schema;
     const registry = getRegistry();
     return registry.parseFunctions(schema.metadata);
   }
@@ -1020,7 +1163,7 @@ export class LocalTable extends Table {
     query: string | IntoVector | MultiVector | FullTextQuery,
     queryType: string = "auto",
     ftsColumns?: string | string[],
-  ): VectorQuery | Query {
+  ): VectorQuery | Query | AutoQuery {
     if (typeof query !== "string" && !instanceOfFullTextQuery(query)) {
       if (queryType === "fts") {
         throw new Error("Cannot perform full text search on a vector query");
@@ -1093,8 +1236,22 @@ export class LocalTable extends Table {
   // TODO: Support BatchUDF
 
   async addColumns(
-    newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema,
+    newColumnTransforms:
+      | AddColumnsSql[]
+      | Field
+      | Field[]
+      | Schema
+      | { computed: AddColumnsSql[] },
   ): Promise<AddColumnsResult> {
+    // Columns defined by an expression are declared, not materialized here.
+    if (
+      typeof newColumnTransforms === "object" &&
+      !Array.isArray(newColumnTransforms) &&
+      "computed" in newColumnTransforms
+    ) {
+      return await this.inner.addComputedColumns(newColumnTransforms.computed);
+    }
+
     // Handle single Field -> convert to array of Fields
     if (newColumnTransforms instanceof Field) {
       newColumnTransforms = [newColumnTransforms];
@@ -1129,6 +1286,21 @@ export class LocalTable extends Table {
     throw new Error("Invalid input type for addColumns");
   }
 
+  async refreshColumn(column: string): Promise<RefreshColumnResult> {
+    return await this.inner.refreshColumn(column);
+  }
+
+  async refreshColumnAsync(column: string): Promise<Job> {
+    return await this.inner.refreshColumnAsync(column);
+  }
+
+  async refreshMaterializedView(
+    full?: boolean,
+    sourceVersion?: number,
+  ): Promise<RefreshMaterializedViewResult> {
+    return await this.inner.refreshMaterializedView(full, sourceVersion);
+  }
+
   async alterColumns(
     columnAlterations: ColumnAlteration[],
   ): Promise<AlterColumnsResult> {
@@ -1191,6 +1363,24 @@ export class LocalTable extends Table {
     return await this.inner.closeLsmWriters();
   }
 
+  async flushLsm(): Promise<void> {
+    return await this.inner.flushLsm();
+  }
+
+  async compactLsm(): Promise<void> {
+    return await this.inner.compactLsm();
+  }
+
+  async checkpointLsm(): Promise<void> {
+    return await this.inner.checkpointLsm();
+  }
+
+  async getLsmStats(
+    includeGenerationRows: boolean = false,
+  ): Promise<LsmStats | undefined> {
+    return (await this.inner.getLsmStats(includeGenerationRows)) ?? undefined;
+  }
+
   async version(): Promise<number> {
     return await this.inner.version();
   }
@@ -1404,8 +1594,8 @@ export interface BranchRowCountSummary {
   deltaAvailable: boolean;
 }
 
-/** A reason why a branch cannot currently be merged. */
-export interface MergeBlocker {
+/** A reason why a cherry-pick cannot currently land. */
+export interface CherryPickError {
   code: string;
   message: string;
 }
@@ -1425,20 +1615,19 @@ export interface BranchDiff {
   changedColumns: BranchColumnChange[];
   addedIndexes: BranchIndexSummary[];
   removedIndexes: BranchIndexSummary[];
-  mergeable: boolean;
-  mergeBlockers: MergeBlocker[];
+  errors: CherryPickError[];
 }
 
-/** Changes that would be, or were, promoted by a branch merge. */
-export interface MergePreview {
+/** Changes that would be, or were, promoted by a cherry-pick. */
+export interface CherryPickPreview {
   promotedColumns: string[];
 }
 
-/** Result of previewing or attempting a branch merge. */
-export interface MergeBranchResult {
-  status: "ready" | "rejected" | "notImplemented" | "merged" | "unknown";
+/** Result of previewing or attempting a cherry-pick. */
+export interface CherryPickResult {
+  status: "ready" | "failed" | "notImplemented" | "cherryPicked" | "unknown";
   diff: BranchDiff;
-  preview: MergePreview;
+  preview: CherryPickPreview;
   mainVersionAfter?: number;
 }
 
@@ -1501,21 +1690,21 @@ export class Branches {
   }
 
   /**
-   * Merge a branch into main.
+   * Cherry-pick a branch onto main.
    *
-   * Set `dryRun` to `true` to preview the merge. A rejected merge resolves
-   * with `status: "rejected"` instead of throwing.
+   * Set `dryRun` to `true` to preview. A failed cherry-pick resolves
+   * with `status: "failed"` instead of throwing.
    *
-   * @param fromBranch Branch to merge from.
-   * @param dryRun When true, only preview the merge. Defaults to false.
+   * @param fromBranch Branch to cherry-pick from.
+   * @param dryRun When true, only preview. Defaults to false.
    */
-  async merge(
+  async cherryPick(
     fromBranch: string,
     dryRun: boolean = false,
-  ): Promise<MergeBranchResult> {
-    return (await this.#inner.merge(
+  ): Promise<CherryPickResult> {
+    return (await this.#inner.cherryPick(
       fromBranch,
       dryRun,
-    )) as unknown as MergeBranchResult;
+    )) as unknown as CherryPickResult;
   }
 }
diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json
index 3c93ed470..ff6347c4d 100644
--- a/nodejs/npm/darwin-arm64/package.json
+++ b/nodejs/npm/darwin-arm64/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.37.1-beta.0",
+	"version": "0.38.0-beta.10",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",
diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json
index 5ade5aaa3..ed99fee05 100644
--- a/nodejs/npm/linux-arm64-gnu/package.json
+++ b/nodejs/npm/linux-arm64-gnu/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.37.1-beta.0",
+	"version": "0.38.0-beta.10",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",
diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json
index 16bb0edd0..5b215dcc0 100644
--- a/nodejs/npm/linux-arm64-musl/package.json
+++ b/nodejs/npm/linux-arm64-musl/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.37.1-beta.0",
+	"version": "0.38.0-beta.10",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",
diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json
index 6ee11e4bc..e0f5a9f26 100644
--- a/nodejs/npm/linux-x64-gnu/package.json
+++ b/nodejs/npm/linux-x64-gnu/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.37.1-beta.0",
+	"version": "0.38.0-beta.10",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",
diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json
index c2e15bb9f..d42541707 100644
--- a/nodejs/npm/linux-x64-musl/package.json
+++ b/nodejs/npm/linux-x64-musl/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.37.1-beta.0",
+	"version": "0.38.0-beta.10",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",
diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json
index d2820b1a1..496a40720 100644
--- a/nodejs/npm/win32-arm64-msvc/package.json
+++ b/nodejs/npm/win32-arm64-msvc/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.37.1-beta.0",
+  "version": "0.38.0-beta.10",
   "os": [
     "win32"
   ],
diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json
index 601b51380..734013343 100644
--- a/nodejs/npm/win32-x64-msvc/package.json
+++ b/nodejs/npm/win32-x64-msvc/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.37.1-beta.0",
+	"version": "0.38.0-beta.10",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",
diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json
index 8e30b0fab..732a7c01c 100644
--- a/nodejs/package-lock.json
+++ b/nodejs/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@lancedb/lancedb",
-  "version": "0.37.1-beta.0",
+  "version": "0.38.0-beta.10",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@lancedb/lancedb",
-      "version": "0.37.1-beta.0",
+      "version": "0.38.0-beta.10",
       "cpu": [
         "x64",
         "arm64"
@@ -55,7 +55,13 @@
         "openai": "4.29.2"
       },
       "peerDependencies": {
+        "@types/node": ">=18",
         "apache-arrow": ">=15.0.0 <=18.1.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/node": {
+          "optional": true
+        }
       }
     },
     "node_modules/@aws-crypto/crc32": {
diff --git a/nodejs/package.json b/nodejs/package.json
index f3f719af2..262d4c4a7 100644
--- a/nodejs/package.json
+++ b/nodejs/package.json
@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.37.1-beta.0",
+  "version": "0.38.0-beta.10",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -101,6 +101,12 @@
     "openai": "4.29.2"
   },
   "peerDependencies": {
+    "@types/node": ">=18",
     "apache-arrow": ">=15.0.0 <=18.1.0"
+  },
+  "peerDependenciesMeta": {
+    "@types/node": {
+      "optional": true
+    }
   }
 }
diff --git a/nodejs/src/connection.rs b/nodejs/src/connection.rs
index c45321aba..5cf676256 100644
--- a/nodejs/src/connection.rs
+++ b/nodejs/src/connection.rs
@@ -17,6 +17,7 @@ use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection, conne
 
 use lance_namespace::models::{
     CreateNamespaceRequest, DescribeNamespaceRequest, DropNamespaceRequest, ListNamespacesRequest,
+    ListTablesRequest,
 };
 use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
 
@@ -36,6 +37,12 @@ pub struct ListNamespacesResponse {
     pub page_token: Option<String>,
 }
 
+#[napi(object)]
+pub struct ListTablesResponse {
+    pub tables: Vec<String>,
+    pub page_token: Option<String>,
+}
+
 #[napi(object)]
 pub struct CreateNamespaceResponse {
     pub properties: Option<HashMap<String, String>>,
@@ -206,6 +213,33 @@ impl Connection {
         op.execute().await.default_error()
     }
 
+    /// List a page of tables in the database.
+    #[napi(catch_unwind)]
+    pub async fn list_tables(
+        &self,
+        namespace_path: Option<Vec<String>>,
+        page_token: Option<String>,
+        limit: Option<u32>,
+    ) -> napi::Result<ListTablesResponse> {
+        let request = ListTablesRequest {
+            // The root namespace is an empty path, not an absent one: a namespace-backed
+            // database rejects a request that names no namespace.
+            id: Some(namespace_path.unwrap_or_default()),
+            page_token,
+            limit: limit.map(|limit| i32::try_from(limit).unwrap_or(i32::MAX)),
+            ..Default::default()
+        };
+        let response = self
+            .get_inner()?
+            .list_tables(request)
+            .await
+            .default_error()?;
+        Ok(ListTablesResponse {
+            tables: response.tables,
+            page_token: response.page_token,
+        })
+    }
+
     /// Create table from a Apache Arrow IPC (file) buffer.
     ///
     /// Parameters:
@@ -266,6 +300,58 @@ impl Connection {
         Ok(Table::new(tbl))
     }
 
+    #[napi(catch_unwind)]
+    pub async fn create_materialized_view(
+        &self,
+        name: String,
+        source: String,
+        projections: Option<Vec<Vec<String>>>,
+        filter: Option<String>,
+        limit: Option<i64>,
+    ) -> napi::Result<Table> {
+        let mut builder = self.get_inner()?.create_materialized_view(name, source);
+        if let Some(projections) = projections {
+            let mut pairs = Vec::with_capacity(projections.len());
+            for pair in projections {
+                let [output, expression]: [String; 2] = pair.try_into().map_err(|_| {
+                    napi::Error::from_reason("each projection must be an [output, expression] pair")
+                })?;
+                pairs.push((output, expression));
+            }
+            builder = builder.select(pairs);
+        }
+        if let Some(filter) = filter {
+            builder = builder.only_if(filter);
+        }
+        if let Some(limit) = limit {
+            let limit = u64::try_from(limit)
+                .map_err(|_| napi::Error::from_reason("limit must be a non-negative integer"))?;
+            builder = builder.limit(limit);
+        }
+        let view = builder.execute().await.default_error()?;
+        Ok(Table::new(view.table().clone()))
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn open_materialized_view(&self, name: String) -> napi::Result<Table> {
+        let view = self
+            .get_inner()?
+            .open_materialized_view(&name)
+            .await
+            .default_error()?;
+        Ok(Table::new(view.table().clone()))
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn list_materialized_views(&self) -> napi::Result<Vec<String>> {
+        let views = self
+            .get_inner()?
+            .list_materialized_views()
+            .await
+            .default_error()?;
+        Ok(views.into_iter().map(|v| v.name).collect())
+    }
+
     #[napi(catch_unwind)]
     pub async fn open_table(
         &self,
@@ -334,6 +420,22 @@ impl Connection {
             .default_error()
     }
 
+    /// Start dropping a table and return its cleanup job.
+    #[napi(catch_unwind)]
+    pub async fn drop_table_async(
+        &self,
+        name: String,
+        namespace_path: Option<Vec<String>>,
+    ) -> napi::Result<crate::job::Job> {
+        let ns = namespace_path.unwrap_or_default();
+        let job = self
+            .get_inner()?
+            .drop_table_async(&name, &ns)
+            .await
+            .default_error()?;
+        Ok(crate::job::Job::new(job))
+    }
+
     #[napi(catch_unwind)]
     pub async fn drop_all_tables(&self, namespace_path: Option<Vec<String>>) -> napi::Result<()> {
         let ns = namespace_path.unwrap_or_default();
diff --git a/nodejs/src/job.rs b/nodejs/src/job.rs
index 6aaeee174..14013fd27 100644
--- a/nodejs/src/job.rs
+++ b/nodejs/src/job.rs
@@ -14,9 +14,12 @@ pub struct Job {
 }
 
 impl Job {
-    pub(crate) fn new(inner: lancedb::Job) -> Self {
+    pub(crate) fn new<T>(inner: lancedb::Job<T>) -> Self
+    where
+        T: Clone + Send + Sync + 'static,
+    {
         Self {
-            inner: Arc::new(inner),
+            inner: Arc::new(inner.map(|_| ())),
         }
     }
 }
diff --git a/nodejs/src/lib.rs b/nodejs/src/lib.rs
index 312b675bd..1110f6203 100644
--- a/nodejs/src/lib.rs
+++ b/nodejs/src/lib.rs
@@ -1,6 +1,10 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
+// The materialized-view refresh future deepens the type graph past the
+// default trait-recursion depth; same raise as the core crate applies.
+#![recursion_limit = "256"]
+
 use std::collections::HashMap;
 
 use env_logger::Env;
diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs
index 7823bd6e9..db74d38fa 100644
--- a/nodejs/src/table.rs
+++ b/nodejs/src/table.rs
@@ -354,6 +354,60 @@ impl Table {
         Ok(res.into())
     }
 
+    #[napi(catch_unwind)]
+    pub async fn add_computed_columns(
+        &self,
+        columns: Vec<AddColumnsSql>,
+    ) -> napi::Result<AddColumnsResult> {
+        let table = self.inner_ref()?;
+        let mut builder = table.add_columns();
+        for column in columns {
+            builder = builder.computed(column.name, column.value_sql);
+        }
+        let res = builder.execute().await.default_error()?;
+        Ok(res.into())
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn refresh_column(&self, column: String) -> napi::Result<RefreshColumnResult> {
+        let res = self
+            .inner_ref()?
+            .refresh_column(column)
+            .await
+            .default_error()?;
+        Ok(res.into())
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn refresh_column_async(&self, column: String) -> napi::Result<crate::job::Job> {
+        let job = self
+            .inner_ref()?
+            .refresh_column_async(column)
+            .await
+            .default_error()?;
+        Ok(crate::job::Job::new(job))
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn refresh_materialized_view(
+        &self,
+        full: Option<bool>,
+        source_version: Option<i64>,
+    ) -> napi::Result<RefreshMaterializedViewResult> {
+        let view = lancedb::MaterializedView::from_table(self.inner_ref()?.clone())
+            .await
+            .default_error()?;
+        let mut builder = view.refresh().full(full.unwrap_or(false));
+        if let Some(version) = source_version {
+            let version = u64::try_from(version).map_err(|_| {
+                napi::Error::from_reason("sourceVersion must be a non-negative integer")
+            })?;
+            builder = builder.source_version(version);
+        }
+        let result = builder.execute().await.default_error()?;
+        Ok(result.into())
+    }
+
     #[napi(catch_unwind)]
     pub async fn add_columns_with_schema(
         &self,
@@ -470,6 +524,34 @@ impl Table {
         self.inner_ref()?.close_lsm_writers().await.default_error()
     }
 
+    #[napi(catch_unwind)]
+    pub async fn flush_lsm(&self) -> napi::Result<()> {
+        self.inner_ref()?.flush_lsm().await.default_error()
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn compact_lsm(&self) -> napi::Result<()> {
+        self.inner_ref()?.compact_lsm().await.default_error()
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn checkpoint_lsm(&self) -> napi::Result<()> {
+        self.inner_ref()?.checkpoint_lsm().await.default_error()
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn get_lsm_stats(
+        &self,
+        include_generation_rows: bool,
+    ) -> napi::Result<Option<LsmStats>> {
+        let stats = self
+            .inner_ref()?
+            .get_lsm_stats(include_generation_rows)
+            .await
+            .default_error()?;
+        Ok(stats.map(LsmStats::from))
+    }
+
     #[napi(catch_unwind)]
     pub async fn version(&self) -> napi::Result<i64> {
         self.inner_ref()?
@@ -479,6 +561,12 @@ impl Table {
             .default_error()
     }
 
+    #[napi(catch_unwind)]
+    pub async fn checkout_current(&self) -> napi::Result<Self> {
+        let table = self.inner_ref()?.checkout_current().await.default_error()?;
+        Ok(Self::new(table))
+    }
+
     #[napi(catch_unwind)]
     pub async fn checkout(&self, version: i64) -> napi::Result<()> {
         self.inner_ref()?
@@ -779,7 +867,8 @@ pub struct LsmWriteSpec {
     pub column: Option<String>,
     /// Bucket variant: the number of buckets, in `[1, 1024]`.
     pub num_buckets: Option<u32>,
-    /// Names of indexes the MemWAL should keep up to date during writes.
+    /// Indexes the MemWAL keeps up to date. Omitted resolves every
+    /// maintainable index on install; an empty array means none.
     pub maintained_indexes: Option<Vec<String>>,
     /// Default `ShardWriter` configuration recorded in the MemWAL index.
     pub writer_config_defaults: Option<HashMap<String, String>>,
@@ -789,7 +878,6 @@ impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
     type Error = napi::Error;
 
     fn try_from(value: LsmWriteSpec) -> napi::Result<Self> {
-        let maintained = value.maintained_indexes.unwrap_or_default();
         let writer_config_defaults = value.writer_config_defaults.unwrap_or_default();
         let spec = match value.spec_type.as_str() {
             "bucket" => {
@@ -816,7 +904,7 @@ impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
             }
         };
         Ok(spec
-            .with_maintained_indexes(maintained)
+            .with_maintained_indexes(value.maintained_indexes)
             .with_writer_config_defaults(writer_config_defaults))
     }
 }
@@ -834,7 +922,7 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
                 spec_type: "bucket".to_string(),
                 column: Some(column),
                 num_buckets: Some(num_buckets),
-                maintained_indexes: Some(maintained_indexes),
+                maintained_indexes,
                 writer_config_defaults: Some(writer_config_defaults),
             },
             Native::Identity {
@@ -845,7 +933,7 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
                 spec_type: "identity".to_string(),
                 column: Some(column),
                 num_buckets: None,
-                maintained_indexes: Some(maintained_indexes),
+                maintained_indexes,
                 writer_config_defaults: Some(writer_config_defaults),
             },
             Native::Unsharded {
@@ -855,13 +943,136 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
                 spec_type: "unsharded".to_string(),
                 column: None,
                 num_buckets: None,
-                maintained_indexes: Some(maintained_indexes),
+                maintained_indexes,
                 writer_config_defaults: Some(writer_config_defaults),
             },
         }
     }
 }
 
+/// One flushed L0 generation.
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct GenerationStats {
+    /// The generation number. Increases as memtables are sealed into L0.
+    pub generation: i64,
+    /// On-disk size of the generation.
+    pub bytes: i64,
+    /// Present only when `includeGenerationRows` was requested. Off by default
+    /// because each count opens an uncached Lance dataset.
+    pub rows: Option<i64>,
+}
+
+impl From<lancedb::table::GenerationStats> for GenerationStats {
+    fn from(g: lancedb::table::GenerationStats) -> Self {
+        Self {
+            generation: g.generation as i64,
+            bytes: g.bytes as i64,
+            rows: g.rows.map(|r| r as i64),
+        }
+    }
+}
+
+/// One in-memory memtable.
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct MemtableStats {
+    /// The generation this memtable will become once sealed.
+    pub generation: i64,
+    /// Rows currently buffered.
+    pub rows: i64,
+    /// Estimated in-memory size.
+    pub bytes: i64,
+    /// Record batches currently buffered.
+    pub batches: i64,
+    /// Names of the indexes this memtable carries. An absent name is the whole
+    /// answer to "why is my fresh-tier search on that column brute-force".
+    pub indexes: Vec<String>,
+}
+
+impl From<lancedb::table::MemtableStats> for MemtableStats {
+    fn from(m: lancedb::table::MemtableStats) -> Self {
+        Self {
+            generation: m.generation as i64,
+            rows: m.rows as i64,
+            bytes: m.bytes as i64,
+            batches: m.batches as i64,
+            indexes: m.indexes,
+        }
+    }
+}
+
+/// Live state of one bucket. A table is N buckets on one node; flattening to a
+/// single number hides the one hot bucket that is usually why someone opened
+/// this endpoint.
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct BucketStats {
+    /// The shard this bucket writes.
+    pub shard_id: String,
+    /// `"Active"` or `"Sealed"` (drop-table 2PC in flight).
+    pub status: String,
+    /// Epoch of the writer that currently owns the shard.
+    pub writer_epoch: i64,
+    /// Version of the shard manifest these numbers were read from.
+    pub manifest_version: i64,
+    /// The generation the active memtable will become.
+    pub current_generation: i64,
+    /// WAL position replay resumes from.
+    pub replay_after_wal_entry_position: i64,
+    /// Highest WAL position the writer has seen. The difference against
+    /// `replayAfterWalEntryPosition` is the WAL lag.
+    pub wal_entry_position_last_seen: i64,
+    /// Flushed L0 generations not yet merged into the base table.
+    pub generations: Vec<GenerationStats>,
+    /// Whether a pass owns this bucket's compaction latch right now. Says *a*
+    /// driver is running, not *whose*, and the latch is held from dispatch —
+    /// including while the pass queues for a pod-wide compactor permit. Read it
+    /// as "do not pile on", never as "mine is progressing".
+    pub compacting: bool,
+    /// Oldest first, active last. Absent for a `"Sealed"` bucket, whose
+    /// in-memory state is torn down.
+    pub memtables: Option<Vec<MemtableStats>>,
+}
+
+impl From<lancedb::table::BucketStats> for BucketStats {
+    fn from(b: lancedb::table::BucketStats) -> Self {
+        Self {
+            shard_id: b.shard_id,
+            status: b.status,
+            writer_epoch: b.writer_epoch as i64,
+            manifest_version: b.manifest_version as i64,
+            current_generation: b.current_generation as i64,
+            replay_after_wal_entry_position: b.replay_after_wal_entry_position as i64,
+            wal_entry_position_last_seen: b.wal_entry_position_last_seen as i64,
+            generations: b.generations.into_iter().map(Into::into).collect(),
+            compacting: b.compacting,
+            memtables: b
+                .memtables
+                .map(|ms| ms.into_iter().map(Into::into).collect()),
+        }
+    }
+}
+
+/// Live per-bucket LSM state, as returned by `Table#getLsmStats`.
+///
+/// Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are
+/// the caller's to compute.
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct LsmStats {
+    /// One entry per bucket backing this table.
+    pub buckets: Vec<BucketStats>,
+}
+
+impl From<lancedb::table::LsmStats> for LsmStats {
+    fn from(stats: lancedb::table::LsmStats) -> Self {
+        Self {
+            buckets: stats.buckets.into_iter().map(Into::into).collect(),
+        }
+    }
+}
+
 /// Statistics about a compaction operation.
 #[napi(object)]
 #[derive(Clone, Debug)]
@@ -1050,7 +1261,10 @@ impl From<lancedb::index::IndexStatistics> for IndexStatistics {
 
 #[napi(object)]
 pub struct TableStatistics {
-    /// The total number of bytes in the table
+    /// The total size, in bytes, of the table's data files, index files, and
+    /// overlay files
+    ///
+    /// Read from the manifest, so this excludes deletion files and manifests.
     pub total_bytes: i64,
 
     /// The number of rows in the table
@@ -1200,6 +1414,46 @@ pub struct AddColumnsResult {
     pub version: i64,
 }
 
+#[napi(object)]
+pub struct RefreshColumnResult {
+    pub rows_filled: i64,
+    pub version: i64,
+}
+
+#[napi(object)]
+pub struct RefreshMaterializedViewResult {
+    /// How the view was brought up to date: "rebuild", "incremental" or "no_op".
+    pub mode: String,
+    pub rows_written: i64,
+    pub source_version: i64,
+    pub version: i64,
+}
+
+impl From<lancedb::RefreshMaterializedViewResult> for RefreshMaterializedViewResult {
+    fn from(value: lancedb::RefreshMaterializedViewResult) -> Self {
+        let mode = match value.mode {
+            lancedb::RefreshMode::Rebuild => "rebuild",
+            lancedb::RefreshMode::Incremental => "incremental",
+            lancedb::RefreshMode::NoOp => "no_op",
+        };
+        Self {
+            mode: mode.to_string(),
+            rows_written: value.rows_written as i64,
+            source_version: value.source_version as i64,
+            version: value.version as i64,
+        }
+    }
+}
+
+impl From<lancedb::table::RefreshColumnResult> for RefreshColumnResult {
+    fn from(value: lancedb::table::RefreshColumnResult) -> Self {
+        Self {
+            rows_filled: value.rows_filled as i64,
+            version: value.version as i64,
+        }
+    }
+}
+
 impl From<lancedb::table::AddColumnsResult> for AddColumnsResult {
     fn from(value: lancedb::table::AddColumnsResult) -> Self {
         Self {
@@ -1409,18 +1663,18 @@ impl Branches {
     }
 
     #[napi(ts_return_type = "Promise<Record<string, unknown>>")]
-    pub async fn merge(
+    pub async fn cherry_pick(
         &self,
         from_branch: String,
         dry_run: Option<bool>,
     ) -> napi::Result<serde_json::Value> {
         let result = self
             .inner
-            .merge_branch(&from_branch, dry_run.unwrap_or(false))
+            .cherry_pick(&from_branch, dry_run.unwrap_or(false))
             .await
             .default_error()?;
         serde_json::to_value(result).map_err(|err| {
-            napi::Error::from_reason(format!("failed to serialize branch merge result: {err}"))
+            napi::Error::from_reason(format!("failed to serialize cherry-pick result: {err}"))
         })
     }
 }
diff --git a/plugins/lancedb/.claude-plugin/plugin.json b/plugins/lancedb/.claude-plugin/plugin.json
deleted file mode 100644
index 9fe3d4b67..000000000
--- a/plugins/lancedb/.claude-plugin/plugin.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
-  "name": "lancedb",
-  "description": "Write, review, debug, and document LanceDB pipelines in Python and TypeScript that work across local LanceDB OSS tables and remote LanceDB Enterprise/Cloud tables, with idiomatic query/search patterns and performance defaults for ingestion, indexing, filtering, and diagnostics.",
-  "version": "0.1.0",
-  "author": {
-    "name": "LanceDB"
-  },
-  "homepage": "https://www.lancedb.com",
-  "keywords": [
-    "lancedb",
-    "vector-search",
-    "full-text-search",
-    "hybrid-search",
-    "python",
-    "typescript",
-    "pipelines",
-    "ingestion",
-    "indexing",
-    "performance"
-  ]
-}
diff --git a/plugins/lancedb/.codex-plugin/plugin.json b/plugins/lancedb/.codex-plugin/plugin.json
deleted file mode 100644
index bb824ceb4..000000000
--- a/plugins/lancedb/.codex-plugin/plugin.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
-  "name": "lancedb",
-  "version": "0.1.0",
-  "description": "Codex plugin for building LanceDB pipelines in Python and TypeScript.",
-  "author": {
-    "name": "LanceDB"
-  },
-  "keywords": [
-    "lancedb",
-    "vector-search",
-    "full-text-search",
-    "hybrid-search",
-    "python",
-    "typescript",
-    "pipelines"
-  ],
-  "skills": "./skills/",
-  "interface": {
-    "displayName": "LanceDB",
-    "shortDescription": "Build LanceDB pipelines in Python and TypeScript.",
-    "longDescription": "Write, review, debug, and document LanceDB pipelines in Python and TypeScript that work across local LanceDB OSS tables and remote LanceDB Enterprise/Cloud tables, with idiomatic query/search patterns and performance defaults for ingestion, indexing, filtering, and diagnostics.",
-    "developerName": "LanceDB",
-    "websiteURL": "https://www.lancedb.com",
-    "category": "Developer Tools",
-    "capabilities": [
-      "Developer Tools"
-    ],
-    "defaultPrompt": "Create a LanceDB table, embed sample text, and run a vector search.",
-    "composerIcon": "./assets/logo.png",
-    "logo": "./assets/logo.png",
-    "logoDark": "./assets/logo-dark.png"
-  }
-}
diff --git a/plugins/lancedb/assets/logo-dark.png b/plugins/lancedb/assets/logo-dark.png
deleted file mode 100644
index 8fd8f220e..000000000
Binary files a/plugins/lancedb/assets/logo-dark.png and /dev/null differ
diff --git a/plugins/lancedb/assets/logo.png b/plugins/lancedb/assets/logo.png
deleted file mode 100644
index de3e82aef..000000000
Binary files a/plugins/lancedb/assets/logo.png and /dev/null differ
diff --git a/plugins/lancedb/skills/lancedb/SKILL.md b/plugins/lancedb/skills/lancedb/SKILL.md
deleted file mode 100644
index 8b8f761c2..000000000
--- a/plugins/lancedb/skills/lancedb/SKILL.md
+++ /dev/null
@@ -1,87 +0,0 @@
----
-name: lancedb
-description: Use when writing, reviewing, debugging, or documenting LanceDB pipelines in Python or TypeScript, especially code that should work across local LanceDB OSS tables and remote LanceDB Enterprise/Cloud tables. Helps avoid non-portable full-table materialization, choose idiomatic query/search patterns, apply LanceDB performance defaults for ingestion, indexing, filtering, and diagnostics, and resolve connections to the remote server for Enterprise-only operations such as jobs.
----
-
-# Building LanceDB Pipelines
-
-Use this skill to produce LanceDB pipelines that are portable between local and remote tables (for LanceDB Enterprise/Cloud) and idiomatic for the selected SDK.
-
-## LanceDB Table Modes
-
-LanceDB has two common execution modes:
-
-- **Local table**: embedded, open source, in-process LanceDB. The client opens data from a local path or object storage URI and executes queries in the application process.
-- **Remote table**: LanceDB Enterprise/Cloud table opened through a `db://...` URI. The data may be very large, commonly backed by object storage, and queried through a remote service.
-
-Do NOT assume local-only table helpers exist on remote tables. If the user asks for LanceDB Enterprise, Cloud, `db://...`, production remote access, or a remote table, focus on the remote table path: use `search()` / `query()`, keep reads bounded with `select()` and `limit()`, and avoid table-level full materialization APIs.
-
-## Workflow
-
-1. Identify the SDK: Python, TypeScript, or both.
-2. Identify the table mode: local/embedded OSS, remote Enterprise/Cloud, or portable across both. If the user says "LanceDB Enterprise", choose the remote table path. If the task involves jobs in any way (listing, inspecting, creating, or canceling jobs), it is always the remote path and requires a remote server connection — see "Connecting to the LanceDB remote server" below before doing anything else.
-3. Read the matching language branch before writing or changing code:
-   - Python patterns: `references/python/patterns.md`
-   - Python API quick reference: `references/python/api_reference.md`
-   - Python performance guidance: `references/python/performance.md`
-   - TypeScript patterns: `references/typescript/patterns.md`
-   - TypeScript API quick reference: `references/typescript/api_reference.md`
-   - TypeScript performance guidance: `references/typescript/performance.md`
-   - Column metadata authoring (both SDKs): `references/column_metadata.md`
-   - Branch operations (both SDKs): `references/branch_ops.md`
-   - Remote server connection resolution (jobs, raw REST): `references/remote_connect.md`
-   - Job operations REST API (list/describe/cancel/query_events): `references/remote_jobs.md`
-4. Start with `patterns.md` for the selected SDK. Read `api_reference.md` when choosing method names or return collectors. Read `performance.md` when the task involves ingestion, indexing, filtering, query tuning, diagnostics, or large datasets. Read `column_metadata.md` when the task is documenting, tagging, classifying, or grouping table columns (field descriptions, `lancedb:tag:*` tags, logical column families). Read `branch_ops.md` when the task involves branch lifecycle (list/create/delete), writing to a non-main branch, or verifying a change stayed off main. Read `remote_connect.md` when the task involves jobs or direct REST access to an Enterprise deployment, and `remote_jobs.md` for the job REST methods themselves (list, describe, cancel, query_events).
-5. For Python schemas, favor Pydantic models and validate records before writing. Use PyArrow schemas when Arrow-native, streaming, or highly dynamic data makes them materially better suited.
-6. Prefer `search()` or `query()` builders with explicit `select()` and `limit()` for reads.
-7. Avoid table-level full materialization in remote or portable code. This is the main local-vs-remote read pitfall.
-8. After a successful embedded OSS ingestion, call `table.optimize()`. Do not call it for Enterprise/Cloud; remote maintenance is automatic.
-9. For remote Enterprise/Cloud writes, never drop-then-reuse or `mode="overwrite"` the same table name — see "Enterprise: never drop-then-reuse the same table name" below. This is the main local-vs-remote write pitfall.
-10. If reviewing an existing file or repo, run `scripts/check_materialization.py` on the relevant paths and inspect each finding before editing.
-11. Cross-check unfamiliar or non-trivial API claims against the source tree instead of relying on memory.
-
-## Core Portability Rule
-
-Do not write code that assumes a local table API will exist on a remote table. Remote tables can be very large, so whole-table materialization helpers are intentionally unavailable or unsafe.
-
-This does **not** mean result conversion is forbidden. Bounded query/search result collection is normal:
-
-- Python: `table.search(...).select([...]).limit(10).to_pandas()`
-- TypeScript: `await table.search(...).select([...]).limit(10).toArray()`
-
-The unsafe pattern is table-level or unbounded collection, plus local-only dataset escape hatches in remote code:
-
-- Python: `table.to_pandas()`, `table.to_arrow()`, `table.to_polars()`; `table.to_lance()` is local/OSS-only dataset access, not materialization
-- TypeScript: `await table.toArrow()`, `await table.query().toArray()` without `limit()`
-
-## Enterprise: never drop-then-reuse the same table name
-
-LanceDB Enterprise/Cloud splits a **control plane** (DDL: create/drop/rename) from a **data plane** (query nodes that serve reads). Query nodes cache the resolved dataset for a table name for up to `table_cache_ttl` — **default 300 seconds (5 minutes)**. After you drop or overwrite a table, the control plane updates immediately but the data plane keeps serving the *old* dataset until that cache entry expires. During the window the two planes disagree.
-
-The failure this causes: you `drop_table("t")` then immediately `create_table("t", ...)` (or `create_table("t", ..., mode="overwrite")`). The DDL returns success, but every query against `t` returns **`500 Internal Server Error`** (the query node resolves the stale/deleted dataset), and a fresh `describe` may still show the *old* schema/version. It looks like your write silently failed; it didn't — the name is cached.
-
-**`mode="overwrite"` has the same problem** — it is a drop+create of the same name under the hood.
-
-Rules for portable Enterprise ingestion:
-
-1. **Never reuse a table name you just dropped/overwrote within the cache TTL.** Do not use `mode="overwrite"` to replace an existing Enterprise table in place.
-2. To (re)load data, **write to a fresh table name** (e.g. `<table>_v2`, or a run-stamped suffix). A brand-new name has no cached data-plane entry, so writes and reads work immediately.
-3. Before creating, `list_tables()` and **fail loudly if the name already exists** rather than overwriting — prompt for a new name.
-4. To land on a specific final name that is currently occupied by an old table: drop the old table, **wait out the TTL (~5 min), then `rename_table(fresh_name, final_name)`**. Renaming onto a name whose old dataset is still cached hits the same race, so the wait is mandatory. `rename_table` is a supported control-plane op.
-5. When you hand a table name back to a human, tell them which step still needs the propagation wait (usually: "the old `t` was dropped; run the rename in ~5 minutes").
-
-This is Enterprise/Cloud-specific. Local/OSS tables have no separate data plane, so `mode="overwrite"` and immediate same-name reuse are fine there.
-
-## Connecting to the LanceDB remote server
-
-LanceDB Enterprise/Cloud deployments are served by a server implementing the lance-namespace OpenAPI spec (<https://github.com/lance-format/lance-namespace/blob/main/docs/src/spec.yaml>). Every remote (`db://...`) connection talks to such a server, and some operations exist only there. In particular, **all operations around jobs (listing, inspecting, creating, or canceling jobs) run server-side** — there is no local/OSS equivalent. Before any job work, or any direct REST call to an Enterprise deployment, read `references/remote_connect.md` to resolve the base URL, credentials, and database header and to validate the connection. Then use the four job REST methods documented in `references/remote_jobs.md` (list, describe, cancel, query_events).
-
-## Script
-
-Run the scanner when reviewing or modifying an existing codebase:
-
-```bash
-python skills/lancedb/scripts/check_materialization.py path/to/file_or_dir
-```
-
-The script reports likely unsafe full-table materialization in Python and TypeScript. Treat results as review prompts, not automatic proof of a bug.
diff --git a/plugins/lancedb/skills/lancedb/agents/openai.yaml b/plugins/lancedb/skills/lancedb/agents/openai.yaml
deleted file mode 100644
index 5a5b2d5cd..000000000
--- a/plugins/lancedb/skills/lancedb/agents/openai.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-interface:
-  display_name: "LanceDB"
-  short_description: "Build LanceDB pipelines in Python and TypeScript"
-  default_prompt: "Use $lancedb to create a table, embed sample text, and run a vector search."
-  icon_small: "./assets/icon.png"
-  icon_large: "./assets/icon.png"
diff --git a/plugins/lancedb/skills/lancedb/assets/icon.png b/plugins/lancedb/skills/lancedb/assets/icon.png
deleted file mode 100644
index 94cdd637a..000000000
Binary files a/plugins/lancedb/skills/lancedb/assets/icon.png and /dev/null differ
diff --git a/plugins/lancedb/skills/lancedb/references/branch_ops.md b/plugins/lancedb/skills/lancedb/references/branch_ops.md
deleted file mode 100644
index e94c7e6db..000000000
--- a/plugins/lancedb/skills/lancedb/references/branch_ops.md
+++ /dev/null
@@ -1,182 +0,0 @@
-# Branch Operations
-
-Manage branches on a LanceDB table: list what exists, create new ones, delete stale ones, and direct read/write operations at a specific branch without touching main. Use for branch lifecycle tasks, experimental/isolated table versions, targeting an operation at a non-main branch, or confirming a mutation did not affect main.
-
-Works on local/OSS and remote Enterprise/Cloud tables, except merging a branch into main, which is Enterprise-only.
-
-## The branch model (important)
-
-Branches are isolated, writable lines of history forked from another branch (or a specific version). Writes on a branch never affect `main`.
-
-There is **no global "switch branch" state** — you never repoint the whole table at a branch. Instead, **operations are scoped by which table handle you use**:
-
-- The handle you got from `open_table(name)` / `openTable(name)` targets `main`.
-- `branches.create(...)` and `branches.checkout(...)` return a **new table handle scoped to that branch**. Every read/write on that handle (add, update, `update_field_metadata`, `create_index`, search, …) lands on the branch.
-- The original main handle is unaffected — keep it around to verify isolation.
-
-`branches.list()` returns only non-main branches. Main always exists and is not listed.
-
-## Python
-
-`table.branches` is a property returning the branch manager; `table.current_branch()` tells you what a handle is scoped to (`None` = main).
-
-```python
-table = db.open_table("products")            # scoped to main
-
-# list — dict of name -> metadata (parent_branch, parent_version, ...); {} = only main
-table.branches.list()
-
-# create: forks from main by default and returns a handle scoped to the new branch
-exp = table.branches.create("experiment-reindex")
-exp = table.branches.create("exp2", from_ref="main", from_version=None)  # optional fork point
-
-# checkout an existing branch -> branch-scoped handle
-wip = table.branches.checkout("wip-branch")
-# with version= it pins to that version (read-only detached view); omit to track latest, writable
-
-# operate on the branch simply by using its handle
-wip.update_field_metadata(
-    {"path": "category", "metadata": {"lancedb:description": "Product category label."}}
-)
-wip.create_scalar_index("category")
-
-# delete: removes only the branch pointer; main and row data remain intact
-table.branches.delete("stale-2024")
-
-# alternatively, open a branch handle directly from the connection
-wip = db.open_table("products", branch="wip-branch")
-
-exp.current_branch()    # "experiment-reindex"
-table.current_branch()  # None (main)
-```
-
-Async: same shape — `table.branches` returns `AsyncBranches`; `await table.branches.create(...)` etc.
-
-## TypeScript
-
-`table.branches()` is an **async method** returning the `Branches` manager; `table.currentBranch()` returns the scoped branch or `null` for main.
-
-```typescript
-const table = await db.openTable("products"); // scoped to main
-const branches = await table.branches();
-
-// list — Record<string, BranchContents>; {} = only main
-await branches.list();
-
-// create: forks from main by default, returns a Table scoped to the new branch
-const exp = await branches.create("experiment-reindex");
-const exp2 = await branches.create("exp2", "main" /* fromRef */, undefined /* fromVersion */);
-
-// checkout an existing branch -> branch-scoped Table
-const wip = await branches.checkout("wip-branch");
-// with a version arg it pins (read-only detached view); omit to track latest, writable
-
-// operate on the branch simply by using its handle
-await wip.updateFieldMetadata([
-  { path: "category", metadata: { "lancedb:description": "Product category label." } },
-]);
-await wip.createIndex("category");
-
-// delete: removes only the branch pointer; main and row data remain intact
-await branches.delete("stale-2024");
-
-// alternatively, open a branch handle directly from the connection
-const wip2 = await db.openTable("products", { branch: "wip-branch" });
-
-exp.currentBranch();   // "experiment-reindex"
-table.currentBranch(); // null (main)
-```
-
-## Verifying isolation
-
-After writing to a branch, confirm the change did NOT land on main by reading through both handles:
-
-```python
-wip = table.branches.checkout("wip-branch")
-wip.update_field_metadata({"path": "category", "metadata": {"lancedb:description": "..."}})
-
-assert b"lancedb:description" in (wip.schema.field("category").metadata or {})
-assert b"lancedb:description" not in (table.schema.field("category").metadata or {})  # main untouched
-```
-
-Two handles on the same branch see each other's writes (e.g. `table.branches.create("exp")` and `db.open_table(name, branch="exp")`); main stays isolated.
-
-## Merging a branch into main (Enterprise only)
-
-Merge is available through the SDKs (`table.branches.merge(...)`) on **Enterprise tables only** — it is not supported on Cloud or local/OSS tables, which raise `NotSupported`.
-
-`merge` takes the branch to merge **from** and a `dry_run` flag. Both the SDK method and the underlying REST endpoint **actually merge by default** (`dry_run=False`); pass `dry_run=True` to only preview. A rejected merge is **not an exception** — it returns a result with `status="rejected"` rather than raising, so inspect the return value. Use `branches.diff(from_branch)` to inspect a branch's pending diff without attempting a merge.
-
-```python
-exp = "experiment-reindex"
-
-# preview only — returns status="ready" if it would merge cleanly
-preview = table.branches.merge(exp, dry_run=True)
-
-# actually merge (default)
-result = table.branches.merge(exp)
-if result["status"] == "merged":
-    print("landed at", result["mainVersionAfter"])
-elif result["status"] == "rejected":
-    print(result["diff"]["mergeBlockers"])   # why it was refused
-
-# inspect a branch's pending diff without merging
-diff = table.branches.diff(exp)
-```
-
-Async: `await table.branches.merge(exp)`, `await table.branches.diff(exp)`.
-
-```typescript
-const branches = await table.branches();
-const exp = "experiment-reindex";
-
-// preview only (second arg is dryRun)
-const preview = await branches.merge(exp, true);
-
-// actually merge (default)
-const result = await branches.merge(exp);
-if (result.status === "merged") {
-  console.log("landed at", result.mainVersionAfter);
-} else if (result.status === "rejected") {
-  console.log(result.diff.mergeBlockers);
-}
-
-const diff = await branches.diff(exp);
-```
-
-The result is the wire JSON, containing `status` (`ready` on a passing dry run, `merged` on success, `rejected` when refused — also `notImplemented`/`unknown`), the branch `diff` (including `mergeBlockers` explaining any rejection), a `preview` of the columns that would be promoted, and — after a real merge — `mainVersionAfter`.
-
-### Merge preconditions
-
-Merge only **promotes newly added columns** onto main; it does not replay arbitrary commits. Practically, a branch is mergeable only if it has **exactly one commit since it was created, and that commit added a column**. The merge is rejected (`status: "rejected"`, with `mergeBlockers` set) if:
-
-- the branch was forked from another branch rather than directly from main
-- main has advanced since the branch was forked
-- the branch's rows changed since the fork (row counts must match main exactly)
-- the branch removed columns or changed a column's type/nullability
-- the branch added no columns (index-only changes are not merged)
-
-### Adding a column in a single commit
-
-Because the branch must contain just one column-adding commit, add the column with its values in one operation rather than add-then-backfill:
-
-1. **SQL transformation** — `add_columns` with a SQL expression computed from existing columns, so the column lands populated in one commit.
-2. **Precompute the values** — compute the column's values externally, then add the fully-populated column in a single operation (e.g. via `merge_insert`/`add_columns` with the data ready).
-3. **Lance-format-level data evolution (pylance)** — use Lance's data evolution with backfill, documented at <https://lance.org/guide/data_evolution/#with-data-backfill>.
-
-## Quick reference
-
-| Goal | Python | TypeScript |
-|------|--------|------------|
-| List branches (non-main) | `table.branches.list()` | `await (await table.branches()).list()` |
-| Create branch (off main) | `table.branches.create(name)` → branch handle | `await branches.create(name)` → branch `Table` |
-| Create from a fork point | `table.branches.create(name, from_ref=..., from_version=...)` | `await branches.create(name, fromRef, fromVersion)` |
-| Get a branch handle | `table.branches.checkout(name)` or `db.open_table(t, branch=name)` | `await branches.checkout(name)` or `await db.openTable(t, { branch: name })` |
-| Pin to a branch version (read-only) | `table.branches.checkout(name, version=v)` | `await branches.checkout(name, v)` |
-| Delete branch | `table.branches.delete(name)` | `await branches.delete(name)` |
-| Which branch is this handle on? | `table.current_branch()` (`None` = main) | `table.currentBranch()` (`null` = main) |
-| Target main | use the original (non-branch) handle | use the original (non-branch) handle |
-| Merge branch into main (Enterprise only) | `table.branches.merge(from_branch, dry_run=False)` | `await branches.merge(fromBranch, dryRun)` |
-| Preview a branch's pending diff (Enterprise only) | `table.branches.diff(from_branch)` | `await branches.diff(fromBranch)` |
-
-Branch names must be non-empty; empty names raise a validation error.
diff --git a/plugins/lancedb/skills/lancedb/references/column_metadata.md b/plugins/lancedb/skills/lancedb/references/column_metadata.md
deleted file mode 100644
index 2fe1c7c5d..000000000
--- a/plugins/lancedb/skills/lancedb/references/column_metadata.md
+++ /dev/null
@@ -1,183 +0,0 @@
-# Column Metadata Authoring
-
-Write column-level descriptions, tags, and logical groupings onto a LanceDB table's schema. Use this when the user wants to document, annotate, tag, or classify what their table columns ARE (embeddings vs labels vs eval metrics, model provenance, version families, etc.).
-
-Works on local/OSS and remote Enterprise/Cloud tables alike — read the schema through the table handle, write through `update_field_metadata` (Python) / `updateFieldMetadata` (TypeScript).
-
-## Metadata key conventions
-
-All metadata uses namespaced keys:
-
-| Key | Purpose | Example value |
-|-----|---------|---------------|
-| `lancedb:description` | Human-readable explanation of what the column contains | `"CLIP ViT-L/14 image embedding, L2-normalized (768-dim)"` |
-| `lancedb:tag:<name>` | Flexible key-value tag; the suffix names the tag category | `lancedb:tag:field_type: "embedding"`, `lancedb:tag:model: "clip"`, `lancedb:tag:project_id: "foo"` |
-| `lancedb:logical-column` | Logical group/family this column belongs to | `"clip_features"` |
-
-Tags are open-ended — use whatever key suffix and value make sense given the user's intent. The tag suffix should describe *what is being classified* (e.g., `field_type`, `model`, `project_id`) and the value describes *how*. Multiple tags on the same column are fine — each is a separate key. All values are strings.
-
-## Step 1: Read the schema and existing metadata
-
-Read existing metadata before writing, to avoid redundant updates.
-
-Python — `table.schema` (sync property; async: `await table.schema()`) returns a `pyarrow.Schema`. **Arrow field metadata is bytes-keyed in Python**:
-
-```python
-schema = table.schema
-for field in schema:
-    meta = field.metadata or {}  # dict[bytes, bytes], e.g. {b"lancedb:description": b"..."}
-    print(field.name, field.type, field.nullable, meta)
-```
-
-TypeScript — `await table.schema()` returns an Arrow `Schema`; field metadata is a `Map<string, string>`:
-
-```typescript
-const schema = await table.schema();
-for (const field of schema.fields) {
-  console.log(field.name, field.type, field.nullable, field.metadata); // Map
-  // field.metadata.get("lancedb:description")
-}
-```
-
-For struct/nested fields, recurse into the field's children and address them as dot-paths (e.g., `parent.child`).
-
-If the user hasn't specified which columns to update, work with all columns.
-
-## Step 2: Generate metadata
-
-Decide what to generate based on the user's request.
-
-### Descriptions (`lancedb:description`)
-
-Base descriptions on:
-- The column name and Arrow type (e.g., `FixedSizeList` of floats → likely an embedding)
-- User-supplied context (upstream pipeline, sample values, domain knowledge)
-- Name patterns: `_embedding`/`_vec`/`_embed` → vector; `_label`/`_class` → label; `_score`/`_eval`/`_metric` → evaluation metric
-
-Be specific and concise. Good: `"Sentence-BERT embedding of the query text (768-dim)."` Not: `"An embedding column."`
-
-### Tags (`lancedb:tag:<name>`)
-
-Choose tag key names that match what the user asked to annotate. Common patterns:
-
-- Semantic field type → `lancedb:tag:field_type: "embedding"` / `"text"` / `"image"` / `"label"` / `"eval"` / `"id"` / `"metadata"`
-- Model or source → `lancedb:tag:model: "clip"` / `"bert"` / `"vit"`
-- Project affiliation → `lancedb:tag:project_id: "<name>"`
-- Version → `lancedb:tag:version: "v3"` (and `lancedb:tag:latest: "true"` for the newest)
-
-Use Arrow type as a hint: `FixedSizeList` + float → embedding; `Utf8`/`LargeUtf8` → text; `Binary` → image or blob.
-
-### Logical groupings (`lancedb:logical-column`)
-
-Look for naming patterns across columns:
-- `clip_v1`, `clip_v2`, `clip_v3` → logical column `"clip"`, latest is `v3`
-- `text_embed_20240101`, `text_embed_20240601` → logical column `"text_embed"`, latest is the most recent date suffix
-
-Write `lancedb:logical-column` on all members of a group. Mark the newest with `lancedb:tag:latest: "true"` (in addition to its version tag).
-
-## Step 3: Write the metadata
-
-Each update names a field by dot-path and carries a metadata map. Semantics (identical in both SDKs):
-
-- **Merge by default** (`replace` omitted/false) — preserves existing metadata the user didn't ask to change
-- `replace: true` swaps the field's entire metadata map — only if the user explicitly asks to overwrite
-- A value of `None`/`null` deletes that specific key
-- Batch all field updates into a single call when possible
-- Returns the new table version
-
-Python (sync and async take one dict per field, as varargs):
-
-```python
-res = table.update_field_metadata(
-    {
-        "path": "clip_v3",
-        "metadata": {
-            "lancedb:description": "CLIP ViT-L/14 image embedding, L2-normalized (1024-dim).",
-            "lancedb:tag:field_type": "embedding",
-            "lancedb:tag:model": "clip",
-            "lancedb:tag:version": "v3",
-            "lancedb:tag:latest": "true",
-            "lancedb:logical-column": "clip",
-        },
-    },
-    {
-        "path": "clip_v2",
-        "metadata": {
-            "lancedb:description": "CLIP ViT-B/32 image embedding (768-dim), superseded by v3.",
-            "lancedb:tag:field_type": "embedding",
-            "lancedb:tag:model": "clip",
-            "lancedb:tag:version": "v2",
-            "lancedb:logical-column": "clip",
-        },
-    },
-)
-print(res.version)  # new table version
-
-# merge semantics: add a key, delete one via None, keep the rest
-table.update_field_metadata(
-    {"path": "clip_v2", "metadata": {"lancedb:tag:archived": "true", "lancedb:tag:latest": None}}
-)
-```
-
-(`replace_field_metadata` is deprecated — use `update_field_metadata`.)
-
-TypeScript (takes an array of `FieldMetadataUpdate`):
-
-```typescript
-const res = await table.updateFieldMetadata([
-  {
-    path: "clip_v3",
-    metadata: {
-      "lancedb:description": "CLIP ViT-L/14 image embedding, L2-normalized (1024-dim).",
-      "lancedb:tag:field_type": "embedding",
-      "lancedb:tag:model": "clip",
-      "lancedb:tag:version": "v3",
-      "lancedb:tag:latest": "true",
-      "lancedb:logical-column": "clip",
-    },
-  },
-  {
-    path: "clip_v2",
-    metadata: {
-      "lancedb:description": "CLIP ViT-B/32 image embedding (768-dim), superseded by v3.",
-      "lancedb:tag:field_type": "embedding",
-      "lancedb:tag:model": "clip",
-      "lancedb:tag:version": "v2",
-      "lancedb:logical-column": "clip",
-    },
-  },
-]);
-console.log(res.version); // new table version
-
-// merge semantics: add a key, delete one via null, keep the rest
-await table.updateFieldMetadata([
-  { path: "clip_v2", metadata: { "lancedb:tag:archived": "true", "lancedb:tag:latest": null } },
-]);
-```
-
-## Step 4: Confirm
-
-Report back:
-- Which columns were updated and what was written
-- The new table version number (from the result)
-- Any columns skipped (e.g., already had up-to-date metadata)
-
-## Quick examples
-
-**"Write descriptions for all columns in the `product_embeddings` table"**
-1. Read `table.schema` → all fields + existing metadata
-2. Generate a `lancedb:description` for each column based on name + type
-3. One `update_field_metadata` call with all descriptions
-4. Report
-
-**"Tag the columns in `model_outputs` with their field type and model"**
-1. Read the schema
-2. For each field, classify by name + Arrow type → set `lancedb:tag:field_type` and `lancedb:tag:model` where applicable
-3. Write in one batched call
-4. Report
-
-**"Group the feature columns in `training_features` into logical families and mark the latest version"**
-1. Read the schema
-2. Find version patterns → assign `lancedb:logical-column` and `lancedb:tag:version`; mark newest with `lancedb:tag:latest: "true"`
-3. Write in one batched call
-4. Show the grouping
diff --git a/plugins/lancedb/skills/lancedb/references/python/api_reference.md b/plugins/lancedb/skills/lancedb/references/python/api_reference.md
deleted file mode 100644
index bbb209630..000000000
--- a/plugins/lancedb/skills/lancedb/references/python/api_reference.md
+++ /dev/null
@@ -1,138 +0,0 @@
-# Python API Reference
-
-Quick method reference for Python LanceDB code. Cross-check source for non-trivial claims.
-
-## Connect
-
-If you're connecting to a remote database, use this:
-```python
-import lancedb
-
-db = lancedb.connect("db://my-db", api_key=api_key, host_override=host_override) # remote
-```
-(values may be found in LANCEDB_API_KEY and LANCEDB_HOST_OVERRIDE, either in env vars or a .env file)
-
-If you're connecting to a local table using OSS LanceDB, use this:
-```python
-db = lancedb.connect("./camelot-db") # local/OSS
-```
-If you're not sure which, or if you can't find the api_key or host_override params, ask the user.
-
-**Place the local database directory next to the script/entrypoint that opens it** (i.e. resolve the path relative to the script, `Path(__file__).parent / "camelot-db"`), not buried under a shared `data/` folder. The Lance dataset is the database, not a data file — keeping it beside its code makes ownership obvious and paths stable regardless of the working directory the script is launched from.
-
-**Do not name the directory `lancedb`** (e.g. `./lancedb`, `./data/lancedb`). It collides with the imported `lancedb` package name, which is confusing to read and easy to shadow in scripts. Give it a name derived from the repo or dataset with a clear prefix/suffix — for example `./<dataset>-db`, `./<repo>_lancedb`, or `./vectordb`.
-
-Async:
-
-```python
-db = await lancedb.connect_async("./camelot-db")
-```
-
-## Table Reads
-
-| Task | Preferred API |
-| --- | --- |
-| Vector search | `table.search(query_vector).limit(k)` |
-| Full scan with filters/projection (sync) | `table.search().where(...).select(...).limit(...)` |
-| Full scan with filters/projection (async) | `table.query().where(...).select(...).limit(...)` |
-| Filter | `.where("col > 10")` |
-| Projection | `.select(["id", "text"])` |
-| Bound result count | `.limit(20)` |
-| Collect bounded result as Python objects (default, no extra deps) | `.to_list()` on query/search result |
-| Collect bounded result as Arrow (default, `pyarrow` always available) | `.to_arrow()` on query/search result |
-| Collect bounded result as pandas (only if project uses pandas) | `.to_pandas()` on query/search result |
-| Collect bounded result as Polars (only if project uses polars) | `.to_polars()` on query/search result |
-
-## Sync vs Async Scan API
-
-The plain-scan entry point differs between the sync and async clients. **Verified against `lancedb` 0.34.0** — re-check if the pinned version changes:
-
-- **Sync** (`lancedb.connect(...)`): the table has **no `.query()` method**. Use `.search()` with no argument for a plain scan; it returns a query builder that supports `.where()`, `.select()`, `.limit()`, and the `.to_list()` / `.to_arrow()` / `.to_pandas()` / `.to_polars()` collectors.
-  ```python
-  rows = table.search().where("status = 'ready'").select(["id", "text"]).limit(20).to_list()
-  ```
-- **Async** (`lancedb.connect_async(...)`): the table has **both** `.query()` and `.search()`. Use `.query()` for a plain scan.
-  ```python
-  rows = await async_table.query().where("status = 'ready'").select(["id", "text"]).limit(20).to_list()
-  ```
-
-Do not call `table.query()` on a sync table — it raises `AttributeError`.
-
-## Local vs Remote Table Methods
-
-| API | Local table | Remote table | Agent guidance |
-| --- | --- | --- | --- |
-| `table.search(...)` | Yes | Yes | Preferred read path (sync + async) |
-| `table.query()` | Async only | Async only | Sync scan path is `table.search()`; `.query()` is the async scan builder |
-| `table.to_pandas()` | Yes | No / unsafe for portability | Avoid in portable code |
-| `table.to_arrow()` | Yes | No / unsafe for portability | Avoid in portable code |
-| `table.to_polars()` | Yes | No / unsafe for portability | Avoid in portable code |
-| `table.to_lance()` | Yes | No | Local/OSS escape hatch only |
-
-## Indexes
-
-Use `create_index(...)` for vector indexes and modern index configs. Use scalar indexes for filtered or merge keys.
-
-Common calls:
-
-```python
-table.create_index("vector")
-table.create_scalar_index("status")
-table.create_fts_index("text")
-```
-
-Check source docs before specifying advanced index config names or parameters.
-
-## Filtering And Recall Knobs
-
-```python
-table.search(query_vector).where("status = 'ready'")  # pre-filter by default
-table.search(query_vector).where("status = 'ready'", prefilter=False)
-table.search(query_vector).limit(10).refine_factor(20)
-table.search(query_vector).limit(10).nprobes(50)
-```
-
-Use post-filtering only when fewer than `limit` results are acceptable.
-
-## Diagnostics
-
-```python
-print(table.search(query_vector).where("year > 2000").limit(10).analyze_plan())
-print(table.index_stats("vector_idx"))
-```
-
-Use these before changing indexes or search tuning.
-
-## Column (Field) Metadata
-
-```python
-schema = table.schema                       # sync property; async: await table.schema()
-meta = schema.field("category").metadata    # dict[bytes, bytes] — Arrow metadata is bytes-keyed
-res = table.update_field_metadata(          # varargs: one dict per field; works local + remote
-    {"path": "category", "metadata": {"lancedb:description": "...", "lancedb:tag:field_type": "label"}}
-)
-res.version  # new table version
-```
-
-Merges by default; a `None` value deletes that key; `"replace": True` swaps the whole map. Nested fields use dot-paths (`"a.b.c"`). `replace_field_metadata` is deprecated. See `references/column_metadata.md` for key conventions (`lancedb:description`, `lancedb:tag:<name>`, `lancedb:logical-column`) and the authoring workflow.
-
-## Branches
-
-```python
-table.branches.list()                          # non-main branches; {} = only main
-exp = table.branches.create("exp")             # fork off main -> handle scoped to the branch
-wip = table.branches.checkout("wip")           # existing branch -> scoped handle (version= pins read-only)
-wip = db.open_table("t", branch="wip")         # or open scoped directly
-table.branches.delete("stale")                 # removes only the branch pointer
-table.current_branch()                         # None = main
-```
-
-There is no global switch — scoping is per table handle: any read/write on a branch handle lands on that branch; the original handle keeps targeting main. See `references/branch_ops.md` for the model and isolation checks.
-
-## Maintenance
-
-```python
-table.optimize()
-```
-
-Call this after every successful local/OSS ingestion. It handles compaction, cleanup of old versions according to retention, and index optimization. Do not add this for LanceDB Enterprise/Cloud remote tables; Enterprise handles compaction and cleanup automatically from cluster configuration.
diff --git a/plugins/lancedb/skills/lancedb/references/python/patterns.md b/plugins/lancedb/skills/lancedb/references/python/patterns.md
deleted file mode 100644
index 4d6be43ef..000000000
--- a/plugins/lancedb/skills/lancedb/references/python/patterns.md
+++ /dev/null
@@ -1,173 +0,0 @@
-# Python Patterns
-
-Use these patterns when writing Python code with `lancedb`.
-
-## Before Writing Code
-
-Choose the output type from what the project actually depends on. **Do not assume `pandas` or `polars` is installed** — they are heavy dependencies that many LanceDB projects do not use. `pyarrow`, by contrast, ships as a LanceDB dependency and is always available, so it is a safe default to lean on.
-
-Default output (after applying `select()` and `limit()`):
-
-- **Python objects**: `.to_list()` — a list of dicts, no extra dependencies. Prefer this for scripts, examples, and agent-generated code unless there is a reason to do otherwise.
-- **PyArrow**: `.to_arrow()` — a `pyarrow.Table`, when the surrounding code is Arrow-native or you need columnar/zero-copy handoff.
-
-Only reach for a DataFrame when the project *already* declares that dependency:
-
-- Pandas projects (pandas in `pyproject.toml`/requirements): `.to_pandas()`.
-- Polars projects (polars declared): `.to_polars()`.
-
-If unsure, check the dependency manifest or the imports in surrounding files. When in doubt, use `.to_list()` or `.to_arrow()`.
-
-## Schema Design and Validation
-
-Favor `LanceModel` and Pydantic validation for Python schemas. They keep field
-types readable, validate source records before a write, and map directly to a
-LanceDB schema. Use `Vector(dimension)` for fixed-size vectors:
-
-```python
-from lancedb.pydantic import LanceModel, Vector
-
-class Document(LanceModel):
-    id: int
-    text: str
-    vector: Vector(384, nullable=False)
-
-rows = [Document.model_validate(row) for row in source_rows]
-table = db.create_table("documents", schema=Document)
-table.add(rows)
-```
-
-Use PyArrow schemas instead when the pipeline is already Arrow-native, needs
-record-batch streaming, or has runtime schema requirements that would make a
-Pydantic model harder to understand. Declare Pydantic as a direct project
-dependency when application code imports it, even if LanceDB also depends on it.
-
-## Recommended Patterns
-
-### Bounded search or query
-
-Use this for application reads, examples, notebooks, and agent-generated scripts:
-
-```python
-results = (
-    table.search(query_vector)
-    .where("status = 'ready'")
-    .select(["id", "text"])
-    .limit(20)
-    .to_list()  # or .to_arrow(); .to_pandas()/.to_polars() only if the project uses them
-)
-```
-
-Why: `search()` works across local and remote tables and on both the sync and async clients. `select()` avoids fetching unused columns. `limit()` prevents accidental full-table reads. `.to_list()` and `.to_arrow()` avoid assuming pandas/polars is installed (see "Before Writing Code").
-
-For a **plain scan** (no query vector), the entry point differs by client:
-
-```python
-# Sync client: no .query() method — use .search() with no argument.
-rows = table.search().where("status = 'ready'").select(["id", "text"]).limit(20).to_list()
-
-# Async client: use .query().
-rows = await async_table.query().where("status = 'ready'").select(["id", "text"]).limit(20).to_list()
-```
-
-`table.query()` on a sync table raises `AttributeError` (verified on `lancedb` 0.34.0). See the "Sync vs Async Scan API" section in `api_reference.md`.
-
-### Bounded query result conversion
-
-It is fine to collect bounded query/search results:
-
-```python
-arrow_table = table.search().select(["id"]).limit(100).to_arrow()  # sync plain scan
-rows = table.search(query_vector).limit(10).to_list()
-df = table.search(query_vector).limit(10).to_pandas()  # only if pandas is a project dep
-```
-
-### Local-only Lance dataset API
-
-`table.to_lance()` does not itself materialize the full dataset. It returns the underlying `lance.LanceDataset`, making the table accessible through the PyLance dataset API. Use it when the task is explicitly local/OSS and needs Lance dataset methods not exposed by LanceDB:
-
-```python
-# Local/OSS only: RemoteTable does not expose table.to_lance().
-ds = table.to_lance()
-for batch in ds.to_batches(columns=["id", "text"], batch_size=10_000):
-    process(batch)
-```
-
-### Async Python
-
-Keep the same shape and bound the result before collecting:
-
-```python
-results = await (
-    async_table.query()
-    .where("status = 'ready'")
-    .select(["id", "text"])
-    .limit(20)
-    .to_list()  # or .to_arrow()
-)
-```
-
-## Anti-Patterns
-
-**Avoid the following anti-patterns in your code.**
-
-### Table-level full materialization
-
-Avoid whole-table collectors in portable or large-table code:
-
-```python
-df = table.to_pandas()
-arrow_table = table.to_arrow()
-polars_df = table.to_polars()
-```
-
-Why: local tables expose these whole-table collectors, but remote tables intentionally do not — a remote production table can be far larger than a local development table, so it is easy to accidentally pull the entire table into memory.
-
-`table.to_lance()` is different: it is not a full materialization call, but it is still local/OSS-only and should not appear in code meant to run against remote Enterprise tables.
-
-### Unbounded result collection
-
-Avoid query/search collection without a meaningful limit:
-
-```python
-rows = table.search().to_list()               # unbounded plain scan
-rows = table.search(query_vector).to_list()   # unbounded vector search
-```
-
-Prefer `select(...).limit(...)` before collecting; for large reads, stream in batches instead.
-
-### Per-row writes
-
-Avoid loops that write one row per call:
-
-```python
-for row in rows:
-    table.add([row])  # one commit + fragment per row
-```
-
-Each `add()` creates a new version and fragment. Pass the whole batch in a single call, or chunk very large inputs:
-
-```python
-table.add(rows)  # single commit
-# for very large inputs, add batches of several thousand rows
-```
-
-After the final successful write to an embedded OSS table, call
-`table.optimize()`. Skip this for Enterprise/Cloud tables because their
-maintenance is automatic.
-
-### Drop-then-reuse the same table name (Enterprise/Cloud)
-
-Avoid dropping or overwriting a remote table and then reusing that name right away:
-
-```python
-db.drop_table("my_table")
-table = db.create_table("my_table", data=rows)          # reads 500 for ~5 min
-table = db.create_table("my_table", data=rows, mode="overwrite")  # same problem
-```
-
-Why: Enterprise/Cloud splits DDL (control plane) from query serving (data plane). The data plane caches the dataset behind a table name for up to `table_cache_ttl` (default 300s / 5 min), so after a drop/overwrite the DDL succeeds but queries against the reused name return `500 Internal Server Error` until the cache expires — and a fresh `describe` may still show the old schema. Instead, write to a **fresh name**, use `list_tables()` and fail if it already exists, then `rename_table(fresh, final)` onto the final name only after the old table's drop has propagated (~5 min). See the "Enterprise: never drop-then-reuse the same table name" section in `SKILL.md`. Local/OSS tables have no separate data plane — overwrite freely there.
-
-### Guessing performance fixes
-
-Avoid changing `nprobes`, `refine_factor`, or index types before checking the query plan and index stats. Diagnose first, then tune one knob at a time.
diff --git a/plugins/lancedb/skills/lancedb/references/python/performance.md b/plugins/lancedb/skills/lancedb/references/python/performance.md
deleted file mode 100644
index 5fd27440b..000000000
--- a/plugins/lancedb/skills/lancedb/references/python/performance.md
+++ /dev/null
@@ -1,131 +0,0 @@
-# Python Performance Guidance
-
-Use this when writing Python code that ingests data, queries large tables, builds indexes, or investigates latency.
-
-## Ingestion
-
-### Recommended: validate schemas and records with Pydantic
-
-Favor `LanceModel` for readable Python schema definitions and validate source
-records before writing. Use PyArrow directly for Arrow-native or streaming
-pipelines where it is the clearer representation.
-
-```python
-from lancedb.pydantic import LanceModel, Vector
-
-class Document(LanceModel):
-    id: int
-    text: str
-    vector: Vector(384, nullable=False)
-
-rows = [Document.model_validate(row) for row in source_rows]
-table = db.create_table("documents", schema=Document)
-table.add(rows)
-```
-
-### Recommended: bulk ingestion for materialized data
-
-```python
-table.add(arrow_table)
-table.add(df)
-table.add(pa.dataset("data/", format="parquet"))
-```
-
-For very large initial loads, create the table empty first, then call `add(...)`. Passing data directly to `create_table(name, data)` can skip the auto-parallel write path.
-
-### Recommended: iterator ingestion for generated or streamed data
-
-```python
-def batches():
-    for raw in source:
-        vectors = model.encode(raw["text"])
-        yield pa.RecordBatch.from_pydict({**raw, "vector": vectors})
-
-table.add(batches())
-```
-
-Use chunks of several thousand rows or more when practical. Tiny batches and per-row writes create many small fragments.
-
-### Anti-pattern: per-row `add()`
-
-```python
-for row in rows:
-    table.add([row])
-```
-
-Each call creates a version and fragment. This slows ingestion and later queries.
-
-## Indexing
-
-- Build a vector index once brute-force vector search becomes too slow. As a rule of thumb, local brute force is fine below roughly 100K vectors; beyond that, build an index.
-- Use `IVF_PQ` as the general-purpose default. Enterprise builds this automatically.
-- Use scalar indexes for filtered columns and merge/upsert keys.
-- Use `BTREE` for mostly distinct numeric/string/temporal columns, `BITMAP` for booleans and low-cardinality columns, and `LABEL_LIST` for list membership queries.
-- Keep full-text defaults unless phrase queries require position data.
-
-## Querying
-
-Always be explicit:
-
-```python
-table.search(query_vector).select(["id", "title"]).limit(20)
-```
-
-- `select()` reduces bytes read and transferred.
-- `limit()` prevents accidental full-table materialization.
-- Pre-filtering is the default and guarantees returned rows satisfy the predicate.
-- Use post-filtering only when fewer than `limit` results are acceptable.
-
-## Recall Tuning
-
-Tune one knob at a time:
-
-- Quantized indexes: raise `refine_factor` to rescore more candidates on full vectors.
-- HNSW-backed indexes: raise `ef`; start around `1.5 * k`, increase toward `10 * k` if recall is short.
-- IVF candidate breadth: `nprobes` is auto-tuned; override only when a selective pre-filter leaves too few neighbors.
-
-## Maintenance
-
-After every successful embedded OSS/local ingestion, call `table.optimize()`.
-Do not add this to LanceDB Enterprise/Cloud remote table code; remote compaction
-and cleanup are handled automatically based on the Enterprise cluster
-configuration.
-
-Why local maintenance is needed:
-
-- Frequent writes can create many small fragments. Queries then need to scan across more files, which can increase latency.
-- Updates, deletes, and appends create new table versions. Old versions are retained for time travel and rollback, which can grow disk usage.
-- Indexes may have newly added rows that are not yet fully optimized into the index structure.
-
-For local/OSS tables, run `optimize()` after the final successful ingestion
-write. Also run it after later batches of update/delete operations or on a
-regular maintenance schedule:
-
-```python
-table.optimize()
-```
-
-If the user wants more aggressive local disk cleanup, pass a shorter cleanup retention window:
-
-```python
-from datetime import timedelta
-
-table.optimize(cleanup_older_than=timedelta(days=1))
-```
-
-Do not use very short cleanup windows when the application depends on time travel, rollback, or old versions.
-
-## Diagnostics
-
-Before changing code or indexes, inspect:
-
-```python
-print(table.search(query_vector).where("year > 2000").limit(10).analyze_plan())
-print(table.index_stats("vector_idx"))
-```
-
-Look for high scan bytes, missing indexes, fragmented data, and unindexed rows.
-
-## Python Multiprocessing
-
-When using multiprocessing, use `spawn` rather than `fork`. LanceDB is multi-threaded internally, and `fork` plus a multi-threaded process is unsafe.
diff --git a/plugins/lancedb/skills/lancedb/references/remote_connect.md b/plugins/lancedb/skills/lancedb/references/remote_connect.md
deleted file mode 100644
index 670242c94..000000000
--- a/plugins/lancedb/skills/lancedb/references/remote_connect.md
+++ /dev/null
@@ -1,45 +0,0 @@
-# Connecting to a LanceDB remote server
-
-LanceDB Enterprise/Cloud deployments are served by a server implementing the
-lance-namespace OpenAPI spec
-(<https://github.com/lance-format/lance-namespace/blob/main/docs/src/spec.yaml>).
-Every remote (`db://...`) connection talks to such a server, and some operations
-exist only there. In particular, all operations around jobs (listing, inspecting,
-creating, or canceling jobs) run server-side — there is no local/OSS equivalent, so
-resolve a server connection before attempting any job work. The job REST methods
-themselves are documented in `references/remote_jobs.md`.
-
-Every request needs two things:
-
-1. **Base URL** — the server endpoint
-2. **Credentials** — an API key (`x-api-key` header over REST), and usually a database name (`x-lancedb-database` header)
-
-## Resolution steps
-
-1. If the user already gave a URL and API key (or said which environment they're working against), use that.
-2. Otherwise, look for credentials already available in the environment:
-   - Env vars like `LANCEDB_URI` / `LANCEDB_HOST` / `LANCEDB_API_KEY`
-   - A server endpoint already running or port-forwarded locally (the REST default port is 2333, i.e. `http://localhost:2333`)
-3. If you didn't find both pieces, ask the user directly: **"What's your LanceDB endpoint's URL, and what's your API key?"** Also ask which database to use if it isn't obvious. Don't guess or probe further — the user knows their deployment.
-
-## Validating the connection
-
-Make a cheap authenticated request and check the status before starting real work:
-
-```bash
-curl -s -w "\n%{http_code}" "{base_url}/v1/table/?limit=1" \
-  -H "x-api-key: <key>" \
-  -H "x-lancedb-database: <database>"
-```
-
-- `200` — connection, key, and database header all good
-- `401` — API key missing or wrong
-- `400` mentioning a database header — this deployment expects `x-lancedb-database`
-
-## Non-REST equivalents
-
-The same credentials work through the SDKs and CLI:
-
-- Python SDK: `lancedb.connect("db://<database>", api_key="<key>", host_override="<base_url>")`
-- TypeScript SDK: `await lancedb.connect("db://<database>", { apiKey: "<key>", hostOverride: "<base_url>" })`
-- `lancedb` CLI: a `[profiles.<name>]` entry in `~/.lancedb/config.toml` with `http_server_url`, `api_key`, `database`
diff --git a/plugins/lancedb/skills/lancedb/references/remote_jobs.md b/plugins/lancedb/skills/lancedb/references/remote_jobs.md
deleted file mode 100644
index 175f0ce51..000000000
--- a/plugins/lancedb/skills/lancedb/references/remote_jobs.md
+++ /dev/null
@@ -1,151 +0,0 @@
-# Job operations over the LanceDB remote server REST API
-
-Jobs are server-side background operations on LanceDB Enterprise/Cloud — index builds,
-column backfills, materialized view refreshes, and similar async work. Endpoints that
-trigger async work (e.g. the column backfill or materialized view refresh endpoints)
-return a `job_id`; these four methods are how you track and manage those jobs.
-
-Resolve the connection first — see `references/remote_connect.md`. All four methods
-are **POST** requests under `{base_url}/v1/jobs/` with JSON bodies, and take the usual
-`x-api-key` / `x-lancedb-database` headers. If every job call returns `501`, job APIs
-are disabled on that deployment (the server has no job registry configured) — report
-that rather than retrying.
-
-## 1. List jobs — `POST /v1/jobs/list`
-
-The body is optional; an empty body lists everything. All fields are filters:
-
-```json
-{
-  "limit": 100,
-  "table_name": "my_table",
-  "job_type": "...",
-  "job_subtype": "...",
-  "state": "...",
-  "page_token": "..."
-}
-```
-
-```bash
-curl -s -X POST "{base_url}/v1/jobs/list" \
-  -H "x-api-key: <key>" -H "x-lancedb-database: <database>" \
-  -H "content-type: application/json" \
-  -d '{"table_name": "my_table"}'
-```
-
-Response:
-
-```json
-{
-  "jobs": [
-    {
-      "job_id": "...",
-      "table": "my_table",
-      "job_type": "...",
-      "job_subtype": "...",
-      "state": "done",
-      "created_at_millis": 1720000000000
-    }
-  ],
-  "page_token": "..."
-}
-```
-
-A `page_token` in the response means there are more results — pass it back in the next
-request to continue. Note list rows use a lowercase `state` string, while describe uses
-an uppercase `job_state`.
-
-## 2. Describe a job — `POST /v1/jobs/describe`
-
-Body: `{"job_id": "<id>"}`. Returns full detail for one job:
-
-```json
-{
-  "job_id": "...",
-  "job_type": "...",
-  "job_subtype": "...",
-  "job_state": "IN_PROGRESS",
-  "creation_ms": 1720000000000,
-  "spec": {},
-  "status": {}
-}
-```
-
-`job_state` is one of `IN_PROGRESS`, `CANCELLED`, `FAILED`, `DONE`. `spec` and `status`
-are job-type-specific JSON objects (the job's input specification and its current
-progress/status). Returns `404` for an unknown job id.
-
-## 3. Cancel a job — `POST /v1/jobs/cancel`
-
-Body: `{"job_id": "<id>"}`; response echoes `{"job_id": "<id>"}`. Cancellation is a
-service-level operation requiring the same administrative authorization as the
-`/admin` routes — a database-scoped API key that can list and describe jobs may still
-get a permission error here. Other errors: `404` unknown job, `409` state conflict
-(e.g. already in a terminal state), `429` too much write contention (safe to retry).
-
-## 4. Query job event history — `POST /v1/jobs/query_events`
-
-Returns the event history (state transitions, progress updates) for one or more jobs.
-Body: `{"job_id": "<id>"}` for one job, or `{"job_ids": ["<id>", ...]}` for a batch.
-Optional fields: `limit` (max event rows), `limit_per_job` (per job in a batch query),
-and `filter` — a SQL-like expression over the columns `state`, `updated_by`,
-`owner_component`, and `claim_entity`. (`full_text_search` is reserved and currently
-rejected as not implemented.)
-
-The response is **not JSON** — it is an Arrow IPC stream
-(`content-type: application/vnd.apache.arrow.stream`). Decode it, e.g. in Python:
-
-```python
-import pyarrow.ipc
-import requests
-
-resp = requests.post(
-    f"{base_url}/v1/jobs/query_events",
-    headers={"x-api-key": key, "x-lancedb-database": database},
-    json={"job_id": job_id},
-)
-resp.raise_for_status()
-events = pyarrow.ipc.open_stream(resp.content).read_all()
-```
-
-## Feature engineering (Geneva) jobs
-
-Feature engineering jobs — UDF column backfills and materialized view refreshes run
-through Geneva — are tracked **separately** from the `/v1/jobs` registry above. Their
-records live in a `geneva_jobs` table inside the database itself (in the `__system`
-namespace), and you access them through a Python `geneva` connection rather than the
-REST endpoints above:
-
-```python
-import geneva
-from geneva.jobs import JobStateManager
-
-# Same credentials as lancedb.connect / the REST API
-conn = geneva.connect("db://<database>", api_key="<key>", host_override="<base_url>")
-jsm = JobStateManager(conn)
-
-# List jobs. NOTE: status defaults to "RUNNING"; pass status=None for all jobs.
-# Statuses: PENDING | RUNNING | DONE | FAILED | CANCELLED
-jobs = jsm.list_jobs(table_name="my_table", status=None)
-
-# Fetch one job by id (returns a list of JobRecord)
-records = jsm.get("<job_id>")
-```
-
-Each `JobRecord` has `table_name`, `column_name`, `job_id`, `job_type`, `status`,
-`launched_at`, `completed_at`, `config`, `launched_by`, `manifest_id`, `cluster_name`,
-`metrics` (progress counters), `events` (human-readable history), and `updated_at`.
-For filters `list_jobs` doesn't support (e.g. time ranges), query the underlying table
-directly: `jsm.get_table(True).search().where("launched_at >= TIMESTAMP '...'")` —
-pass `True` to check out the latest version, since other processes update job state.
-
-Stale-status caveat: nothing reaps dead Geneva jobs, so a job can sit in
-`RUNNING`/`PENDING` forever if its worker died. Treat a job as effectively `FAILED`
-when it has been running longer than ~36 hours, or its `updated_at` is more than ~2
-hours old (this matches the heuristic the Geneva console UI applies on read).
-
-## Workflow tips
-
-- To wait for async work (a backfill, an index build), poll `describe` until
-  `job_state` leaves `IN_PROGRESS`; on `FAILED`, pull `status` and `query_events` for
-  the failure detail.
diff --git a/plugins/lancedb/skills/lancedb/references/typescript/api_reference.md b/plugins/lancedb/skills/lancedb/references/typescript/api_reference.md
deleted file mode 100644
index c98f39a0c..000000000
--- a/plugins/lancedb/skills/lancedb/references/typescript/api_reference.md
+++ /dev/null
@@ -1,105 +0,0 @@
-# TypeScript API Reference
-
-Quick method reference for TypeScript LanceDB code. Cross-check source for non-trivial claims.
-
-## Connect
-
-```typescript
-import * as lancedb from "@lancedb/lancedb";
-
-const db = await lancedb.connect("./camelot-db");
-```
-
-**Place the local database directory next to the script/entrypoint that opens it** (resolve the path relative to the module, e.g. via `import.meta.dirname` / `__dirname`), not buried under a shared `data/` folder. The Lance dataset is the database, not a data file — keeping it beside its code makes ownership obvious and paths stable regardless of the working directory the script is launched from.
-
-**Do not name the directory `lancedb`** (e.g. `./lancedb`, `./data/lancedb`). It collides with the imported `lancedb` package/namespace, which is confusing to read. Give it a name derived from the repo or dataset with a clear prefix/suffix — for example `./<dataset>-db`, `./<repo>_lancedb`, or `./vectordb`.
-
-Remote connections use `db://...` plus Enterprise/Cloud credentials and deployment settings. Check current source/docs for exact connection options.
-
-## Table Reads
-
-| Task | Preferred API |
-| --- | --- |
-| Vector search | `table.search(queryVector).limit(k)` |
-| Full scan with filters/projection | `table.query().where(...).select(...).limit(...)` |
-| Filter | `.where("col > 10")` |
-| Projection | `.select(["id", "text"])` |
-| Bound result count | `.limit(20)` |
-| Collect bounded result as objects | `.toArray()` on query/search result |
-| Collect bounded result as Arrow | `.toArrow()` on query/search result |
-| Stream result batches | `for await (const batch of table.query()...)` |
-
-## Local vs Remote Safety
-
-| API | Agent guidance |
-| --- | --- |
-| `table.search(...)` | Preferred read path |
-| `table.query()` | Preferred scan/filter path |
-| `await table.toArrow()` | Avoid in portable or large-table code |
-| `await table.query().toArray()` with no `limit()` | Avoid; unbounded collection |
-| `await table.query().toArrow()` with no `limit()` | Avoid; unbounded collection |
-
-## Indexes
-
-```typescript
-await table.createIndex("vector");
-await table.createIndex("status");
-```
-
-Use vector indexes for large vector search workloads and scalar indexes for filtered columns or merge/upsert keys. Check source/docs before specifying advanced index options.
-
-## Filtering And Recall Knobs
-
-```typescript
-await table.search(queryVector).where("status = 'ready'").limit(10).toArray();
-await table.search(queryVector).limit(10).refineFactor(20).toArray();
-await table.search(queryVector).limit(10).nprobes(50).toArray();
-await table.search(queryVector).limit(10).ef(100).toArray();
-await table.search(queryVector).where("status = 'ready'").postfilter().limit(10).toArray();
-```
-
-Use `postfilter()` only when fewer than `limit` results are acceptable.
-
-## Diagnostics
-
-```typescript
-console.log(await table.search(queryVector).where("year > 2000").limit(10).analyzePlan());
-console.log(await table.indexStats("vector_idx"));
-```
-
-Use these before changing indexes or search tuning.
-
-## Column (Field) Metadata
-
-```typescript
-const schema = await table.schema();
-const meta = schema.fields.find((f) => f.name === "category")?.metadata; // Map<string, string>
-const res = await table.updateFieldMetadata([
-  { path: "category", metadata: { "lancedb:description": "...", "lancedb:tag:field_type": "label" } },
-]);
-res.version; // new table version
-```
-
-Merges by default; a `null` value deletes that key; `replace: true` swaps the whole map. Nested fields use dot-paths (`"a.b.c"`). See `references/column_metadata.md` for key conventions (`lancedb:description`, `lancedb:tag:<name>`, `lancedb:logical-column`) and the authoring workflow.
-
-## Branches
-
-```typescript
-const branches = await table.branches();       // async manager
-await branches.list();                         // non-main branches; {} = only main
-const exp = await branches.create("exp");      // fork off main -> Table scoped to the branch
-const wip = await branches.checkout("wip");    // existing branch -> scoped Table (version arg pins read-only)
-const wip2 = await db.openTable("t", { branch: "wip" }); // or open scoped directly
-await branches.delete("stale");                // removes only the branch pointer
-table.currentBranch();                         // null = main
-```
-
-There is no global switch — scoping is per table handle: any read/write on a branch handle lands on that branch; the original handle keeps targeting main. See `references/branch_ops.md` for the model and isolation checks.
-
-## Maintenance
-
-```typescript
-await table.optimize();
-```
-
-Call this after every successful local/OSS ingestion. It handles compaction, cleanup of old versions according to retention, and index optimization. Do not add this for LanceDB Enterprise/Cloud remote tables; Enterprise handles compaction and cleanup automatically from cluster configuration.
diff --git a/plugins/lancedb/skills/lancedb/references/typescript/patterns.md b/plugins/lancedb/skills/lancedb/references/typescript/patterns.md
deleted file mode 100644
index 1aa380968..000000000
--- a/plugins/lancedb/skills/lancedb/references/typescript/patterns.md
+++ /dev/null
@@ -1,100 +0,0 @@
-# TypeScript Patterns
-
-Use these patterns when writing TypeScript code with `@lancedb/lancedb`.
-
-## Recommended Patterns
-
-### Bounded query
-
-Use this for application reads, scripts, and examples:
-
-```typescript
-const rows = await table
-  .query()
-  .where("status = 'ready'")
-  .select(["id", "text"])
-  .limit(20)
-  .toArray();
-```
-
-### Bounded vector search
-
-```typescript
-const rows = await table
-  .search(queryVector)
-  .select(["id", "text"])
-  .limit(20)
-  .toArray();
-```
-
-### Batch streaming for larger reads
-
-When the task needs many rows, avoid collecting everything at once:
-
-```typescript
-for await (const batch of table
-  .query()
-  .where("status = 'ready'")
-  .select(["id", "text"])
-  .limit(10_000)) {
-  process(batch);
-}
-```
-
-## Anti-Patterns
-
-**Avoid the following anti-patterns in your code.**
-
-### Table-level full materialization
-
-Avoid whole-table collectors in portable or large-table code:
-
-```typescript
-const tableArrow = await table.toArrow();
-```
-
-Why: local tables expose these whole-table collectors, but remote tables intentionally do not — a remote production table can be far larger than a local development table, so it is easy to accidentally pull the entire table into memory.
-
-### Unbounded result collection
-
-Avoid query/search collection without a meaningful limit:
-
-```typescript
-const rows = await table.query().toArray();              // unbounded plain scan
-const rows = await table.search(queryVector).toArray();  // unbounded vector search
-```
-
-Prefer `select(...).limit(...)` before collecting; for large reads, stream in batches instead.
-
-### Per-row writes
-
-Avoid loops that write one row per call:
-
-```typescript
-for (const row of rows) {
-  await table.add([row]); // one commit + fragment per row
-}
-```
-
-Each `add()` creates a new version and fragment. Pass the whole batch in a single call, or chunk very large inputs:
-
-```typescript
-await table.add(rows); // single commit
-// for very large inputs, add in chunks of several thousand rows
-```
-
-### Drop-then-reuse the same table name (Enterprise/Cloud)
-
-Avoid dropping or overwriting a remote table and then reusing that name right away:
-
-```typescript
-await db.dropTable("my_table");
-const table = await db.createTable("my_table", rows);                    // reads 500 for ~5 min
-const table = await db.createTable("my_table", rows, { mode: "overwrite" }); // same problem
-```
-
-Why: Enterprise/Cloud splits DDL (control plane) from query serving (data plane). The data plane caches the dataset behind a table name for up to `table_cache_ttl` (default 300s / 5 min), so after a drop/overwrite the DDL succeeds but queries against the reused name return `500 Internal Server Error` until the cache expires — and a fresh `describe` may still show the old schema. Instead, write to a **fresh name**, use `tableNames()` and fail if it already exists, then `renameTable(fresh, final)` onto the final name only after the old table's drop has propagated (~5 min). See the "Enterprise: never drop-then-reuse the same table name" section in `SKILL.md`. Local/OSS tables have no separate data plane — overwrite freely there.
-
-### Guessing performance fixes
-
-Avoid changing `nprobes`, `refineFactor`, `ef`, or index settings before checking `analyzePlan()` and `indexStats(...)`. Diagnose first, then tune one knob at a time.
diff --git a/plugins/lancedb/skills/lancedb/references/typescript/performance.md b/plugins/lancedb/skills/lancedb/references/typescript/performance.md
deleted file mode 100644
index 9bf07e9ae..000000000
--- a/plugins/lancedb/skills/lancedb/references/typescript/performance.md
+++ /dev/null
@@ -1,78 +0,0 @@
-# TypeScript Performance Guidance
-
-Use this when writing TypeScript code that ingests data, queries large tables, builds indexes, or investigates latency.
-
-## Ingestion
-
-- Prefer bulk or batched writes.
-- Avoid per-row write loops; they create many small commits/fragments.
-- For generated data, accumulate reasonable batches before adding.
-- For file-backed data, prefer APIs that stream from Arrow/Parquet-style inputs when available.
-
-## Indexing
-
-- Build a vector index once brute-force vector search becomes too slow. As a rule of thumb, local brute force is fine below roughly 100K vectors; beyond that, build an index.
-- Use the general-purpose vector index defaults unless the task has explicit recall/latency requirements.
-- Build scalar indexes for filtered columns and merge/upsert keys.
-- Use full-text index phrase options only when phrase queries require them.
-
-## Querying
-
-Always be explicit:
-
-```typescript
-await table.search(queryVector).select(["id", "title"]).limit(20).toArray();
-```
-
-- `select()` reduces bytes read and transferred.
-- `limit()` prevents accidental full-table collection.
-- Pre-filtering is the default behavior. Use `postfilter()` only when fewer than `limit` results are acceptable.
-
-## Recall Tuning
-
-Tune one knob at a time:
-
-- Quantized indexes: raise `refineFactor(...)` to rescore more candidates on full vectors.
-- HNSW-backed indexes: raise `ef(...)`; start around `1.5 * k`, increase toward `10 * k` if recall is short.
-- IVF candidate breadth: `nprobes(...)` is usually auto-tuned; override only when a selective pre-filter leaves too few neighbors.
-
-## Maintenance
-
-After every successful embedded OSS/local ingestion, call `table.optimize()`.
-Do not add this to LanceDB Enterprise/Cloud remote table code; remote compaction
-and cleanup are handled automatically based on the Enterprise cluster
-configuration.
-
-Why local maintenance is needed:
-
-- Frequent writes can create many small fragments. Queries then need to scan across more files, which can increase latency.
-- Updates, deletes, and appends create new table versions. Old versions are retained for time travel and rollback, which can grow disk usage.
-- Indexes may have newly added rows that are not yet fully optimized into the index structure.
-
-For local/OSS tables, run `optimize()` after the final successful ingestion
-write. Also run it after later batches of update/delete operations or on a
-regular maintenance schedule:
-
-```typescript
-await table.optimize();
-```
-
-If the user wants more aggressive local disk cleanup, pass a shorter cleanup retention window:
-
-```typescript
-const olderThan = new Date(Date.now() - 24 * 60 * 60 * 1000);
-await table.optimize({ cleanupOlderThan: olderThan });
-```
-
-Do not use very short cleanup windows when the application depends on time travel, rollback, or old versions.
-
-## Diagnostics
-
-Before changing code or indexes, inspect:
-
-```typescript
-console.log(await table.search(queryVector).where("year > 2000").limit(10).analyzePlan());
-console.log(await table.indexStats("vector_idx"));
-```
-
-Look for high scan cost, missing indexes, fragmented data, and unindexed rows.
diff --git a/plugins/lancedb/skills/lancedb/scripts/check_materialization.py b/plugins/lancedb/skills/lancedb/scripts/check_materialization.py
deleted file mode 100644
index cbd8abc04..000000000
--- a/plugins/lancedb/skills/lancedb/scripts/check_materialization.py
+++ /dev/null
@@ -1,135 +0,0 @@
-#!/usr/bin/env python3
-"""Scan Python and TypeScript for likely unsafe LanceDB materialization."""
-
-from __future__ import annotations
-
-import argparse
-import re
-import sys
-from dataclasses import dataclass
-from pathlib import Path
-
-
-PY_FULL_TABLE = re.compile(r"\b\w+\.(to_pandas|to_arrow|to_polars)\s*\(")
-TS_TABLE_TO_ARROW = re.compile(r"\b\w+\.toArrow\s*\(")
-TS_QUERY_COLLECTOR = re.compile(r"\.query\s*\(\s*\)[\s\S]*?\.to(Array|Arrow)\s*\(")
-
-
-@dataclass
-class Finding:
-    path: Path
-    line: int
-    message: str
-    text: str
-
-
-def iter_files(paths: list[Path]) -> list[Path]:
-    files: list[Path] = []
-    for path in paths:
-        if path.is_dir():
-            files.extend(
-                p
-                for p in path.rglob("*")
-                if p.suffix in {".py", ".ts", ".tsx"} and "node_modules" not in p.parts
-            )
-        elif path.suffix in {".py", ".ts", ".tsx"}:
-            files.append(path)
-    return sorted(set(files))
-
-
-def line_number(text: str, offset: int) -> int:
-    return text.count("\n", 0, offset) + 1
-
-
-def scan_python(path: Path, text: str) -> list[Finding]:
-    findings: list[Finding] = []
-    for match in PY_FULL_TABLE.finditer(text):
-        line_start = text.rfind("\n", 0, match.start()) + 1
-        line_end = text.find("\n", match.start())
-        if line_end == -1:
-            line_end = len(text)
-        line = text[line_start:line_end].strip()
-        if ".search(" in line or ".query(" in line:
-            continue
-        findings.append(
-            Finding(
-                path,
-                line_number(text, match.start()),
-                f"Review Python `{match.group(1)}()` call; table-level materialization is not portable to remote tables.",
-                line,
-            )
-        )
-    return findings
-
-
-def statement_around(text: str, start: int, end: int) -> str:
-    before = max(text.rfind(";", 0, start), text.rfind("\n\n", 0, start))
-    after_candidates = [pos for pos in (text.find(";", end), text.find("\n\n", end)) if pos != -1]
-    after = min(after_candidates) if after_candidates else len(text)
-    return text[before + 1 : after].strip()
-
-
-def scan_typescript(path: Path, text: str) -> list[Finding]:
-    findings: list[Finding] = []
-    for match in TS_TABLE_TO_ARROW.finditer(text):
-        stmt = statement_around(text, match.start(), match.end())
-        if ".query(" in stmt or ".search(" in stmt:
-            continue
-        findings.append(
-            Finding(
-                path,
-                line_number(text, match.start()),
-                "Review TypeScript `table.toArrow()`-style call; table-level materialization is not portable for large/remote tables.",
-                stmt.splitlines()[0].strip(),
-            )
-        )
-
-    for match in TS_QUERY_COLLECTOR.finditer(text):
-        stmt = statement_around(text, match.start(), match.end())
-        if ".limit(" in stmt:
-            continue
-        findings.append(
-            Finding(
-                path,
-                line_number(text, match.start()),
-                "Review unbounded TypeScript query collection; add `limit()` or stream batches.",
-                stmt.splitlines()[0].strip(),
-            )
-        )
-    return findings
-
-
-def scan_file(path: Path) -> list[Finding]:
-    text = path.read_text(encoding="utf-8", errors="replace")
-    if path.suffix == ".py":
-        return scan_python(path, text)
-    if path.suffix in {".ts", ".tsx"}:
-        return scan_typescript(path, text)
-    return []
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("paths", nargs="+", type=Path)
-    parser.add_argument(
-        "--no-fail", action="store_true", help="Always exit 0 after reporting findings."
-    )
-    args = parser.parse_args()
-
-    findings: list[Finding] = []
-    for path in iter_files(args.paths):
-        findings.extend(scan_file(path))
-
-    for finding in findings:
-        print(f"{finding.path}:{finding.line}: {finding.message}")
-        print(f"  {finding.text}")
-
-    if findings:
-        print(
-            f"\n{len(findings)} finding(s). Review manually; bounded query result conversion may be OK."
-        )
-    return 0 if args.no_fail or not findings else 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())
diff --git a/python/Cargo.toml b/python/Cargo.toml
index cc706e712..3a8a05522 100644
--- a/python/Cargo.toml
+++ b/python/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.37.1-beta.0"
+version = "0.38.0-beta.10"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"
@@ -15,10 +15,10 @@ name = "_lancedb"
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow = { version = "58.0.0", features = ["pyarrow"] }
-async-trait = "0.1"
-bytes = "1"
-lancedb = { path = "../rust/lancedb", default-features = false }
+arrow = { workspace = true, features = ["pyarrow"] }
+async-trait.workspace = true
+bytes.workspace = true
+lancedb.workspace = true
 datafusion-common.workspace = true
 lance-core.workspace = true
 lance-namespace.workspace = true
@@ -26,25 +26,24 @@ lance-namespace-impls.workspace = true
 lance-io.workspace = true
 env_logger.workspace = true
 log.workspace = true
-pyo3 = { version = "0.28", features = ["extension-module", "abi3-py39", "chrono"] }
-chrono = { version = "0.4", default-features = false, features = ["clock"] }
+# Maturin enables extension-module mode for Python builds. Keeping it out of
+# Cargo features lets Rust unit tests link against libpython.
+pyo3 = { version = "0.28", features = ["abi3-py310", "chrono"] }
+chrono.workspace = true
 pyo3-async-runtimes = { version = "0.28", features = [
     "attributes",
     "tokio-runtime",
 ] }
-pin-project = "1.1.5"
+pin-project.workspace = true
 futures.workspace = true
-serde = "1"
-serde_json = "1"
+serde.workspace = true
+serde_json.workspace = true
 snafu.workspace = true
-tokio = { version = "1.40", features = ["sync", "rt-multi-thread"] }
+tokio.workspace = true
 libc = "0.2"
 
 [build-dependencies]
-pyo3-build-config = { version = "0.28", features = [
-    "extension-module",
-    "abi3-py39",
-] }
+pyo3-build-config = { version = "0.28", features = ["abi3-py310"] }
 
 [features]
 default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/cos", "lancedb/goosefs", "lancedb/metrics-otel"]
diff --git a/python/README.md b/python/README.md
index 550698500..3a81c486a 100644
--- a/python/README.md
+++ b/python/README.md
@@ -38,6 +38,25 @@ Stable releases are created about every 2 weeks. For the latest features and bug
 pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb
 ```
 
+### Threading in CPU-limited containers
+
+LanceDB uses separate pools for compute work and storage I/O. On a container with
+two visible CPUs, current releases intentionally use one compute worker by default;
+no manual configuration is needed. If every query logs an I/O core reservation
+warning on a two-CPU container, upgrade from LanceDB 0.21.1 or earlier.
+
+The two commonly tuned environment variables control different resources:
+
+- `LANCE_CPU_THREADS` overrides the number of compute workers. One worker is the
+  appropriate setting for a two-CPU container when an explicit override is needed.
+- `LANCE_IO_THREADS` controls concurrent storage operations, not reserved CPU
+  cores. Its default can be greater than the number of CPUs because I/O workers
+  spend much of their time waiting for storage.
+
+Keep the defaults unless measurements show that the workload benefits from an
+override. See the [Lance threading model](https://lance.org/guide/performance/#threading-model)
+for the current defaults and tuning guidance.
+
 ## Usage
 
 ### Basic Example
diff --git a/python/pyproject.toml b/python/pyproject.toml
index cb175bd6d..22a41a8a9 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -8,7 +8,7 @@ dependencies = [
     "overrides>=0.7; python_version<'3.12'",
     "packaging>=23.0",
     "pyarrow>=16",
-    "pydantic>=1.10",
+    "pydantic>=2.7.4,<3",
     "tqdm>=4.27.0",
     "lance-namespace>=0.3.2"
 ]
@@ -60,7 +60,7 @@ tests = [
     "pytest-asyncio>=0.21",
     "duckdb>=0.9.0",
     "pytz>=2023.3",
-    "polars>=0.19, <=1.3.0",
+    "polars>=0.19, <=1.32.3",
     "pyarrow<25",
     "pyarrow-stubs>=16.0",
     "pylance==9.0.0rc1",
@@ -101,9 +101,12 @@ azure = ["adlfs>=2024.2.0"]
 [tool.maturin]
 python-source = "python"
 module-name = "lancedb._lancedb"
+# uv installs the project as an editable package before `uv run`, so keep that
+# bootstrap build consistent with `maturin develop`.
+editable-profile = "dev"
 
 [build-system]
-requires = ["maturin>=1.4"]
+requires = ["maturin>=1.10"]
 build-backend = "maturin"
 
 [tool.ruff.lint]
@@ -140,6 +143,7 @@ include = [
     "python/lancedb/remote/errors.py",
     "python/lancedb/embeddings/__init__.py",
     "python/lancedb/_lancedb.pyi",
+    "python/type_tests/connect.py",
 ]
 exclude = ["python/tests/"]
 pythonVersion = "3.13"
diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py
index 235049f97..8cb85a3ed 100644
--- a/python/python/lancedb/__init__.py
+++ b/python/python/lancedb/__init__.py
@@ -12,6 +12,7 @@ __version__ = importlib.metadata.version("lancedb")
 
 from ._lancedb import connect as lancedb_connect
 from ._lancedb import FtsToken
+from ._lancedb import LsmWriteSpec
 from ._lancedb import tokenize as _tokenize
 from .common import URI, sanitize_uri
 from urllib.parse import urlparse
@@ -21,6 +22,22 @@ from .remote.db import RemoteDBConnection
 from .expr import Expr, col, lit, func
 from .schema import blob, vector, BlobType
 from .job import AsyncJob, Job
+from .functions import (
+    FunctionArtifactRequest as FunctionArtifactRequest,
+    FunctionApplication as FunctionApplication,
+    FunctionBinding as FunctionBinding,
+    FunctionRegistrationRequest as FunctionRegistrationRequest,
+    FunctionVersion as FunctionVersion,
+    PythonRuntimeSpec as PythonRuntimeSpec,
+    RefreshColumnResult as RefreshColumnResult,
+    UdfDefinition as UdfDefinition,
+    udf as udf,
+)
+from .materialized_view import (
+    AsyncMaterializedView,
+    MaterializedView,
+    MaterializedViewDefinition,
+)
 from .table import AsyncTable, Table
 from .types import BaseTokenizerType
 from ._lancedb import Session
@@ -162,6 +179,18 @@ def connect(
     ...     },
     ... )
 
+    For Azure Blob Storage, credentials can be passed directly without setting
+    environment variables:
+
+    >>> azure_storage_options = {
+    ...     "account_name": "some-account",
+    ...     "account_key": "some-key",
+    ... }
+    >>> db = lancedb.connect(  # doctest: +SKIP
+    ...     "az://my-container/my-database",
+    ...     storage_options=azure_storage_options,
+    ... )
+
     For tests and temporary data, use an in-memory database:
 
     >>> db = lancedb.connect("memory://")
@@ -448,6 +477,10 @@ async def connect_async(
     --------
 
     >>> import lancedb
+    >>> azure_storage_options = {
+    ...     "account_name": "some-account",
+    ...     "account_key": "some-key",
+    ... }
     >>> async def doctest_example():
     ...     # For a local directory, provide a path to the database
     ...     db = await lancedb.connect_async("~/.lancedb")
@@ -455,6 +488,11 @@ async def connect_async(
     ...     db = await lancedb.connect_async("s3://my-bucket/lancedb",
     ...                                      storage_options={
     ...                                          "aws_access_key_id": "***"})
+    ...     # Azure credentials can also be passed directly
+    ...     db = await lancedb.connect_async(
+    ...         "az://my-container/my-database",
+    ...         storage_options=azure_storage_options,
+    ...     )
     ...     # For tests and temporary data, use an in-memory database
     ...     db = await lancedb.connect_async("memory://")
     ...     # Connect to LanceDB cloud
@@ -495,6 +533,9 @@ async def connect_async(
 
 
 __all__ = [
+    "AsyncMaterializedView",
+    "MaterializedView",
+    "MaterializedViewDefinition",
     "connect",
     "connect_async",
     "tokenize",
@@ -518,6 +559,7 @@ __all__ = [
     "Job",
     "LanceDBConnection",
     "LanceNamespaceDBConnection",
+    "LsmWriteSpec",
     "RemoteDBConnection",
     "Session",
     "Table",
diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi
index 47e727f99..593bceffa 100644
--- a/python/python/lancedb/_lancedb.pyi
+++ b/python/python/lancedb/_lancedb.pyi
@@ -147,6 +147,8 @@ class Connection(object):
         limit: Optional[int],
     ) -> list[str]: ...  # Deprecated: Use list_tables instead
     def job(self, job_id: str) -> Job: ...
+    async def create_function_async(self, request_json: str) -> Job: ...
+    async def get_function(self, name: str, version: str) -> str: ...
     async def list_jobs(self) -> List[JobInfo]: ...
     async def get_job(self, job_id: str) -> Optional[JobDescription]: ...
     async def cancel_job(self, job_id: str) -> bool: ...
@@ -195,9 +197,21 @@ class Connection(object):
         cur_namespace_path: Optional[List[str]] = None,
         new_namespace_path: Optional[List[str]] = None,
     ) -> None: ...
+    async def create_materialized_view(
+        self,
+        name: str,
+        source: str,
+        projections: Optional[List[Tuple[str, str]]] = None,
+        filter: Optional[str] = None,
+        limit: Optional[int] = None,
+    ) -> Table: ...
+    async def list_materialized_views(self) -> List[str]: ...
     async def drop_table(
         self, name: str, namespace_path: Optional[List[str]] = None
     ) -> None: ...
+    async def drop_table_async(
+        self, name: str, namespace_path: Optional[List[str]] = None
+    ) -> Job: ...
     async def drop_all_tables(
         self, namespace_path: Optional[List[str]] = None
     ) -> None: ...
@@ -220,7 +234,7 @@ class Job:
     @property
     def id(self) -> Optional[str]: ...
     async def status(self) -> str: ...
-    async def wait(self) -> None: ...
+    async def wait(self) -> Optional[str]: ...
     async def cancel(self) -> None: ...
 
 class JobInfo:
@@ -269,6 +283,7 @@ class Table:
         mode: Literal["append", "overwrite"],
         progress: Optional[Any] = None,
         write_parallelism: Optional[int] = None,
+        allow_external_blob_outside_bases: bool = False,
     ) -> AddResult: ...
     async def update(
         self, updates: Dict[str, str], where: Optional[str]
@@ -335,6 +350,17 @@ class Table:
     ) -> list[FtsToken]: ...
     async def delete(self, filter: Union[str, PyExpr]) -> DeleteResult: ...
     async def add_columns(self, columns: list[tuple[str, str]]) -> AddColumnsResult: ...
+    async def add_computed_columns(
+        self, columns: list[tuple[str, str]]
+    ) -> AddColumnsResult: ...
+    async def add_function_columns(
+        self, application_json: str, output_name: Optional[str]
+    ) -> AddColumnsResult: ...
+    async def refresh_column(self, column: str) -> RefreshColumnResult: ...
+    async def refresh_column_async(self, column: str) -> Job: ...
+    async def refresh_materialized_view(
+        self, full: bool = False, source_version: Optional[int] = None
+    ) -> RefreshMaterializedViewResult: ...
     async def add_columns_with_schema(self, schema: pa.Schema) -> AddColumnsResult: ...
     async def alter_columns(
         self, columns: list[dict[str, Any]]
@@ -355,6 +381,10 @@ class Table:
     async def set_lsm_write_spec(self, spec: LsmWriteSpec) -> None: ...
     async def unset_lsm_write_spec(self) -> None: ...
     async def get_lsm_write_spec(self) -> Optional[LsmWriteSpec]: ...
+    async def checkpoint_lsm(self) -> None: ...
+    async def flush_lsm(self) -> None: ...
+    async def compact_lsm(self) -> None: ...
+    async def get_lsm_stats(self, include_generation_rows: bool) -> Optional[dict]: ...
     async def close_lsm_writers(self) -> None: ...
     @property
     def tags(self) -> Tags: ...
@@ -396,7 +426,7 @@ class Branches:
     async def checkout(self, name: str, version: Optional[int] = None) -> Table: ...
     async def delete(self, name: str) -> None: ...
     async def diff(self, from_branch: str) -> Dict[str, Any]: ...
-    async def merge(
+    async def cherry_pick(
         self, from_branch: str, dry_run: bool = False
     ) -> Dict[str, Any]: ...
 
@@ -649,9 +679,10 @@ class LsmWriteSpec:
     def identity(column: str) -> "LsmWriteSpec": ...
     @staticmethod
     def unsharded() -> "LsmWriteSpec": ...
-    def with_maintained_indexes(self, indexes: List[str]) -> "LsmWriteSpec":
-        """Return a copy of this spec asking the MemWAL to keep the named
-        indexes up to date as rows are appended."""
+    def with_maintained_indexes(self, indexes: Optional[List[str]]) -> "LsmWriteSpec":
+        """Set which indexes the MemWAL keeps up to date. None resolves every
+        index on the table at install, failing if one cannot be maintained;
+        a list is verbatim, empty means none."""
         ...
     def with_writer_config_defaults(self, defaults: Dict[str, str]) -> "LsmWriteSpec":
         """Return a copy of this spec recording the given default
@@ -666,13 +697,25 @@ class LsmWriteSpec:
     @property
     def num_buckets(self) -> Optional[int]: ...
     @property
-    def maintained_indexes(self) -> List[str]: ...
+    def maintained_indexes(self) -> Optional[List[str]]:
+        """Indexes the MemWAL keeps up to date, or None for every supported one."""
+        ...
     @property
     def writer_config_defaults(self) -> Dict[str, str]: ...
 
 class AddColumnsResult:
     version: int
 
+class RefreshColumnResult:
+    rows_filled: int
+    version: int
+
+class RefreshMaterializedViewResult:
+    mode: str
+    rows_written: int
+    source_version: int
+    version: int
+
 class AlterColumnsResult:
     version: int
 
diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py
index eeae8bf50..51b8d9993 100644
--- a/python/python/lancedb/db.py
+++ b/python/python/lancedb/db.py
@@ -45,7 +45,14 @@ from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
 
 from . import __version__
 from ._lancedb import connect as lancedb_connect  # type: ignore
-from .job import AsyncJob, Job
+from .functions import FunctionVersion, UdfDefinition
+from .job import AsyncJob, Job, _typed_job
+from .materialized_view import (
+    AsyncMaterializedView,
+    MaterializedView,
+    SelectArg,
+    normalize_select,
+)
 from .table import (
     AsyncTable,
     LanceTable,
@@ -509,6 +516,70 @@ class DBConnection(EnforceOverrides):
         """
         raise NotImplementedError
 
+    def create_materialized_view(
+        self,
+        name: str,
+        source: str,
+        *,
+        select: SelectArg = None,
+        where: Optional[str] = None,
+        limit: Optional[int] = None,
+    ) -> MaterializedView:
+        """Define a materialized view named ``name`` over the table ``source``.
+
+        The view is created empty, with the query recorded in its schema
+        metadata; ``view.refresh()`` computes the rows. The view is a normal
+        table: it can be queried, indexed and searched, and it appears in
+        ``table_names``. Local databases only.
+
+        The source table must have stable row ids (create it with the
+        ``new_table_enable_stable_row_ids`` storage option): they keep the
+        view's provenance valid across source compactions, and cannot be
+        enabled after a table exists.
+
+        Parameters
+        ----------
+        name: str
+            The name of the view.
+        source: str
+            The name of the source table, in this database.
+        select: list or dict, optional
+            The view's columns: column names, ``(alias, SQL expression)``
+            pairs, or a dict of the same. Omitting it selects every source
+            column, expanded against the source schema at creation time.
+        where: str, optional
+            SQL predicate; only matching source rows appear in the view.
+        limit: int, optional
+            Cap the view at this many rows, in materialization order.
+
+        Returns
+        -------
+        MaterializedView
+        """
+        raise NotImplementedError(
+            "materialized views are not supported on this connection type"
+        )
+
+    def open_materialized_view(self, name: str) -> MaterializedView:
+        """Open the materialized view named ``name``.
+
+        Raises ``ValueError`` if the table exists but is not a materialized
+        view.
+        """
+        raise NotImplementedError(
+            "materialized views are not supported on this connection type"
+        )
+
+    def list_materialized_views(self) -> List[str]:
+        """The names of the materialized views in this database.
+
+        Found by reading every table's schema, so this costs an open per
+        table.
+        """
+        raise NotImplementedError(
+            "materialized views are not supported on this connection type"
+        )
+
     def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
         """Drop a table from the database.
 
@@ -524,6 +595,12 @@ class DBConnection(EnforceOverrides):
             namespace_path = []
         raise NotImplementedError
 
+    def drop_table_async(
+        self, name: str, namespace_path: Optional[List[str]] = None
+    ) -> Job:
+        """Start dropping a table and return its cleanup job."""
+        raise NotImplementedError
+
     def rename_table(
         self,
         cur_name: str,
@@ -610,6 +687,31 @@ class DBConnection(EnforceOverrides):
         """
         raise NotImplementedError("serialize is not supported for this connection type")
 
+    def create_function(self, definition: UdfDefinition) -> FunctionVersion:
+        """Register a scalar Python UDF and wait for its immutable version.
+
+        This is the blocking counterpart of :meth:`create_function_async`.
+        Local connections raise ``NotImplementedError``.
+        """
+        return self.create_function_async(definition).wait()
+
+    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
+        """Register a scalar Python UDF through the remote Function catalog.
+
+        Submission returns a typed job. The immutable Function version becomes
+        available only when :meth:`Job.wait` succeeds. Local connections raise
+        ``NotImplementedError``.
+        """
+        raise NotImplementedError(
+            "Function catalog operations are not supported for this connection type"
+        )
+
+    def get_function(self, name: str, *, version: str) -> FunctionVersion:
+        """Open one exact immutable Function version from the remote catalog."""
+        raise NotImplementedError(
+            "Function catalog operations are not supported for this connection type"
+        )
+
     def job(self, job_id: str) -> Job:
         """A [Job][lancedb.job.Job] handle for a server-side job by id.
 
@@ -1104,6 +1206,58 @@ class LanceDBConnection(DBConnection):
             tbl.checkout(version)
         return tbl
 
+    @override
+    def create_materialized_view(
+        self,
+        name: str,
+        source: str,
+        *,
+        select: SelectArg = None,
+        where: Optional[str] = None,
+        limit: Optional[int] = None,
+    ) -> MaterializedView:
+        """Define a materialized view named ``name`` over the table ``source``.
+        See
+        [DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view].
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> db = lancedb.connect(
+        ...     "./.lancedb",
+        ...     storage_options={"new_table_enable_stable_row_ids": "true"},
+        ... )
+        >>> data = [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]
+        >>> table = db.create_table("people", data)
+        >>> view = db.create_materialized_view(
+        ...     "adults",
+        ...     "people",
+        ...     select=["name", ("shout", "upper(name)")],
+        ...     where="age >= 18",
+        ... )
+        >>> result = view.refresh()
+        >>> result.rows_written
+        1
+        """
+        LOOP.run(
+            self._conn.create_materialized_view(
+                name, source, select=select, where=where, limit=limit
+            )
+        )
+        return MaterializedView(self.open_table(name))
+
+    @override
+    def open_materialized_view(self, name: str) -> MaterializedView:
+        """Open the materialized view named ``name``."""
+        view = MaterializedView(self.open_table(name))
+        view.definition
+        return view
+
+    @override
+    def list_materialized_views(self) -> List[str]:
+        """The names of the materialized views in this database."""
+        return LOOP.run(self._conn.list_materialized_views())
+
     def clone_table(
         self,
         target_table_name: str,
@@ -1186,6 +1340,20 @@ class LanceDBConnection(DBConnection):
             )
         )
 
+    @override
+    def drop_table_async(
+        self, name: str, namespace_path: Optional[List[str]] = None
+    ) -> Job:
+        """Start dropping a table and return its cleanup job.
+
+        The table may become unavailable before its data files are removed.
+        Call :meth:`Job.wait` to wait for cleanup to finish.
+        """
+        if namespace_path is None:
+            namespace_path = []
+        job = LOOP.run(self._conn.drop_table_async(name, namespace_path=namespace_path))
+        return Job(job if isinstance(job, AsyncJob) else AsyncJob(job))
+
     @override
     def drop_all_tables(self, namespace_path: Optional[List[str]] = None):
         if namespace_path is None:
@@ -1236,6 +1404,15 @@ class LanceDBConnection(DBConnection):
         """
         return Job(self._conn.job(job_id))
 
+    @override
+    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
+        job = LOOP.run(self._conn.create_function_async(definition))
+        return Job(job)
+
+    @override
+    def get_function(self, name: str, *, version: str) -> FunctionVersion:
+        return LOOP.run(self._conn.get_function(name, version=version))
+
     @override
     def list_jobs(self) -> List[JobInfo]:
         """List server-side jobs across the database's tables."""
@@ -1851,6 +2028,50 @@ class AsyncConnection(object):
             await tbl.checkout(version)
         return tbl
 
+    async def create_materialized_view(
+        self,
+        name: str,
+        source: str,
+        *,
+        select: SelectArg = None,
+        where: Optional[str] = None,
+        limit: Optional[int] = None,
+    ) -> AsyncMaterializedView:
+        """Define a materialized view named ``name`` over the table ``source``.
+        See
+        [DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view].
+        """
+        inner = await self._inner.create_materialized_view(
+            name,
+            source,
+            projections=normalize_select(select),
+            filter=where,
+            limit=limit,
+        )
+        return AsyncMaterializedView(AsyncTable(inner))
+
+    async def open_materialized_view(self, name: str) -> AsyncMaterializedView:
+        """Open the materialized view named ``name``.
+
+        Raises ``ValueError`` if the table exists but is not a materialized
+        view.
+        """
+        if self.uri.startswith("db://"):
+            raise NotImplementedError(
+                "materialized views are supported only on local databases"
+            )
+        view = AsyncMaterializedView(await self.open_table(name))
+        await view.definition()
+        return view
+
+    async def list_materialized_views(self) -> List[str]:
+        """The names of the materialized views in this database.
+
+        Found by reading every table's schema, so this costs an open per
+        table.
+        """
+        return await self._inner.list_materialized_views()
+
     async def clone_table(
         self,
         target_table_name: str,
@@ -1963,6 +2184,23 @@ class AsyncConnection(object):
             if f"Table '{name}' was not found" not in str(e):
                 raise e
 
+    async def drop_table_async(
+        self,
+        name: str,
+        *,
+        namespace_path: Optional[List[str]] = None,
+    ) -> AsyncJob:
+        """Start dropping a table and return its cleanup job.
+
+        The table may become unavailable before its data files are removed.
+        Await :meth:`AsyncJob.wait` to wait for cleanup to finish.
+        """
+        if namespace_path is None:
+            namespace_path = []
+        return AsyncJob(
+            await self._inner.drop_table_async(name, namespace_path=namespace_path)
+        )
+
     async def drop_all_tables(self, namespace_path: Optional[List[str]] = None):
         """Drop all tables from the database.
 
@@ -1986,6 +2224,25 @@ class AsyncConnection(object):
         """
         return AsyncJob(self._inner.job(job_id))
 
+    async def create_function_async(
+        self, definition: UdfDefinition
+    ) -> AsyncJob[FunctionVersion]:
+        """Register a scalar Python UDF through the remote Function catalog.
+
+        The returned typed job resolves to the immutable Function version.
+        Local connections raise ``NotImplementedError``.
+        """
+        if not isinstance(definition, UdfDefinition):
+            raise TypeError("create_function_async requires a @udf definition")
+        inner = await self._inner.create_function_async(
+            definition.registration_request.to_canonical_json()
+        )
+        return _typed_job(inner, FunctionVersion.from_json)
+
+    async def get_function(self, name: str, *, version: str) -> FunctionVersion:
+        """Open one exact immutable Function version from the remote catalog."""
+        return FunctionVersion.from_json(await self._inner.get_function(name, version))
+
     async def list_jobs(self) -> List[JobInfo]:
         """List server-side jobs across the database's tables."""
         return await self._inner.list_jobs()
diff --git a/python/python/lancedb/embeddings/base.py b/python/python/lancedb/embeddings/base.py
index f711e5b7d..149b9c089 100644
--- a/python/python/lancedb/embeddings/base.py
+++ b/python/python/lancedb/embeddings/base.py
@@ -26,7 +26,6 @@ class EmbeddingFunction(BaseModel, ABC):
     3. ndims() which returns the number of dimensions of the vector column
     """
 
-    __slots__ = ("__weakref__",)  # pydantic 1.x compatibility
     max_retries: int = (
         7  # Setting 0 disables retires. Maybe this should not be enabled by default,
     )
diff --git a/python/python/lancedb/embeddings/bedrock.py b/python/python/lancedb/embeddings/bedrock.py
index dc2badceb..dc93a2da9 100644
--- a/python/python/lancedb/embeddings/bedrock.py
+++ b/python/python/lancedb/embeddings/bedrock.py
@@ -7,8 +7,7 @@ from functools import cached_property
 from typing import List, Union
 
 import numpy as np
-
-from lancedb.pydantic import PYDANTIC_VERSION
+from pydantic import ConfigDict
 
 from ..util import attempt_import_or_raise
 from .base import TextEmbeddingFunction
@@ -67,13 +66,7 @@ class BedRockText(TextEmbeddingFunction):
     source_input_type: str = "search_document"
     query_input_type: str = "search_query"
 
-    if PYDANTIC_VERSION.major < 2:  # Pydantic 1.x compat
-
-        class Config:
-            keep_untouched = (cached_property,)
-    else:
-        model_config = dict()
-        model_config["ignored_types"] = (cached_property,)
+    model_config = ConfigDict(ignored_types=(cached_property,))
 
     def ndims(self):
         # return len(self._generate_embedding("test"))
diff --git a/python/python/lancedb/embeddings/gemini_text.py b/python/python/lancedb/embeddings/gemini_text.py
index 32f2d4d04..d3bf79af4 100644
--- a/python/python/lancedb/embeddings/gemini_text.py
+++ b/python/python/lancedb/embeddings/gemini_text.py
@@ -7,8 +7,7 @@ from functools import cached_property
 from typing import List, Optional, Union
 
 import numpy as np
-
-from lancedb.pydantic import PYDANTIC_VERSION
+from pydantic import ConfigDict
 
 from ..util import attempt_import_or_raise
 from .base import TextEmbeddingFunction
@@ -87,13 +86,7 @@ class GeminiText(TextEmbeddingFunction):
     query_task_type: str = "retrieval_query"
     source_task_type: str = "retrieval_document"
 
-    if PYDANTIC_VERSION.major < 2:  # Pydantic 1.x compat
-
-        class Config:
-            keep_untouched = (cached_property,)
-    else:
-        model_config = dict()
-        model_config["ignored_types"] = (cached_property,)
+    model_config = ConfigDict(ignored_types=(cached_property,))
 
     def ndims(self):
         if self.dim:
diff --git a/python/python/lancedb/embeddings/imagebind.py b/python/python/lancedb/embeddings/imagebind.py
index 84bb0a123..c8051f14d 100644
--- a/python/python/lancedb/embeddings/imagebind.py
+++ b/python/python/lancedb/embeddings/imagebind.py
@@ -7,14 +7,13 @@ from typing import List, Union
 
 import numpy as np
 import pyarrow as pa
+from pydantic import ConfigDict
 
 from ..util import attempt_import_or_raise
 from .base import EmbeddingFunction
 from .registry import register
 from .utils import AUDIO, IMAGES, TEXT
 
-from lancedb.pydantic import PYDANTIC_VERSION
-
 
 @register("imagebind")
 class ImageBindEmbeddings(EmbeddingFunction):
@@ -31,13 +30,7 @@ class ImageBindEmbeddings(EmbeddingFunction):
     device: str = "cpu"
     normalize: bool = False
 
-    if PYDANTIC_VERSION.major < 2:  # Pydantic 1.x compat
-
-        class Config:
-            keep_untouched = (cached_property,)
-    else:
-        model_config = dict()
-        model_config["ignored_types"] = (cached_property,)
+    model_config = ConfigDict(ignored_types=(cached_property,))
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
diff --git a/python/python/lancedb/embeddings/instructor.py b/python/python/lancedb/embeddings/instructor.py
index 675a0139c..37ae1c296 100644
--- a/python/python/lancedb/embeddings/instructor.py
+++ b/python/python/lancedb/embeddings/instructor.py
@@ -101,8 +101,7 @@ class InstructorEmbeddingFunction(TextEmbeddingFunction):
 
     @weak_lru(maxsize=1)
     def ndims(self):
-        model = self.get_model()
-        return model.encode("foo").shape[0]
+        return len(self.generate_embeddings([[self.source_instruction, "foo"]])[0])
 
     def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]:
         return self.generate_embeddings([[self.query_instruction, query]])
diff --git a/python/python/lancedb/embeddings/jinaai.py b/python/python/lancedb/embeddings/jinaai.py
index 9656f041f..f6ab601b3 100644
--- a/python/python/lancedb/embeddings/jinaai.py
+++ b/python/python/lancedb/embeddings/jinaai.py
@@ -87,12 +87,13 @@ class JinaEmbeddings(EmbeddingFunction):
         if isinstance(image, bytes):
             image_dict = {"image": base64.b64encode(image).decode("utf-8")}
         elif isinstance(image, (str, Path)):
-            parsed = urlparse.urlparse(image)
-            # TODO handle drive letter on windows.
+            parsed = urlparse(str(image))
             PIL_Image = attempt_import_or_raise("PIL.Image", "pillow")
             if parsed.scheme == "file":
                 pil_image = PIL_Image.open(parsed.path)
-            elif parsed.scheme == "":
+            elif parsed.scheme == "" or (os.name == "nt" and len(parsed.scheme) == 1):
+                # A Windows drive letter parses as a one-character scheme
+                # ("C:\\img.png" -> scheme="c"), so treat it as a local path.
                 pil_image = PIL_Image.open(image if os.name == "nt" else parsed.path)
             elif parsed.scheme.startswith("http"):
                 pil_image = PIL_Image.open(io.BytesIO(url_retrieve(image)))
diff --git a/python/python/lancedb/embeddings/transformers.py b/python/python/lancedb/embeddings/transformers.py
index c8a65b9ca..3f42edfdd 100644
--- a/python/python/lancedb/embeddings/transformers.py
+++ b/python/python/lancedb/embeddings/transformers.py
@@ -7,8 +7,7 @@ from typing import List, Any
 
 import numpy as np
 
-from pydantic import PrivateAttr
-from lancedb.pydantic import PYDANTIC_VERSION
+from pydantic import ConfigDict, PrivateAttr
 
 from ..util import attempt_import_or_raise
 from .base import EmbeddingFunction
@@ -59,13 +58,7 @@ class TransformersEmbeddingFunction(EmbeddingFunction):
         )
         self._model.to(self.device)
 
-    if PYDANTIC_VERSION.major < 2:  # Pydantic 1.x compat
-
-        class Config:
-            keep_untouched = (cached_property,)
-    else:
-        model_config = dict()
-        model_config["ignored_types"] = (cached_property,)
+    model_config = ConfigDict(ignored_types=(cached_property,))
 
     def ndims(self):
         self._ndims = self._model.config.hidden_size
diff --git a/python/python/lancedb/expr.py b/python/python/lancedb/expr.py
index e8b2d63a4..d16ba95d7 100644
--- a/python/python/lancedb/expr.py
+++ b/python/python/lancedb/expr.py
@@ -85,8 +85,9 @@ class Expr:
     # for dict keys / set membership.
     __hash__ = None  # type: ignore[assignment]
 
-    def __init__(self, inner: PyExpr) -> None:
+    def __init__(self, inner: PyExpr, *, column_path: str | None = None) -> None:
         self._inner = inner
+        self._column_path = column_path
 
     # ── comparisons ──────────────────────────────────────────────────────────
 
@@ -273,7 +274,7 @@ def col(name: str) -> Expr:
     >>> col("age") > lit(18)
     Expr((age > 18))
     """
-    return Expr(expr_col(name))
+    return Expr(expr_col(name), column_path=name)
 
 
 def lit(value: Union[bool, int, float, str, bytes, date, datetime, Decimal]) -> Expr:
diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py
new file mode 100644
index 000000000..237ed73c2
--- /dev/null
+++ b/python/python/lancedb/functions.py
@@ -0,0 +1,1081 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+"""Canonical Function values exchanged with LanceDB Enterprise services.
+
+These immutable models contain client/wire state only. Catalog persistence,
+environment bake, and execution are owned by Sophon.
+``RefreshColumnResult`` is also the backend-neutral result of a local
+expression-backed refresh job.
+"""
+
+from __future__ import annotations
+
+import ast
+import builtins
+import base64
+import functools
+import hashlib
+import importlib
+import inspect
+import symtable
+import json
+import math
+import re
+import sys
+import textwrap
+import types
+from collections.abc import Mapping
+from datetime import date, datetime
+from typing import (
+    Annotated,
+    Any,
+    Callable,
+    Optional,
+    Union,
+    get_args,
+    get_origin,
+    get_type_hints,
+    overload,
+)
+
+import pyarrow as pa
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    conint,
+    field_validator,
+    model_validator,
+)
+
+_Int32 = conint(strict=True, ge=-(2**31), le=2**31 - 1)
+_UInt32 = conint(strict=True, ge=0, le=2**32 - 1)
+_UInt64 = conint(strict=True, ge=0, le=2**64 - 1)
+
+
+class _FrozenDict(dict):
+    def _immutable(self, *args, **kwargs):
+        raise TypeError("remote canonical values are immutable")
+
+    __setitem__ = _immutable
+    __delitem__ = _immutable
+    clear = _immutable
+    pop = _immutable
+    popitem = _immutable
+    setdefault = _immutable
+    update = _immutable
+
+    def __ior__(self, other):
+        self._immutable()
+
+
+def _freeze_value(value):
+    if isinstance(value, Mapping):
+        return _FrozenDict({key: _freeze_value(child) for key, child in value.items()})
+    if isinstance(value, (list, tuple)):
+        return tuple(_freeze_value(child) for child in value)
+    return value
+
+
+def _validate_literal(value):
+    if isinstance(value, float):
+        raise ValueError(
+            "floating-point Function literals are not part of the Slice 1 "
+            "canonical wire contract"
+        )
+    if isinstance(value, int) and not isinstance(value, bool):
+        if not -(2**63) <= value <= 2**64 - 1:
+            raise ValueError(
+                "Function integer literal is outside the canonical JSON range"
+            )
+    elif isinstance(value, Mapping):
+        for child in value.values():
+            _validate_literal(child)
+    elif isinstance(value, (list, tuple)):
+        for child in value:
+            _validate_literal(child)
+    return value
+
+
+def _known_wire_value(value):
+    if isinstance(value, _RemoteValue):
+        return value._known_dict()
+    if isinstance(value, Mapping):
+        return {key: _known_wire_value(child) for key, child in value.items()}
+    if isinstance(value, (list, tuple)):
+        return [_known_wire_value(child) for child in value]
+    return value
+
+
+class _RemoteValue(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    @model_validator(mode="after")
+    def _freeze_mappings(self):
+        for name, value in self.__dict__.items():
+            object.__setattr__(self, name, _freeze_value(value))
+        return self
+
+    @classmethod
+    def from_json(cls, payload: str):
+        return cls.model_validate_json(payload)
+
+    def _known_dict(self) -> dict[str, Any]:
+        known = {}
+        for name, field in self.__class__.model_fields.items():
+            value = getattr(self, name)
+            if value is None:
+                continue
+            if not field.is_required():
+                default_factory = field.default_factory
+                if default_factory is not None and value == default_factory():
+                    continue
+                if default_factory is None and value == field.default:
+                    continue
+            known[name] = _known_wire_value(value)
+        return known
+
+    def _copy(self, *, update: Mapping[str, Any]):
+        update = {name: _freeze_value(value) for name, value in update.items()}
+        return self.model_copy(update=update)
+
+    def to_canonical_json(self) -> str:
+        return json.dumps(
+            self._known_dict(),
+            ensure_ascii=False,
+            allow_nan=False,
+            sort_keys=True,
+            separators=(",", ":"),
+        )
+
+
+class _OpenRemoteValue(_RemoteValue):
+    """Forward-readable value whose extras stay out of canonical encoding."""
+
+    model_config = ConfigDict(extra="allow", frozen=True)
+
+    def _unknown_field_names(self) -> set[str]:
+        return set((self.__pydantic_extra__ or {}).keys())
+
+
+class FunctionArtifact(_RemoteValue):
+    """Content-addressed Python artifact identity."""
+
+    kind: str
+    digest: str
+    entrypoint: str
+
+
+class FunctionArtifactContent(_RemoteValue):
+    """Encoded artifact bytes uploaded during remote registration."""
+
+    encoding: str
+    data: str
+
+
+class PythonAdapterSpec(_RemoteValue):
+    """Internal scalar-callable to Arrow-batch adapter selection."""
+
+    kind: str
+    version: _UInt32
+
+
+class FunctionArtifactRequest(_RemoteValue):
+    """Source artifact uploaded while registering a Function."""
+
+    kind: str
+    digest: str
+    entrypoint: str
+    content: FunctionArtifactContent
+    adapter: PythonAdapterSpec
+
+
+class FunctionParameter(_RemoteValue):
+    name: str
+    arrow_type: str
+    nullable: bool
+
+
+class FunctionResultField(_OpenRemoteValue):
+    name: str
+    arrow_type: str
+    nullable: bool
+
+
+class FunctionOutput(_OpenRemoteValue):
+    """Scalar or ordered named-struct output; unknown kinds remain decodable."""
+
+    kind: str
+    arrow_type: Optional[str] = None
+    nullable: Optional[bool] = None
+    fields: tuple[FunctionResultField, ...] = ()
+
+
+class FunctionSignature(_RemoteValue):
+    inputs: tuple[FunctionParameter, ...]
+    output: FunctionOutput
+
+
+class PythonEnvironmentSpec(_RemoteValue):
+    """One Sophon-managed Python environment source."""
+
+    kind: str
+    packages: tuple[str, ...] = ()
+    path: Optional[str] = None
+    modules: tuple[str, ...] = ()
+    image: Optional[str] = None
+
+
+class PythonRuntimeSpec(_RemoteValue):
+    """Remote runtime definition with environment values.
+
+    V1 supports ``kind="python"``. Newer runtime kinds remain readable, while
+    their unknown payload fields are intentionally not retained by the client.
+    """
+
+    kind: str
+    python_version: Optional[str] = None
+    environment: Optional[PythonEnvironmentSpec] = None
+    env: Optional[Mapping[str, str]] = None
+
+    @model_validator(mode="after")
+    def _validate_runtime_kind(self):
+        if self.kind == "python":
+            if self.python_version is None:
+                raise ValueError("python runtime requires python_version")
+            if self.environment is None:
+                raise ValueError("python runtime requires environment")
+        else:
+            object.__setattr__(self, "python_version", None)
+            object.__setattr__(self, "environment", None)
+            object.__setattr__(self, "env", None)
+        return self
+
+
+class FunctionVersion(_RemoteValue):
+    """An exact immutable Function version returned by Enterprise.
+
+    Scheduling resources, priority, concurrency, and retry policy belong to
+    the submitting Job and are not part of this identity.
+    """
+
+    name: str
+    version: str
+    artifact: FunctionArtifact
+    signature: FunctionSignature
+    runtime: PythonRuntimeSpec
+    runtime_digest: str
+    environment_digest: str
+    created_at: str
+
+    def __call__(self, **inputs: Any) -> FunctionApplication:
+        """Bind this exact version to named table columns.
+
+        Every input must be a direct [lancedb.col][lancedb.expr.col]
+        reference. The returned application is immutable and retains a
+        named-struct output as one binding, so every row's sibling values
+        come from one logical Function evaluation. Map result fields to table
+        columns with
+        [FunctionApplication.rename][lancedb.functions.FunctionApplication.rename],
+        then pass the application to
+        [Table.add_columns][lancedb.table.Table.add_columns].
+
+        Examples
+        --------
+        >>> from lancedb import col
+        >>> application = function(  # doctest: +SKIP
+        ...     title=col("title"),
+        ...     body=col("body"),
+        ... ).rename(columns={
+        ...     "normalized_text": "search_text",
+        ...     "token_count": "search_token_count",
+        ... })
+        >>> table.add_columns(application)  # doctest: +SKIP
+        """
+        from lancedb.expr import Expr
+
+        parameters = tuple(parameter.name for parameter in self.signature.inputs)
+        missing = [parameter for parameter in parameters if parameter not in inputs]
+        unknown = sorted(set(inputs) - set(parameters))
+        if missing or unknown:
+            details = []
+            if missing:
+                details.append(f"missing inputs: {missing!r}")
+            if unknown:
+                details.append(f"unknown inputs: {unknown!r}")
+            raise TypeError("invalid Function inputs (" + "; ".join(details) + ")")
+
+        bindings = []
+        for parameter in parameters:
+            value = inputs[parameter]
+            if not isinstance(value, Expr) or value._column_path is None:
+                raise TypeError(
+                    f"Function input {parameter!r} must be a direct col(...) reference"
+                )
+            bindings.append(
+                ApplicationInput(
+                    parameter=parameter,
+                    kind="column",
+                    value={"path": value._column_path},
+                )
+            )
+        return FunctionApplication(
+            function=FunctionVersionRef(name=self.name, version=self.version),
+            inputs=tuple(bindings),
+            output=self.signature.output,
+        )
+
+
+class FunctionRegistrationRequest(_RemoteValue):
+    """Stable remote registration envelope produced by :func:`udf`."""
+
+    name: str
+    artifact: FunctionArtifactRequest
+    signature: FunctionSignature
+    runtime: PythonRuntimeSpec
+
+
+class FunctionVersionRef(_OpenRemoteValue):
+    name: str
+    version: str
+
+
+class ApplicationInput(_OpenRemoteValue):
+    """One parameter value.
+
+    Slice 1 freezes integers, strings, booleans, nulls, arrays, and objects.
+    Floating-point literal encoding is deferred until Python authoring is
+    introduced with a language-neutral numeric representation.
+    """
+
+    parameter: str
+    kind: str
+    value: Any
+
+    @field_validator("value")
+    @classmethod
+    def _validate_value(cls, value):
+        return _validate_literal(value)
+
+
+class FunctionApplication(_OpenRemoteValue):
+    """Immutable pre-declaration application of an exact Function version.
+
+    A named-struct output remains one application through table
+    declaration and execution.
+    [FunctionApplication.rename][lancedb.functions.FunctionApplication.rename]
+    records the result-field to table-column mapping without splitting sibling
+    outputs into separate UDF calls.
+    """
+
+    function: FunctionVersionRef
+    inputs: tuple[ApplicationInput, ...]
+    output: FunctionOutput
+    columns: Mapping[str, str] = Field(default_factory=dict)
+
+    def _known_dict(self) -> dict[str, Any]:
+        value = super()._known_dict()
+        for name in self._unknown_field_names():
+            value.pop(name, None)
+        return value
+
+    def _ensure_declarable(self) -> None:
+        unknown = {f"application.{name}" for name in self._unknown_field_names()}
+        unknown.update(
+            f"function.{name}" for name in self.function._unknown_field_names()
+        )
+        for index, input_value in enumerate(self.inputs):
+            unknown.update(
+                f"inputs[{index}].{name}" for name in input_value._unknown_field_names()
+            )
+        unknown.update(f"output.{name}" for name in self.output._unknown_field_names())
+        for index, field in enumerate(self.output.fields):
+            unknown.update(
+                f"output.fields[{index}].{name}"
+                for name in field._unknown_field_names()
+            )
+        if unknown:
+            raise ValueError(
+                "Function application contains fields from a newer contract: "
+                f"{sorted(unknown)!r}"
+            )
+
+    def rename(self, *, columns: Mapping[str, str]) -> FunctionApplication:
+        """Return a copy with result-field to table-column aliases."""
+        if self.output.kind != "named_struct":
+            raise ValueError("rename(columns=...) requires a named-struct application")
+        result_fields = {field.name for field in self.output.fields}
+        unknown = set(columns) - result_fields
+        if unknown:
+            raise ValueError(f"unknown Function result fields: {sorted(unknown)!r}")
+        merged = dict(self.columns)
+        merged.update(columns)
+        destinations = tuple(
+            merged.get(field.name, field.name) for field in self.output.fields
+        )
+        if len(set(destinations)) != len(destinations):
+            raise ValueError("FunctionApplication rename destinations must be unique")
+        return self._copy(update={"columns": merged})
+
+
+class InputBinding(_RemoteValue):
+    parameter: str
+    field_id: _Int32
+    field_path: str
+    arrow_type: str
+    nullable: bool
+
+
+class OutputMapping(_RemoteValue):
+    """One stable result-field mapping.
+
+    Assignment state is outside the Slice 1 client contract. During the NULL
+    transition Lance exposes no public cell-flag identifier to persist here.
+    """
+
+    result_field: str
+    output_name: str
+    output_field_id: _Int32
+    output_ordinal: _UInt32
+    arrow_type: str
+    nullable: bool
+
+
+class FunctionBinding(_RemoteValue):
+    """Immutable Function binding persisted by the Enterprise table service."""
+
+    binding_id: str
+    function: FunctionVersionRef
+    inputs: tuple[InputBinding, ...]
+    outputs: tuple[OutputMapping, ...]
+    input_schema: Optional[Mapping[str, Any]] = None
+    output_schema: Optional[Mapping[str, Any]] = None
+
+
+class RefreshColumnResult(_RemoteValue):
+    """Terminal result of an expression-backed or Function-backed refresh Job.
+
+    Local jobs produce this value in process. LanceDB Cloud and Enterprise
+    decode the same value from the durable server-job terminal payload.
+    """
+
+    rows_assigned: _UInt64
+    rows_failed: _UInt64
+    rows_remaining: _UInt64
+    source_version: _UInt64
+    published_version: Optional[_UInt64] = None
+
+    @property
+    def rows_filled(self) -> int:
+        """Deprecated compatibility alias for :attr:`rows_assigned`."""
+        return self.rows_assigned
+
+    @property
+    def version(self) -> Optional[int]:
+        """Deprecated compatibility alias for :attr:`published_version`."""
+        return self.published_version
+
+
+_FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$")
+
+
+_GRAMMAR_PRIMITIVES = (
+    (pa.bool_(), "bool"),
+    (pa.int8(), "int8"),
+    (pa.int16(), "int16"),
+    (pa.int32(), "int32"),
+    (pa.int64(), "int64"),
+    (pa.uint8(), "uint8"),
+    (pa.uint16(), "uint16"),
+    (pa.uint32(), "uint32"),
+    (pa.uint64(), "uint64"),
+    (pa.float16(), "float16"),
+    (pa.float32(), "float32"),
+    (pa.float64(), "float64"),
+    (pa.string(), "utf8"),
+    (pa.binary(), "binary"),
+    (pa.date32(), "date32"),
+    (pa.date64(), "date64"),
+)
+
+
+def _canonical_arrow_type(data_type: pa.DataType) -> str:
+    """The server's V1 Function type grammar. Anything outside it is rejected
+    here rather than at registration."""
+    for candidate, name in _GRAMMAR_PRIMITIVES:
+        if data_type == candidate:
+            return name
+    if pa.types.is_list(data_type) or pa.types.is_large_list(data_type):
+        prefix = "list" if pa.types.is_list(data_type) else "large_list"
+        return f"{prefix}<{_canonical_list_item(data_type)}>"
+    if pa.types.is_fixed_size_list(data_type) and data_type.list_size > 0:
+        return (
+            f"fixed_size_list<{_canonical_list_item(data_type)}, {data_type.list_size}>"
+        )
+    raise TypeError(f"unsupported Arrow type for Function signature: {data_type}")
+
+
+def _canonical_list_item(data_type: pa.DataType) -> str:
+    """The grammar names only the item type; it always means a non-nullable
+    child called `item`, so any other child metadata cannot be represented."""
+    child = data_type.value_field
+    if child.name != "item" or child.nullable or child.metadata:
+        raise TypeError(
+            "unsupported Arrow type for Function signature: list items must be a "
+            f"non-nullable field named 'item', got {child}"
+        )
+    return _canonical_arrow_type(child.type)
+
+
+def _list_of(item: pa.DataType) -> pa.DataType:
+    return pa.list_(pa.field("item", item, nullable=False))
+
+
+def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]:
+    nullable = False
+    origin = get_origin(annotation)
+    if origin in (Union, types.UnionType):
+        arguments = get_args(annotation)
+        non_none = tuple(
+            argument for argument in arguments if argument is not type(None)
+        )
+        if len(non_none) != 1 or len(non_none) == len(arguments):
+            raise TypeError(f"unsupported union annotation: {annotation!r}")
+        annotation = non_none[0]
+        nullable = True
+
+    origin = get_origin(annotation)
+    if origin is Annotated:
+        base, *metadata = get_args(annotation)
+        arrow_types = [value for value in metadata if isinstance(value, pa.DataType)]
+        if len(arrow_types) != 1:
+            raise TypeError(
+                "Annotated Function types require exactly one PyArrow DataType"
+            )
+        _, base_nullable = _annotation_type(base)
+        return arrow_types[0], nullable or base_nullable
+
+    if isinstance(annotation, pa.DataType):
+        return annotation, nullable
+    if annotation is bool:
+        return pa.bool_(), nullable
+    if annotation is int:
+        return pa.int64(), nullable
+    if annotation is float:
+        return pa.float64(), nullable
+    if annotation is str:
+        return pa.string(), nullable
+    if annotation is bytes:
+        return pa.binary(), nullable
+    if annotation is date:
+        return pa.date32(), nullable
+    if annotation is datetime:
+        return pa.timestamp("us"), nullable
+    if get_origin(annotation) is list:
+        arguments = get_args(annotation)
+        if len(arguments) != 1:
+            raise TypeError(f"unsupported list annotation: {annotation!r}")
+        value_type, value_nullable = _annotation_type(arguments[0])
+        if value_nullable:
+            raise TypeError("nullable Function list elements are not supported")
+        return _list_of(value_type), nullable
+    raise TypeError(f"unsupported Function annotation: {annotation!r}")
+
+
+def _callable_parameters(function: Callable[..., Any]) -> tuple[inspect.Parameter, ...]:
+    parameters = tuple(inspect.signature(function).parameters.values())
+    for parameter in parameters:
+        if parameter.kind in (
+            inspect.Parameter.POSITIONAL_ONLY,
+            inspect.Parameter.VAR_POSITIONAL,
+            inspect.Parameter.VAR_KEYWORD,
+        ):
+            raise TypeError("Function callables require named, non-variadic parameters")
+        if parameter.default is not inspect.Parameter.empty:
+            raise TypeError("Function callable defaults are not supported")
+    return parameters
+
+
+def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutput:
+    if isinstance(output, pa.Schema):
+        fields = tuple(output)
+    elif isinstance(output, pa.Field) and pa.types.is_struct(output.type):
+        if output.nullable:
+            raise ValueError("Function output must be non-nullable")
+        fields = tuple(output.type)
+    elif isinstance(output, pa.DataType) and pa.types.is_struct(output):
+        fields = tuple(output)
+    else:
+        field = (
+            output
+            if isinstance(output, pa.Field)
+            else pa.field("result", output, nullable=False)
+        )
+        if not isinstance(field, pa.Field):
+            raise TypeError(
+                "output_schema must be a PyArrow DataType, Field, or Schema"
+            )
+        if field.nullable:
+            raise ValueError("Function output must be non-nullable")
+        return FunctionOutput(
+            kind="scalar",
+            arrow_type=_canonical_arrow_type(field.type),
+            nullable=False,
+        )
+
+    if not fields:
+        raise ValueError("named-struct Function output must contain at least one field")
+    if any(field.nullable for field in fields):
+        raise ValueError("Function output fields must be non-nullable")
+    names = [field.name for field in fields]
+    if len(set(names)) != len(names):
+        raise ValueError("Function output field names must be unique")
+    return FunctionOutput(
+        kind="named_struct",
+        fields=tuple(
+            FunctionResultField(
+                name=field.name,
+                arrow_type=_canonical_arrow_type(field.type),
+                nullable=False,
+            )
+            for field in fields
+        ),
+    )
+
+
+def _infer_signature(
+    function: Callable[..., Any],
+    input_schema: Optional[pa.Schema],
+    output_schema: Optional[pa.DataType | pa.Field | pa.Schema],
+) -> FunctionSignature:
+    parameters = _callable_parameters(function)
+    if (input_schema is None) != (output_schema is None):
+        raise ValueError("input_schema and output_schema must be provided together")
+
+    if input_schema is not None:
+        if not isinstance(input_schema, pa.Schema):
+            raise TypeError("input_schema must be a PyArrow Schema")
+        expected = tuple(parameter.name for parameter in parameters)
+        actual = tuple(input_schema.names)
+        if actual != expected:
+            raise ValueError(
+                "input_schema fields must exactly match callable parameters in order: "
+                f"expected {expected!r}, got {actual!r}"
+            )
+        inputs = tuple(
+            FunctionParameter(
+                name=field.name,
+                arrow_type=_canonical_arrow_type(field.type),
+                nullable=field.nullable,
+            )
+            for field in input_schema
+        )
+        return FunctionSignature(inputs=inputs, output=_function_output(output_schema))
+
+    try:
+        annotations = get_type_hints(function, include_extras=True)
+    except Exception as error:
+        raise TypeError(f"failed to resolve Function annotations: {error}") from error
+    missing = [
+        parameter.name for parameter in parameters if parameter.name not in annotations
+    ]
+    if missing or "return" not in annotations:
+        names = missing + ([] if "return" in annotations else ["return"])
+        raise TypeError(f"missing Function annotations: {names!r}")
+    inputs = []
+    for parameter in parameters:
+        data_type, nullable = _annotation_type(annotations[parameter.name])
+        inputs.append(
+            FunctionParameter(
+                name=parameter.name,
+                arrow_type=_canonical_arrow_type(data_type),
+                nullable=nullable,
+            )
+        )
+    output_type, output_nullable = _annotation_type(annotations["return"])
+    if output_nullable:
+        raise ValueError("Function output must be non-nullable")
+    return FunctionSignature(
+        inputs=tuple(inputs),
+        output=_function_output(pa.field("result", output_type, nullable=False)),
+    )
+
+
+def _is_udf_decorator(node: ast.expr) -> bool:
+    if isinstance(node, ast.Call):
+        node = node.func
+    return (isinstance(node, ast.Name) and node.id == "udf") or (
+        isinstance(node, ast.Attribute) and node.attr == "udf"
+    )
+
+
+def _literal_source(value: Any) -> str:
+    if value is None or type(value) in (bool, int, str, bytes):
+        return repr(value)
+    if type(value) is float and math.isfinite(value):
+        return repr(value)
+    if type(value) is tuple:
+        children = ", ".join(_literal_source(child) for child in value)
+        if len(value) == 1:
+            children += ","
+        return f"({children})"
+    raise TypeError(
+        "Function source references an unsupported global value of type "
+        f"{type(value).__name__}"
+    )
+
+
+_DYNAMIC_NAMESPACE_ACCESS = frozenset(
+    {"globals", "locals", "vars", "eval", "exec", "compile", "__import__"}
+)
+# Modules that hand out namespaces (`sys.modules`, `builtins`, importers,
+# introspection). The artifact's module namespace holds only the names it was
+# packaged with, so reaching around it cannot be represented.
+_NAMESPACE_MODULES = frozenset(
+    {"sys", "builtins", "importlib", "inspect", "gc", "ctypes", "types"}
+)
+
+
+def _namespace_acquisition(
+    definition: ast.FunctionDef, references: set[str]
+) -> list[str]:
+    found = set(references & _DYNAMIC_NAMESPACE_ACCESS)
+    for node in ast.walk(definition):
+        if isinstance(node, ast.Import):
+            found.update(
+                alias.name
+                for alias in node.names
+                if alias.name.split(".")[0] in _NAMESPACE_MODULES
+            )
+        elif isinstance(node, ast.ImportFrom) and node.module:
+            if node.module.split(".")[0] in _NAMESPACE_MODULES:
+                found.add(node.module)
+    return sorted(found)
+
+
+def _module_references(module_source: str) -> set[str]:
+    """Names any scope in `module_source` binds or loads at module scope.
+    Python's own scope analysis on the exact text that ships: free variables
+    belong to an enclosing scope inside the function, and postponed
+    annotations are not runtime loads."""
+
+    def visit(table: symtable.SymbolTable, found: set[str]) -> None:
+        for symbol in table.get_symbols():
+            if symbol.is_global() and (
+                symbol.is_referenced() or symbol.is_declared_global()
+            ):
+                found.add(symbol.get_name())
+        for child in table.get_children():
+            visit(child, found)
+
+    found: set[str] = set()
+    for table in symtable.symtable(module_source, "<udf>", "exec").get_children():
+        visit(table, found)
+    return found
+
+
+def _global_source(name: str, value: Any) -> str:
+    """One module-level line that rebinds `name` to `value` in the artifact:
+    an import for modules and importable classes/functions, a literal otherwise."""
+    if isinstance(value, types.ModuleType):
+        if value.__name__.split(".")[0] in _NAMESPACE_MODULES:
+            raise ValueError(
+                f"@udf cannot package dynamic namespace access: {value.__name__!r}"
+            )
+        try:
+            imported = importlib.import_module(value.__name__)
+        except ImportError:
+            imported = None
+        if imported is not value:
+            raise TypeError(
+                f"Function source references module {name!r} that does not import "
+                f"as {value.__name__!r}"
+            )
+        return f"import {value.__name__} as {name}"
+    module_name = getattr(value, "__module__", None)
+    qualname = getattr(value, "__qualname__", None)
+    if (
+        isinstance(module_name, str)
+        and isinstance(qualname, str)
+        and module_name != "__main__"
+        and "." not in qualname
+        and "<" not in qualname
+    ):
+        try:
+            imported = getattr(importlib.import_module(module_name), qualname)
+        except (ImportError, AttributeError):
+            imported = None
+        if imported is value:
+            return f"from {module_name} import {qualname} as {name}"
+    return f"{name} = {_literal_source(value)}"
+
+
+def _is_recursive_reference(function: Callable[..., Any], name: str) -> bool:
+    """`name` inside the body means the function itself unless the module has
+    since bound it to something else."""
+    if name != function.__name__:
+        return False
+    bound = function.__globals__.get(name, function)
+    if bound is function:
+        return True
+    # The decorator's own result is the one wrapper known to call `function`
+    # unchanged; any other binding may behave differently from a self-call.
+    return type(bound) is UdfDefinition and bound._function is function
+
+
+def _package_source(function: Callable[..., Any]) -> bytes:
+    if not inspect.isfunction(function) or inspect.iscoroutinefunction(function):
+        raise TypeError("@udf requires a synchronous Python function")
+    try:
+        source = textwrap.dedent(inspect.getsource(function))
+    except (OSError, TypeError) as error:
+        raise ValueError("@udf requires inspectable Python source") from error
+    module = ast.parse(source)
+    definitions = [
+        node
+        for node in module.body
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
+        and node.name == function.__name__
+    ]
+    if len(definitions) != 1 or not isinstance(definitions[0], ast.FunctionDef):
+        raise ValueError("@udf source must contain exactly one synchronous function")
+    definition = definitions[0]
+    if any(not _is_udf_decorator(decorator) for decorator in definition.decorator_list):
+        raise ValueError("@udf cannot package additional Python decorators")
+    definition.decorator_list = []
+
+    closure = inspect.getclosurevars(function)
+    if closure.nonlocals:
+        raise ValueError("@udf cannot package functions that capture closure values")
+    function_source = ast.unparse(definition)
+    module_header = "from __future__ import annotations"
+    references = _module_references(f"{module_header}\n\n{function_source}\n")
+    dynamic = _namespace_acquisition(definition, references)
+    if dynamic:
+        raise ValueError(f"@udf cannot package dynamic namespace access: {dynamic!r}")
+    # Resolve every module-scope reference the way the interpreter would: the
+    # function's own globals first (a module global may shadow a builtin, and
+    # nested scopes are not visible to getclosurevars), then its builtins.
+    # The artifact runs under the standard builtins; only the exact mapping is
+    # provably equivalent (a subclass or copy can change lookups and hooks).
+    if function.__builtins__ is not vars(builtins):
+        raise ValueError("@udf cannot package a non-standard builtins environment")
+    globals_source = []
+    unresolved = []
+    for name in sorted(references):
+        if name == function.__name__:
+            if not _is_recursive_reference(function, name):
+                raise ValueError(
+                    f"@udf cannot package {name!r}: the module binds that name to "
+                    "another value, which the artifact's own definition would shadow"
+                )
+            continue
+        if name in function.__globals__:
+            globals_source.append(_global_source(name, function.__globals__[name]))
+        elif hasattr(builtins, name):
+            pass
+        else:
+            unresolved.append(name)
+    if unresolved:
+        raise ValueError(
+            f"@udf source contains unresolved global names: {unresolved!r}"
+        )
+
+    parts = [module_header]
+    if globals_source:
+        parts.extend(["", *globals_source])
+    parts.extend(["", function_source, ""])
+    packaged = "\n".join(parts)
+    return packaged.encode("utf-8")
+
+
+class UdfDefinition:
+    """A scalar Python callable prepared for remote Function registration.
+
+    Instances are created with :func:`udf`. Calling an instance executes the
+    original scalar Python function, which keeps local unit testing ordinary.
+    Remote execution adapts that scalar callable to the internal Arrow batch
+    ABI described by the registration artifact.
+    """
+
+    def __init__(
+        self,
+        function: Callable[..., Any],
+        *,
+        name: Optional[str],
+        input_schema: Optional[pa.Schema],
+        output_schema: Optional[pa.DataType | pa.Field | pa.Schema],
+        pip: tuple[str, ...],
+        env: Mapping[str, str],
+        python_version: Optional[str],
+    ):
+        function_name = name or function.__name__
+        if not _FUNCTION_NAME.fullmatch(function_name):
+            raise ValueError(f"invalid Function name: {function_name!r}")
+        packages = tuple(sorted(set(pip)))
+        if any(not package or package != package.strip() for package in packages):
+            raise ValueError("pip requirements must be non-empty and trimmed")
+        environment = dict(env)
+        if any(
+            not isinstance(key, str) or not isinstance(value, str)
+            for key, value in environment.items()
+        ):
+            raise TypeError("Function env keys and values must be strings")
+        signature = _infer_signature(function, input_schema, output_schema)
+        source = _package_source(function)
+        digest = f"sha256:{hashlib.sha256(source).hexdigest()}"
+        runtime = PythonRuntimeSpec(
+            kind="python",
+            python_version=python_version
+            or f"{sys.version_info.major}.{sys.version_info.minor}",
+            environment=PythonEnvironmentSpec(kind="pip", packages=packages),
+            env=environment,
+        )
+        self._function = function
+        self._request = FunctionRegistrationRequest(
+            name=function_name,
+            artifact=FunctionArtifactRequest(
+                kind="python_callable",
+                digest=digest,
+                entrypoint=function.__name__,
+                content=FunctionArtifactContent(
+                    encoding="base64",
+                    data=base64.b64encode(source).decode("ascii"),
+                ),
+                adapter=PythonAdapterSpec(
+                    kind="scalar_to_arrow_batch",
+                    version=1,
+                ),
+            ),
+            signature=signature,
+            runtime=runtime,
+        )
+        functools.update_wrapper(self, function)
+
+    @property
+    def registration_request(self) -> FunctionRegistrationRequest:
+        """The immutable request sent by ``create_function_async``."""
+        return self._request
+
+    def __call__(self, *args, **kwargs):
+        return self._function(*args, **kwargs)
+
+
+@overload
+def udf(function: Callable[..., Any]) -> UdfDefinition: ...
+
+
+@overload
+def udf(
+    function: None = None,
+    *,
+    name: Optional[str] = None,
+    input_schema: Optional[pa.Schema] = None,
+    output_schema: Optional[pa.DataType | pa.Field | pa.Schema] = None,
+    pip: tuple[str, ...] | list[str] = (),
+    env: Optional[Mapping[str, str]] = None,
+    python_version: Optional[str] = None,
+) -> Callable[[Callable[..., Any]], UdfDefinition]: ...
+
+
+def udf(
+    function: Optional[Callable[..., Any]] = None,
+    *,
+    name: Optional[str] = None,
+    input_schema: Optional[pa.Schema] = None,
+    output_schema: Optional[pa.DataType | pa.Field | pa.Schema] = None,
+    pip: tuple[str, ...] | list[str] = (),
+    env: Optional[Mapping[str, str]] = None,
+    python_version: Optional[str] = None,
+):
+    """Prepare a scalar Python callable for remote Function registration.
+
+    Input and output signatures are inferred from supported annotations. For
+    Arrow types annotations cannot express precisely, pass ``input_schema``
+    and ``output_schema`` together. Nullable outputs are rejected because V1
+    uses physical NULL to represent unassigned computed-column rows.
+
+    Parameters
+    ----------
+    function : Callable, optional
+        The synchronous scalar callable to package.
+    name : str, optional
+        The remote Function name. Defaults to the callable name.
+    input_schema : pyarrow.Schema, optional
+        Explicit input fields in the exact order of the callable parameters.
+        Must be provided together with ``output_schema``.
+    output_schema : pyarrow.DataType, pyarrow.Field, or pyarrow.Schema, optional
+        Explicit scalar or named-struct output. Must be non-nullable and be
+        provided together with ``input_schema``.
+    pip : sequence of str, optional
+        Pip requirements for the remote environment.
+    env : mapping of str to str, optional
+        Environment variables included in the Function definition.
+    python_version : str, optional
+        Remote Python major/minor version. Defaults to the client version.
+
+    The packaged artifact is a snapshot: the function source plus exactly
+    the module-level names it references (modules as imports, importable
+    classes and functions as imports, literals inline). Code that reaches the
+    module namespace another way -- ``globals()``/``eval``, ``sys.modules``,
+    ``builtins`` -- is rejected where it can be seen and otherwise
+    unsupported; closures and a non-standard ``__builtins__`` are rejected.
+
+    Returns
+    -------
+    UdfDefinition
+        A callable definition accepted by
+        :meth:`lancedb.db.DBConnection.create_function`,
+        :meth:`lancedb.db.AsyncConnection.create_function_async` and
+        :meth:`lancedb.db.DBConnection.create_function_async`.
+
+    Examples
+    --------
+    >>> from lancedb import udf
+    >>> @udf(pip=["numpy==2.2.0"])
+    ... def score(value: float) -> float:
+    ...     return value * 2
+    >>> score(1.5)
+    3.0
+    """
+
+    def decorate(target: Callable[..., Any]) -> UdfDefinition:
+        return UdfDefinition(
+            target,
+            name=name,
+            input_schema=input_schema,
+            output_schema=output_schema,
+            pip=tuple(pip),
+            env={} if env is None else env,
+            python_version=python_version,
+        )
+
+    if function is None:
+        return decorate
+    return decorate(function)
+
+
+__all__ = [
+    "ApplicationInput",
+    "FunctionApplication",
+    "FunctionArtifact",
+    "FunctionArtifactContent",
+    "FunctionArtifactRequest",
+    "FunctionBinding",
+    "FunctionOutput",
+    "FunctionParameter",
+    "FunctionRegistrationRequest",
+    "FunctionResultField",
+    "FunctionSignature",
+    "FunctionVersion",
+    "FunctionVersionRef",
+    "InputBinding",
+    "OutputMapping",
+    "PythonEnvironmentSpec",
+    "PythonAdapterSpec",
+    "PythonRuntimeSpec",
+    "RefreshColumnResult",
+    "UdfDefinition",
+    "udf",
+]
diff --git a/python/python/lancedb/index.py b/python/python/lancedb/index.py
index aa7846892..d2b63baf6 100644
--- a/python/python/lancedb/index.py
+++ b/python/python/lancedb/index.py
@@ -163,6 +163,15 @@ class FTS:
         The number of documents per compressed posting block. Supported values
         are 128 and 256. A value of 256 uses the experimental FTS V3 format
         and may introduce breaking changes.
+    memory_limit : int, optional
+        The total memory limit in MiB for the local FTS build stage. The limit
+        is divided evenly among indexing workers. This build-only setting is
+        not persisted with the index and does not apply to remote tables.
+    num_workers : int, optional
+        The number of workers for a local FTS build. By default Lance uses
+        roughly half of the available CPU cores. The effective value is
+        limited by the available compute capacity. This build-only setting is
+        not persisted with the index and does not apply to remote tables.
 
     Notes
     -----
@@ -185,6 +194,8 @@ class FTS:
     prefix_only: bool = False
     block_size: int = 128
     custom_stop_words: Optional[List[str]] = None
+    memory_limit: Optional[int] = None
+    num_workers: Optional[int] = None
 
 
 @dataclass
diff --git a/python/python/lancedb/job.py b/python/python/lancedb/job.py
index d33b62cbf..f688768cb 100644
--- a/python/python/lancedb/job.py
+++ b/python/python/lancedb/job.py
@@ -5,21 +5,30 @@
 
 import asyncio
 from datetime import timedelta
-from typing import Optional
+from typing import Any, Callable, Generic, Optional, TypeVar, cast
 
 from lancedb.background_loop import LOOP
 
 from . import _lancedb
 
+T = TypeVar("T")
 
-class AsyncJob:
+
+class AsyncJob(Generic[T]):
     """A handle to an operation that may still be running.
 
-    The operation may already be complete when the handle is created.
+    The operation may already be complete when the handle is created. ``T``
+    is the endpoint's terminal result type; unit-result jobs resolve to
+    ``None``.
     """
 
-    def __init__(self, inner: Optional["_lancedb.Job"]):
+    def __init__(
+        self,
+        inner: Optional[Any],
+        result_decoder: Optional[Callable[[Any], T]] = None,
+    ):
         self._inner = inner
+        self._result_decoder = result_decoder
 
     @property
     def id(self) -> Optional[str]:
@@ -44,18 +53,24 @@ class AsyncJob:
             return "finished"
         return await self._inner.status()
 
-    async def wait(self, timeout: Optional[timedelta] = None):
+    async def wait(self, timeout: Optional[timedelta] = None) -> T:
         """Wait until the operation reaches a terminal state.
 
+        Returns the endpoint's typed result, or ``None`` for a unit-result
+        job.
+
         Raises `JobFailedError` if the operation failed, `JobCancelledError`
         if it was cancelled, and `TimeoutError` if `timeout` elapses first.
         """
         if self._inner is None:
-            return
+            return cast(T, None)
         if timeout is None:
-            await self._inner.wait()
+            result = await self._inner.wait()
         else:
-            await asyncio.wait_for(self._inner.wait(), timeout.total_seconds())
+            result = await asyncio.wait_for(self._inner.wait(), timeout.total_seconds())
+        if self._result_decoder is not None:
+            return self._result_decoder(result)
+        return cast(T, result)
 
     async def cancel(self):
         """Request cancellation. Cancelling a finished operation is a no-op."""
@@ -64,10 +79,10 @@ class AsyncJob:
         await self._inner.cancel()
 
 
-class Job:
-    """Synchronous counterpart of `AsyncJob`."""
+class Job(Generic[T]):
+    """Synchronous counterpart of `AsyncJob` with the same result type."""
 
-    def __init__(self, inner: Optional[AsyncJob]):
+    def __init__(self, inner: Optional[AsyncJob[T]]):
         self._inner = inner
 
     @property
@@ -88,18 +103,28 @@ class Job:
             return "finished"
         return LOOP.run(self._inner.status())
 
-    def wait(self, timeout: Optional[timedelta] = None):
+    def wait(self, timeout: Optional[timedelta] = None) -> T:
         """Block until the operation reaches a terminal state.
 
+        Returns the endpoint's typed result, or ``None`` for a unit-result
+        job.
+
         Raises `JobFailedError` if the operation failed, `JobCancelledError`
         if it was cancelled, and `TimeoutError` if `timeout` elapses first.
         """
         if self._inner is None:
-            return
-        LOOP.run(self._inner.wait(timeout))
+            return cast(T, None)
+        return LOOP.run(self._inner.wait(timeout))
 
     def cancel(self):
         """Request cancellation. Cancelling a finished operation is a no-op."""
         if self._inner is None:
             return
         LOOP.run(self._inner.cancel())
+
+
+def _typed_job(
+    inner: "_lancedb.Job", result_decoder: Callable[[str], T]
+) -> AsyncJob[T]:
+    """Bind an internal JSON-producing job to its public result model."""
+    return AsyncJob(inner, result_decoder)
diff --git a/python/python/lancedb/materialized_view.py b/python/python/lancedb/materialized_view.py
new file mode 100644
index 000000000..5abb44dc0
--- /dev/null
+++ b/python/python/lancedb/materialized_view.py
@@ -0,0 +1,178 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+"""Materialized views: tables defined by a query over a source table and
+maintained by refresh. See ``DBConnection.create_materialized_view``."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Tuple, Union
+
+from .background_loop import LOOP
+
+if TYPE_CHECKING:
+    import pyarrow as pa
+
+    from ._lancedb import RefreshMaterializedViewResult
+    from .table import AsyncTable, LanceTable
+
+DEFINITION_META_KEY = b"mv.definition"
+
+SelectArg = Union[
+    str,
+    Sequence[Union[str, Tuple[str, str]]],
+    Dict[str, str],
+    None,
+]
+
+
+@dataclass
+class MaterializedViewDefinition:
+    """The query that defines a materialized view."""
+
+    source_table: str
+    """Name of the source table, in the same database as the view."""
+    projections: List[Tuple[str, str]]
+    """``(output column, SQL expression)`` pairs, in view schema order."""
+    filter: Optional[str] = None
+    """SQL predicate selecting the source rows the view holds."""
+    limit: Optional[int] = None
+    """Cap on the number of rows the view holds."""
+    inputs: List[str] = field(default_factory=list)
+    """Source columns the projections and filter read."""
+
+
+def _definition_from_schema(
+    schema: "pa.Schema", name: str
+) -> MaterializedViewDefinition:
+    metadata = schema.metadata or {}
+    raw = metadata.get(DEFINITION_META_KEY)
+    if raw is None:
+        raise ValueError(f"Table '{name}' is not a materialized view")
+    value = json.loads(raw)
+    kind = value.get("kind")
+    if kind != "select":
+        raise NotImplementedError(
+            f"materialized view '{name}' is defined by '{kind}', which this "
+            "version of lancedb cannot refresh"
+        )
+    return MaterializedViewDefinition(
+        source_table=value["source_table"],
+        projections=[
+            (p["output"], p["expression"]) for p in value.get("projections", [])
+        ],
+        filter=value.get("filter"),
+        limit=value.get("limit"),
+        inputs=value.get("inputs", []),
+    )
+
+
+def _quote_identifier(name: str) -> str:
+    """Quote a column name as a Lance SQL identifier (backticks)."""
+    escaped = name.replace("`", "``")
+    return f"`{escaped}`"
+
+
+def normalize_select(select: SelectArg) -> Optional[List[Tuple[str, str]]]:
+    """``select`` items may be a column name, an ``(alias, expression)`` pair,
+    or a dict of the same. A bare name projects itself and is quoted, so any
+    valid column name works; dict and pair entries are kept verbatim because
+    their right side is an expression.
+
+    A lone string is one column, not a sequence of its characters."""
+    if select is None:
+        return None
+    if isinstance(select, str):
+        select = [select]
+    if isinstance(select, dict):
+        return list(select.items())
+    normalized = []
+    for item in select:
+        if isinstance(item, str):
+            normalized.append((item, _quote_identifier(item)))
+        else:
+            alias, expression = item
+            normalized.append((alias, expression))
+    return normalized
+
+
+class AsyncMaterializedView:
+    """A handle on a materialized view: its table plus its definition.
+
+    Obtained from ``AsyncConnection.create_materialized_view`` or
+    ``AsyncConnection.open_materialized_view``.
+    """
+
+    def __init__(self, table: "AsyncTable"):
+        self._table = table
+
+    def __repr__(self) -> str:
+        return f"AsyncMaterializedView(name={self.name!r})"
+
+    @property
+    def name(self) -> str:
+        return self._table.name
+
+    @property
+    def table(self) -> "AsyncTable":
+        """The view, as the table it is. Queries, indexes and search all
+        apply; writes are not blocked, but a rebuild replaces them."""
+        return self._table
+
+    async def definition(self) -> MaterializedViewDefinition:
+        """The query that defines the view, read from its stored schema."""
+        return _definition_from_schema(await self._table.schema(), self.name)
+
+    async def refresh(
+        self, *, full: bool = False, source_version: Optional[int] = None
+    ) -> "RefreshMaterializedViewResult":
+        """Recompute the view from its source.
+
+        The refresh is incremental when the source's changes can be
+        reconciled into the view -- rows added, changed or removed since the
+        last one -- and otherwise rebuilds. ``full=True`` forces a rebuild;
+        ``source_version`` refreshes to that source version instead of the
+        latest.
+
+        Concurrent refreshes of one view do not duplicate its rows. Two that
+        plan the same source rows conflict on commit, and the loser raises
+        rather than writing them a second time.
+        """
+        return await self._table._inner.refresh_materialized_view(
+            full=full, source_version=source_version
+        )
+
+
+class MaterializedView:
+    """Synchronous variant of
+    [AsyncMaterializedView][lancedb.materialized_view.AsyncMaterializedView]."""
+
+    def __init__(self, table: "LanceTable"):
+        self._table = table
+        self._async = AsyncMaterializedView(table._table)
+
+    def __repr__(self) -> str:
+        return f"MaterializedView(name={self.name!r})"
+
+    @property
+    def name(self) -> str:
+        return self._table.name
+
+    @property
+    def table(self) -> "LanceTable":
+        """The view, as the table it is."""
+        return self._table
+
+    @property
+    def definition(self) -> MaterializedViewDefinition:
+        """The query that defines the view, read from its stored schema."""
+        return _definition_from_schema(self._table.schema, self.name)
+
+    def refresh(
+        self, *, full: bool = False, source_version: Optional[int] = None
+    ) -> "RefreshMaterializedViewResult":
+        """Recompute the view from its source. See
+        [AsyncMaterializedView.refresh][lancedb.materialized_view.AsyncMaterializedView.refresh]."""
+        return LOOP.run(self._async.refresh(full=full, source_version=source_version))
diff --git a/python/python/lancedb/namespace.py b/python/python/lancedb/namespace.py
index b151395cc..f2e553321 100644
--- a/python/python/lancedb/namespace.py
+++ b/python/python/lancedb/namespace.py
@@ -49,6 +49,7 @@ from lancedb._lancedb import (
 )
 from lancedb.background_loop import LOOP
 from lancedb.db import AsyncConnection, DBConnection
+from lancedb.job import AsyncJob, Job
 from lance_namespace import (
     LanceNamespace,
     connect as namespace_connect,
@@ -60,6 +61,11 @@ from lance_namespace import (
     NamespaceExistsRequest,
     TableExistsRequest,
 )
+from lancedb.materialized_view import (
+    AsyncMaterializedView,
+    MaterializedView,
+    SelectArg,
+)
 from lancedb.table import AsyncTable, LanceTable, Table
 from lancedb.util import validate_table_name
 from lancedb.common import DATA
@@ -618,12 +624,60 @@ class LanceNamespaceDBConnection(DBConnection):
             tbl.checkout(version)
         return tbl
 
+    @override
+    def create_materialized_view(
+        self,
+        name: str,
+        source: str,
+        *,
+        select: "SelectArg" = None,
+        where: Optional[str] = None,
+        limit: Optional[int] = None,
+    ) -> "MaterializedView":
+        """Define a materialized view over a table in the root namespace.
+        See
+        [DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view].
+        """
+        return MaterializedView(
+            self.open_table(
+                LOOP.run(
+                    self._inner.create_materialized_view(
+                        name, source, select=select, where=where, limit=limit
+                    )
+                ).name
+            )
+        )
+
+    @override
+    def open_materialized_view(self, name: str) -> "MaterializedView":
+        """Open the materialized view named ``name``."""
+        view = MaterializedView(self.open_table(name))
+        view.definition
+        return view
+
+    @override
+    def list_materialized_views(self) -> List[str]:
+        """The names of the materialized views in the root namespace."""
+        return LOOP.run(self._inner.list_materialized_views())
+
     @override
     def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
         if namespace_path is None:
             namespace_path = []
         LOOP.run(self._inner.drop_table(name, namespace_path=namespace_path))
 
+    @override
+    def drop_table_async(
+        self, name: str, namespace_path: Optional[List[str]] = None
+    ) -> Job:
+        """Start dropping a table and return its cleanup job."""
+        if namespace_path is None:
+            namespace_path = []
+        job = LOOP.run(
+            self._inner.drop_table_async(name, namespace_path=namespace_path)
+        )
+        return Job(job if isinstance(job, AsyncJob) else AsyncJob(job))
+
     @override
     def rename_table(
         self,
@@ -1128,12 +1182,47 @@ class AsyncLanceNamespaceDBConnection:
             route_pushdown_to_rust=self._route_pushdown_to_rust,
         )
 
+    async def create_materialized_view(
+        self,
+        name: str,
+        source: str,
+        *,
+        select: "SelectArg" = None,
+        where: Optional[str] = None,
+        limit: Optional[int] = None,
+    ) -> "AsyncMaterializedView":
+        """Define a materialized view over a table in the root namespace."""
+        view = await self._inner.create_materialized_view(
+            name, source, select=select, where=where, limit=limit
+        )
+        # Reopen through the namespace so the view's table carries the
+        # namespace client and pushdown configuration a bare inner table lacks.
+        return AsyncMaterializedView(await self.open_table(view.name))
+
+    async def open_materialized_view(self, name: str) -> "AsyncMaterializedView":
+        """Open the materialized view named ``name``."""
+        view = AsyncMaterializedView(await self.open_table(name))
+        await view.definition()
+        return view
+
+    async def list_materialized_views(self) -> List[str]:
+        """The names of the materialized views in the root namespace."""
+        return await self._inner.list_materialized_views()
+
     async def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
         """Drop a table from the namespace."""
         if namespace_path is None:
             namespace_path = []
         await self._inner.drop_table(name, namespace_path=namespace_path)
 
+    async def drop_table_async(
+        self, name: str, namespace_path: Optional[List[str]] = None
+    ) -> AsyncJob:
+        """Start dropping a table and return its cleanup job."""
+        if namespace_path is None:
+            namespace_path = []
+        return await self._inner.drop_table_async(name, namespace_path=namespace_path)
+
     async def rename_table(
         self,
         cur_name: str,
diff --git a/python/python/lancedb/permutation.py b/python/python/lancedb/permutation.py
index bcf84bf3a..8287ef2fe 100644
--- a/python/python/lancedb/permutation.py
+++ b/python/python/lancedb/permutation.py
@@ -41,21 +41,15 @@ class PermutationBuilder:
     The permutation is stored in memory and will be lost when the program exits.
     """
 
-    def __init__(self, table: LanceTable):
+    def __init__(self, table: Table):
         """
         Creates a new permutation builder for the given table.
 
         By default, the permutation builder will create a single split that contains all
         rows in the same order as the base table.
+
+        Tables with an LSM write spec are rejected: unflushed rows have no row id.
         """
-        if not hasattr(table, "_inner"):
-            raise TypeError(
-                f"PermutationBuilder requires a local LanceTable, "
-                f"got {type(table).__name__}. "
-                "The permutation API is not supported on remote tables. "
-                "Remote tables connect to LanceDB Cloud or Enterprise and do not have "
-                "direct access to the underlying Lance dataset needed for permutations."
-            )
         self._async = async_permutation_builder(table)
 
     def split_random(
@@ -231,7 +225,7 @@ class PermutationBuilder:
         return LOOP.run(do_execute())
 
 
-def permutation_builder(table: LanceTable) -> PermutationBuilder:
+def permutation_builder(table: Table) -> PermutationBuilder:
     return PermutationBuilder(table)
 
 
@@ -248,7 +242,7 @@ class Permutations:
 
     Attributes
     ----------
-    base_table: LanceTable
+    base_table: Table
         The base table that the permutations are based on.
     permutation_table: LanceTable
         The permutation table that defines the splits.
@@ -282,7 +276,7 @@ class Permutations:
     {'train': 0, 'test': 1}
     """
 
-    def __init__(self, base_table: LanceTable, permutation_table: LanceTable):
+    def __init__(self, base_table: Table, permutation_table: LanceTable):
         self.base_table = base_table
         self.permutation_table = permutation_table
 
@@ -397,6 +391,15 @@ def _table_to_pickle_state(table: Table) -> dict[str, Any]:
     }
 
 
+def _drop_base_version(permutation_data: pa.Table) -> pa.Table:
+    """Strip the recorded base version so the reader leaves the base table unpinned."""
+    metadata = dict(permutation_data.schema.metadata or {})
+    if metadata.pop(b"base_version", None) is None:
+        return permutation_data
+    metadata.pop(b"base_branch", None)
+    return permutation_data.replace_schema_metadata(metadata)
+
+
 def _table_from_pickle_state(state: dict[str, Any]) -> Table:
     from . import connect
 
@@ -685,11 +688,15 @@ class Permutation:
         from . import connect
 
         connection_factory = state["connection_factory"]
+        rebuilt_base = False
         if connection_factory is not None:
             base_table = connection_factory(state["base_table_name"])
         elif "base_table_state" in state:
-            base_table = _table_from_pickle_state(state["base_table_state"])
+            base_state = state["base_table_state"]
+            rebuilt_base = base_state["kind"] == "memory"
+            base_table = _table_from_pickle_state(base_state)
         elif "base_table_data" in state:
+            rebuilt_base = True
             # In-memory base table inlined into the pickle; rebuild the same
             # way we rebuild the in-memory permutation table.
             mem_db = connect("memory://")
@@ -707,11 +714,14 @@ class Permutation:
             )
 
         permutation_table: Optional[Table] = None
-        if state["permutation_data"] is not None:
+        permutation_data = state["permutation_data"]
+        if permutation_data is not None:
+            if rebuilt_base:
+                # The base table was materialized from Arrow, so it is a fresh
+                # single-version dataset and the recorded pin cannot resolve on it.
+                permutation_data = _drop_base_version(permutation_data)
             mem_db = connect("memory://")
-            permutation_table = mem_db.create_table(
-                "permutation", state["permutation_data"]
-            )
+            permutation_table = mem_db.create_table("permutation", permutation_data)
 
         self.base_table = base_table
         self.permutation_table = permutation_table
diff --git a/python/python/lancedb/py.typed b/python/python/lancedb/py.typed
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/python/python/lancedb/py.typed
@@ -0,0 +1 @@
+
diff --git a/python/python/lancedb/pydantic.py b/python/python/lancedb/pydantic.py
index c4dedc0e6..528d865d7 100644
--- a/python/python/lancedb/pydantic.py
+++ b/python/python/lancedb/pydantic.py
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-"""Pydantic (v1 / v2) adapter for LanceDB"""
+"""Pydantic adapter for LanceDB."""
 
 from __future__ import annotations
 
@@ -14,9 +14,6 @@ from enum import Enum
 from typing import (
     TYPE_CHECKING,
     Any,
-    Callable,
-    Dict,
-    Generator,
     List,
     Type,
     Union,
@@ -24,17 +21,9 @@ from typing import (
     GenericAlias,
 )
 
-import numpy as np
 import pyarrow as pa
 import pydantic
-from packaging.version import Version
-
-PYDANTIC_VERSION = Version(pydantic.__version__)
-try:
-    from pydantic_core import CoreSchema, core_schema
-except ImportError:
-    if PYDANTIC_VERSION.major >= 2:
-        raise
+from pydantic_core import CoreSchema, core_schema
 
 if TYPE_CHECKING:
     from pydantic.fields import FieldInfo
@@ -131,28 +120,18 @@ def Vector(
                 ),
             )
 
-        @classmethod
-        def __get_validators__(cls) -> Generator[Callable, None, None]:
-            yield cls.validate
-
-        # For pydantic v1
-        @classmethod
-        def validate(cls, v):
-            if not isinstance(v, (list, range, np.ndarray)) or len(v) != dim:
-                raise TypeError("A list of numbers or numpy.ndarray is needed")
-            return cls(v)
-
-        if PYDANTIC_VERSION.major < 2:
-
-            @classmethod
-            def __modify_schema__(cls, field_schema: Dict[str, Any]):
-                field_schema["items"] = {"type": "number"}
-                field_schema["maxItems"] = dim
-                field_schema["minItems"] = dim
-
     return FixedSizeList
 
 
+def _raise_bare_vector_error(*_args):
+    raise TypeError("Vector must be parameterized with a dimension, e.g. Vector(128).")
+
+
+# Pydantic otherwise inspects the bare factory as a field type and produces
+# misleading errors about its internal annotations.
+setattr(Vector, "__get_pydantic_core_schema__", _raise_bare_vector_error)
+
+
 def MultiVector(
     dim: int, value_type: pa.DataType = pa.float32(), nullable: bool = True
 ) -> Type:
@@ -223,31 +202,6 @@ def MultiVector(
                 ),
             )
 
-        @classmethod
-        def __get_validators__(cls) -> Generator[Callable, None, None]:
-            yield cls.validate
-
-        # For pydantic v1
-        @classmethod
-        def validate(cls, v):
-            if not isinstance(v, (list, range)):
-                raise TypeError("A list of vectors is needed")
-            for vec in v:
-                if not isinstance(vec, (list, range, np.ndarray)) or len(vec) != dim:
-                    raise TypeError(f"Each vector must be a list of {dim} numbers")
-            return cls(v)
-
-        if PYDANTIC_VERSION.major < 2:
-
-            @classmethod
-            def __modify_schema__(cls, field_schema: Dict[str, Any]):
-                field_schema["items"] = {
-                    "type": "array",
-                    "items": {"type": "number"},
-                    "minItems": dim,
-                    "maxItems": dim,
-                }
-
     return MultiVectorList
 
 
@@ -293,20 +247,10 @@ def _py_type_to_arrow_type(py_type: Type[Any], field: FieldInfo) -> pa.DataType:
     )
 
 
-if PYDANTIC_VERSION.major < 2:
-
-    def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
-        return [
-            _pydantic_to_field(name, field) for name, field in model.__fields__.items()
-        ]
-
-else:
-
-    def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
-        return [
-            _pydantic_to_field(name, field)
-            for name, field in model.model_fields.items()
-        ]
+def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
+    return [
+        _pydantic_to_field(name, field) for name, field in model.model_fields.items()
+    ]
 
 
 def _pydantic_type_to_arrow_type(tp: Any, field: FieldInfo) -> pa.DataType:
@@ -499,8 +443,6 @@ class LanceModel(pydantic.BaseModel):
 
     @classmethod
     def safe_get_fields(cls):
-        if PYDANTIC_VERSION.major < 2:
-            return cls.__fields__
         return cls.model_fields
 
     @classmethod
@@ -538,23 +480,9 @@ def get_extras(field_info: FieldInfo, key: str) -> Any:
     """
     Get the extra metadata from a Pydantic FieldInfo.
     """
-    if PYDANTIC_VERSION.major >= 2:
-        return (field_info.json_schema_extra or {}).get(key)
-    return (field_info.field_info.extra or {}).get("json_schema_extra", {}).get(key)
+    return (field_info.json_schema_extra or {}).get(key)
 
 
-if PYDANTIC_VERSION.major < 2:
-
-    def model_to_dict(model: pydantic.BaseModel) -> Dict[str, Any]:
-        """
-        Convert a Pydantic model to a dictionary.
-        """
-        return model.dict()
-
-else:
-
-    def model_to_dict(model: pydantic.BaseModel) -> Dict[str, Any]:
-        """
-        Convert a Pydantic model to a dictionary.
-        """
-        return model.model_dump()
+def model_to_dict(model: pydantic.BaseModel) -> dict[str, Any]:
+    """Convert a Pydantic model to a dictionary."""
+    return model.model_dump()
diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py
index 095a7b5ff..bd5e2cea2 100644
--- a/python/python/lancedb/query.py
+++ b/python/python/lancedb/query.py
@@ -32,8 +32,6 @@ from typing_extensions import Annotated
 
 from lancedb._lancedb import fts_query_to_json
 from lancedb.background_loop import LOOP
-from lancedb.pydantic import PYDANTIC_VERSION
-
 from . import __version__
 from .arrow import AsyncRecordBatchReader
 from .dependencies import pandas as pd
@@ -827,12 +825,7 @@ class Query(pydantic.BaseModel):
 
     # This tells pydantic to allow custom types (needed for the `vector` query since
     # pa.Array wouln't be allowed otherwise)
-    if PYDANTIC_VERSION.major < 2:  # Pydantic 1.x compat
-
-        class Config:
-            arbitrary_types_allowed = True
-    else:
-        model_config = {"arbitrary_types_allowed": True}
+    model_config = pydantic.ConfigDict(arbitrary_types_allowed=True)
 
 
 class LanceQueryBuilder(ABC):
@@ -2235,6 +2228,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
             reranker=self._reranker,
             limit=self._limit,
             with_row_ids=True,
+            offset=self._offset,
         )
         return self._finish_hybrid_results(results)
 
@@ -2256,6 +2250,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         reranker,
         limit: int,
         with_row_ids: bool,
+        offset: Optional[int] = None,
     ) -> pa.Table:
         if norm == "rank":
             vector_results = LanceHybridQueryBuilder._rank(vector_results, "_distance")
@@ -2332,7 +2327,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
             score_i = results.column_names.index("_score")
             results = results.set_column(score_i, "_score", original_scores)
 
-        results = results.slice(length=limit)
+        results = results.slice(offset=offset or 0, length=limit)
 
         if not with_row_ids:
             results = results.drop(["_rowid"])
@@ -2679,8 +2674,12 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
 
         # Apply common configurations
         if self._limit:
-            self._vector_query.limit(self._limit)
-            self._fts_query.limit(self._limit)
+            # The final offset/limit window is sliced out of the combined,
+            # reranked results, so each sub-query must fetch enough rows to
+            # cover the skipped prefix as well as the window itself.
+            sub_query_limit = self._limit + (self._offset or 0)
+            self._vector_query.limit(sub_query_limit)
+            self._fts_query.limit(sub_query_limit)
         if self._columns:
             self._vector_query.select(self._columns)
             self._fts_query.select(self._columns)
@@ -3245,12 +3244,7 @@ class AsyncStandardQuery(AsyncQueryBase):
         if ordering is None:
             self._inner.order_by(None)
         else:
-            self._inner.order_by(
-                [
-                    o.model_dump() if hasattr(o, "model_dump") else o.dict()
-                    for o in ordering
-                ]
-            )
+            self._inner.order_by([o.model_dump() for o in ordering])
         return self
 
     def fast_search(self) -> Self:
diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py
index 332886590..b228cfb5b 100644
--- a/python/python/lancedb/remote/db.py
+++ b/python/python/lancedb/remote/db.py
@@ -23,7 +23,9 @@ import pyarrow as pa
 
 from ..common import DATA
 from ..db import DBConnection, LOOP
-from ..job import Job
+from ..functions import FunctionVersion, UdfDefinition
+from ..job import AsyncJob, Job
+from ..materialized_view import MaterializedView, SelectArg
 
 if TYPE_CHECKING:
     from .._lancedb import JobDescription, JobInfo
@@ -647,6 +649,32 @@ class RemoteDBConnection(DBConnection):
             namespace_path=namespace_path,
         )
 
+    @override
+    def create_materialized_view(
+        self,
+        name: str,
+        source: str,
+        *,
+        select: SelectArg = None,
+        where: Optional[str] = None,
+        limit: Optional[int] = None,
+    ) -> MaterializedView:
+        raise NotImplementedError(
+            "materialized views are supported only on local databases"
+        )
+
+    @override
+    def open_materialized_view(self, name: str) -> MaterializedView:
+        raise NotImplementedError(
+            "materialized views are supported only on local databases"
+        )
+
+    @override
+    def list_materialized_views(self) -> List[str]:
+        raise NotImplementedError(
+            "materialized views are supported only on local databases"
+        )
+
     @override
     def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
         """Drop a table from the database.
@@ -663,6 +691,16 @@ class RemoteDBConnection(DBConnection):
             namespace_path = []
         LOOP.run(self._conn.drop_table(name, namespace_path=namespace_path))
 
+    @override
+    def drop_table_async(
+        self, name: str, namespace_path: Optional[List[str]] = None
+    ) -> Job:
+        """Start dropping a table and return its cleanup job."""
+        if namespace_path is None:
+            namespace_path = []
+        job = LOOP.run(self._conn.drop_table_async(name, namespace_path=namespace_path))
+        return Job(job if isinstance(job, AsyncJob) else AsyncJob(job))
+
     @override
     def rename_table(
         self,
@@ -703,6 +741,14 @@ class RemoteDBConnection(DBConnection):
         """
         return Job(self._conn.job(job_id))
 
+    @override
+    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
+        return Job(LOOP.run(self._conn.create_function_async(definition)))
+
+    @override
+    def get_function(self, name: str, *, version: str) -> FunctionVersion:
+        return LOOP.run(self._conn.get_function(name, version=version))
+
     @override
     def list_jobs(self) -> List["JobInfo"]:
         """List server-side jobs across the database's tables."""
diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py
index acc2f4c9d..d0bf9f67a 100644
--- a/python/python/lancedb/remote/table.py
+++ b/python/python/lancedb/remote/table.py
@@ -49,6 +49,7 @@ from lancedb.index import (
     LabelList,
 )
 from lancedb.job import Job
+from lancedb.functions import FunctionApplication, RefreshColumnResult
 from lancedb.remote.db import LOOP
 from lancedb.table import IndexConfigType, KNOWN_METRICS
 import pyarrow as pa
@@ -609,6 +610,7 @@ class RemoteTable(Table):
         fill_value: float = 0.0,
         progress: Optional[Union[bool, Callable, Any]] = None,
         write_parallelism: Optional[int] = None,
+        allow_external_blob_outside_bases: bool = False,
     ) -> AddResult:
         """Add more data to the [Table][lancedb.table.Table].
 
@@ -641,6 +643,8 @@ class RemoteTable(Table):
             data in flight. Defaults to an estimate based on the data size,
             capped at the number of CPU cores. Lower this if bulk ingestion is
             using too much memory.
+        allow_external_blob_outside_bases: bool, default False
+            Not supported on LanceDB Cloud. Setting this raises.
 
         Returns
         -------
@@ -657,6 +661,7 @@ class RemoteTable(Table):
                     fill_value=fill_value,
                     progress=progress,
                     write_parallelism=write_parallelism,
+                    allow_external_blob_outside_bases=allow_external_blob_outside_bases,
                 )
             )
         finally:
@@ -958,8 +963,21 @@ class RemoteTable(Table):
     def count_rows(self, filter: Optional[str] = None) -> int:
         return LOOP.run(self._table.count_rows(filter))
 
-    def add_columns(self, transforms: Dict[str, str]) -> AddColumnsResult:
-        return LOOP.run(self._table.add_columns(transforms))
+    def add_columns(
+        self,
+        transforms: Dict[str, str | FunctionApplication]
+        | FunctionApplication
+        | None = None,
+        *,
+        computed: Dict[str, str] | None = None,
+    ) -> AddColumnsResult:
+        return LOOP.run(self._table.add_columns(transforms, computed=computed))
+
+    def refresh_column(self, column: str):
+        return LOOP.run(self._table.refresh_column(column))
+
+    def refresh_column_async(self, column: str) -> Job[RefreshColumnResult]:
+        return Job(LOOP.run(self._table.refresh_column_async(column)))
 
     def alter_columns(
         self, *alterations: Iterable[Dict[str, str]]
@@ -979,17 +997,39 @@ class RemoteTable(Table):
         return LOOP.run(self._table.set_unenforced_primary_key(columns))
 
     def set_lsm_write_spec(self, spec: "LsmWriteSpec") -> None:
-        """Not supported on LanceDB Cloud."""
+        """Install an LsmWriteSpec."""
         return LOOP.run(self._table.set_lsm_write_spec(spec))
 
     def unset_lsm_write_spec(self) -> None:
-        """Not supported on LanceDB Cloud."""
+        """Remove the LsmWriteSpec."""
         return LOOP.run(self._table.unset_lsm_write_spec())
 
     def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]:
         """Read the installed LsmWriteSpec, or ``None``."""
         return LOOP.run(self._table.get_lsm_write_spec())
 
+    def checkpoint_lsm(self) -> None:
+        """Synchronous version of
+        [`AsyncTable.checkpoint_lsm`][lancedb.AsyncTable.checkpoint_lsm]."""
+        return LOOP.run(self._table.checkpoint_lsm())
+
+    def flush_lsm(self) -> None:
+        """Synchronous version of
+        [`AsyncTable.flush_lsm`][lancedb.AsyncTable.flush_lsm]."""
+        return LOOP.run(self._table.flush_lsm())
+
+    def compact_lsm(self) -> None:
+        """Synchronous version of
+        [`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm]."""
+        return LOOP.run(self._table.compact_lsm())
+
+    def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]:
+        """Synchronous version of
+        [`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats]."""
+        return LOOP.run(
+            self._table.get_lsm_stats(include_generation_rows=include_generation_rows)
+        )
+
     def close_lsm_writers(self) -> None:
         """No-op on LanceDB Cloud (no local shard writers)."""
         return LOOP.run(self._table.close_lsm_writers())
diff --git a/python/python/lancedb/streaming.py b/python/python/lancedb/streaming.py
index 525ed3d63..76b3702a1 100644
--- a/python/python/lancedb/streaming.py
+++ b/python/python/lancedb/streaming.py
@@ -11,25 +11,37 @@ Provides StreamingDataset, a PyTorch IterableDataset that guarantees:
 - **Resumability**: state_dict / load_state_dict capture per-split consumption
   counts so training can resume from an exact mid-epoch position even when the
   distributed topology changes between runs.
+
+Transform failures on bad rows (e.g. nulls or NaNs from incomplete data) can
+be tolerated with ``on_transform_error="skip"``; see the parameter
+documentation on StreamingDataset for how this interacts with the guarantees
+above.
 """
 
 import ctypes
+import heapq
 import logging
 import os
 import random
 import threading
 import time
+import warnings
 from collections import deque
 from concurrent.futures import ThreadPoolExecutor
+from copy import deepcopy
 from multiprocessing import RawArray
-from typing import Any, Callable, Iterator, Optional
+from typing import Any, Callable, cast, Iterator, Literal, NamedTuple, Optional, Union
 
-from torch.utils.data import IterableDataset, get_worker_info
+import pyarrow as pa
+import pyarrow.compute as pc
+import torch
+from torch.utils.data import DataLoader, IterableDataset, get_worker_info
 
 from .permutation import (
     Permutation,
     Transforms,
     permutation_builder,
+    _drop_base_version,
     _table_from_pickle_state,
     _table_to_pickle_state,
 )
@@ -45,6 +57,155 @@ DEFAULT_READ_BATCH_SIZE = 64
 DEFAULT_PREFETCH_BATCHES = 4
 
 
+class _WorkerSample(NamedTuple):
+    data: Any
+    dataset: "StreamingDataset"
+
+
+class _WorkerBatch(NamedTuple):
+    data: Any
+    state: dict
+
+
+class _ConsumerIteratorLease(NamedTuple):
+    owner_token: int
+    owner_thread: int
+
+
+class _CheckpointCollate:
+    """Attach the worker's post-fetch state to a collated batch."""
+
+    def __init__(self, collate_fn: Callable):
+        self._collate_fn = collate_fn
+
+    def __call__(self, samples):
+        try:
+            if isinstance(samples, list):
+                if not samples:
+                    return _WorkerBatch(self._collate_fn(samples), {})
+                worker_samples = samples
+                data = self._collate_fn([sample.data for sample in worker_samples])
+                dataset = worker_samples[-1].dataset
+            else:
+                data = self._collate_fn(samples.data)
+                dataset = samples.dataset
+        except StopIteration as exc:
+            raise RuntimeError(
+                "collate_fn raised StopIteration before returning a batch"
+            ) from exc
+        return _WorkerBatch(data, dataset._checkpoint_snapshot())
+
+
+class _StreamingDatasetAdapter(IterableDataset):
+    """Yield private sample wrappers for :class:`StreamingDataLoader`."""
+
+    def __init__(self, dataset: "StreamingDataset"):
+        super().__init__()
+        self.dataset = dataset
+
+    def __iter__(self):
+        for sample in self.dataset._iter(consumer_checkpoint_transport=True):
+            yield _WorkerSample(sample, self.dataset)
+
+    def __getattr__(self, name):
+        dataset = self.__dict__.get("dataset")
+        if dataset is None:
+            raise AttributeError(name)
+        return getattr(dataset, name)
+
+
+class _ConsumerCommitIterator:
+    def __init__(
+        self,
+        iterator,
+        dataset: "StreamingDataset",
+        *,
+        owner_token: int,
+        require_uniform: bool,
+    ):
+        self._iterator = iterator
+        self._dataset = dataset
+        self._owner_token = owner_token
+        self._require_uniform = require_uniform
+        self._released = False
+        self._terminal = False
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        if self._terminal:
+            raise StopIteration
+        try:
+            batch = next(self._iterator)
+        except StopIteration:
+            self._terminal = True
+            self._release()
+            raise
+        except BaseException as exc:
+            self._dataset._invalidate_checkpoint(
+                f"a DataLoader batch failed before it was returned: {exc}"
+            )
+            raise
+        try:
+            if not isinstance(batch, _WorkerBatch):
+                raise RuntimeError(
+                    "StreamingDataLoader did not receive worker checkpoint metadata"
+                )
+            self._dataset._commit_worker_state(
+                batch.state, require_uniform=self._require_uniform
+            )
+            return batch.data
+        except BaseException as exc:
+            self._dataset._invalidate_checkpoint(
+                f"a DataLoader batch failed before it was returned: {exc}"
+            )
+            raise
+
+    def _release(self) -> None:
+        if self.__dict__.get("_released", True):
+            return
+        self._released = True
+        dataset = self.__dict__.get("_dataset")
+        if dataset is not None:
+            dataset._release_consumer_iterator(self._owner_token)
+
+    def _shutdown_workers(self):
+        if self.__dict__.get("_released", True):
+            return None
+        self._terminal = True
+        iterator = self.__dict__.get("_iterator")
+        shutdown = getattr(iterator, "_shutdown_workers", None)
+        try:
+            if shutdown is not None:
+                shutdown()
+            else:
+                fetcher = getattr(iterator, "_dataset_fetcher", None)
+                dataset_iterator = getattr(fetcher, "dataset_iter", None)
+                close = getattr(dataset_iterator, "close", None)
+                if close is None:
+                    raise RuntimeError(
+                        "StreamingDataLoader could not close its inner iterator"
+                    )
+                close()
+        except BaseException as exc:
+            self._dataset._invalidate_checkpoint(
+                f"a DataLoader iterator could not be shut down safely: {exc}"
+            )
+            raise
+        else:
+            self._release()
+
+    def __del__(self):
+        try:
+            self._shutdown_workers()
+        except BaseException:
+            pass
+
+    def __getattr__(self, name):
+        return getattr(self._iterator, name)
+
+
 class StreamingDataset(IterableDataset):
     """An elastic, resumable PyTorch IterableDataset backed by a LanceDB table.
 
@@ -56,7 +217,7 @@ class StreamingDataset(IterableDataset):
 
     Internally ``__iter__`` runs a two-stage pipeline:
 
-    - **Stage 1 (I/O)**: one thread pool with ``num_splits * prefetch_batches``
+    - **Stage 1 (I/O)**: one thread pool with ``num_splits * io_queue_depth``
       workers fetches raw ``RecordBatch`` objects from LanceDB in parallel
       across all splits and places them in a per-split raw-batch queue.
     - **Stage 2 (transform)**: a second thread pool with
@@ -99,11 +260,11 @@ class StreamingDataset(IterableDataset):
         call.  Larger values amortise per-request overhead (critical on object
         storage) at the cost of higher memory usage per split buffer.  Defaults
         to ``DEFAULT_READ_BATCH_SIZE`` (64).
-    prefetch_batches:
+    io_queue_depth:
         Number of I/O batches to keep in flight per split.  Higher values
         overlap storage latency with transform and training compute at the cost
-        of more memory and threads.  Defaults to ``DEFAULT_PREFETCH_BATCHES``
-        (4).
+        of more memory and threads.  Must be greater than zero.  Defaults to
+        ``DEFAULT_PREFETCH_BATCHES`` (4).
     columns:
         Optional list of column names to read.  When set, only those columns
         are fetched from storage; all others are omitted.  ``None`` (the
@@ -127,6 +288,92 @@ class StreamingDataset(IterableDataset):
         Maximum number of transforms to run concurrently.  Must be greater
         than zero.  When ``None`` (the default), uses ``os.cpu_count()`` or 1
         when the CPU count is unavailable.
+    pack_sequences:
+        Sequence-packing mode: token lists from consecutive documents are
+        joined with ``eos_id`` and sliced into blocks of this many tokens.
+        Each item is then a dict of two ``(pack_sequences,)`` LongTensors —
+        ``input_ids`` and ``doc_ids`` (per-position document index within
+        the block, for block-diagonal masks or position-id resets).
+        * Packing happens independently per owned split and preserves per-split
+        resume state.
+        * When a split cannot fill a real block for a cycle but an owned sibling
+        still can, or when only a short tail remains at epoch end, the short buffer
+        is padded to ``pack_sequences`` with ``pad_id`` so every local cycle emits
+        one block per owned split.
+        * ``eos_id``, ``pad_id``, and ``columns`` naming a single integer-list
+        column are required; incompatible with ``transform``.
+    eos_id:
+        Separator token id between packed documents.  Required with
+        ``pack_sequences``, ignored otherwise.
+    pad_id:
+        Padding token id used to complete blocks when a split runs out of
+        real tokens mid-cycle or at epoch end.  Required with
+        ``pack_sequences``, ignored otherwise. It must be reserved for padding:
+        padding positions retain the preceding document's ``doc_id`` (or zero
+        in an all-padding block), so callers must mask them separately using
+        ``input_ids == pad_id``.
+    blocks_per_epoch:
+        Total number of packed blocks emitted globally per epoch. Required with
+        ``pack_sequences``. An integer must be divisible by ``num_splits``.
+        Every logical split emits exactly ``blocks_per_epoch / num_splits``
+        blocks: exhausted splits emit padding, while tokens beyond the budget
+        are left out of the epoch. This fixed per-split budget keeps packed
+        iteration and checkpoints independent of rank topology.
+        Pass ``"auto"`` to estimate a corpus-level budget from a bounded sample
+        of token lists. The estimate may be inaccurate.
+    on_transform_error:
+        What to do when the transform raises an exception:
+
+        - ``"raise"`` (the default): the exception propagates and iteration
+          aborts.
+        - ``"skip"``: the failing rows are dropped and iteration continues.
+        - ``"warn"``: like ``"skip"``, but a warning is logged for each
+          failing batch.
+        - a callable ``handler(exc) -> bool``: called with the exception;
+          return ``True`` to skip the failing rows or ``False`` to re-raise.
+          Useful to skip only expected error types (compatible with
+          ``webdataset.handlers`` style handlers).
+
+        When a batch fails, the transform is re-invoked on each single-row
+        slice of the batch so that only the rows that actually fail are
+        dropped.  Transforms should therefore be deterministic and accept
+        batches of any size (including one row).  Skipped rows are counted in
+        ``rows_skipped``.
+
+        Skipping weakens the elastic-determinism guarantee at the end of the
+        epoch: splits that lose more rows than others run dry earlier, and
+        each rank's iterator ends at the last cycle where every split *it
+        owns* still has a row.  Because bad rows are not distributed evenly
+        across splits, this means one rank's iterator can yield noticeably
+        fewer or more steps than another rank's *in the same run* — there is
+        no cross-rank coordination that stops every rank at the same global
+        step.  This is generally safe for asynchronous or single-rank use,
+        but synchronous distributed training (e.g. ranks that call
+        ``all_reduce`` every step) can hang or deadlock if one rank's
+        iterator is exhausted while others are still stepping; callers doing
+        synchronous multi-rank training with ``on_transform_error != "raise"``
+        are responsible for their own cross-rank stopping mechanism (e.g.
+        broadcasting a stop signal on ``StopIteration``).  The final few
+        global steps can also differ across topologies (bounded by the skew
+        in bad-row counts across splits).  The sequence of samples yielded
+        from each split remains deterministic.  Mid-epoch
+        checkpoints remain exact provided the transform fails
+        deterministically; in multi-rank training each rank must save its
+        own ``state_dict`` and the states must be combined with
+        ``merge_state_dicts`` before resuming on a different topology.
+        Prefer the ``filter`` parameter when bad rows can be expressed as a
+        SQL predicate (e.g. ``"col IS NOT NULL"``) — filtering happens before
+        splits are built, so every guarantee is fully preserved.
+    transform_queue_depth:
+        Number of transform-result batches to buffer per split in the
+        post-transform queue before backpressure is applied to the transform
+        stage.  When the combined count of in-flight transform futures and
+        already-buffered rows for a split reaches
+        ``transform_queue_depth * read_batch_size``, no new transforms are
+        submitted for that split until the consumer catches up.  Useful for
+        capping peak memory when the consumer (e.g. a GPU training step) is
+        slower than the transform stage.  Must be greater than zero.
+        ``None`` (the default) imposes no limit.
     worker_info_override:
         If set, used in place of ``torch.utils.data.get_worker_info()`` to
         determine the DataLoader worker assignment.  Intended for unit tests
@@ -146,16 +393,30 @@ class StreamingDataset(IterableDataset):
         rank: int = 0,
         world_size: int = 1,
         read_batch_size: int = DEFAULT_READ_BATCH_SIZE,
-        prefetch_batches: int = DEFAULT_PREFETCH_BATCHES,
+        io_queue_depth: int = DEFAULT_PREFETCH_BATCHES,
         columns: Optional[list[str]] = None,
         shuffle_clump_size: Optional[int] = None,
         filter: Optional[str] = None,
         transform: Optional[Callable] = None,
         transform_parallelism: Optional[int] = None,
+        pack_sequences: Optional[int] = None,
+        eos_id: Optional[int] = None,
+        pad_id: Optional[int] = None,
+        blocks_per_epoch: Optional[Union[int, Literal["auto"]]] = None,
+        on_transform_error: Union[str, Callable[[Exception], bool]] = "raise",
+        transform_queue_depth: Optional[int] = None,
         connection_factory: Optional[Callable[[str], Any]] = None,
         worker_info_override=None,
+        # Deprecated; use io_queue_depth instead.
+        prefetch_batches: Optional[int] = None,
     ):
         super().__init__()
+        if prefetch_batches is not None:
+            logger.warning(
+                "prefetch_batches is deprecated and will be removed in a future "
+                "version; use io_queue_depth instead"
+            )
+            io_queue_depth = prefetch_batches
         if num_splits is None:
             num_splits = world_size
         if shuffle_seed is None:
@@ -165,8 +426,68 @@ class StreamingDataset(IterableDataset):
                 f"num_splits ({num_splits}) must be divisible by "
                 f"world_size ({world_size})"
             )
+        if io_queue_depth <= 0:
+            raise ValueError("io_queue_depth must be greater than 0")
         if transform_parallelism is not None and transform_parallelism <= 0:
             raise ValueError("transform_parallelism must be greater than 0")
+        if pack_sequences is not None:
+            if pack_sequences <= 0:
+                raise ValueError("pack_sequences must be greater than 0")
+            if eos_id is None:
+                raise ValueError("eos_id is required when pack_sequences is set")
+            if pad_id is None:
+                raise ValueError("pad_id is required when pack_sequences is set")
+            if blocks_per_epoch is None:
+                raise ValueError(
+                    "blocks_per_epoch is required when pack_sequences is set"
+                )
+            if blocks_per_epoch != "auto":
+                if not isinstance(blocks_per_epoch, int) or isinstance(
+                    blocks_per_epoch, bool
+                ):
+                    raise ValueError(
+                        "blocks_per_epoch must be a positive integer or 'auto'"
+                    )
+                if blocks_per_epoch <= 0:
+                    raise ValueError("blocks_per_epoch must be greater than 0")
+                if blocks_per_epoch % num_splits != 0:
+                    raise ValueError(
+                        f"blocks_per_epoch ({blocks_per_epoch}) must be divisible by "
+                        f"num_splits ({num_splits})"
+                    )
+            if transform is not None:
+                raise ValueError("transform cannot be combined with pack_sequences")
+            if columns is None or len(columns) != 1:
+                raise ValueError(
+                    "pack_sequences requires columns to name exactly one "
+                    "list-typed column of token ids"
+                )
+            field = table.schema.field(columns[0])
+            if not (
+                pa.types.is_list(field.type)
+                or pa.types.is_large_list(field.type)
+                or pa.types.is_fixed_size_list(field.type)
+            ):
+                raise ValueError(
+                    f"pack_sequences requires a list-typed token column; "
+                    f"{columns[0]} has type {field.type}"
+                )
+            if not pa.types.is_integer(field.type.value_type):
+                raise ValueError(
+                    "pack_sequences requires a token column with integer values; "
+                    f"{columns[0]} has value type {field.type.value_type}"
+                )
+        elif blocks_per_epoch is not None:
+            raise ValueError("blocks_per_epoch requires pack_sequences")
+        if on_transform_error not in ("raise", "skip", "warn") and not callable(
+            on_transform_error
+        ):
+            raise ValueError(
+                "on_transform_error must be 'raise', 'skip', 'warn', or a "
+                f"callable, got {on_transform_error!r}"
+            )
+        if transform_queue_depth is not None and transform_queue_depth <= 0:
+            raise ValueError("transform_queue_depth must be greater than 0")
 
         self._table = table
         self._num_splits = num_splits
@@ -176,15 +497,26 @@ class StreamingDataset(IterableDataset):
         self._rank = rank
         self._world_size = world_size
         self._read_batch_size = read_batch_size
-        self._prefetch_batches = prefetch_batches
+        self._io_queue_depth = io_queue_depth
         self._columns = columns
         self._shuffle_clump_size = shuffle_clump_size
         self._filter = filter
         self._transform = transform
         self._transform_parallelism = transform_parallelism
+        self._pack_sequences = pack_sequences
+        self._eos_id = eos_id
+        self._pad_id = pad_id
+        self._blocks_per_epoch = blocks_per_epoch
+        self._on_transform_error = on_transform_error
+        self._transform_queue_depth = transform_queue_depth
         self._connection_factory = connection_factory
         self._worker_info_override = worker_info_override
 
+        # Packing resume state: permutation positions and partial-block buffers.
+        self._pack_consumed: list[int] = [0] * num_splits
+        self._pack_buffers: dict[int, dict[str, list[int]]] = {}
+        self._pack_blocks_emitted: list[int] = [0] * num_splits
+
         # Live references to pipeline state, set only while __iter__ is running
         # in the same process.  Used by the observability properties when the
         # DataLoader runs with num_workers=0.
@@ -199,19 +531,48 @@ class StreamingDataset(IterableDataset):
         # in the main process.  RawArray is picklable via the forkserver
         # reduction protocol so it survives the dataset pickle round-trip.
         # Layout: [unscanned_rows, raw_rows, cooked_rows, consumed_rows,
-        #          bytes_loaded, fetch_time_us, transform_time_us]
-        self._worker_stats: RawArray = RawArray(ctypes.c_int64, 7)
+        #          bytes_loaded, fetch_time_us, transform_time_us,
+        #          rows_skipped]
+        self._worker_stats: RawArray = RawArray(ctypes.c_int64, 8)
+
+        # A standard multi-process DataLoader cannot report which prefetched
+        # batches were actually returned to its consumer.  Workers set this
+        # shared flag so state_dict() can reject a stale parent checkpoint
+        # unless StreamingDataLoader installed the consumer-commit transport.
+        self._untracked_worker_iteration: RawArray = RawArray(ctypes.c_int64, 1)
+
+        # Parent-side checkpoint lifecycle.  A failed DataLoader task creates
+        # a permanent hole in that iterator's delivery stream, while a
+        # multi-worker checkpoint is safe to restore only after all splits
+        # reach the same logical step boundary.
+        self._checkpoint_invalid_reason: Optional[str] = None
+        self._consumer_checkpoint_requires_uniform = False
+        self._consumer_iterator_lock = threading.Lock()
+        self._consumer_iterator_generation = 0
+        self._consumer_iterator_lease: Optional[_ConsumerIteratorLease] = None
 
         # Cumulative bytes of Arrow buffer data fetched across all iterations.
         self._bytes_loaded: int = 0
         # Cumulative seconds spent in LanceDB I/O and in transform functions.
         self._fetch_time: float = 0.0
         self._transform_time: float = 0.0
+        # Cumulative rows dropped by on_transform_error across all iterations.
+        self._rows_skipped: int = 0
 
         # Number of samples each split has already been consumed.  At global
         # step boundaries all splits have consumed this many samples, so a
         # single scalar captures the topology-independent checkpoint state.
         self._resume_offset: int = 0
+        # Exact yielded-sample counts for splits this process has advanced.
+        # Missing entries use _resume_offset, which remains the lower-bound
+        # checkpoint inherited from an earlier uniform/global state.
+        self._resume_samples: dict[int, int] = {}
+        # Permutation position each split has consumed through, keyed by
+        # global split index.  Equal to _resume_offset for every split unless
+        # on_transform_error skipped rows, in which case skipped positions
+        # push the watermark of the affected splits further ahead.  Splits
+        # this instance has never iterated have no entry.
+        self._resume_positions: dict[int, int] = {}
 
         # Build the permutation table once, deterministically.
         builder = permutation_builder(table)
@@ -225,6 +586,9 @@ class StreamingDataset(IterableDataset):
         else:
             self._perm_table = builder.split_sequential(fixed=num_splits).execute()
 
+        if self._blocks_per_epoch == "auto":
+            self._blocks_per_epoch = self._estimate_blocks_per_epoch()
+
         # Contiguous block of global split indices assigned to this rank.
         splits_per_rank = num_splits // world_size
         rank_start = rank * splits_per_rank
@@ -232,6 +596,71 @@ class StreamingDataset(IterableDataset):
             range(rank_start, rank_start + splits_per_rank)
         )
 
+    def _estimate_blocks_per_epoch(self) -> int:
+        """Estimate a fixed packed-block budget from a bounded token sample."""
+        # TODO: Replace this fallback with Lance's dedicated exact token-count
+        # estimation API once it is available.
+        if self._pack_sequences is None or not self._columns:
+            raise RuntimeError(
+                "packing must be configured before estimating its budget"
+            )
+
+        pack_len = self._pack_sequences
+        token_column = self._columns[0]
+        sample_cap_per_split = max(1, 100_000 // self._num_splits)
+        sampled_tokens = 0
+        total_sampled = 0
+        total_rows = 0
+        rng = random.Random(self._shuffle_seed)
+
+        warnings.warn(
+            "blocks_per_epoch='auto' uses an approximate token-count sample; "
+            "pass an explicit value for exact epoch sizing",
+        )
+
+        for split in range(self._num_splits):
+            permutation = Permutation.from_tables(
+                self._table, self._perm_table, split=split
+            )
+            permutation = permutation.select_columns([token_column])
+            permutation = permutation.with_transform(Transforms.arrow2arrow)
+            split_rows = permutation.num_rows
+            if split_rows == 0:
+                raise ValueError(
+                    "blocks_per_epoch='auto' cannot estimate an empty dataset"
+                )
+
+            # Sample roughly 1% from each logical split, with at least one row
+            # per split and a global target cap of 100,000 rows.
+            sample_rows = min(
+                split_rows,
+                max(1, min((split_rows + 99) // 100, sample_cap_per_split)),
+            )
+            sample_offsets = sorted(rng.sample(range(split_rows), sample_rows))
+            sample_batch_size = max(1, self._read_batch_size)
+            for start in range(0, sample_rows, sample_batch_size):
+                batch = permutation.__getitems__(
+                    sample_offsets[start : start + sample_batch_size]
+                )
+                lengths = pc.list_value_length(batch.column(0))
+                if lengths.null_count:
+                    raise ValueError("pack_sequences does not support null token lists")
+                sampled_tokens += int(pc.sum(lengths).as_py())
+
+            total_sampled += sample_rows
+            total_rows += split_rows
+
+        # Pool the samples into one global average. Each document contributes
+        # one EOS token.
+        estimated_tokens = (
+            (sampled_tokens + total_sampled) * total_rows // total_sampled
+        )
+        blocks = estimated_tokens // pack_len
+        return max(
+            self._num_splits,
+            blocks - blocks % self._num_splits,
+        )
+
     def _resolve_my_splits(self) -> list[int]:
         """Return the split indices this instance should read in __iter__."""
         torch_worker_info = get_worker_info()
@@ -263,11 +692,45 @@ class StreamingDataset(IterableDataset):
         return self._rank_splits[start : start + splits_per_worker]
 
     def __iter__(self) -> Iterator[dict[str, Any]]:
+        return self._iter()
+
+    def _iter(
+        self, *, consumer_checkpoint_transport: bool = False
+    ) -> Iterator[dict[str, Any]]:
+        owner_token = None
+        previous_lease = self._consumer_iterator_lease
+        if consumer_checkpoint_transport:
+            if not self._consumer_iterator_active:
+                raise RuntimeError(
+                    "StreamingDataLoader worker transport requires an active "
+                    "parent iterator reservation"
+                )
+        else:
+            try:
+                owner_token = self._acquire_consumer_iterator()
+            except BaseException:
+                self._release_consumer_iterator_after_failed_acquire(previous_lease)
+                raise
+        try:
+            yield from self._iter_owned(
+                consumer_checkpoint_transport=consumer_checkpoint_transport
+            )
+        finally:
+            if owner_token is not None:
+                self._release_consumer_iterator(owner_token)
+
+    def _iter_owned(
+        self, *, consumer_checkpoint_transport: bool
+    ) -> Iterator[dict[str, Any]]:
         if self._raw_batches_ref is not None:
             raise RuntimeError(
                 "StreamingDataset does not support concurrent iteration. "
                 "Only one active iterator per dataset instance is allowed."
             )
+        real_worker = get_worker_info() is not None
+        if real_worker and not consumer_checkpoint_transport:
+            self._untracked_worker_iteration[0] = 1
+
         my_splits = self._resolve_my_splits()
         if not my_splits:
             return
@@ -275,39 +738,78 @@ class StreamingDataset(IterableDataset):
         # Set identity transform on each Permutation so __getitems__ returns
         # the raw RecordBatch.  Stage 2 applies the real transform.
         permutations: list[Permutation] = []
+        initial_samples: list[int] = []
+        initial_positions: list[int] = []
         for split_idx in my_splits:
             perm = Permutation.from_tables(
                 self._table, self._perm_table, split=split_idx
             )
             if self._columns is not None:
                 perm = perm.select_columns(self._columns)
-            perm = perm.with_transform(lambda batch: batch)
-            if self._resume_offset > 0:
-                perm = perm.with_skip(self._resume_offset)
+            perm = perm.with_transform(Transforms.arrow2arrow)
+            sample_count = self._resume_samples.get(split_idx, self._resume_offset)
+            # Both modes resume from absolute permutation positions. Packing
+            # stores them separately because it also checkpoints partial blocks.
+            start_pos = (
+                self._pack_consumed[split_idx]
+                if self._pack_sequences is not None
+                else self._resume_positions.get(split_idx, sample_count)
+            )
+            if start_pos > 0:
+                perm = perm.with_skip(start_pos)
+            initial_samples.append(sample_count)
+            initial_positions.append(start_pos)
             permutations.append(perm)
 
         n = len(permutations)
         split_sizes = [perm.num_rows for perm in permutations]
-        initial_offset = self._resume_offset
         local_consumed = [0] * n
+        # Permutation position each split has consumed through (absolute,
+        # i.e. counted from the start of the unskipped split).  Runs ahead of
+        # initial + local_consumed when rows are skipped.
+        pos_consumed = list(initial_positions)
 
         batch_size = self._read_batch_size
-        max_prefetch = self._prefetch_batches
+        io_queue_depth = self._io_queue_depth
         transform_workers = (
             self._transform_parallelism
             if self._transform_parallelism is not None
             else (os.cpu_count() or 1)
         )
-        final_transform = (
-            self._transform if self._transform is not None else Transforms.arrow2python
+        final_transform: Callable[[pa.RecordBatch], Any]
+        if self._pack_sequences is not None:
+            # Packing consumes raw token lists, one per document.
+            def arrow_tokens(batch: pa.RecordBatch) -> list[list[int]]:
+                token_column = batch.column(0)
+                if token_column.null_count or token_column.flatten().null_count:
+                    raise ValueError(
+                        "pack_sequences does not support null token lists or values"
+                    )
+                return cast(list[list[int]], token_column.to_pylist())
+
+            final_transform = arrow_tokens
+        else:
+            final_transform = (
+                self._transform
+                if self._transform is not None
+                else Transforms.arrow2python
+            )
+        # None means no limit; otherwise cap rows per split to
+        # transform_queue_depth batches worth (including in-flight transforms).
+        max_cooked_rows = (
+            self._transform_queue_depth * batch_size
+            if self._transform_queue_depth is not None
+            else None
         )
 
-        # Per-split pipeline state.
+        # Per-split pipeline state.  Batches are paired with the absolute
+        # permutation position of their first row so that skipped rows can be
+        # accounted for in pos_consumed.
         fetch_head = [0] * n
-        io_pending = [deque() for _ in range(n)]  # Future[RecordBatch]
-        raw_batches = [deque() for _ in range(n)]  # RecordBatch — fetched, awaiting tx
-        tx_pending = [deque() for _ in range(n)]  # Future[list[Any]]
-        cooked = [deque() for _ in range(n)]  # rows ready to yield
+        io_pending = [deque() for _ in range(n)]  # (abs_start, Future[RecordBatch])
+        raw_batches = [deque() for _ in range(n)]  # (abs_start, RecordBatch)
+        tx_pending = [deque() for _ in range(n)]  # Future[list[(abs_pos, row)]]
+        cooked = [deque() for _ in range(n)]  # (abs_pos, row) ready to yield
 
         # Limit simultaneous transforms to transform_workers across all splits.
         tx_semaphore = threading.Semaphore(transform_workers)
@@ -330,23 +832,83 @@ class StreamingDataset(IterableDataset):
             fetch_head[i] += fetch
             perm_i = permutations[i]
             indices = list(range(start, start + fetch))
-            io_pending[i].append(io_pool.submit(_io_call, perm_i, indices))
+            abs_start = initial_positions[i] + start
+            io_pending[i].append((abs_start, io_pool.submit(_io_call, perm_i, indices)))
 
         def _fill_io(i: int) -> None:
-            while len(io_pending[i]) < max_prefetch and fetch_head[i] < split_sizes[i]:
+            while (
+                len(io_pending[i]) < io_queue_depth and fetch_head[i] < split_sizes[i]
+            ):
                 _submit_io(i)
 
         def _drain_io(i: int) -> None:
             """Move completed I/O futures into raw_batches non-blockingly."""
-            while io_pending[i] and io_pending[i][0].done():
-                raw_batches[i].append(io_pending[i].popleft().result())
+            while io_pending[i] and io_pending[i][0][1].done():
+                abs_start, fut = io_pending[i].popleft()
+                raw_batches[i].append((abs_start, fut.result()))
 
         # ── Stage 2 helpers ───────────────────────────────────────────────────
 
-        def _tx_call_guarded(batch):
+        on_error = self._on_transform_error
+
+        def _should_skip(exc: Exception) -> bool:
+            if on_error == "raise":
+                return False
+            if callable(on_error):
+                return bool(on_error(exc))
+            return True  # "skip" or "warn"
+
+        def _check_row_count(rows: list, num_rows: int) -> None:
+            if len(rows) != num_rows:
+                raise ValueError(
+                    f"transform returned {len(rows)} rows for a batch of "
+                    f"{num_rows}; transforms must return exactly one output "
+                    "row per input row.  To drop bad rows, raise inside the "
+                    "transform and pass on_transform_error='skip'."
+                )
+
+        def _transform_isolated(abs_start, batch, batch_exc):
+            """Re-run the transform on single-row slices, dropping failures."""
+            out = []
+            skipped = 0
+            first_exc = None
+            for j in range(batch.num_rows):
+                try:
+                    rows = list(final_transform(batch.slice(j, 1)))
+                except Exception as exc:
+                    if not _should_skip(exc):
+                        raise
+                    skipped += 1
+                    if first_exc is None:
+                        first_exc = exc
+                    continue
+                _check_row_count(rows, 1)
+                out.append((abs_start + j, rows[0]))
+            self._rows_skipped += skipped
+            if skipped and on_error == "warn":
+                logger.warning(
+                    "Skipped %d of %d rows whose transform failed (first error: %r)",
+                    skipped,
+                    batch.num_rows,
+                    first_exc if first_exc is not None else batch_exc,
+                )
+            return out
+
+        def _transform_batch(abs_start, batch):
+            """Apply the transform, returning [(abs_pos, row), ...]."""
+            try:
+                rows = list(final_transform(batch))
+            except Exception as exc:
+                if not _should_skip(exc):
+                    raise
+                return _transform_isolated(abs_start, batch, exc)
+            _check_row_count(rows, batch.num_rows)
+            return [(abs_start + j, row) for j, row in enumerate(rows)]
+
+        def _tx_call_guarded(abs_start, batch):
             try:
                 t0 = time.perf_counter()
-                result = final_transform(batch)
+                result = _transform_batch(abs_start, batch)
                 self._transform_time += time.perf_counter() - t0
                 return result
             finally:
@@ -354,9 +916,21 @@ class StreamingDataset(IterableDataset):
 
         def _try_submit_tx(i: int) -> None:
             """Submit transforms for raw_batches[i] up to available capacity."""
-            while raw_batches[i] and tx_semaphore.acquire(blocking=False):
-                batch = raw_batches[i].popleft()
-                tx_pending[i].append(tx_pool.submit(_tx_call_guarded, batch))
+            while raw_batches[i]:
+                # Backpressure: only submit a new transform when there is room
+                # for a full batch in the post-transform queue.  Checking for
+                # a full batch prevents submitting a transform that would
+                # overflow the limit mid-batch (e.g. 990 rows queued with a
+                # capacity of 1000 and a batch_size of 128 must wait until
+                # 128 rows have been consumed, not just 1).
+                if max_cooked_rows is not None:
+                    in_pipeline = len(cooked[i]) + len(tx_pending[i]) * batch_size
+                    if in_pipeline + batch_size > max_cooked_rows:
+                        break
+                if not tx_semaphore.acquire(blocking=False):
+                    break
+                abs_start, batch = raw_batches[i].popleft()
+                tx_pending[i].append(tx_pool.submit(_tx_call_guarded, abs_start, batch))
 
         def _drain_tx(i: int) -> None:
             """Move completed transform futures into cooked non-blockingly."""
@@ -384,61 +958,222 @@ class StreamingDataset(IterableDataset):
                     # Acquire a transform slot (may block briefly if all
                     # transform_workers are busy with other splits).
                     tx_semaphore.acquire()
-                    batch = raw_batches[i].popleft()
-                    tx_pending[i].append(tx_pool.submit(_tx_call_guarded, batch))
+                    abs_start, batch = raw_batches[i].popleft()
+                    tx_pending[i].append(
+                        tx_pool.submit(_tx_call_guarded, abs_start, batch)
+                    )
                 elif io_pending[i]:
                     # Block on the oldest in-flight I/O fetch.
-                    raw_batches[i].append(io_pending[i].popleft().result())
+                    abs_start, fut = io_pending[i].popleft()
+                    raw_batches[i].append((abs_start, fut.result()))
                     _advance(i)
                 else:
                     break  # split exhausted
 
+        def _update_stats(*, idle: bool = False) -> None:
+            """Refresh pipeline statistics visible to the parent process."""
+            ws = self._worker_stats
+            ws[0] = sum(split_sizes[j] - fetch_head[j] for j in range(n))
+            ws[1] = (
+                0
+                if idle
+                else sum(batch.num_rows for q in raw_batches for _, batch in q)
+            )
+            ws[2] = 0 if idle else sum(len(q) for q in cooked)
+            ws[3] = sum(local_consumed)
+            ws[4] = self._bytes_loaded
+            ws[5] = int(self._fetch_time * 1_000_000)
+            ws[6] = int(self._transform_time * 1_000_000)
+            ws[7] = self._rows_skipped
+
+        # Sequence-packing helpers
+        pack_len = cast(int, self._pack_sequences)
+        eos_id = cast(int, self._eos_id)
+        pad_id = cast(int, self._pad_id)
+        blocks_per_split = (
+            cast(int, self._blocks_per_epoch) // self._num_splits
+            if self._pack_sequences is not None
+            else 0
+        )
+        pack_consumed = list(self._pack_consumed)
+        pack_buffers = deepcopy(self._pack_buffers)
+        pack_blocks_emitted = list(self._pack_blocks_emitted)
+
+        def _pack_buffer(i: int) -> dict[str, list[int]]:
+            return pack_buffers.setdefault(my_splits[i], {"tokens": [], "starts": []})
+
+        def _fill_block(i: int) -> None:
+            """Fill split i's buffer to one block or exhaust the split."""
+            buf = _pack_buffer(i)
+            while len(buf["tokens"]) < pack_len:
+                _ensure_cooked(i)
+                if not cooked[i]:
+                    return
+                buf["starts"].append(len(buf["tokens"]))
+                pos, tokens = cooked[i].popleft()
+                buf["tokens"].extend(tokens)
+                buf["tokens"].append(eos_id)
+                pack_consumed[my_splits[i]] = pos + 1
+                local_consumed[i] += 1
+                _advance(i)
+
+        def _emit_block(i: int) -> dict[str, Any]:
+            buf = _pack_buffer(i)
+            tokens, starts = buf["tokens"], buf["starts"]
+            # doc_ids label document segments within the block; 0 also covers
+            # the continuation of a document begun in a prior block.
+            doc_ids = torch.zeros(pack_len, dtype=torch.int64)
+            doc_starts = [s for s in starts if 0 < s < pack_len]
+            doc_ids[doc_starts] = 1
+            doc_ids.cumsum_(dim=0)  # cumulative sum marks document boundaries
+            block = {
+                "input_ids": torch.tensor(tokens[:pack_len], dtype=torch.int64),
+                "doc_ids": doc_ids,
+            }
+            del tokens[:pack_len]
+            # Shift start boundaries for the next call.
+            buf["starts"] = [s - pack_len for s in starts if s >= pack_len]
+            return block
+
+        def _commit_pack_state() -> None:
+            self._pack_consumed = list(pack_consumed)
+            self._pack_buffers = {
+                split: {
+                    "tokens": list(buffer["tokens"]),
+                    "starts": list(buffer["starts"]),
+                }
+                for split, buffer in pack_buffers.items()
+            }
+            self._pack_blocks_emitted = list(pack_blocks_emitted)
+
         # ── Main loop ─────────────────────────────────────────────────────────
 
-        with ThreadPoolExecutor(max_workers=n * max_prefetch) as io_pool:
+        with ThreadPoolExecutor(max_workers=n * io_queue_depth) as io_pool:
             with ThreadPoolExecutor(max_workers=transform_workers) as tx_pool:
                 self._raw_batches_ref = raw_batches
                 self._cooked_ref = cooked
                 self._fetch_head_ref = fetch_head
                 self._split_sizes_ref = split_sizes
                 self._local_consumed_ref = local_consumed
+
                 try:
                     for i in range(n):
                         _fill_io(i)
 
+                    def _yield_row(i: int):
+                        pos, row = cooked[i].popleft()
+                        # Surface any completed prefetched failure before the
+                        # current row becomes durable checkpoint progress.
+                        _advance(i)
+                        local_consumed[i] += 1
+                        pos_consumed[i] = pos + 1
+                        split_idx = my_splits[i]
+                        self._resume_samples[split_idx] = (
+                            initial_samples[i] + local_consumed[i]
+                        )
+                        self._resume_positions[split_idx] = pos_consumed[i]
+                        return row
+
+                    def _update_progress_stats() -> None:
+                        if not real_worker:
+                            self._resume_offset = min(
+                                initial_samples[j] + local_consumed[j] for j in range(n)
+                            )
+                        _update_stats()
+
+                    if self._pack_sequences is not None:
+                        first_count = pack_blocks_emitted[my_splits[0]]
+                        if any(
+                            pack_blocks_emitted[split] != first_count
+                            for split in my_splits[1:]
+                        ):
+                            raise ValueError(
+                                "Packed checkpoint is not aligned across the splits "
+                                "owned by this iterator; merge every rank "
+                                "state with merge_state_dicts before resuming on a "
+                                "different topology"
+                            )
+
+                        while pack_blocks_emitted[my_splits[0]] < blocks_per_split:
+                            # Each logical split gets one block per cycle. Exhausted
+                            # splits are padded through the fixed global budget.
+                            for i in range(n):
+                                _fill_block(i)
+
+                            for i in range(n):
+                                tokens = _pack_buffer(i)["tokens"]
+                                if len(tokens) < pack_len:
+                                    tokens.extend([pad_id] * (pack_len - len(tokens)))
+                                block = _emit_block(i)
+                                pack_blocks_emitted[my_splits[i]] += 1
+                                # Checkpoint state must advance before yielding so
+                                # StreamingDataLoader can attach the exact state to
+                                # the batch it transports to the parent process.
+                                _commit_pack_state()
+                                if i == n - 1:
+                                    _update_stats()
+                                yield block
+                        return
+
+                    # A checkpoint taken between round-robin split turns has
+                    # non-uniform counts.  Resume lagging splits first so the
+                    # exact canonical sequence continues without replaying
+                    # already-consumed rows.
+                    if len(set(initial_samples)) > 1:
+                        catch_up_to = max(initial_samples)
+                        pending = [
+                            (initial_samples[i], my_splits[i], i)
+                            for i in range(n)
+                            if initial_samples[i] < catch_up_to
+                        ]
+                        heapq.heapify(pending)
+                        while pending:
+                            consumed, _, i = heapq.heappop(pending)
+                            _ensure_cooked(i)
+                            if not cooked[i]:
+                                return
+                            row = _yield_row(i)
+                            if consumed + 1 < catch_up_to:
+                                heapq.heappush(pending, (consumed + 1, my_splits[i], i))
+                            _update_progress_stats()
+                            yield row
+
                     while True:
-                        # Stop when any split is exhausted (all exhaust
-                        # simultaneously: equal split sizes + round-robin).
-                        if any(local_consumed[i] >= split_sizes[i] for i in range(n)):
+                        # A cycle only runs if every split can still produce a
+                        # row.  Without skips all splits exhaust simultaneously
+                        # (equal split sizes + round-robin); when
+                        # on_transform_error drops rows a split can run dry
+                        # early, ending the epoch at the last complete cycle.
+                        # This check only sees splits owned by this rank/worker
+                        # (my_splits) — there is no cross-rank coordination, so
+                        # a different rank with fewer skipped rows keeps going;
+                        # see the on_transform_error docstring.
+                        exhausted = False
+                        for i in range(n):
+                            _ensure_cooked(i)
+                            if not cooked[i]:
+                                exhausted = True
+                                break
+                        if exhausted:
                             break
 
                         for i in range(n):
-                            _ensure_cooked(i)
-                            row = cooked[i].popleft()
-                            local_consumed[i] += 1
-                            _advance(i)
+                            row = _yield_row(i)
 
                             # After the last split in each cycle: update the
                             # global offset and refresh the shared-memory stats
                             # so the main process can observe pipeline depth
                             # even when __iter__ runs in a worker process.
                             if i == n - 1:
-                                self._resume_offset = initial_offset + local_consumed[i]
-                                ws = self._worker_stats
-                                ws[0] = sum(
-                                    split_sizes[j] - fetch_head[j] for j in range(n)
-                                )
-                                ws[1] = sum(
-                                    batch.num_rows for q in raw_batches for batch in q
-                                )
-                                ws[2] = sum(len(q) for q in cooked)
-                                ws[3] = sum(local_consumed)
-                                ws[4] = self._bytes_loaded
-                                ws[5] = int(self._fetch_time * 1_000_000)
-                                ws[6] = int(self._transform_time * 1_000_000)
+                                _update_progress_stats()
 
                             yield row
                 finally:
+                    # Final stats flush: the per-cycle write above never runs
+                    # when iteration ends mid-cycle (e.g. a split whose rows
+                    # were all skipped before completing a single cycle), so
+                    # counters like rows_skipped would otherwise be stale.
+                    _update_stats(idle=True)
                     self._raw_batches_ref = None
                     self._cooked_ref = None
                     self._fetch_head_ref = None
@@ -492,7 +1227,7 @@ class StreamingDataset(IterableDataset):
         batches.  Returns 0 when not iterating.
         """
         if self._raw_batches_ref is not None:
-            return sum(batch.num_rows for q in self._raw_batches_ref for batch in q)
+            return sum(batch.num_rows for q in self._raw_batches_ref for _, batch in q)
         return int(self._worker_stats[1])
 
     @property
@@ -522,6 +1257,19 @@ class StreamingDataset(IterableDataset):
             )
         return int(self._worker_stats[0])
 
+    @property
+    def rows_skipped(self) -> int:
+        """Number of rows dropped because their transform raised an exception.
+
+        Only ever non-zero when ``on_transform_error`` is set to ``"skip"``,
+        ``"warn"``, or a callable that returned ``True``.  Accumulates across
+        multiple iterations of the same dataset instance and is never reset
+        automatically.
+        """
+        if self._raw_batches_ref is not None:
+            return self._rows_skipped
+        return int(self._worker_stats[7])
+
     @property
     def consumed_rows(self) -> int:
         """Number of rows already yielded to the caller across all splits.
@@ -564,6 +1312,7 @@ class StreamingDataset(IterableDataset):
             "_local_consumed_ref",
         ):
             state[key] = None
+        state["_consumer_iterator_lock"] = None
         return state
 
     def __setstate__(self, state):
@@ -574,33 +1323,232 @@ class StreamingDataset(IterableDataset):
         table_state = state.pop("_table")
         perm_name, perm_data = state.pop("_perm_table")
         self.__dict__.update(state)
+        self._consumer_iterator_lock = threading.Lock()
         if self._connection_factory is not None:
             self._table = self._connection_factory(table_name)
         else:
             self._table = _table_from_pickle_state(table_state)
+            if table_state["kind"] == "memory":
+                # Rebuilt from Arrow, so the recorded pin cannot resolve on it.
+                perm_data = _drop_base_version(perm_data)
         self._perm_table = _connect("memory://").create_table(perm_name, perm_data)
 
     def state_dict(self) -> dict:
         """Snapshot the dataset's consumption state.
 
-        The returned dict is topology-independent: at global step boundaries
-        every split has been consumed the same number of times (by the
-        round-robin design), so the per-split count is a single uniform value
-        that is identical across all ranks and DataLoader workers.
+        When using DataLoader workers, construct a
+        [StreamingDataLoader][lancedb.streaming.StreamingDataLoader].  It
+        commits worker state only when a prefetched batch is returned to the
+        trainer.  A standard multi-process ``DataLoader`` cannot expose that
+        boundary, so calling this method after one has started raises
+        ``RuntimeError`` instead of returning stale producer state.
+
+        In row mode, the returned dict is topology-independent at global step
+        boundaries. ``positions_consumed_per_split`` records how far each
+        split's permutation has advanced, which can differ from the sample
+        count when ``on_transform_error`` skips rows. ``StreamingDataLoader``
+        combines worker state in its parent process. Combine state dicts from
+        every rank with
+        [merge_state_dicts][lancedb.streaming.StreamingDataset.merge_state_dicts]
+        before resuming on a different topology.
+
+        Packed state includes partial token buffers and emitted block counts
+        for every logical split. When packing is sharded, merge every rank
+        state with ``merge_state_dicts`` before loading it.
         """
+        if self._untracked_worker_iteration[0] and get_worker_info() is None:
+            raise RuntimeError(
+                "StreamingDataset cannot checkpoint a standard DataLoader with "
+                "num_workers > 0 because prefetched worker progress is not "
+                "consumer-committed. Use StreamingDataLoader instead."
+            )
+        if self._checkpoint_invalid_reason is not None:
+            raise RuntimeError(
+                "StreamingDataset checkpointing is invalid because "
+                f"{self._checkpoint_invalid_reason}. Load the last valid "
+                "checkpoint into a fresh dataset before continuing."
+            )
+        state = self._checkpoint_snapshot()
+        if self._pack_sequences is not None:
+            rank_blocks = [
+                state["blocks_emitted_per_split"][split] for split in self._rank_splits
+            ]
+            if len(set(rank_blocks)) > 1:
+                raise RuntimeError(
+                    "Packed StreamingDataset checkpointing is only safe at a "
+                    "complete logical step boundary, when every split assigned "
+                    "to this rank has emitted the same block count. Consume more "
+                    "batches before calling state_dict()."
+                )
+        elif self._consumer_checkpoint_requires_uniform:
+            samples = state["samples_consumed_per_split"]
+            rank_samples = [samples[split] for split in self._rank_splits]
+            if len(set(rank_samples)) > 1:
+                raise RuntimeError(
+                    "StreamingDataLoader checkpointing with multiple workers is "
+                    "only safe at a complete logical step boundary, when every "
+                    "split assigned to this rank has the same consumed-sample "
+                    "count. Consume more batches before calling state_dict()."
+                )
+        return state
+
+    def _checkpoint_snapshot(self) -> dict:
+        if self._pack_sequences is not None:
+            return {
+                "shuffle_seed": self._shuffle_seed,
+                "num_splits": self._num_splits,
+                "epoch": self._epoch,
+                "pack_sequences": self._pack_sequences,
+                "eos_id": self._eos_id,
+                "pad_id": self._pad_id,
+                "blocks_per_epoch": self._blocks_per_epoch,
+                "samples_consumed_per_split": list(self._pack_consumed),
+                "blocks_emitted_per_split": list(self._pack_blocks_emitted),
+                "pack_buffers": deepcopy(self._pack_buffers),
+            }
+        samples = [
+            self._resume_samples.get(split, self._resume_offset)
+            for split in range(self._num_splits)
+        ]
+        positions = [
+            self._resume_positions.get(split, samples[split])
+            for split in range(self._num_splits)
+        ]
         return {
             "shuffle_seed": self._shuffle_seed,
             "num_splits": self._num_splits,
             "epoch": self._epoch,
-            "samples_consumed_per_split": [self._resume_offset] * self._num_splits,
+            "samples_consumed_per_split": samples,
+            "positions_consumed_per_split": positions,
         }
 
+    def _invalidate_checkpoint(self, reason: str) -> None:
+        if self._checkpoint_invalid_reason is None:
+            self._checkpoint_invalid_reason = reason
+
+    @property
+    def _consumer_iterator_active(self) -> bool:
+        return self._consumer_iterator_lease is not None
+
+    @property
+    def _consumer_iterator_owner(self) -> Optional[int]:
+        lease = self._consumer_iterator_lease
+        return lease.owner_token if lease is not None else None
+
+    @property
+    def _consumer_iterator_owner_thread(self) -> Optional[int]:
+        lease = self._consumer_iterator_lease
+        return lease.owner_thread if lease is not None else None
+
+    def _acquire_consumer_iterator(self) -> int:
+        """Reserve this parent dataset for one checkpoint-aware iterator."""
+        with self._consumer_iterator_lock:
+            if self._consumer_iterator_active or self._raw_batches_ref is not None:
+                raise RuntimeError(
+                    "StreamingDataset does not support concurrent iteration. "
+                    "Only one active iterator per dataset instance is allowed."
+                )
+            owner_thread = threading.get_ident()
+            owner_token = self._consumer_iterator_generation + 1
+            lease = _ConsumerIteratorLease(owner_token, owner_thread)
+            self._consumer_iterator_generation = owner_token
+            self._consumer_iterator_lease = lease
+            return owner_token
+
+    def _release_consumer_iterator(self, owner_token: int) -> None:
+        with self._consumer_iterator_lock:
+            lease = self._consumer_iterator_lease
+            if lease is not None and lease.owner_token == owner_token:
+                self._consumer_iterator_lease = None
+
+    def _release_consumer_iterator_after_failed_acquire(
+        self, previous_lease: Optional[_ConsumerIteratorLease]
+    ) -> None:
+        """Clean up when an interrupted acquire set a lease but did not return it."""
+        owner_thread = threading.current_thread().ident
+        with self._consumer_iterator_lock:
+            lease = self._consumer_iterator_lease
+            if (
+                lease is not None
+                and lease is not previous_lease
+                and lease.owner_thread == owner_thread
+            ):
+                self._consumer_iterator_lease = None
+
+    def _commit_worker_state(self, state: dict, *, require_uniform: bool) -> None:
+        """Merge one trainer-consumed worker batch into parent state."""
+        for key, expected in (
+            ("shuffle_seed", self._shuffle_seed),
+            ("num_splits", self._num_splits),
+            ("epoch", self._epoch),
+        ):
+            if state.get(key) != expected:
+                raise ValueError(
+                    f"{key} mismatch in worker checkpoint: "
+                    f"{state.get(key)} != {expected}"
+                )
+        packed = "pack_buffers" in state
+        if packed != (self._pack_sequences is not None):
+            raise ValueError("worker checkpoint mode does not match the dataset")
+        if packed:
+            for key in ("pack_sequences", "eos_id", "pad_id", "blocks_per_epoch"):
+                expected = getattr(self, f"_{key}")
+                if state.get(key) != expected:
+                    raise ValueError(
+                        f"{key} mismatch in worker checkpoint: "
+                        f"{state.get(key)} != {expected}"
+                    )
+            samples = state["samples_consumed_per_split"]
+            emitted = state["blocks_emitted_per_split"]
+            if len(samples) != self._num_splits or len(emitted) != self._num_splits:
+                raise ValueError(
+                    "packed worker checkpoint must contain one entry per split"
+                )
+            buffers = state["pack_buffers"]
+            for split, (count, blocks) in enumerate(zip(samples, emitted)):
+                incoming = (int(blocks), int(count))
+                current = (
+                    self._pack_blocks_emitted[split],
+                    self._pack_consumed[split],
+                )
+                if incoming > current:
+                    self._pack_blocks_emitted[split] = incoming[0]
+                    self._pack_consumed[split] = incoming[1]
+                    buffer = buffers.get(split, buffers.get(str(split)))
+                    if buffer is None:
+                        self._pack_buffers.pop(split, None)
+                    else:
+                        self._pack_buffers[split] = {
+                            "tokens": list(buffer["tokens"]),
+                            "starts": list(buffer["starts"]),
+                        }
+            self._consumer_checkpoint_requires_uniform |= require_uniform
+            return
+
+        samples = state["samples_consumed_per_split"]
+        positions = state.get("positions_consumed_per_split", samples)
+        for split, count in enumerate(samples):
+            current = self._resume_samples.get(split, self._resume_offset)
+            self._resume_samples[split] = max(current, int(count))
+        for split, position in enumerate(positions):
+            current = self._resume_positions.get(
+                split, self._resume_samples.get(split, self._resume_offset)
+            )
+            self._resume_positions[split] = max(current, int(position))
+        self._resume_offset = min(
+            self._resume_samples.get(split, self._resume_offset)
+            for split in range(self._num_splits)
+        )
+        self._consumer_checkpoint_requires_uniform |= require_uniform
+
     def load_state_dict(self, state: dict) -> None:
         """Resume from a previously snapshotted state.
 
         Raises ``ValueError`` if ``num_splits`` or ``shuffle_seed`` differ
         from the checkpoint, since a different split structure or shuffle order
-        makes mid-epoch resumption meaningless.
+        makes mid-epoch resumption meaningless.  Packed checkpoints
+        pin ``pack_sequences``, ``eos_id``, ``pad_id``,
+        ``blocks_per_epoch``, and ``epoch``.
         """
         if state["num_splits"] != self._num_splits:
             raise ValueError(
@@ -612,9 +1560,295 @@ class StreamingDataset(IterableDataset):
                 f"shuffle_seed mismatch: checkpoint has {state['shuffle_seed']}, "
                 f"current dataset has {self._shuffle_seed}"
             )
+        self._consumer_checkpoint_requires_uniform = False
+
+        if "pack_buffers" in state or self._pack_sequences is not None:
+            for key in (
+                "pack_sequences",
+                "eos_id",
+                "pad_id",
+                "blocks_per_epoch",
+                "epoch",
+            ):
+                ours = getattr(self, f"_{key}")
+                if state.get(key) != ours:
+                    raise ValueError(
+                        f"{key} mismatch: checkpoint has {state.get(key)}, "
+                        f"current dataset has {ours}"
+                    )
+            self._pack_consumed = [int(c) for c in state["samples_consumed_per_split"]]
+            self._pack_blocks_emitted = [
+                int(c) for c in state["blocks_emitted_per_split"]
+            ]
+            self._pack_buffers = {
+                int(g): {"tokens": list(b["tokens"]), "starts": list(b["starts"])}
+                for g, b in state["pack_buffers"].items()
+            }
+            return
+
         consumed = state["samples_consumed_per_split"]
-        # All entries are equal at step boundaries; use the first.
         if isinstance(consumed, list):
-            self._resume_offset = consumed[0] if consumed else 0
+            self._resume_offset = min(consumed) if consumed else 0
+            self._resume_samples = {
+                split: int(count) for split, count in enumerate(consumed)
+            }
         else:
             self._resume_offset = int(consumed)
+            self._resume_samples = {}
+        # Older checkpoints predate positions_consumed_per_split; without
+        # skipped rows positions equal sample counts, so falling back to
+        # the per-split sample count (the .get default in __iter__) is exact.
+        positions = state.get("positions_consumed_per_split")
+        if positions is None:
+            self._resume_positions = {}
+        else:
+            self._resume_positions = {
+                split: int(pos) for split, pos in enumerate(positions)
+            }
+
+    @staticmethod
+    def merge_state_dicts(states: list[dict]) -> dict:
+        """Merge state dicts saved by different ranks into one exact state.
+
+        In row mode, each rank records exact consumer-committed progress for
+        its own splits and lower bounds for the rest, so elementwise maxima
+        recover both sample counts and permutation positions. In packed mode,
+        the state that emitted the most blocks for each logical split supplies
+        that split's permutation position and partial token buffer. Packed
+        states must cover every rank at the same global step.
+
+        Raises ``ValueError`` if the states are empty, were not produced by
+        the same run, or do not represent the same global step.
+
+        The merge is always all-to-all and topology-agnostic: collect the
+        ``state_dict()`` from every rank of the *previous* run into
+        one list, merge that whole list, and hand the identical merged result
+        to every rank of the *next* run — regardless of whether the
+        topology grew, shrank, or stayed the same. There is no pairwise or
+        subset merging step, because each split's exact state is only known to
+        whichever iterator owned that split.
+
+        For example, checkpointing 8 ranks and resuming on 4 (the same
+        pattern applies when growing, e.g. 4 ranks resuming on 8)::
+
+            states = [ds.state_dict() for ds in previous_run_datasets]  # 8
+            merged = StreamingDataset.merge_state_dicts(states)
+            for ds in resumed_datasets:  # now only 4 ranks
+                ds.load_state_dict(merged)  # same dict on every rank
+
+        The rank count on either side never affects the merge itself, since
+        ``merge_state_dicts`` only cares about the list of states it is
+        given.  Each split's position is recovered by elementwise maximum;
+        here rank 0 owned split 0 (and skipped two rows there) while rank 1
+        owned split 1 (and skipped one row):
+
+        >>> rank0 = {
+        ...     "shuffle_seed": 0, "num_splits": 2, "epoch": 0,
+        ...     "samples_consumed_per_split": [3, 3],
+        ...     "positions_consumed_per_split": [5, 3],
+        ... }
+        >>> rank1 = {
+        ...     "shuffle_seed": 0, "num_splits": 2, "epoch": 0,
+        ...     "samples_consumed_per_split": [3, 3],
+        ...     "positions_consumed_per_split": [3, 4],
+        ... }
+        >>> merged = StreamingDataset.merge_state_dicts([rank0, rank1])
+        >>> merged["positions_consumed_per_split"]
+        [5, 4]
+        """
+        if not states:
+            raise ValueError("merge_state_dicts requires at least one state dict")
+        first = states[0]
+        packed = "pack_buffers" in first
+        config_keys = ["shuffle_seed", "num_splits", "epoch"]
+        if packed:
+            config_keys.extend(
+                ["pack_sequences", "eos_id", "pad_id", "blocks_per_epoch"]
+            )
+
+        for state in states[1:]:
+            if ("pack_buffers" in state) != packed:
+                raise ValueError("cannot merge packed and unpacked state dicts")
+            for key in config_keys:
+                if state[key] != first[key]:
+                    raise ValueError(
+                        f"{key} mismatch across state dicts: "
+                        f"{state[key]} != {first[key]}"
+                    )
+
+        if packed:
+            num_splits = first["num_splits"]
+            for state in states:
+                for key in (
+                    "samples_consumed_per_split",
+                    "blocks_emitted_per_split",
+                ):
+                    if len(state[key]) != num_splits:
+                        raise ValueError(
+                            f"{key} must contain one entry per logical split"
+                        )
+
+            merged_consumed = []
+            merged_emitted = []
+            merged_buffers = {}
+            for split in range(num_splits):
+                owner = states[0]
+                owner_progress = (
+                    owner["blocks_emitted_per_split"][split],
+                    owner["samples_consumed_per_split"][split],
+                )
+                for state in states[1:]:
+                    progress = (
+                        state["blocks_emitted_per_split"][split],
+                        state["samples_consumed_per_split"][split],
+                    )
+                    if progress > owner_progress:
+                        owner = state
+                        owner_progress = progress
+                merged_consumed.append(owner["samples_consumed_per_split"][split])
+                merged_emitted.append(owner["blocks_emitted_per_split"][split])
+                buffer = owner["pack_buffers"].get(
+                    split, owner["pack_buffers"].get(str(split))
+                )
+                if buffer is not None:
+                    merged_buffers[split] = deepcopy(buffer)
+
+            if len(set(merged_emitted)) > 1:
+                raise ValueError(
+                    "packed state dicts were not captured at the same global "
+                    "step or do not cover every rank"
+                )
+
+            merged = dict(first)
+            merged["samples_consumed_per_split"] = merged_consumed
+            merged["blocks_emitted_per_split"] = merged_emitted
+            merged["pack_buffers"] = merged_buffers
+            return merged
+
+        merged = dict(first)
+        merged["samples_consumed_per_split"] = [
+            max(per_split)
+            for per_split in zip(
+                *(state["samples_consumed_per_split"] for state in states)
+            )
+        ]
+        all_positions = [
+            state.get(
+                "positions_consumed_per_split", state["samples_consumed_per_split"]
+            )
+            for state in states
+        ]
+        merged["positions_consumed_per_split"] = [
+            max(per_split) for per_split in zip(*all_positions)
+        ]
+        return merged
+
+
+class StreamingDataLoader(DataLoader):
+    """A PyTorch DataLoader with consumer-committed dataset checkpoints.
+
+    PyTorch workers prefetch batches ahead of the trainer, so worker-local
+    producer progress is not a safe checkpoint.  This loader carries a state
+    snapshot alongside every internal batch and applies it to the parent
+    [StreamingDataset][lancedb.streaming.StreamingDataset] only when that batch
+    is returned by ``next()``.
+    The trainer receives the same collated batch it would receive from a
+    standard ``torch.utils.data.DataLoader``.
+
+    With more than one worker, row-mode ``state_dict()`` is available only at
+    complete logical step boundaries, when every split assigned to the rank has
+    the same consumed-sample count. Packed checkpoints require equal emitted-block
+    counts across the rank's splits for any worker count. ``persistent_workers=True``
+    is not supported because prefetched worker copies cannot be restored from
+    parent-committed state. If batch collation raises, checkpointing remains
+    invalid for that dataset instance; restore the last valid checkpoint into a
+    fresh dataset before continuing.
+    Only one active iterator may own a dataset at a time, including when worker
+    processes are used. Exhausting or explicitly shutting down the iterator
+    releases that ownership. ``drop_last=True`` is not supported because worker
+    replicas discard incomplete tails independently, which cannot produce a
+    topology-independent checkpoint.
+
+    Parameters are the same as ``torch.utils.data.DataLoader`` except that
+    ``dataset`` must be a
+    [StreamingDataset][lancedb.streaming.StreamingDataset].
+    Subclasses that override ``StreamingDataset.__iter__`` are not supported
+    because the custom iterator cannot provide the exact per-yield checkpoint
+    snapshots required by this loader.
+
+    Examples
+    --------
+    >>> # dataset = StreamingDataset(table, num_splits=2)
+    >>> # loader = StreamingDataLoader(dataset, batch_size=8, num_workers=2)
+    >>> # batch = next(iter(loader))
+    >>> # checkpoint = dataset.state_dict()
+    """
+
+    def __init__(self, dataset: StreamingDataset, *args, **kwargs):
+        if not isinstance(dataset, StreamingDataset):
+            raise TypeError("StreamingDataLoader requires a StreamingDataset")
+        if type(dataset).__iter__ is not StreamingDataset.__iter__:
+            raise TypeError(
+                "StreamingDataLoader does not support StreamingDataset subclasses "
+                "that override __iter__ because they cannot provide exact "
+                "per-yield checkpoint state"
+            )
+        if kwargs.get("in_order", True) is False:
+            raise ValueError(
+                "StreamingDataLoader requires in_order=True for deterministic "
+                "consumer checkpoints"
+            )
+        if kwargs.get("persistent_workers", False):
+            raise ValueError(
+                "StreamingDataLoader does not support persistent_workers=True "
+                "because worker prefetch state cannot be reset from a checkpoint"
+            )
+        self._streaming_dataset = dataset
+        super().__init__(_StreamingDatasetAdapter(dataset), *args, **kwargs)
+        if self.drop_last:
+            raise ValueError(
+                "StreamingDataLoader does not support drop_last=True because "
+                "discarded worker tails cannot be checkpointed "
+                "topology-independently"
+            )
+        self.collate_fn = _CheckpointCollate(self.collate_fn)
+
+    def __iter__(self):
+        dataset = self._streaming_dataset
+        previous_lease = dataset._consumer_iterator_lease
+        owner_token = None
+        try:
+            owner_token = dataset._acquire_consumer_iterator()
+            state = dataset._checkpoint_snapshot()
+            packed = dataset._pack_sequences is not None
+            if packed:
+                blocks = state["blocks_emitted_per_split"]
+                rank_blocks = [blocks[split] for split in dataset._rank_splits]
+                if len(set(rank_blocks)) > 1:
+                    raise RuntimeError(
+                        "StreamingDataLoader cannot start from a partial packed "
+                        "logical step; resume from a checkpoint whose splits "
+                        "assigned to this rank have equal emitted-block counts"
+                    )
+            elif self.num_workers > 1:
+                samples = state["samples_consumed_per_split"]
+                rank_samples = [samples[split] for split in dataset._rank_splits]
+                if len(set(rank_samples)) > 1:
+                    raise RuntimeError(
+                        "StreamingDataLoader cannot start multiple workers from a "
+                        "partial logical step; resume from a checkpoint whose "
+                        "splits assigned to this rank have equal consumed-sample "
+                        "counts"
+                    )
+            return _ConsumerCommitIterator(
+                super().__iter__(),
+                dataset,
+                owner_token=owner_token,
+                require_uniform=self.num_workers > 1 or packed,
+            )
+        except BaseException:
+            if owner_token is not None:
+                dataset._release_consumer_iterator(owner_token)
+            else:
+                dataset._release_consumer_iterator_after_failed_acquire(previous_lease)
+            raise
diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py
index ae36bac7a..b3cab006e 100644
--- a/python/python/lancedb/table.py
+++ b/python/python/lancedb/table.py
@@ -40,7 +40,7 @@ from ._blob import (
 from .types import BlobMode
 from lancedb.arrow import peek_reader
 from lancedb.background_loop import LOOP, embedding_executor
-from lancedb.job import AsyncJob, Job
+from lancedb.job import AsyncJob, Job, _typed_job
 from .dependencies import (
     _check_for_hugging_face,
     _check_for_lance,
@@ -72,6 +72,10 @@ from .index import (
     FTS,
 )
 from .expr import Expr
+from .functions import (
+    FunctionApplication,
+    RefreshColumnResult as RefreshColumnJobResult,
+)
 from .merge import LanceMergeInsertBuilder
 from .pydantic import LanceModel, model_to_dict
 from .query import (
@@ -108,6 +112,11 @@ def _should_push_down_query_table(
     return namespace_client is not None and "QueryTable" in pushdown_operations
 
 
+def _polars_predicate_pushdown_barrier(frame: Any) -> Any:
+    """Return a Polars frame unchanged while blocking predicate pushdown."""
+    return frame
+
+
 _MODEL_BACKED_TOKENIZER_PREFIXES = ("jieba", "lindera")
 _MODEL_BACKED_TOKENIZER_ERRORS = (
     "unknown base tokenizer",
@@ -171,6 +180,7 @@ if TYPE_CHECKING:
         CompactionStats,
         Tag,
         AddColumnsResult,
+        RefreshColumnResult,
         AddResult,
         AlterColumnsResult,
         UpdateFieldMetadataResult,
@@ -427,6 +437,20 @@ def _cast_to_target_schema(
     return pa.RecordBatchReader.from_batches(reordered_schema, gen())
 
 
+def _field_extension_name(field: pa.Field) -> Optional[str]:
+    extension_name = getattr(field.type, "extension_name", None)
+    if extension_name is not None:
+        return extension_name
+
+    metadata = field.metadata or {}
+    extension_name = metadata.get(b"ARROW:extension:name") or metadata.get(
+        "ARROW:extension:name"
+    )
+    if isinstance(extension_name, bytes):
+        return extension_name.decode()
+    return extension_name
+
+
 def _align_field_types(
     fields: List[pa.Field],
     target_fields: List[pa.Field],
@@ -439,6 +463,16 @@ def _align_field_types(
         target_field = next((f for f in target_fields if f.name == field.name), None)
         if target_field is None:
             raise ValueError(f"Field '{field.name}' not found in target schema")
+        # Preserve arrow.json input until it reaches Lance. LanceDB exposes stored
+        # JSON columns as lance.json (JSONB-backed LargeBinary), but casting the
+        # input to that storage type here merely relabels the raw JSON bytes as
+        # JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
+        if (
+            _field_extension_name(field) == "arrow.json"
+            and _field_extension_name(target_field) == "lance.json"
+        ):
+            new_fields.append(field)
+            continue
         if pa.types.is_struct(target_field.type):
             if pa.types.is_struct(field.type):
                 new_type = pa.struct(
@@ -864,12 +898,18 @@ class Table(ABC):
         """
         raise NotImplementedError
 
-    def to_polars(self, **kwargs) -> "pl.DataFrame":
-        """Return the table as a polars.DataFrame.
+    def to_polars(self, **kwargs) -> "pl.LazyFrame":
+        """Return the table as a Polars LazyFrame.
+
+        Note
+        ----
+        The Polars streaming engine is not supported because it does not currently
+        implement Python PyArrow dataset scans. Use the default engine when collecting
+        this LazyFrame.
 
         Returns
         -------
-        polars.DataFrame
+        polars.LazyFrame
         """
         raise NotImplementedError
 
@@ -1229,6 +1269,7 @@ class Table(ABC):
         fill_value: float = 0.0,
         progress: Optional[Union[bool, Callable, Any]] = None,
         write_parallelism: Optional[int] = None,
+        allow_external_blob_outside_bases: bool = False,
     ) -> AddResult:
         """Add more data to the [Table][lancedb.table.Table].
 
@@ -1280,6 +1321,10 @@ class Table(ABC):
             data in flight. Defaults to an estimate based on the data size,
             capped at the number of CPU cores. Lower this if bulk ingestion is
             using too much memory.
+        allow_external_blob_outside_bases: bool, default False
+            Store blob URIs that sit outside registered blob bases. The row
+            keeps a reference, so the object has to stay readable. Local
+            tables only.
 
         Returns
         -------
@@ -1905,14 +1950,23 @@ class Table(ABC):
 
     @abstractmethod
     def add_columns(
-        self, transforms: Dict[str, str] | pa.Field | List[pa.Field] | pa.Schema
+        self,
+        transforms: Dict[str, str | FunctionApplication]
+        | FunctionApplication
+        | pa.Field
+        | List[pa.Field]
+        | pa.Schema
+        | None = None,
+        *,
+        computed: Dict[str, str] | None = None,
     ):
         """
         Add new columns with defined values.
 
         Parameters
         ----------
-        transforms: Dict[str, str], pa.Field, List[pa.Field], pa.Schema
+        transforms: Dict[str, str | FunctionApplication], FunctionApplication,
+            pa.Field, List[pa.Field], pa.Schema
             A map of column name to a SQL expression to use to calculate the
             value of the new column. These expressions will be evaluated for
             each row in the table, and can reference existing columns.
@@ -1920,10 +1974,109 @@ class Table(ABC):
             new columns with the specified data types. The new columns will
             be initialized with null values.
 
+            A mapping with one ``FunctionApplication`` value keeps its scalar
+            or named-struct result in the named table column. A bare
+            named-struct application expands its ordered result fields as one
+            atomic binding; aliases come from ``rename(columns=...)``.
+            Function columns are supported only on LanceDB Cloud and
+            Enterprise.
+        computed: Dict[str, str], optional
+            A map of column name to a SQL expression defining the column. The
+            column's type and inputs are derived from the expression, so no
+            data type is supplied.
+
+            Unlike ``transforms``, the expression is stored rather than
+            evaluated now: the column is committed with no values, and rows get
+            them from [`refresh_column`][lancedb.table.Table.refresh_column].
+            Declaring one therefore costs the same on a large table as on an
+            empty one.
+
+            A refresh does not revisit rows it has already filled, so mutating
+            an input leaves the value computed at fill time; recomputing means
+            dropping the column and declaring it again. While a declaration
+            reads a column, that column cannot be renamed, retyped or dropped.
+
+            On LanceDB Cloud and Enterprise the expression is planned by the
+            server, and the refresh runs as a server job -- see
+            [`refresh_column_async`][lancedb.table.Table.refresh_column_async].
+            Cannot be combined with ``transforms``.
+
         Returns
         -------
         AddColumnsResult
             version: the new version number of the table after adding columns.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> db = lancedb.connect("./.lancedb")
+        >>> table = db.create_table("computed_demo", [{"x": 1}, {"x": 2}])
+        >>> table.add_columns(computed={"doubled": "x * 2"})
+        AddColumnsResult(version=2)
+        >>> table.refresh_column("doubled")
+        RefreshColumnResult(rows_filled=2, version=3)
+        >>> table.to_arrow().sort_by("x").to_pandas()
+           x  doubled
+        0  1        2
+        1  2        4
+        """
+
+    @abstractmethod
+    def refresh_column(self, column: str) -> "RefreshColumnResult":
+        """
+        Fill the rows of a computed column that hold no value yet.
+
+        Declared with ``add_columns(computed=...)``, a column starts empty and
+        gets its values here. Rows appended since the last refresh are filled
+        by the next one; rows already filled are left as they are, so the call
+        is idempotent and does not observe a mutated input.
+
+        Local tables only: a remote refresh runs as a server job, through
+        [`refresh_column_async`][lancedb.table.Table.refresh_column_async].
+
+        Parameters
+        ----------
+        column: str
+            The name of the computed column to fill.
+
+        Returns
+        -------
+        RefreshColumnResult
+            rows_filled: the number of rows given a value.
+            version: the new version number of the table.
+        """
+
+    @abstractmethod
+    def refresh_column_async(self, column: str) -> Job[RefreshColumnJobResult]:
+        """
+        Like :meth:`refresh_column`, but returns a handle to the refresh job
+        instead of blocking until it completes.
+
+        The job may already be complete when returned; callers must not assume
+        the column is filled until :meth:`Job.wait` returns. Invalid input --
+        an unknown column, or one that is not computed -- raises here rather
+        than failing the job. On local tables the job runs in-process; on
+        LanceDB Cloud and Enterprise it is the server's backfill job.
+
+        Returns
+        -------
+        Job[RefreshColumnResult]
+            A job whose successful ``wait`` returns row counts plus the source
+            and published table versions.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> db = lancedb.connect("./.lancedb")
+        >>> table = db.create_table("computed_job_demo", [{"x": 1}, {"x": 2}])
+        >>> table.add_columns(computed={"doubled": "x * 2"})
+        AddColumnsResult(version=2)
+        >>> job = table.refresh_column_async("doubled")
+        >>> result = job.wait()
+        >>> result.rows_assigned
+        2
+        >>> job.status()
+        'finished'
         """
 
     @abstractmethod
@@ -2569,6 +2722,9 @@ class LanceTable(Table):
         2. Currently we've disabled push-down of the filters from polars
            because polars pushdown into pyarrow uses pyarrow compute
            expressions rather than SQl strings (which LanceDB supports)
+        3. The Polars streaming engine is not supported because it does not
+           currently implement Python PyArrow dataset scans. Use the default
+           engine when collecting this LazyFrame.
 
         Returns
         -------
@@ -2577,8 +2733,12 @@ class LanceTable(Table):
         from lancedb.integrations.pyarrow import PyarrowDatasetAdapter
 
         dataset = PyarrowDatasetAdapter(self)
-        return pl.scan_pyarrow_dataset(
-            dataset, allow_pyarrow_filter=False, batch_size=batch_size
+        # Polars 1.32's non-PyArrow callback path passes batch_size twice.  Keep
+        # the compatible PyArrow path, but block predicates because this adapter
+        # cannot translate PyArrow expressions into LanceDB filters.
+        return pl.scan_pyarrow_dataset(dataset, batch_size=batch_size).map_batches(
+            _polars_predicate_pushdown_barrier,
+            predicate_pushdown=False,
         )
 
     # New unified API overload
@@ -3254,6 +3414,7 @@ class LanceTable(Table):
         fill_value: float = 0.0,
         progress: Optional[Union[bool, Callable, Any]] = None,
         write_parallelism: Optional[int] = None,
+        allow_external_blob_outside_bases: bool = False,
     ) -> AddResult:
         """Add data to the table.
         If vector columns are missing and the table
@@ -3281,6 +3442,9 @@ class LanceTable(Table):
             data in flight. Defaults to an estimate based on the data size,
             capped at the number of CPU cores. Lower this if bulk ingestion is
             using too much memory.
+        allow_external_blob_outside_bases: bool, default False
+            Allow blob URIs outside registered bases. See :meth:`Table.add`.
+            Local tables only.
 
         Returns
         -------
@@ -3297,6 +3461,7 @@ class LanceTable(Table):
                     fill_value=fill_value,
                     progress=progress,
                     write_parallelism=write_parallelism,
+                    allow_external_blob_outside_bases=allow_external_blob_outside_bases,
                 )
             )
         finally:
@@ -3921,9 +4086,29 @@ class LanceTable(Table):
         return LOOP.run(self._table.index_stats(index_name))
 
     def add_columns(
-        self, transforms: Dict[str, str] | pa.field | List[pa.field] | pa.Schema
+        self,
+        transforms: Dict[str, str | FunctionApplication]
+        | FunctionApplication
+        | pa.Field
+        | List[pa.Field]
+        | pa.Schema
+        | None = None,
+        *,
+        computed: Dict[str, str] | None = None,
     ) -> AddColumnsResult:
-        return LOOP.run(self._table.add_columns(transforms))
+        return LOOP.run(self._table.add_columns(transforms, computed=computed))
+
+    def refresh_column(self, column: str) -> "RefreshColumnResult":
+        """Fill a computed column's unfilled rows. See
+        [`AsyncTable.refresh_column`][lancedb.AsyncTable.refresh_column]."""
+        return LOOP.run(self._table.refresh_column(column))
+
+    def refresh_column_async(self, column: str) -> Job[RefreshColumnJobResult]:
+        """Fill a computed column's unfilled rows, returning a handle to the
+        refresh job. See
+        [`Table.refresh_column_async`][lancedb.table.Table.refresh_column_async].
+        """
+        return Job(LOOP.run(self._table.refresh_column_async(column)))
 
     def alter_columns(
         self, *alterations: Iterable[Dict[str, str]]
@@ -3958,6 +4143,28 @@ class LanceTable(Table):
         [`AsyncTable.get_lsm_write_spec`][lancedb.AsyncTable.get_lsm_write_spec]."""
         return LOOP.run(self._table.get_lsm_write_spec())
 
+    def checkpoint_lsm(self) -> None:
+        """Synchronous version of
+        [`AsyncTable.checkpoint_lsm`][lancedb.AsyncTable.checkpoint_lsm]."""
+        return LOOP.run(self._table.checkpoint_lsm())
+
+    def flush_lsm(self) -> None:
+        """Synchronous version of
+        [`AsyncTable.flush_lsm`][lancedb.AsyncTable.flush_lsm]."""
+        return LOOP.run(self._table.flush_lsm())
+
+    def compact_lsm(self) -> None:
+        """Synchronous version of
+        [`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm]."""
+        return LOOP.run(self._table.compact_lsm())
+
+    def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]:
+        """Synchronous version of
+        [`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats]."""
+        return LOOP.run(
+            self._table.get_lsm_stats(include_generation_rows=include_generation_rows)
+        )
+
     def close_lsm_writers(self) -> None:
         """Close cached MemWAL shard writers. See
         [`AsyncTable.close_lsm_writers`][lancedb.AsyncTable.close_lsm_writers]."""
@@ -4636,6 +4843,13 @@ class AsyncTable:
         via [`set_unenforced_primary_key`]; bucket sharding additionally
         requires it to be the single column being bucketed.
 
+        By default the MemWAL maintains every index on the table, resolved
+        here — a snapshot, so an index created afterwards needs the spec unset
+        and set again. This fails if one cannot be maintained; name the set
+        with ``with_maintained_indexes`` to install anyway. That pins an exact
+        set (a still-building index is rejected, not omitted); ``[]`` maintains
+        none.
+
         Parameters
         ----------
         spec : LsmWriteSpec
@@ -4643,7 +4857,7 @@ class AsyncTable:
 
         Examples
         --------
-        >>> from lancedb._lancedb import LsmWriteSpec
+        >>> from lancedb import LsmWriteSpec
         >>> # table.set_unenforced_primary_key("id")
         >>> # table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 16))
         """
@@ -4662,12 +4876,73 @@ class AsyncTable:
 
         Returns ``None`` when the MemWAL LSM write path is not enabled (no
         spec has been set, or it was removed with `unset_lsm_write_spec`).
-        The returned spec — including its ``maintained_indexes`` and
-        ``writer_config_defaults`` — mirrors what was passed to
-        `set_lsm_write_spec`.
+        The returned spec mirrors what was passed to `set_lsm_write_spec`,
+        except that ``maintained_indexes`` always reports the concrete list
+        resolved when the spec was set — ``None`` never round-trips.
         """
         return await self._inner.get_lsm_write_spec()
 
+    async def checkpoint_lsm(self) -> None:
+        """Converge this table's LSM write path into its base table.
+
+        One flush, sealing every memtable into L0, then compaction triggers
+        until every generation that existed at that moment has reached base.
+        The loop runs client-side, reading progress from ``get_lsm_stats``.
+
+        Best-effort: generations created *while* it runs are deliberately not
+        waited on, which is what lets it terminate on a table taking writes.
+        Idempotent and safe on a cadence.
+
+        There is no deadline, and the caller owns that. It returns when the
+        target generations are gone, raises on a terminal server fault, and
+        otherwise waits however long the server takes. A slow table and a
+        stuck one are the same picture from the client: the compactor pool is
+        shared across every table on the node, so a checkpoint queued behind
+        unrelated work looks exactly like one that is merging. Wrap this in
+        ``asyncio.wait_for`` for a wall-clock bound; abandoning it partway
+        costs nothing.
+        """
+        await self._inner.checkpoint_lsm()
+
+    async def flush_lsm(self) -> None:
+        """Seal every bucket's active memtable into L0.
+
+        Does not touch the base table — moving L0 into base is
+        `compact_lsm`. On a node that has not claimed this table, this claims
+        it and replays its WAL log first.
+        """
+        await self._inner.flush_lsm()
+
+    async def compact_lsm(self) -> None:
+        """Trigger a background L0 to base compaction pass per bucket.
+
+        Returns once the passes are dispatched, not once they finish: watch
+        ``get_lsm_stats`` for progress, or use ``checkpoint_lsm`` to loop
+        until the current L0 has reached base.
+        """
+        await self._inner.compact_lsm()
+
+    async def get_lsm_stats(
+        self, *, include_generation_rows: bool = False
+    ) -> Optional[dict]:
+        """Read live per-bucket LSM state.
+
+        Answers "how far behind is my fresh tier", "which bucket is hot", and
+        "why is my fresh-tier vector search brute-force". Mutates no table
+        state, though on a node that has not claimed this table it claims it,
+        exactly as a read would.
+
+        Returns ``None`` only when the LSM write path is not enabled.
+
+        Parameters
+        ----------
+        include_generation_rows
+            Report a row count per L0 generation. Off by default: each count
+            opens an uncached Lance dataset, and ``checkpoint_lsm`` polls this
+            needing only generation numbers.
+        """
+        return await self._inner.get_lsm_stats(include_generation_rows)
+
     async def close_lsm_writers(self) -> None:
         """Drain and close any cached MemWAL shard writers for this table.
 
@@ -5100,6 +5375,7 @@ class AsyncTable:
         fill_value: Optional[float] = None,
         progress: Optional[Union[bool, Callable, Any]] = None,
         write_parallelism: Optional[int] = None,
+        allow_external_blob_outside_bases: bool = False,
     ) -> AddResult:
         """Add more data to the [AsyncTable][lancedb.table.AsyncTable].
 
@@ -5130,6 +5406,9 @@ class AsyncTable:
             data in flight. Defaults to an estimate based on the data size,
             capped at the number of CPU cores. Lower this if bulk ingestion is
             using too much memory.
+        allow_external_blob_outside_bases: bool, default False
+            Allow blob URIs outside registered bases. See :meth:`Table.add`.
+            Local tables only.
 
         """
         schema = await self.schema()
@@ -5166,6 +5445,7 @@ class AsyncTable:
                 mode or "append",
                 progress=progress,
                 write_parallelism=write_parallelism,
+                allow_external_blob_outside_bases=allow_external_blob_outside_bases,
             )
         except RuntimeError as e:
             if "Cast error" in str(e):
@@ -5748,37 +6028,166 @@ class AsyncTable:
         return await self._inner.update(updates_sql, where)
 
     async def add_columns(
-        self, transforms: dict[str, str] | pa.field | List[pa.field] | pa.Schema
+        self,
+        transforms: dict[str, str | FunctionApplication]
+        | FunctionApplication
+        | pa.Field
+        | List[pa.Field]
+        | pa.Schema
+        | None = None,
+        *,
+        computed: dict[str, str] | None = None,
     ) -> AddColumnsResult:
         """
         Add new columns with defined values.
 
         Parameters
         ----------
-        transforms: Dict[str, str]
+        transforms: Dict[str, str | FunctionApplication] or FunctionApplication
             A map of column name to a SQL expression to use to calculate the
             value of the new column. These expressions will be evaluated for
             each row in the table, and can reference existing columns.
             Alternatively, you can pass a pyarrow field or schema to add
             new columns with NULLs.
 
+            A mapping with one ``FunctionApplication`` value keeps its scalar
+            or named-struct result in the named table column. A bare
+            named-struct application expands its ordered result fields as one
+            atomic binding; aliases come from ``rename(columns=...)``.
+            Function columns are supported only on LanceDB Cloud and
+            Enterprise.
+        computed: Dict[str, str], optional
+            A map of column name to a SQL expression defining the column. The
+            column's type and inputs are derived from the expression.
+
+            Unlike ``transforms``, the expression is stored rather than
+            evaluated now: the column is committed with no values, and rows get
+            them from
+            [`refresh_column`][lancedb.table.AsyncTable.refresh_column].
+
+            A refresh does not revisit rows it has already filled, so mutating
+            an input leaves the value computed at fill time. While a
+            declaration reads a column, that column cannot be renamed, retyped
+            or dropped.
+
+            On LanceDB Cloud and Enterprise the expression is planned by
+            the server. Cannot be combined with ``transforms``.
+
         Returns
         -------
         AddColumnsResult
             version: the new version number of the table after adding columns.
 
         """
+        function_application = None
+        function_output_name = None
+        if isinstance(transforms, FunctionApplication):
+            function_application = transforms
+        elif isinstance(transforms, dict) and any(
+            isinstance(value, FunctionApplication) for value in transforms.values()
+        ):
+            if len(transforms) != 1 or not all(
+                isinstance(value, FunctionApplication) for value in transforms.values()
+            ):
+                raise ValueError(
+                    "one add_columns call declares exactly one Function binding"
+                )
+            function_output_name, function_application = next(iter(transforms.items()))
+
+        if function_application is not None:
+            if computed:
+                raise ValueError(
+                    "add_columns cannot mix a Function application with SQL "
+                    "computed columns"
+                )
+            function_application._ensure_declarable()
+            return await self._inner.add_function_columns(
+                function_application.to_canonical_json(), function_output_name
+            )
+
         if isinstance(transforms, pa.Field):
             transforms = [transforms]
         if isinstance(transforms, list) and all(
             {isinstance(f, pa.Field) for f in transforms}
         ):
             transforms = pa.schema(transforms)
+        if computed:
+            if transforms:
+                raise ValueError(
+                    "add_columns cannot take both transforms and computed columns"
+                )
+            return await self._inner.add_computed_columns(list(computed.items()))
+        if transforms is None:
+            raise ValueError("add_columns requires transforms or computed columns")
         if isinstance(transforms, pa.Schema):
             return await self._inner.add_columns_with_schema(transforms)
         else:
             return await self._inner.add_columns(list(transforms.items()))
 
+    async def refresh_column(self, column: str) -> RefreshColumnResult:
+        """
+        Fill the rows of a computed column that hold no value yet.
+
+        Declared with ``add_columns(computed=...)``, a column starts empty and
+        gets its values here. Rows appended since the last refresh are filled
+        by the next one; rows already filled are left as they are, so the call
+        is idempotent and does not observe a mutated input.
+
+        Local tables only: a remote refresh runs as a server job, through
+        [`refresh_column_async`][lancedb.table.Table.refresh_column_async].
+
+        Parameters
+        ----------
+        column: str
+            The name of the computed column to fill.
+
+        Returns
+        -------
+        RefreshColumnResult
+            The number of rows filled and the new version of the table.
+        """
+        return await self._inner.refresh_column(column)
+
+    async def refresh_column_async(
+        self, column: str
+    ) -> AsyncJob[RefreshColumnJobResult]:
+        """
+        Like :meth:`refresh_column`, but returns a handle to the refresh job
+        instead of blocking until it completes.
+
+        The job may already be complete when returned; callers must not assume
+        the column is filled until :meth:`AsyncJob.wait` resolves. Invalid
+        input -- an unknown column, or one that is not computed -- raises here
+        rather than failing the job. On local tables the job runs
+        in-process; on LanceDB Cloud and Enterprise it is the server's
+        backfill job.
+
+        Returns
+        -------
+        AsyncJob[RefreshColumnResult]
+            A job whose successful ``wait`` returns row counts plus the source
+            and published table versions.
+
+        Examples
+        --------
+        >>> import asyncio
+        >>> import lancedb
+        >>> async def refresh_in_background():
+        ...     db = await lancedb.connect_async("./.lancedb")
+        ...     table = await db.create_table("computed_job_async_demo", [{"x": 1}])
+        ...     await table.add_columns(computed={"doubled": "x * 2"})
+        ...     job = await table.refresh_column_async("doubled")
+        ...     result = await job.wait()
+        ...     assert result.rows_assigned == 1
+        ...     return await job.status()
+        >>> asyncio.run(refresh_in_background())
+        'finished'
+        """
+        return _typed_job(
+            await self._inner.refresh_column_async(column),
+            RefreshColumnJobResult.from_json,
+        )
+
     async def alter_columns(
         self, *alterations: Iterable[dict[str, Any]]
     ) -> AlterColumnsResult:
@@ -6233,7 +6642,9 @@ class TableStatistics:
     Attributes
     ----------
     total_bytes: int
-        The total number of bytes in the table.
+        The total size, in bytes, of the table's data files, index files, and
+        overlay files. Read from the manifest, so this excludes deletion files
+        and manifests.
     num_rows: int
         The total number of rows in the table.
     num_indices: int
@@ -6428,21 +6839,21 @@ class Branches:
         """Diff a branch against main."""
         return LOOP.run(self._table.branches.diff(from_branch))
 
-    def merge(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]:
-        """Merge a branch into main, or dry-run.
+    def cherry_pick(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]:
+        """Cherry-pick a branch onto main, or dry-run.
 
         Parameters
         ----------
         from_branch: str
-            Branch to merge from.
+            Branch to cherry-pick from.
         dry_run: bool, default False
-            When True, only preview. When False, attempt the merge.
+            When True, only preview. When False, attempt the cherry-pick.
 
         Notes
         -----
-        A rejected merge returns ``status="rejected"`` instead of raising.
+        A failed cherry-pick returns ``status="failed"`` instead of raising.
         """
-        return LOOP.run(self._table.branches.merge(from_branch, dry_run))
+        return LOOP.run(self._table.branches.cherry_pick(from_branch, dry_run))
 
     def _wrap(
         self, async_table: "AsyncTable", version: Optional[int] = None
@@ -6578,9 +6989,11 @@ class AsyncBranches:
         """Diff a branch against main."""
         return await self._table.branches.diff(from_branch)
 
-    async def merge(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]:
-        """Merge a branch into main, or dry-run.
+    async def cherry_pick(
+        self, from_branch: str, dry_run: bool = False
+    ) -> Dict[str, Any]:
+        """Cherry-pick a branch onto main, or dry-run.
 
-        A rejected merge returns ``status="rejected"`` instead of raising.
+        A failed cherry-pick returns ``status="failed"`` instead of raising.
         """
-        return await self._table.branches.merge(from_branch, dry_run)
+        return await self._table.branches.cherry_pick(from_branch, dry_run)
diff --git a/python/python/lancedb/util.py b/python/python/lancedb/util.py
index f582be7b4..dbc52bff6 100644
--- a/python/python/lancedb/util.py
+++ b/python/python/lancedb/util.py
@@ -395,6 +395,11 @@ def _(value: dict):
     )
 
 
+@value_to_sql.register(pa.Scalar)
+def _(value: pa.Scalar):
+    return value_to_sql(value.as_py())
+
+
 @value_to_sql.register(np.ndarray)
 def _(value: np.ndarray):
     return value_to_sql(value.tolist())
diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py
index b205c20a6..5d7682f24 100644
--- a/python/python/tests/test_blob.py
+++ b/python/python/tests/test_blob.py
@@ -617,3 +617,71 @@ def test_fetch_blobs_nested_path_survives_sort_after_query():
 def _identifiable_payload(size: int) -> bytes:
     block = 256
     return b"".join(bytes([i % 256]) * block for i in range(size // block))
+
+
+def _external_uri_blob_array(uris):
+    blob_type = lancedb.blob("image").type
+    storage_type = blob_type.storage_type
+    child_names = [field.name for field in storage_type]
+    assert "uri" in child_names, "blob layout no longer has a uri child"
+    children = [
+        pa.array(uris if field.name == "uri" else [None] * len(uris), type=field.type)
+        for field in storage_type
+    ]
+    storage = pa.StructArray.from_arrays(children, fields=list(storage_type))
+    return pa.ExtensionArray.from_storage(blob_type, storage)
+
+
+def _external_uri_table_and_rows(name, uris):
+    db = lancedb.connect("memory:///")
+    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
+    table = db.create_table(name, schema=schema)
+    rows = pa.Table.from_arrays(
+        [
+            pa.array(range(len(uris)), type=pa.int64()),
+            _external_uri_blob_array(uris),
+        ],
+        schema=schema,
+    )
+    return table, rows
+
+
+def test_add_external_uri_struct_round_trips_with_flag(tmp_path):
+    payload = b"external-uri-bytes"
+    blob_path = tmp_path / "payload.bin"
+    blob_path.write_bytes(payload)
+
+    table, rows = _external_uri_table_and_rows("external_struct", [blob_path.as_uri()])
+    table.add(rows, allow_external_blob_outside_bases=True)
+
+    hits = table.search().to_arrow()
+    blobs = table.fetch_blobs("image", hits)
+    assert blobs[0].as_py() == payload
+
+
+def test_add_external_uri_without_flag_raises(tmp_path):
+    blob_path = tmp_path / "payload.bin"
+    blob_path.write_bytes(b"unreachable")
+
+    table, rows = _external_uri_table_and_rows("external_no_flag", [blob_path.as_uri()])
+    with pytest.raises(ValueError, match="allow_external_blob_outside_bases"):
+        table.add(rows)
+    assert table.count_rows() == 0
+
+
+def test_add_external_uri_string_round_trips_with_flag(tmp_path):
+    payload = b"external-uri-bytes"
+    blob_path = tmp_path / "payload.bin"
+    blob_path.write_bytes(payload)
+
+    db = lancedb.connect("memory:///")
+    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
+    table = db.create_table("external_string", schema=schema)
+    table.add(
+        [{"id": 1, "image": blob_path.as_uri()}],
+        allow_external_blob_outside_bases=True,
+    )
+
+    hits = table.search().to_arrow()
+    blobs = table.fetch_blobs("image", hits)
+    assert blobs[0].as_py() == payload
diff --git a/python/python/tests/test_db.py b/python/python/tests/test_db.py
index 93b791650..aeb9feeb8 100644
--- a/python/python/tests/test_db.py
+++ b/python/python/tests/test_db.py
@@ -2,9 +2,11 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 
+import inspect
 import re
 import sys
 from datetime import timedelta
+from importlib import resources
 import os
 from types import SimpleNamespace
 
@@ -17,6 +19,10 @@ from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
 from lancedb.pydantic import LanceModel, Vector
 
 
+def test_package_includes_pep_561_marker():
+    assert resources.files(lancedb).joinpath("py.typed").is_file()
+
+
 def test_basic(tmp_path):
     db = lancedb.connect(tmp_path)
 
@@ -62,17 +68,23 @@ def test_basic(tmp_path):
     assert db.open_table("test").name == db["test"].name
 
 
-def test_sync_repr_does_not_use_background_loop(tmp_path, monkeypatch):
+def test_sync_debugger_inspection_does_not_use_background_loop(tmp_path, monkeypatch):
     from lancedb.background_loop import LOOP
 
     db = lancedb.connect(tmp_path)
     table = db.create_table("test", data=[{"id": 1}])
 
     def fail_run(*args, **kwargs):
-        raise AssertionError("repr should not use the Python background loop")
+        raise AssertionError("debugger inspection should not use the background loop")
 
     monkeypatch.setattr(LOOP, "run", fail_run)
 
+    # Debuggers enumerate and evaluate every exposed attribute when expanding a
+    # variable. This must remain safe while their breakpoint suspends LOOP's thread.
+    members = dict(inspect.getmembers(db))
+
+    assert members["uri"] == str(tmp_path)
+    assert members["read_consistency_interval"] is None
     assert repr(db) == f"LanceDBConnection(uri={str(tmp_path)!r})"
     assert repr(table) == f"LanceTable(name='test', _conn={db!r})"
 
@@ -743,8 +755,7 @@ def test_delete_table(tmp_db: lancedb.DBConnection):
     assert tmp_db.table_names() == []
 
 
-@pytest.mark.asyncio
-async def test_delete_table_async(tmp_db: lancedb.DBConnection):
+def test_drop_table_async(tmp_db: lancedb.DBConnection):
     data = pd.DataFrame(
         {
             "vector": [[3.1, 4.1], [5.9, 26.5]],
@@ -760,7 +771,10 @@ async def test_delete_table_async(tmp_db: lancedb.DBConnection):
 
     assert tmp_db.table_names() == ["test"]
 
-    tmp_db.drop_table("test")
+    job = tmp_db.drop_table_async("test")
+    assert job.id is None
+    assert job.status() == "finished"
+    assert job.wait() is None
     assert tmp_db.table_names() == []
 
     tmp_db.create_table("test", data=data)
@@ -769,6 +783,17 @@ async def test_delete_table_async(tmp_db: lancedb.DBConnection):
     tmp_db.drop_table("does_not_exist", ignore_missing=True)
 
 
+@pytest.mark.asyncio
+async def test_drop_table_async_connection(tmp_db_async: lancedb.AsyncConnection):
+    await tmp_db_async.create_table("test", data=pa.table({"id": [1, 2]}))
+
+    job = await tmp_db_async.drop_table_async("test")
+    assert job.id is None
+    assert await job.status() == "finished"
+    assert await job.wait() is None
+    assert await tmp_db_async.table_names() == []
+
+
 def test_drop_database(tmp_db: lancedb.DBConnection):
     data = pd.DataFrame(
         {
diff --git a/python/python/tests/test_elastic_dataloader.py b/python/python/tests/test_elastic_dataloader.py
index 22918082b..da27e5bfc 100644
--- a/python/python/tests/test_elastic_dataloader.py
+++ b/python/python/tests/test_elastic_dataloader.py
@@ -32,15 +32,22 @@ Parameters used throughout:
 
 import dataclasses
 import logging
+import threading
 from unittest.mock import patch
 
 import lancedb
 import pyarrow as pa
 import pytest
+from utils import (
+    MockPermutationServer,
+    assert_server_safe_row_id_requests,
+    mock_remote_table,
+)
 
 torch = pytest.importorskip("torch")
 streaming = pytest.importorskip("lancedb.streaming")
 StreamingDataset = streaming.StreamingDataset
+StreamingDataLoader = streaming.StreamingDataLoader
 
 # ---------------------------------------------------------------------------
 # Dataset parameters
@@ -87,6 +94,27 @@ class FakeWorkerInfo:
     num_workers: int
 
 
+def _collate_with_first_batch_error(samples):
+    ids = [sample["id"] for sample in samples]
+    if ids == [0, 1]:
+        raise ValueError("first batch fails")
+    return ids
+
+
+def _collate_with_first_batch_stop(samples):
+    ids = [sample["id"] for sample in samples]
+    if ids == [0, 1]:
+        raise StopIteration("first batch stopped")
+    return ids
+
+
+def _collate_with_first_batch_interrupt(samples):
+    ids = [sample["id"] for sample in samples]
+    if ids == [0, 1]:
+        raise KeyboardInterrupt("first batch interrupted")
+    return ids
+
+
 # ---------------------------------------------------------------------------
 # Fixtures
 # ---------------------------------------------------------------------------
@@ -1003,6 +1031,565 @@ def test_multi_worker_elastic_det_across_worker_counts(lance_table):
 # ── Resumability with num_workers ─────────────────────────────────────────────
 
 
+def test_streaming_dataloader_commits_only_consumed_worker_batches(tmp_path):
+    """Prefetched worker state is committed only as the trainer receives it."""
+    db = lancedb.connect(tmp_path)
+    table = db.create_table(
+        "worker_commit", pa.table({"id": [1, 2, 3, 4, 10, 20, 30, 40]})
+    )
+    dataset = StreamingDataset(table, num_splits=2, shuffle=False)
+    loader = StreamingDataLoader(
+        dataset,
+        batch_size=2,
+        num_workers=2,
+        multiprocessing_context="spawn",
+        prefetch_factor=4,
+    )
+    iterator = iter(loader)
+    try:
+        first = next(iterator)["id"].tolist()
+
+        assert first == [1, 2]
+        assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [2, 0]
+        with pytest.raises(RuntimeError, match="complete logical step boundary"):
+            dataset.state_dict()
+
+        second = next(iterator)["id"].tolist()
+        assert second == [10, 20]
+        checkpoint = dataset.state_dict()
+        assert checkpoint["samples_consumed_per_split"] == [2, 2]
+        uninterrupted = [batch["id"].tolist() for batch in iterator]
+    finally:
+        iterator._shutdown_workers()
+
+    resumed = StreamingDataset(table, num_splits=2, shuffle=False)
+    resumed.load_state_dict(checkpoint)
+    resumed_loader = StreamingDataLoader(
+        resumed,
+        batch_size=2,
+        num_workers=2,
+        multiprocessing_context="spawn",
+        prefetch_factor=4,
+    )
+    resumed_iterator = iter(resumed_loader)
+    try:
+        remaining = [batch["id"].tolist() for batch in resumed_iterator]
+    finally:
+        resumed_iterator._shutdown_workers()
+    assert remaining == uninterrupted == [[3, 4], [30, 40]]
+
+
+def test_distributed_checkpoint_uses_rank_local_worker_boundary(tmp_path):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("rank_boundary", pa.table({"id": list(range(8))}))
+    dataset = StreamingDataset(
+        table,
+        num_splits=4,
+        shuffle=False,
+        rank=0,
+        world_size=2,
+    )
+    loader = StreamingDataLoader(
+        dataset,
+        batch_size=1,
+        num_workers=2,
+        multiprocessing_context="spawn",
+    )
+    iterator = iter(loader)
+    try:
+        assert next(iterator)["id"].tolist() == [0]
+        assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [
+            1,
+            0,
+            0,
+            0,
+        ]
+        with pytest.raises(RuntimeError, match="complete logical step boundary"):
+            dataset.state_dict()
+
+        assert next(iterator)["id"].tolist() == [2]
+        checkpoint = dataset.state_dict()
+        remaining = [batch["id"].tolist() for batch in iterator]
+    finally:
+        iterator._shutdown_workers()
+
+    assert checkpoint["samples_consumed_per_split"] == [1, 1, 0, 0]
+    assert remaining == [[1], [3]]
+
+
+def test_standard_dataloader_rejects_stale_parent_checkpoint(tmp_path):
+    """A standard DataLoader must not expose prefetched producer progress."""
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("untracked_workers", pa.table({"id": [1, 2, 10, 20]}))
+    dataset = StreamingDataset(table, num_splits=2, shuffle=False)
+    # Merely constructing the checkpoint-aware loader must not authorize a
+    # later plain DataLoader's worker progress.
+    StreamingDataLoader(dataset, batch_size=2, num_workers=0)
+    loader = torch.utils.data.DataLoader(
+        dataset,
+        batch_size=2,
+        num_workers=2,
+        multiprocessing_context="spawn",
+    )
+    iterator = iter(loader)
+    try:
+        assert next(iterator)["id"].tolist() == [1, 2]
+        with pytest.raises(RuntimeError, match="Use StreamingDataLoader"):
+            dataset.state_dict()
+        list(iterator)
+    finally:
+        iterator._shutdown_workers()
+
+
+def test_streaming_dataloader_rejects_persistent_workers(tmp_path):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("persistent_workers", pa.table({"id": [1, 2]}))
+    dataset = StreamingDataset(table, num_splits=2, shuffle=False)
+
+    with pytest.raises(ValueError, match="persistent_workers=True"):
+        StreamingDataLoader(
+            dataset,
+            batch_size=1,
+            num_workers=2,
+            persistent_workers=True,
+        )
+
+
+def test_collate_failure_invalidates_consumer_checkpoint(tmp_path):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table(
+        "collate_failure", pa.table({"id": [0, 1, 2, 3, 100, 101, 102, 103]})
+    )
+    dataset = StreamingDataset(table, num_splits=2, shuffle=False)
+    loader = StreamingDataLoader(
+        dataset,
+        batch_size=2,
+        num_workers=2,
+        multiprocessing_context="spawn",
+        collate_fn=_collate_with_first_batch_error,
+        prefetch_factor=2,
+    )
+    iterator = iter(loader)
+    try:
+        with pytest.raises(ValueError, match="first batch fails"):
+            next(iterator)
+        assert next(iterator) == [100, 101]
+        assert next(iterator) == [2, 3]
+        with pytest.raises(RuntimeError, match="failed before it was returned"):
+            dataset.state_dict()
+        list(iterator)
+    finally:
+        iterator._shutdown_workers()
+
+
+def test_collate_stop_iteration_invalidates_consumer_checkpoint(tmp_path):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("collate_stop", pa.table({"id": list(range(6))}))
+    dataset = StreamingDataset(table, num_splits=1, shuffle=False)
+    loader = StreamingDataLoader(
+        dataset,
+        batch_size=2,
+        num_workers=0,
+        collate_fn=_collate_with_first_batch_stop,
+    )
+    iterator = iter(loader)
+
+    with pytest.raises(RuntimeError, match="collate_fn raised StopIteration"):
+        next(iterator)
+    assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [2]
+    with pytest.raises(RuntimeError, match="failed before it was returned"):
+        dataset.state_dict()
+    assert list(iterator) == [[2, 3], [4, 5]]
+
+
+def test_batch_base_exception_invalidates_consumer_checkpoint(tmp_path):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("collate_interrupt", pa.table({"id": list(range(6))}))
+    dataset = StreamingDataset(table, num_splits=1, shuffle=False)
+    loader = StreamingDataLoader(
+        dataset,
+        batch_size=2,
+        num_workers=0,
+        collate_fn=_collate_with_first_batch_interrupt,
+    )
+    iterator = iter(loader)
+
+    with pytest.raises(KeyboardInterrupt, match="first batch interrupted"):
+        next(iterator)
+    assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [2]
+    with pytest.raises(RuntimeError, match="failed before it was returned"):
+        dataset.state_dict()
+    assert list(iterator) == [[2, 3], [4, 5]]
+
+
+def test_parent_commit_base_exception_invalidates_consumer_checkpoint(tmp_path):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("commit_interrupt", pa.table({"id": list(range(4))}))
+    dataset = StreamingDataset(table, num_splits=1, shuffle=False)
+    loader = StreamingDataLoader(dataset, batch_size=2, num_workers=0)
+    iterator = iter(loader)
+    real_commit = dataset._commit_worker_state
+
+    def interrupt_after_commit(state, *, require_uniform):
+        real_commit(state, require_uniform=require_uniform)
+        raise KeyboardInterrupt("after parent commit")
+
+    with patch.object(
+        dataset, "_commit_worker_state", side_effect=interrupt_after_commit
+    ):
+        with pytest.raises(KeyboardInterrupt, match="after parent commit"):
+            next(iterator)
+
+    assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [2]
+    with pytest.raises(RuntimeError, match="failed before it was returned"):
+        dataset.state_dict()
+
+
+def test_direct_iteration_surfaces_prefetch_failure_before_committing_row(
+    tmp_path, monkeypatch
+):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("prefetch_failure", pa.table({"id": list(range(4))}))
+    release = threading.Event()
+    failed = threading.Event()
+    real_getitems = streaming.Permutation.__getitems__
+
+    def controlled_getitems(permutation, indices):
+        if indices and indices[0] >= 2:
+            assert release.wait(timeout=5)
+            failed.set()
+            raise RuntimeError("later prefetched I/O failed")
+        return real_getitems(permutation, indices)
+
+    class SignalDict(dict):
+        def __setitem__(self, key, value):
+            super().__setitem__(key, value)
+            release.set()
+            assert failed.wait(timeout=5)
+
+    monkeypatch.setattr(streaming.Permutation, "__getitems__", controlled_getitems)
+    dataset = StreamingDataset(
+        table,
+        num_splits=1,
+        shuffle=False,
+        read_batch_size=2,
+        io_queue_depth=2,
+    )
+    dataset._resume_positions = SignalDict()
+    iterator = iter(dataset)
+
+    assert next(iterator)["id"] == 0
+    with pytest.raises(RuntimeError, match="later prefetched I/O failed"):
+        next(iterator)
+
+    checkpoint = dataset.state_dict()
+    assert checkpoint["samples_consumed_per_split"] == [1]
+    assert checkpoint["positions_consumed_per_split"] == [1]
+
+
+@pytest.mark.parametrize("workers", [0, 1, 2])
+def test_streaming_dataloader_rejects_drop_last(tmp_path, workers):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("drop_last", pa.table({"id": [0, 1, 2]}))
+    dataset = StreamingDataset(table, num_splits=1, shuffle=False)
+    worker_options = {"multiprocessing_context": "spawn"} if workers else {}
+
+    with pytest.raises(ValueError, match="drop_last=True"):
+        StreamingDataLoader(
+            dataset,
+            batch_size=2,
+            num_workers=workers,
+            drop_last=True,
+            **worker_options,
+        )
+
+
+def test_streaming_dataloader_owns_one_iterator_until_teardown(tmp_path):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("iterator_owner", pa.table({"id": list(range(4))}))
+    dataset = StreamingDataset(table, num_splits=1, shuffle=False)
+    loader = StreamingDataLoader(
+        dataset,
+        batch_size=2,
+        num_workers=1,
+        multiprocessing_context="spawn",
+    )
+
+    first = iter(loader)
+    try:
+        assert next(first)["id"].tolist() == [0, 1]
+        with pytest.raises(RuntimeError, match="concurrent iteration"):
+            iter(loader)
+    finally:
+        first._shutdown_workers()
+
+    second = iter(loader)
+    try:
+        assert [batch["id"].tolist() for batch in second] == [[2, 3]]
+    except BaseException:
+        second._shutdown_workers()
+        raise
+
+    # Natural exhaustion releases ownership too.
+    third = iter(loader)
+    try:
+        assert list(third) == []
+    finally:
+        third._shutdown_workers()
+
+
+def test_zero_worker_shutdown_closes_inner_iterator_before_release(tmp_path):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("zero_worker_shutdown", pa.table({"id": list(range(6))}))
+    dataset = StreamingDataset(table, num_splits=1, shuffle=False)
+    loader = StreamingDataLoader(dataset, batch_size=2, num_workers=0)
+
+    first = iter(loader)
+    assert next(first)["id"].tolist() == [0, 1]
+    first._shutdown_workers()
+
+    assert dataset._consumer_iterator_active is False
+    assert dataset._raw_batches_ref is None
+    second = iter(loader)
+    try:
+        with pytest.raises(StopIteration):
+            next(first)
+        assert next(second)["id"].tolist() == [2, 3]
+    finally:
+        second._shutdown_workers()
+
+
+def test_direct_and_loader_admission_share_one_atomic_lease(tmp_path, monkeypatch):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("direct_loader_lease", pa.table({"id": list(range(4))}))
+    dataset = StreamingDataset(table, num_splits=1, shuffle=False)
+    loader = StreamingDataLoader(dataset, batch_size=2, num_workers=0)
+    entered = threading.Event()
+    release = threading.Event()
+    direct_result = []
+    direct_error = []
+    contender = []
+    real_resolve = dataset._resolve_my_splits
+
+    def controlled_resolve():
+        if threading.current_thread().name == "direct-start":
+            entered.set()
+            assert release.wait(timeout=5)
+        return real_resolve()
+
+    def advance_direct(iterator):
+        try:
+            direct_result.append(next(iterator)["id"])
+        except BaseException as exc:
+            direct_error.append(exc)
+
+    monkeypatch.setattr(dataset, "_resolve_my_splits", controlled_resolve)
+    direct = iter(dataset)
+    thread = threading.Thread(
+        target=advance_direct, args=(direct,), name="direct-start"
+    )
+    thread.start()
+    assert entered.wait(timeout=5)
+    try:
+        with pytest.raises(RuntimeError, match="concurrent iteration"):
+            contender.append(iter(loader))
+    finally:
+        release.set()
+        thread.join(timeout=5)
+        if contender:
+            contender[0]._shutdown_workers()
+        direct.close()
+
+    assert not thread.is_alive()
+    assert direct_error == []
+    assert direct_result == [0]
+
+
+def test_loader_acquires_before_snapshot_and_cleans_interrupted_acquire(
+    tmp_path, monkeypatch
+):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("lease_snapshot", pa.table({"id": list(range(4))}))
+    dataset = StreamingDataset(table, num_splits=1, shuffle=False)
+    loader = StreamingDataLoader(dataset, batch_size=2, num_workers=0)
+    first = iter(loader)
+    assert next(first)["id"].tolist() == [0, 1]
+
+    entered = threading.Event()
+    release = threading.Event()
+    pending = []
+    pending_errors = []
+    observed_snapshots = []
+    real_acquire = dataset._acquire_consumer_iterator
+    real_snapshot = dataset._checkpoint_snapshot
+
+    def controlled_acquire():
+        if threading.current_thread().name == "stale-start":
+            entered.set()
+            assert release.wait(timeout=5)
+        return real_acquire()
+
+    def recording_snapshot():
+        state = real_snapshot()
+        if threading.current_thread().name == "stale-start":
+            observed_snapshots.append(state["samples_consumed_per_split"])
+        return state
+
+    def create_pending_iterator():
+        try:
+            pending.append(iter(loader))
+        except BaseException as exc:
+            pending_errors.append(exc)
+
+    monkeypatch.setattr(dataset, "_acquire_consumer_iterator", controlled_acquire)
+    monkeypatch.setattr(dataset, "_checkpoint_snapshot", recording_snapshot)
+    thread = threading.Thread(target=create_pending_iterator, name="stale-start")
+    thread.start()
+    assert entered.wait(timeout=5)
+    assert next(first)["id"].tolist() == [2, 3]
+    with pytest.raises(StopIteration):
+        next(first)
+    release.set()
+    thread.join(timeout=5)
+
+    assert not thread.is_alive()
+    assert pending_errors == []
+    assert observed_snapshots == [[4]]
+    assert len(pending) == 1
+    assert list(pending[0]) == []
+    assert dataset.state_dict()["samples_consumed_per_split"] == [4]
+
+    def interrupted_acquire():
+        real_acquire()
+        raise KeyboardInterrupt("after acquire")
+
+    monkeypatch.setattr(dataset, "_acquire_consumer_iterator", interrupted_acquire)
+    with pytest.raises(KeyboardInterrupt, match="after acquire"):
+        iter(loader)
+    assert dataset._consumer_iterator_active is False
+
+
+def test_consumer_iterator_lease_publication_is_atomic(tmp_path, monkeypatch):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("atomic_lease", pa.table({"id": [0, 1]}))
+    dataset = StreamingDataset(table, num_splits=1, shuffle=False)
+    loader = StreamingDataLoader(dataset, batch_size=1, num_workers=0)
+    real_get_ident = streaming.threading.get_ident
+    calls = 0
+
+    def interrupt_during_publication():
+        nonlocal calls
+        calls += 1
+        if calls == 1:
+            raise KeyboardInterrupt("during lease mutation")
+        return real_get_ident()
+
+    monkeypatch.setattr(streaming.threading, "get_ident", interrupt_during_publication)
+    with pytest.raises(KeyboardInterrupt, match="during lease mutation"):
+        iter(loader)
+    monkeypatch.setattr(streaming.threading, "get_ident", real_get_ident)
+
+    assert dataset._consumer_iterator_active is False
+    iterator = iter(loader)
+    try:
+        assert next(iterator)["id"].tolist() == [0]
+    finally:
+        iterator._shutdown_workers()
+
+
+def test_streaming_dataloader_rejects_dataset_iter_override(tmp_path):
+    class CustomizedDataset(StreamingDataset):
+        def __iter__(self):
+            return iter([1000, 1001])
+
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("custom_iteration", pa.table({"id": [0, 1, 2]}))
+    dataset = CustomizedDataset(table, num_splits=1, shuffle=False)
+
+    assert list(dataset) == [1000, 1001]
+    with pytest.raises(TypeError, match="override __iter__"):
+        StreamingDataLoader(
+            dataset,
+            batch_size=2,
+            num_workers=0,
+            collate_fn=list,
+        )
+
+
+def test_interleaved_adapters_do_not_authorize_plain_iteration(tmp_path):
+    db = lancedb.connect(tmp_path)
+    table_a = db.create_table("adapter_a", pa.table({"id": [0, 1]}))
+    table_b = db.create_table("adapter_b", pa.table({"id": [10, 11]}))
+    dataset_a = StreamingDataset(table_a, num_splits=1, shuffle=False)
+    dataset_b = StreamingDataset(table_b, num_splits=1, shuffle=False)
+    initial_state = dataset_a.state_dict()
+
+    owner_a = dataset_a._acquire_consumer_iterator()
+    owner_b = dataset_b._acquire_consumer_iterator()
+    try:
+        iterator_a = iter(streaming._StreamingDatasetAdapter(dataset_a))
+        iterator_b = iter(streaming._StreamingDatasetAdapter(dataset_b))
+        assert next(iterator_a).data["id"] == 0
+        assert next(iterator_b).data["id"] == 10
+        assert [sample.data["id"] for sample in iterator_a] == [1]
+        assert [sample.data["id"] for sample in iterator_b] == [11]
+    finally:
+        dataset_a._release_consumer_iterator(owner_a)
+        dataset_b._release_consumer_iterator(owner_b)
+
+    dataset_a.load_state_dict(initial_state)
+    with patch(
+        "lancedb.streaming.get_worker_info",
+        return_value=FakeWorkerInfo(id=0, num_workers=1),
+    ):
+        plain_iterator = iter(dataset_a)
+        assert next(plain_iterator)["id"] == 0
+        plain_iterator.close()
+
+    assert dataset_a._untracked_worker_iteration[0] == 1
+    with pytest.raises(RuntimeError, match="Use StreamingDataLoader"):
+        dataset_a.state_dict()
+
+
+def test_resume_from_partial_split_cycle_preserves_remaining_order(tmp_path):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("partial_cycle", pa.table({"id": [1, 2, 10, 20]}))
+    dataset = StreamingDataset(table, num_splits=2, shuffle=False)
+    iterator = iter(dataset)
+
+    assert next(iterator)["id"] == 1
+    checkpoint = dataset.state_dict()
+    iterator.close()
+    assert checkpoint["samples_consumed_per_split"] == [1, 0]
+
+    resumed = StreamingDataset(table, num_splits=2, shuffle=False)
+    resumed.load_state_dict(checkpoint)
+    assert [row["id"] for row in resumed] == [10, 2, 20]
+
+
+def test_partial_cycle_resume_preserves_skip_truncation(tmp_path):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table(
+        "partial_skip", pa.table({"id": [0, 1, 2, 3, 100, 101, 102, 103]})
+    )
+    kwargs = dict(
+        num_splits=2,
+        shuffle=False,
+        transform=_failing_transform({1, 2, 3}),
+        on_transform_error="skip",
+    )
+    dataset = StreamingDataset(table, **kwargs)
+    iterator = iter(dataset)
+
+    assert next(iterator)["id"] == 0
+    checkpoint = dataset.state_dict()
+    uninterrupted = [row["id"] for row in iterator]
+
+    resumed = StreamingDataset(table, **kwargs)
+    resumed.load_state_dict(checkpoint)
+    assert [row["id"] for row in resumed] == uninterrupted == [100]
+
+
 def test_multi_worker_resumability_same_topology(lance_table):
     """Checkpoint with num_workers=2, resume with num_workers=2: exact continuation."""
     world_size = 1
@@ -1369,6 +1956,188 @@ def test_transform_parallelism_must_be_positive(lance_table, transform_paralleli
         )
 
 
+# ---------------------------------------------------------------------------
+# Backpressure / transform_queue_depth tests
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("transform_queue_depth", [0, -1])
+def test_transform_queue_depth_must_be_positive(lance_table, transform_queue_depth):
+    """transform_queue_depth=0 or negative must raise ValueError."""
+    with pytest.raises(
+        ValueError, match="transform_queue_depth must be greater than 0"
+    ):
+        StreamingDataset(
+            lance_table,
+            num_splits=NUM_SPLITS,
+            transform_queue_depth=transform_queue_depth,
+        )
+
+
+@pytest.mark.parametrize("transform_queue_depth", [1, 2, 4])
+def test_transform_queue_depth_correctness(lance_table, transform_queue_depth):
+    """With backpressure enabled, every row is still yielded exactly once."""
+    ds = StreamingDataset(
+        lance_table,
+        num_splits=NUM_SPLITS,
+        shuffle_seed=SHUFFLE_SEED,
+        transform_queue_depth=transform_queue_depth,
+        read_batch_size=8,
+    )
+    items = list(ds)
+    assert sorted(item["id"] for item in items) == list(range(NUM_ROWS))
+
+
+def test_transform_queue_depth_matches_no_backpressure(lance_table):
+    """With backpressure enabled the same samples are produced as without it."""
+    ds_unlimited = StreamingDataset(
+        lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED
+    )
+    ds_limited = StreamingDataset(
+        lance_table,
+        num_splits=NUM_SPLITS,
+        shuffle_seed=SHUFFLE_SEED,
+        transform_queue_depth=1,
+    )
+    assert [item["id"] for item in ds_unlimited] == [
+        item["id"] for item in ds_limited
+    ], "transform_queue_depth must not affect the sample ordering or set"
+
+
+def test_transform_queue_depth_bounds_cooked_rows(lance_table):
+    """prefetch_queue_depth stays within transform_queue_depth * read_batch_size
+    per split when observed from the main thread during iteration."""
+    n_splits = 4
+    batch_size = 8
+    cooked_depth = 2
+    # max cooked rows across all 4 splits: 4 * 2 * 8 = 64
+    max_allowed = n_splits * cooked_depth * batch_size
+
+    ds = StreamingDataset(
+        lance_table,
+        num_splits=n_splits,
+        shuffle_seed=SHUFFLE_SEED,
+        transform_queue_depth=cooked_depth,
+        read_batch_size=batch_size,
+        transform_parallelism=1,
+        world_size=1,
+    )
+
+    peak = 0
+    for _ in ds:
+        depth = ds.prefetch_queue_depth
+        if depth > peak:
+            peak = depth
+
+    # The main thread observes depth *after* popping a row, so the peak is at
+    # most max_allowed (one row already popped from the split just served).
+    assert peak <= max_allowed, (
+        f"prefetch_queue_depth peaked at {peak}, expected <= {max_allowed}"
+    )
+
+
+def test_transform_queue_depth_does_not_admit_at_capacity_minus_one(tmp_path):
+    """Admission requires a full read_batch_size of free space, not just one slot.
+
+    The test intercepts ThreadPoolExecutor.submit to make I/O calls execute
+    synchronously on the main thread.  This ensures all raw batches land in
+    raw_batches (via _drain_io) before _try_submit_tx evaluates the admission
+    predicate for the first time.  Without this, the I/O future for batch N+1
+    might still be in io_pending at the capacity-minus-one transition, leaving
+    raw_batches empty and causing _try_submit_tx to skip the admission check
+    entirely — so both the correct and the broken predicate produce depth=0
+    observations and the test cannot distinguish them.
+
+    With all raw batches pre-loaded in raw_batches the 4→3 cooked transition
+    (consuming one row from a full cooked queue) always triggers _try_submit_tx
+    against a non-empty raw_batches.
+
+    With transform_queue_depth=1 and batch_size=4, max_cooked_rows=4.
+    A transform may only be submitted when in_pipeline + batch_size <= 4, i.e.
+    when in_pipeline == 0 (cooked is completely empty).  Under the old broken
+    predicate (in_pipeline >= max_cooked_rows) the second transform would be
+    admitted with cooked containing batch_size-1 rows still unconsumed.
+    """
+    import concurrent.futures as cf
+    from concurrent.futures import ThreadPoolExecutor
+    from unittest.mock import patch
+
+    db = lancedb.connect(tmp_path)
+    batch_size = 4
+    # Four full batches → four transform submissions to observe.
+    table = db.create_table("t", pa.table({"id": list(range(batch_size * 4))}))
+
+    cooked_at_submit: list[int] = []
+
+    original_submit = ThreadPoolExecutor.submit
+
+    def tracking_submit(self, fn, *args, **kwargs):
+        name = getattr(fn, "__name__", "")
+        if name == "_io_call":
+            # Run I/O synchronously on the calling (main) thread and return an
+            # already-completed Future.  _drain_io checks fut.done(), so a
+            # completed Future is moved to raw_batches immediately on the next
+            # _advance call — making raw-batch readiness deterministic at the
+            # capacity-minus-one transition instead of depending on I/O thread
+            # scheduling.
+            fut = cf.Future()
+            try:
+                fut.set_result(fn(*args, **kwargs))
+            except Exception as exc:
+                fut.set_exception(exc)
+            return fut
+        if name == "_tx_call_guarded":
+            # Capture cooked depth synchronously on the main thread before the
+            # transform worker can drain the queue.
+            ref = ds._cooked_ref
+            cooked_at_submit.append(len(ref[0]) if ref is not None else -1)
+        return original_submit(self, fn, *args, **kwargs)
+
+    with patch.object(ThreadPoolExecutor, "submit", tracking_submit):
+        ds = StreamingDataset(
+            table,
+            num_splits=1,
+            shuffle_seed=42,
+            read_batch_size=batch_size,
+            transform_queue_depth=1,
+            transform_parallelism=1,
+        )
+        list(ds)
+
+    assert len(cooked_at_submit) == 4, (
+        f"Expected 4 transform submissions (one per batch), got {len(cooked_at_submit)}"
+    )
+    # With full-batch backpressure each transform is only admitted when the
+    # cooked queue is completely empty (depth == 0).  The old broken predicate
+    # would admit at depth == batch_size - 1 == 3.
+    assert all(depth == 0 for depth in cooked_at_submit), (
+        "Transform admitted with non-empty cooked queue; full-batch backpressure "
+        "requires in_pipeline + batch_size <= max_cooked_rows before admission. "
+        f"Cooked depths at each submission: {cooked_at_submit}"
+    )
+
+
+# ---------------------------------------------------------------------------
+# Deprecated parameter name tests
+# ---------------------------------------------------------------------------
+
+
+def test_prefetch_batches_deprecated_warns(lance_table, caplog):
+    """prefetch_batches logs a deprecation warning and behaves like io_queue_depth."""
+    with caplog.at_level(logging.WARNING, logger="lancedb.streaming"):
+        ds = StreamingDataset(
+            lance_table,
+            num_splits=NUM_SPLITS,
+            shuffle_seed=SHUFFLE_SEED,
+            prefetch_batches=2,
+        )
+    messages = [r.message for r in caplog.records if r.levelno >= logging.WARNING]
+    assert any("deprecated" in m.lower() and "io_queue_depth" in m for m in messages), (
+        f"Expected deprecation warning mentioning io_queue_depth; got: {messages}"
+    )
+    assert sorted(item["id"] for item in ds) == list(range(NUM_ROWS))
+
+
 def test_filter_limits_rows(tmp_path):
     """A filter expression is applied to the permutation so only matching rows
     are yielded.  IDs 0..59 pass ``id < 60``; the other 60 are excluded."""
@@ -1456,6 +2225,425 @@ def test_shuffle_clump_size_yields_all_rows(lance_table):
     )
 
 
+# ---------------------------------------------------------------------------
+# on_transform_error tests
+# ---------------------------------------------------------------------------
+
+
+class BadRowError(ValueError):
+    """Raised by the failing transforms below when a batch contains a bad id."""
+
+
+def _failing_transform(bad_ids: set):
+    """A transform that raises BadRowError whenever the batch has a bad id.
+
+    Raises on the full batch and on any single-row slice containing a bad id,
+    so per-row isolation drops exactly the bad rows.
+    """
+
+    def transform(batch: pa.RecordBatch) -> list:
+        ids = batch.column("id").to_pylist()
+        bad = sorted(set(ids) & bad_ids)
+        if bad:
+            raise BadRowError(f"bad ids in batch: {bad}")
+        return [{"id": i} for i in ids]
+
+    return transform
+
+
+def _sequential_split_members(table) -> list[list[int]]:
+    """Return each split's ids in yield order for shuffle=False.
+
+    With a single rank and no workers the round-robin yields one row per split
+    per cycle, so item k of a clean run belongs to split k % NUM_SPLITS.
+    """
+    ds = StreamingDataset(table, num_splits=NUM_SPLITS, shuffle=False)
+    members: list[list[int]] = [[] for _ in range(NUM_SPLITS)]
+    for k, row in enumerate(ds):
+        members[k % NUM_SPLITS].append(row["id"])
+    return members
+
+
+def test_on_transform_error_default_raises(lance_table):
+    """By default a transform exception propagates and aborts iteration."""
+    ds = StreamingDataset(
+        lance_table,
+        num_splits=NUM_SPLITS,
+        shuffle_seed=SHUFFLE_SEED,
+        transform=_failing_transform({7}),
+    )
+    with pytest.raises(BadRowError):
+        list(ds)
+
+
+def test_on_transform_error_invalid_value(lance_table):
+    with pytest.raises(ValueError, match="on_transform_error"):
+        StreamingDataset(lance_table, num_splits=NUM_SPLITS, on_transform_error="bogus")
+
+
+def test_on_transform_error_skip_drops_bad_rows(lance_table):
+    """With one bad row per split, 'skip' yields every good row exactly once
+    and counts the dropped rows in rows_skipped."""
+    members = _sequential_split_members(lance_table)
+    bad_ids = {members[i][4] for i in range(NUM_SPLITS)}
+
+    ds = StreamingDataset(
+        lance_table,
+        num_splits=NUM_SPLITS,
+        shuffle=False,
+        transform=_failing_transform(bad_ids),
+        on_transform_error="skip",
+    )
+    assert ds.rows_skipped == 0
+
+    ids = [row["id"] for row in ds]
+
+    assert sorted(ids) == sorted(set(range(NUM_ROWS)) - bad_ids)
+    assert ds.rows_skipped == NUM_SPLITS
+
+
+def test_on_transform_error_skip_uneven_ends_at_last_complete_cycle(lance_table):
+    """When one split loses more rows than the others, the epoch ends at the
+    last cycle where every split still has a row — no crash, no bad rows, and
+    every step remains one sample per split."""
+    members = _sequential_split_members(lance_table)
+    bad_ids = set(members[0][:3])  # all 3 bad rows in split 0
+
+    ds = StreamingDataset(
+        lance_table,
+        num_splits=NUM_SPLITS,
+        shuffle=False,
+        transform=_failing_transform(bad_ids),
+        on_transform_error="skip",
+    )
+    items = [row["id"] for row in ds]
+
+    rows_per_split = NUM_ROWS // NUM_SPLITS
+    expected_cycles = rows_per_split - len(bad_ids)
+    assert len(items) == expected_cycles * NUM_SPLITS
+    assert len(set(items)) == len(items), "duplicate samples yielded"
+    assert not set(items) & bad_ids, "a bad row was yielded"
+    # Split 0 contributed exactly its surviving rows, in order, one per cycle.
+    survivors = [i for i in members[0] if i not in bad_ids]
+    assert items[0::NUM_SPLITS] == survivors[:expected_cycles]
+
+
+def test_on_transform_error_warn_logs(lance_table, caplog):
+    """'warn' skips like 'skip' but logs a warning for the failing batch."""
+    members = _sequential_split_members(lance_table)
+    bad_ids = {members[i][3] for i in range(NUM_SPLITS)}
+
+    ds = StreamingDataset(
+        lance_table,
+        num_splits=NUM_SPLITS,
+        shuffle=False,
+        transform=_failing_transform(bad_ids),
+        on_transform_error="warn",
+    )
+    with caplog.at_level(logging.WARNING, logger="lancedb.streaming"):
+        items = list(ds)
+
+    assert len(items) == NUM_ROWS - NUM_SPLITS
+    assert ds.rows_skipped == NUM_SPLITS
+    assert "Skipped" in caplog.text
+    assert "BadRowError" in caplog.text
+
+
+def test_on_transform_error_callable_selective(lance_table):
+    """A callable handler can skip expected errors and re-raise the rest."""
+    members = _sequential_split_members(lance_table)
+    bad_ids = {members[i][0] for i in range(NUM_SPLITS)}
+
+    handled: list[Exception] = []
+
+    def handler(exc: Exception) -> bool:
+        handled.append(exc)
+        return isinstance(exc, BadRowError)
+
+    ds = StreamingDataset(
+        lance_table,
+        num_splits=NUM_SPLITS,
+        shuffle=False,
+        transform=_failing_transform(bad_ids),
+        on_transform_error=handler,
+    )
+    items = list(ds)
+    assert len(items) == NUM_ROWS - NUM_SPLITS
+    assert handled and all(isinstance(exc, BadRowError) for exc in handled)
+
+    def broken_transform(batch: pa.RecordBatch) -> list:
+        raise TypeError("boom")
+
+    ds2 = StreamingDataset(
+        lance_table,
+        num_splits=NUM_SPLITS,
+        shuffle=False,
+        transform=broken_transform,
+        on_transform_error=handler,
+    )
+    with pytest.raises(TypeError, match="boom"):
+        list(ds2)
+
+
+def test_transform_wrong_row_count_raises(lance_table):
+    """A transform that returns the wrong number of rows is an error even with
+    on_transform_error='skip' — silent shrinkage would corrupt accounting."""
+
+    def drops_rows(batch: pa.RecordBatch) -> list:
+        return batch.column("id").to_pylist()[:-1]
+
+    ds = StreamingDataset(
+        lance_table,
+        num_splits=NUM_SPLITS,
+        shuffle_seed=SHUFFLE_SEED,
+        transform=drops_rows,
+        on_transform_error="skip",
+    )
+    with pytest.raises(ValueError, match="one output row per input row"):
+        list(ds)
+
+
+def test_skip_deterministic_across_runs(lance_table):
+    """With a fixed seed, skipping produces the identical sample sequence on
+    every run — skips are data-dependent, not run-dependent."""
+    bad_ids = {5, 17, 46}
+
+    def run() -> tuple[list[int], int]:
+        ds = StreamingDataset(
+            lance_table,
+            num_splits=NUM_SPLITS,
+            shuffle_seed=SHUFFLE_SEED,
+            transform=_failing_transform(bad_ids),
+            on_transform_error="skip",
+        )
+        return [row["id"] for row in ds], ds.rows_skipped
+
+    ids_a, skipped_a = run()
+    ids_b, skipped_b = run()
+    assert ids_a == ids_b
+    assert skipped_a == skipped_b
+    assert not set(ids_a) & bad_ids
+
+
+def test_skip_elastic_det_across_world_sizes(lance_table):
+    """With equal bad-row counts per split, skipping preserves the full
+    elastic-determinism guarantee: identical global batches at every step for
+    every compatible world_size."""
+    members = _sequential_split_members(lance_table)
+    bad_ids = {members[i][6] for i in range(NUM_SPLITS)}
+
+    def collect(world_size: int) -> list[frozenset[int]]:
+        micro = GLOBAL_BATCH_SIZE // world_size
+        iters = [
+            iter(
+                StreamingDataset(
+                    lance_table,
+                    num_splits=NUM_SPLITS,
+                    shuffle=False,
+                    rank=rank,
+                    world_size=world_size,
+                    transform=_failing_transform(bad_ids),
+                    on_transform_error="skip",
+                )
+            )
+            for rank in range(world_size)
+        ]
+        _STOP = object()
+        batches: list[frozenset[int]] = []
+        while True:
+            step_samples: set[int] = set()
+            exhausted = 0
+            for it in iters:
+                for _ in range(micro):
+                    val = next(it, _STOP)
+                    if val is _STOP:
+                        exhausted += 1
+                        break
+                    step_samples.add(val["id"])
+            if exhausted == len(iters):
+                break
+            assert exhausted == 0, (
+                "Rank iterators exhausted at different steps despite equal "
+                "bad-row counts per split"
+            )
+            batches.append(frozenset(step_samples))
+        return batches
+
+    reference = collect(1)
+    assert len(reference) == NUM_ROWS // NUM_SPLITS - 1
+    for ws in (2, 3, 4):
+        assert collect(ws) == reference, f"world_size={ws} diverged"
+
+
+def test_resumability_with_skips_same_topology(lance_table):
+    """Checkpointing mid-epoch with skipped rows resumes exactly: no sample
+    repeated, no sample lost, skipped rows stay skipped."""
+    members = _sequential_split_members(lance_table)
+    # Uneven skips: positions diverge across splits (2 bad in split 0, 1 in
+    # split 5), which only a position-based checkpoint can resume exactly.
+    bad_ids = {members[0][2], members[0][3], members[5][7]}
+    kwargs = dict(
+        num_splits=NUM_SPLITS,
+        shuffle=False,
+        transform=_failing_transform(bad_ids),
+        on_transform_error="skip",
+    )
+
+    reference = [row["id"] for row in StreamingDataset(lance_table, **kwargs)]
+    rows_per_split = NUM_ROWS // NUM_SPLITS
+    assert len(reference) == (rows_per_split - 2) * NUM_SPLITS
+
+    steps = 3
+    ds = StreamingDataset(lance_table, **kwargs)
+    it = iter(ds)
+    consumed = [next(it)["id"] for _ in range(steps * NUM_SPLITS)]
+    checkpoint = ds.state_dict()
+    it.close()
+
+    # Split 0 skipped positions 2 and 3 within its first 3 yields; split 5's
+    # bad row is beyond the checkpoint.  Everything else is at 3 = the sample
+    # count.
+    positions = checkpoint["positions_consumed_per_split"]
+    assert positions[0] == 5
+    assert positions[1:] == [3] * (NUM_SPLITS - 1)
+    assert checkpoint["samples_consumed_per_split"] == [3] * NUM_SPLITS
+
+    ds2 = StreamingDataset(lance_table, **kwargs)
+    ds2.load_state_dict(checkpoint)
+    resumed = [row["id"] for row in ds2]
+
+    assert consumed == reference[: steps * NUM_SPLITS]
+    assert resumed == reference[steps * NUM_SPLITS :]
+
+
+def test_resumability_with_skips_elastic_merge(lance_table):
+    """Elastic resume with skips: each rank's checkpoint knows exact positions
+    only for its own splits; merge_state_dicts recovers the global state, and
+    a run on a different world_size continues exactly."""
+    members = _sequential_split_members(lance_table)
+    # Bad rows early in split 0 (rank 0) and split 6 (rank 1 of a ws=2 run) so
+    # both ranks' position vectors diverge before the checkpoint.
+    bad_ids = {members[0][0], members[0][2], members[6][1]}
+    kwargs = dict(
+        num_splits=NUM_SPLITS,
+        shuffle=False,
+        transform=_failing_transform(bad_ids),
+        on_transform_error="skip",
+    )
+
+    reference = [row["id"] for row in StreamingDataset(lance_table, **kwargs)]
+
+    steps = 3
+    world_size = 2
+    micro = GLOBAL_BATCH_SIZE // world_size
+    datasets = [
+        StreamingDataset(lance_table, rank=rank, world_size=world_size, **kwargs)
+        for rank in range(world_size)
+    ]
+    iters = [iter(ds) for ds in datasets]
+    seen: list[frozenset[int]] = []
+    for _ in range(steps):
+        step_samples = set()
+        for it in iters:
+            for _ in range(micro):
+                step_samples.add(next(it)["id"])
+        seen.append(frozenset(step_samples))
+    states = [ds.state_dict() for ds in datasets]
+    for it in iters:
+        it.close()
+
+    merged = StreamingDataset.merge_state_dicts(states)
+    expected_positions = [3] * NUM_SPLITS
+    expected_positions[0] = 5  # skipped positions 0 and 2
+    expected_positions[6] = 4  # skipped position 1
+    assert merged["positions_consumed_per_split"] == expected_positions
+
+    # The first 3 global batches match the world_size=1 reference.
+    ref_batches = [
+        frozenset(reference[s * NUM_SPLITS : (s + 1) * NUM_SPLITS])
+        for s in range(len(reference) // NUM_SPLITS)
+    ]
+    assert seen == ref_batches[:steps]
+
+    # Resume on world_size=1 from the merged state.
+    ds_resume = StreamingDataset(lance_table, **kwargs)
+    ds_resume.load_state_dict(merged)
+    resumed = [row["id"] for row in ds_resume]
+    assert resumed == reference[steps * NUM_SPLITS :]
+
+
+def test_rows_skipped_flushed_when_split_entirely_bad(lance_table):
+    """A split whose rows all fail never completes a cycle, so the epoch ends
+    immediately — but rows_skipped must still report the drops after the
+    iterator exits (the shared-memory counter is flushed on exhaustion)."""
+    members = _sequential_split_members(lance_table)
+    bad_ids = set(members[0])  # every row of split 0 is bad
+
+    ds = StreamingDataset(
+        lance_table,
+        num_splits=NUM_SPLITS,
+        shuffle=False,
+        transform=_failing_transform(bad_ids),
+        on_transform_error="skip",
+    )
+    assert list(ds) == []
+    assert ds.rows_skipped == len(bad_ids)
+
+
+def test_merge_state_dicts_validates_consistency(lance_table):
+    ds = StreamingDataset(lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED)
+    state = ds.state_dict()
+    other = dict(state, shuffle_seed=SHUFFLE_SEED + 1)
+    with pytest.raises(ValueError, match="shuffle_seed mismatch"):
+        StreamingDataset.merge_state_dicts([state, other])
+    with pytest.raises(ValueError, match="at least one"):
+        StreamingDataset.merge_state_dicts([])
+
+
+def test_merge_state_dicts_combines_nonuniform_consumer_progress(lance_table):
+    dataset = StreamingDataset(
+        lance_table, num_splits=2, shuffle=False, shuffle_seed=SHUFFLE_SEED
+    )
+    rank0 = dataset.state_dict()
+    rank0["samples_consumed_per_split"] = [2, 0]
+    rank0["positions_consumed_per_split"] = [2, 0]
+    rank1 = dataset.state_dict()
+    rank1["samples_consumed_per_split"] = [0, 2]
+    rank1["positions_consumed_per_split"] = [0, 2]
+
+    merged = StreamingDataset.merge_state_dicts([rank0, rank1])
+
+    assert merged["samples_consumed_per_split"] == [2, 2]
+    assert merged["positions_consumed_per_split"] == [2, 2]
+
+
+def test_load_state_dict_without_positions_key(lance_table):
+    """Checkpoints from before positions_consumed_per_split existed still
+    resume exactly (positions equal sample counts when nothing is skipped)."""
+    reference = [
+        row["id"]
+        for row in StreamingDataset(
+            lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED
+        )
+    ]
+
+    steps = 4
+    ds = StreamingDataset(lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED)
+    it = iter(ds)
+    for _ in range(steps * NUM_SPLITS):
+        next(it)
+    checkpoint = ds.state_dict()
+    it.close()
+    del checkpoint["positions_consumed_per_split"]
+
+    ds2 = StreamingDataset(
+        lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED
+    )
+    ds2.load_state_dict(checkpoint)
+    resumed = [row["id"] for row in ds2]
+    assert resumed == reference[steps * NUM_SPLITS :]
+
+
 def test_num_splits_defaults_to_world_size(lance_table):
     """Omitting num_splits gives world_size splits (one per rank)."""
     ds = StreamingDataset(
@@ -1524,6 +2712,273 @@ def test_shuffle_seed_none_generates_stable_seed(lance_table):
     assert first == second, "Same resolved seed must produce the same ordering"
 
 
+# Sequence packing tests
+
+
+def _create_token_table(tmp_path, documents):
+    db = lancedb.connect(tmp_path)
+    tokens = pa.array(documents, type=pa.list_(pa.int64()))
+    return db.create_table("tokens", pa.table({"tokens": tokens}))
+
+
+def _packed_dataset(table, pack_sequences, *, blocks_per_epoch, pad_id=0, **kwargs):
+    return StreamingDataset(
+        table,
+        shuffle=False,
+        columns=["tokens"],
+        pack_sequences=pack_sequences,
+        eos_id=9,
+        pad_id=pad_id,
+        blocks_per_epoch=blocks_per_epoch,
+        **kwargs,
+    )
+
+
+def test_pack_sequences_emits_blocks_and_pads_final_tail(tmp_path):
+    table = _create_token_table(tmp_path, [[1, 2], [3, 4], [5]])
+    dataset = _packed_dataset(table, 6, blocks_per_epoch=2)
+
+    blocks = list(dataset)
+
+    assert len(blocks) == 2
+    assert blocks[0]["input_ids"].tolist() == [1, 2, 9, 3, 4, 9]
+    assert blocks[0]["doc_ids"].tolist() == [0, 0, 0, 1, 1, 1]
+    assert blocks[1]["input_ids"].tolist() == [5, 9, 0, 0, 0, 0]
+    assert blocks[1]["doc_ids"].tolist() == [0, 0, 0, 0, 0, 0]
+    assert blocks[0]["input_ids"].dtype == torch.int64
+    assert blocks[0]["doc_ids"].dtype == torch.int64
+
+
+def test_pack_sequences_pads_lagging_splits(tmp_path):
+    table = _create_token_table(
+        tmp_path,
+        [[1], [2], [10, 11, 12, 13, 14, 15, 16, 17], [20]],
+    )
+    dataset = _packed_dataset(table, 5, blocks_per_epoch=6, num_splits=2)
+    input_ids = [block["input_ids"].tolist() for block in dataset]
+    # Split 0 has four real tokens including EOS markers, while split 1 has
+    # eleven. Packing must emit three complete two-split cycles.
+    assert input_ids == [
+        [1, 9, 2, 9, 0],
+        [10, 11, 12, 13, 14],
+        [0, 0, 0, 0, 0],
+        [15, 16, 17, 9, 20],
+        [0, 0, 0, 0, 0],
+        [9, 0, 0, 0, 0],
+    ]
+
+    per_rank = []
+    for rank in range(2):
+        rank_dataset = _packed_dataset(
+            table,
+            5,
+            blocks_per_epoch=6,
+            num_splits=2,
+            world_size=2,
+            rank=rank,
+        )
+        per_rank.append([block["input_ids"].tolist() for block in rank_dataset])
+
+    assert [len(blocks) for blocks in per_rank] == [3, 3]
+    sharded = [block for cycle in zip(*per_rank) for block in cycle]
+    assert sharded == input_ids
+
+
+def test_pack_sequences_auto_estimates_filtered_token_column(tmp_path):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table(
+        "tokens",
+        pa.table(
+            {
+                "tokens": pa.array([[1] * 4, [2] * 9], type=pa.list_(pa.int64())),
+                "keep": [True, False],
+            }
+        ),
+    )
+    table.add(
+        pa.table(
+            {
+                "tokens": pa.array([[3] * 4, [4] * 9], type=pa.list_(pa.int64())),
+                "keep": [True, False],
+            }
+        )
+    )
+
+    with pytest.warns(UserWarning, match="approximate token-count sample"):
+        dataset = _packed_dataset(
+            table,
+            5,
+            blocks_per_epoch="auto",
+            num_splits=2,
+            filter="keep",
+        )
+
+    # Two kept documents contain 8 tokens plus 2 EOS tokens: two blocks.
+    assert dataset.state_dict()["blocks_per_epoch"] == 2
+
+
+def test_pack_sequences_checkpoint_resumes_on_new_topology(tmp_path):
+    table = _create_token_table(
+        tmp_path,
+        [[1], [2], [10, 11, 12, 13, 14, 15, 16, 17], [20]],
+    )
+    kwargs = dict(pack_sequences=5, blocks_per_epoch=6, num_splits=2)
+    reference = list(_packed_dataset(table, **kwargs))
+
+    datasets = [
+        _packed_dataset(table, world_size=2, rank=rank, **kwargs) for rank in range(2)
+    ]
+    iterators = [iter(dataset) for dataset in datasets]
+    first_cycle = [next(iterator) for iterator in iterators]
+    checkpoint = StreamingDataset.merge_state_dicts(
+        [dataset.state_dict() for dataset in datasets]
+    )
+    for iterator in iterators:
+        iterator.close()
+
+    resumed = _packed_dataset(table, **kwargs)
+    resumed.load_state_dict(checkpoint)
+    actual_remaining = list(resumed)
+
+    assert [block["input_ids"].tolist() for block in first_cycle] == [
+        [1, 9, 2, 9, 0],
+        [10, 11, 12, 13, 14],
+    ]
+    assert checkpoint["blocks_emitted_per_split"] == [1, 1]
+    assert [block["input_ids"].tolist() for block in actual_remaining] == [
+        block["input_ids"].tolist() for block in reference[2:]
+    ]
+    assert [block["doc_ids"].tolist() for block in actual_remaining] == [
+        block["doc_ids"].tolist() for block in reference[2:]
+    ]
+
+
+def test_packed_checkpoint_requires_complete_split_cycle(tmp_path):
+    table = _create_token_table(tmp_path, [[1], [2], [10], [20]])
+    dataset = _packed_dataset(table, pack_sequences=3, blocks_per_epoch=4, num_splits=2)
+    iterator = iter(dataset)
+
+    next(iterator)
+    with pytest.raises(RuntimeError, match="complete logical step boundary"):
+        dataset.state_dict()
+
+    next(iterator)
+    assert dataset.state_dict()["blocks_emitted_per_split"] == [1, 1]
+    iterator.close()
+
+
+def test_streaming_dataloader_commits_consumed_packed_batches(tmp_path):
+    table = _create_token_table(
+        tmp_path,
+        [[1], [2], [3], [4], [10], [20], [30], [40]],
+    )
+    kwargs = dict(pack_sequences=4, blocks_per_epoch=4, num_splits=2)
+    dataset = _packed_dataset(table, **kwargs)
+    loader = StreamingDataLoader(
+        dataset,
+        batch_size=1,
+        num_workers=2,
+        multiprocessing_context="spawn",
+        prefetch_factor=2,
+    )
+    iterator = iter(loader)
+    try:
+        next(iterator)
+        with pytest.raises(RuntimeError, match="complete logical step boundary"):
+            dataset.state_dict()
+
+        next(iterator)
+        checkpoint = dataset.state_dict()
+        uninterrupted = [batch["input_ids"].tolist() for batch in iterator]
+    finally:
+        iterator._shutdown_workers()
+
+    resumed = _packed_dataset(table, **kwargs)
+    resumed.load_state_dict(checkpoint)
+    resumed_loader = StreamingDataLoader(
+        resumed,
+        batch_size=1,
+        num_workers=2,
+        multiprocessing_context="spawn",
+        prefetch_factor=2,
+    )
+    resumed_iterator = iter(resumed_loader)
+    try:
+        remaining = [batch["input_ids"].tolist() for batch in resumed_iterator]
+    finally:
+        resumed_iterator._shutdown_workers()
+
+    assert checkpoint["blocks_emitted_per_split"] == [1, 1]
+    assert remaining == uninterrupted
+
+
+def test_pack_sequences_validates_configuration_and_tokens(tmp_path):
+    table = _create_token_table(tmp_path, [[1, 2]])
+
+    with pytest.raises(ValueError, match="pad_id is required"):
+        StreamingDataset(
+            table,
+            shuffle=False,
+            columns=["tokens"],
+            pack_sequences=4,
+            eos_id=9,
+        )
+
+    with pytest.raises(ValueError, match="blocks_per_epoch is required"):
+        StreamingDataset(
+            table,
+            shuffle=False,
+            columns=["tokens"],
+            pack_sequences=4,
+            eos_id=9,
+            pad_id=0,
+        )
+
+    with pytest.raises(ValueError, match="must be divisible"):
+        _packed_dataset(table, 4, blocks_per_epoch=3, num_splits=2)
+
+    with pytest.raises(ValueError, match="positive integer or 'auto'"):
+        _packed_dataset(table, 4, blocks_per_epoch="estimate")
+
+    checkpoint = _packed_dataset(table, 4, blocks_per_epoch=1).state_dict()
+    resumed = _packed_dataset(table, 4, blocks_per_epoch=1, pad_id=8)
+    with pytest.raises(ValueError, match="pad_id mismatch"):
+        resumed.load_state_dict(checkpoint)
+
+    float_db = lancedb.connect(tmp_path / "float")
+    float_table = float_db.create_table(
+        "tokens",
+        pa.table({"tokens": pa.array([[1.5, 2.5]], type=pa.list_(pa.float64()))}),
+    )
+    with pytest.raises(ValueError, match="token column with integer values"):
+        _packed_dataset(float_table, 4, blocks_per_epoch=1)
+
+    null_db = lancedb.connect(tmp_path / "null")
+    null_table = null_db.create_table(
+        "tokens",
+        pa.table({"tokens": pa.array([None], type=pa.list_(pa.int64()))}),
+    )
+    with pytest.raises(ValueError, match="does not support null token lists"):
+        list(_packed_dataset(null_table, 4, blocks_per_epoch=1))
+
+    null_value_db = lancedb.connect(tmp_path / "null_value")
+    null_value_table = null_value_db.create_table(
+        "tokens",
+        pa.table(
+            {"tokens": pa.array([[1], [2, None], [3]], type=pa.list_(pa.int64()))}
+        ),
+    )
+    blocks = list(
+        _packed_dataset(
+            null_value_table,
+            2,
+            blocks_per_epoch=2,
+            on_transform_error="skip",
+        )
+    )
+    assert [block["input_ids"].tolist() for block in blocks] == [[1, 9], [3, 9]]
+
+
 # ---------------------------------------------------------------------------
 # Doc examples — each test mirrors the code snippet in index.mdx so that
 # broken doc examples are caught before they ship.
@@ -1547,7 +3002,7 @@ def test_doc_example_basic(tmp_path):
 
 
 def test_doc_example_prefetch_params(tmp_path):
-    """doc: Prefetching — read_batch_size and prefetch_batches still cover all rows."""
+    """doc: Prefetching — read_batch_size and io_queue_depth still cover all rows."""
     db = lancedb.connect(tmp_path)
     table = db.create_table("t", pa.table({"id": list(range(NUM_ROWS))}))
 
@@ -1556,7 +3011,7 @@ def test_doc_example_prefetch_params(tmp_path):
         num_splits=NUM_SPLITS,
         shuffle_seed=SHUFFLE_SEED,
         read_batch_size=8,
-        prefetch_batches=2,
+        io_queue_depth=2,
     )
     assert sorted(s["id"] for s in ds) == list(range(NUM_ROWS))
 
@@ -1716,3 +3171,27 @@ def test_doc_example_checkpoint(lance_table):
     assert sorted(consumed + remaining_original) == list(range(NUM_ROWS)), (
         "Consumed + remaining must cover every row exactly once"
     )
+
+
+# ---------------------------------------------------------------------------
+# Remote tables (LanceDB Cloud / Enterprise)
+# ---------------------------------------------------------------------------
+
+
+def test_streaming_dataset_over_remote_table():
+    """StreamingDataset reads a remote table, with server-safe requests.
+
+    Builds a permutation over a remote table, then fetches batches from it by row id.
+    """
+    server = MockPermutationServer()
+
+    with mock_remote_table(server) as table:
+        ds = StreamingDataset(table, num_splits=2, shuffle_seed=SHUFFLE_SEED)
+        ids = [row["id"] for row in ds]
+
+    assert sorted(ids) == list(range(server.num_rows)), (
+        "Every row of the remote table must be yielded exactly once"
+    )
+    assert len(server.scans) == 1, "the permutation is built with one row-id scan"
+    assert server.takes, "rows must be fetched with row-id takes"
+    assert_server_safe_row_id_requests(server)
diff --git a/python/python/tests/test_embeddings.py b/python/python/tests/test_embeddings.py
index 5efb7d98a..9850669eb 100644
--- a/python/python/tests/test_embeddings.py
+++ b/python/python/tests/test_embeddings.py
@@ -64,6 +64,23 @@ def test_embedding_function(tmp_path):
     assert np.allclose(actual, expected)
 
 
+def test_instructor_ndims_uses_instruction():
+    instructor = get_registry().get("instructor").create()
+    model = MagicMock()
+    model.encode.return_value = np.zeros((1, 384))
+
+    with patch.object(type(instructor), "get_model", return_value=model):
+        assert instructor.ndims() == 384
+
+    model.encode.assert_called_once_with(
+        [[instructor.source_instruction, "foo"]],
+        batch_size=instructor.batch_size,
+        show_progress_bar=instructor.show_progress_bar,
+        normalize_embeddings=instructor.normalize_embeddings,
+        device=instructor.device,
+    )
+
+
 def test_embedding_function_variables():
     @register("variable-testing")
     class VariableTestingFunction(TextEmbeddingFunction):
@@ -115,34 +132,16 @@ def test_embedding_function_variables():
     assert func.safe_model_dump()["secret_key"] == "$var:secret"
 
 
-def test_parse_functions_with_variables():
-    @register("variable-parsing-test")
-    class VariableParsingFunction(TextEmbeddingFunction):
-        api_key: str
-        base_url: Optional[str] = None
-
-        @staticmethod
-        def sensitive_keys():
-            return ["api_key"]
-
-        def ndims(self):
-            return 10
-
-        def generate_embeddings(self, texts):
-            # Mock implementation that just returns random embeddings
-            # In real usage, this would use the api_key to call an API
-            return [np.random.rand(self.ndims()).tolist() for _ in texts]
-
+def test_openai_variables_survive_metadata_round_trip():
     registry = EmbeddingFunctionRegistry.get_instance()
 
     registry.set_var("test_api_key", "sk-test-key-12345")
-    registry.set_var("test_base_url", "https://api.example.com")
 
     conf = EmbeddingFunctionConfig(
         source_column="text",
         vector_column="vector",
-        function=registry.get("variable-parsing-test").create(
-            api_key="$var:test_api_key", base_url="$var:test_base_url"
+        function=registry.get("openai").create(
+            api_key="$var:test_api_key", base_url="https://api.example.com"
         ),
     )
 
@@ -150,7 +149,10 @@ def test_parse_functions_with_variables():
 
     # Create a mock arrow table with the metadata
     schema = pa.schema(
-        [pa.field("text", pa.string()), pa.field("vector", pa.list_(pa.float32(), 10))]
+        [
+            pa.field("text", pa.string()),
+            pa.field("vector", pa.list_(pa.float32(), 1536)),
+        ]
     )
     table = pa.table({"text": [], "vector": []}, schema=schema)
     table = table.replace_schema_metadata(metadata)
@@ -164,13 +166,15 @@ def test_parse_functions_with_variables():
 
     assert parsed_func.api_key == "sk-test-key-12345"
     assert parsed_func.base_url == "https://api.example.com"
-
-    embeddings = parsed_func.generate_embeddings(["test text"])
-    assert len(embeddings) == 1
-    assert len(embeddings[0]) == 10
-
     assert parsed_func.safe_model_dump()["api_key"] == "$var:test_api_key"
 
+    with patch("lancedb.embeddings.openai.attempt_import_or_raise") as import_openai:
+        parsed_func._openai_client
+
+    import_openai.return_value.OpenAI.assert_called_once_with(
+        api_key="sk-test-key-12345", base_url="https://api.example.com"
+    )
+
 
 def test_embedding_with_bad_results(tmp_path):
     @register("null-embedding")
@@ -627,3 +631,23 @@ def test_url_retrieve_downloads_image():
     image_bytes = url_retrieve(image_url)
     img = Image.open(io.BytesIO(image_bytes))
     assert img.size[0] > 0 and img.size[1] > 0
+
+
+def test_jina_generate_image_input_dict_local_path(tmp_path):
+    """
+    JinaEmbeddings._generate_image_input_dict must accept a local image path
+    (str or Path), not just bytes. Previously it crashed with
+    `AttributeError: 'function' object has no attribute 'urlparse'` on any
+    str/Path input because it called `urlparse.urlparse(image)` instead of
+    `urlparse(image)` (urlparse was imported as a function, not a module).
+    """
+    Image = pytest.importorskip("PIL.Image")
+    from lancedb.embeddings.jinaai import JinaEmbeddings
+
+    image_path = tmp_path / "test.png"
+    Image.new("RGB", (4, 4), color="red").save(image_path, format="PNG")
+
+    for image in (str(image_path), image_path):
+        image_dict = JinaEmbeddings._generate_image_input_dict(image)
+        assert "image" in image_dict
+        assert isinstance(image_dict["image"], str) and len(image_dict["image"]) > 0
diff --git a/python/python/tests/test_expr.py b/python/python/tests/test_expr.py
index 6aa78943e..0eb6f8929 100644
--- a/python/python/tests/test_expr.py
+++ b/python/python/tests/test_expr.py
@@ -632,3 +632,101 @@ class TestExprBytesIntegration:
             .to_arrow()
         )
         assert result.num_rows == 2
+
+
+# ── datetime / timezone integration for lit() (issue #3262) ──────────────────
+
+
+class TestExprDatetimeTimezoneIntegration:
+    """Integration coverage for lit(datetime) against table timestamp columns.
+
+    PyArrow stores naive timestamps as UTC wall-clock microseconds. Python's
+    datetime.timestamp() treats naive values as *local* time, which used to
+    shift lit(naive) by the host UTC offset and break equality filters on
+    non-UTC machines. These cases lock the expected semantics.
+    """
+
+    def test_both_naive_match(self, tmp_path):
+        """Table naive + lit naive with the same wall clock must match."""
+        db = lancedb.connect(str(tmp_path / "naive"))
+        ts = datetime(2024, 7, 1, 10, 0, 0)
+        table = db.create_table(
+            "t", [{"id": 1, "ts": ts}, {"id": 2, "ts": datetime(2024, 7, 2, 10, 0, 0)}]
+        )
+        result = table.search().where(col("ts") == lit(ts)).to_list()
+        assert len(result) == 1
+        assert result[0]["id"] == 1
+
+    def test_both_same_timezone_match(self, tmp_path):
+        """Table UTC + lit UTC for the same instant must match."""
+        db = lancedb.connect(str(tmp_path / "utc"))
+        ts = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc)
+        table = db.create_table(
+            "t",
+            pa.table(
+                {
+                    "id": [1, 2],
+                    "ts": pa.array(
+                        [ts, datetime(2024, 7, 2, 10, 0, 0, tzinfo=timezone.utc)],
+                        type=pa.timestamp("us", tz="UTC"),
+                    ),
+                }
+            ),
+        )
+        result = table.search().where(col("ts") == lit(ts)).to_list()
+        assert len(result) == 1
+        assert result[0]["id"] == 1
+
+    def test_different_timezones_same_instant(self, tmp_path):
+        """UTC table row equals lit of the same instant in a different zone."""
+        db = lancedb.connect(str(tmp_path / "diff_tz"))
+        ts_utc = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc)
+        # Same instant as 06:00 in UTC-4
+        ts_est = datetime(2024, 7, 1, 6, 0, 0, tzinfo=timezone(timedelta(hours=-4)))
+        table = db.create_table(
+            "t",
+            pa.table(
+                {
+                    "id": [1],
+                    "ts": pa.array([ts_utc], type=pa.timestamp("us", tz="UTC")),
+                }
+            ),
+        )
+        result = table.search().where(col("ts") == lit(ts_est)).to_list()
+        assert len(result) == 1
+        assert result[0]["id"] == 1
+
+    def test_table_tz_literal_naive(self, tmp_path):
+        """UTC table + naive lit uses wall-clock equality (10:00 == 10:00 UTC)."""
+        db = lancedb.connect(str(tmp_path / "tz_naive"))
+        ts_utc = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc)
+        ts_naive = datetime(2024, 7, 1, 10, 0, 0)
+        table = db.create_table(
+            "t",
+            pa.table(
+                {
+                    "id": [1],
+                    "ts": pa.array([ts_utc], type=pa.timestamp("us", tz="UTC")),
+                }
+            ),
+        )
+        result = table.search().where(col("ts") == lit(ts_naive)).to_list()
+        assert len(result) == 1
+        assert result[0]["id"] == 1
+
+    def test_table_naive_literal_aware(self, tmp_path):
+        """Naive table + UTC lit with the same wall clock must match."""
+        db = lancedb.connect(str(tmp_path / "naive_aware"))
+        ts_naive = datetime(2024, 7, 1, 10, 0, 0)
+        ts_utc = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc)
+        table = db.create_table("t", [{"id": 1, "ts": ts_naive}])
+        result = table.search().where(col("ts") == lit(ts_utc)).to_list()
+        assert len(result) == 1
+        assert result[0]["id"] == 1
+
+    def test_naive_lit_sql_is_wall_clock_not_local_shifted(self):
+        """Regression: naive lit must not apply the host local UTC offset."""
+        ts = datetime(2024, 7, 1, 10, 0, 0)
+        sql = lit(ts).to_sql()
+        # Must encode 10:00 wall clock, not 10:00+local_offset.
+        assert "2024-07-01 10:00:00" in sql
diff --git a/python/python/tests/test_first_class_function_slice1.py b/python/python/tests/test_first_class_function_slice1.py
new file mode 100644
index 000000000..89172ba3f
--- /dev/null
+++ b/python/python/tests/test_first_class_function_slice1.py
@@ -0,0 +1,358 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import json
+from pathlib import Path
+
+import pytest
+
+from lancedb import col
+import lancedb.functions as functions
+from lancedb.functions import (
+    FunctionApplication,
+    FunctionBinding,
+    FunctionVersion,
+    PythonRuntimeSpec,
+    RefreshColumnResult,
+)
+from lancedb.table import AsyncTable
+
+
+FIXTURES = (
+    Path(__file__).parents[3]
+    / "rust"
+    / "lancedb"
+    / "tests"
+    / "fixtures"
+    / "first_class_functions"
+    / "v1"
+)
+
+
+def fixture(name: str) -> str:
+    return (FIXTURES / name).read_text()
+
+
+def job_result(name: str) -> dict:
+    return json.loads(fixture(name))["result"]
+
+
+def test_public_function_values_are_in_api_reference():
+    docs = Path(__file__).parents[3] / "docs" / "src" / "python" / "python.md"
+    rendered = docs.read_text()
+    for name in functions.__all__:
+        assert f"::: lancedb.functions.{name}" in rendered
+
+
+@pytest.mark.parametrize(
+    ("fixture_name", "canonical_name", "model", "nested_result"),
+    [
+        (
+            "remote_function_job.json",
+            "remote_function_version.canonical.json",
+            FunctionVersion,
+            True,
+        ),
+        (
+            "remote_function_application.json",
+            "remote_function_application.canonical.json",
+            FunctionApplication,
+            False,
+        ),
+        (
+            "remote_function_binding.json",
+            "remote_function_binding.canonical.json",
+            FunctionBinding,
+            False,
+        ),
+        (
+            "remote_refresh_job.json",
+            "remote_refresh_result.canonical.json",
+            RefreshColumnResult,
+            True,
+        ),
+        (
+            "remote_refresh_result_without_published_version.json",
+            "remote_refresh_result_without_published_version.canonical.json",
+            RefreshColumnResult,
+            False,
+        ),
+    ],
+)
+def test_python_and_rust_share_remote_canonical_goldens(
+    fixture_name, canonical_name, model, nested_result
+):
+    value = json.loads(fixture(fixture_name))
+    if nested_result:
+        value = value["result"]
+    decoded = model.from_json(json.dumps(value))
+    assert decoded.to_canonical_json() == fixture(canonical_name).strip()
+
+
+def test_function_version_identity_is_immutable_and_exact():
+    value = job_result("remote_function_job.json")
+    version = FunctionVersion.from_json(json.dumps(value))
+    assert version.name == "embed"
+    assert version.version == "fv_01K3EXACT"
+
+    with pytest.raises((TypeError, ValueError)):
+        version.version = "fv_changed"
+    with pytest.raises(TypeError, match="immutable"):
+        version.runtime.env["TOKENIZERS_PARALLELISM"] = "true"
+
+    changed = dict(value)
+    changed["version"] = "fv_changed"
+    assert FunctionVersion(**changed) != version
+
+
+def test_function_version_binds_named_columns_as_one_immutable_application():
+    version = FunctionVersion.from_json(
+        json.dumps(job_result("remote_function_job.json"))
+    )
+
+    application = version(text=col("documents.body"))
+
+    assert application.function.name == version.name
+    assert application.function.version == version.version
+    assert application.output is version.signature.output
+    assert [
+        (value.parameter, value.kind, value.value["path"])
+        for value in application.inputs
+    ] == [("text", "column", "documents.body")]
+
+
+def test_function_version_binding_validates_names_and_direct_columns():
+    version = FunctionVersion.from_json(
+        json.dumps(job_result("remote_function_job.json"))
+    )
+
+    with pytest.raises(TypeError, match=r"missing inputs: \['text'\]"):
+        version()
+    with pytest.raises(TypeError, match=r"unknown inputs: \['body'\]"):
+        version(text=col("text"), body=col("body"))
+    with pytest.raises(TypeError, match="direct col"):
+        version(text=col("text").lower())
+
+
+def test_function_version_keeps_named_struct_outputs_in_one_application():
+    value = job_result("remote_function_job.json")
+    value["name"] = "text_features"
+    value["version"] = "fv_multi_output"
+    value["signature"] = {
+        "inputs": [
+            {"name": "title", "arrow_type": "utf8", "nullable": True},
+            {"name": "body", "arrow_type": "utf8", "nullable": True},
+        ],
+        "output": {
+            "kind": "named_struct",
+            "fields": [
+                {
+                    "name": "normalized_text",
+                    "arrow_type": "utf8",
+                    "nullable": False,
+                },
+                {
+                    "name": "token_count",
+                    "arrow_type": "int64",
+                    "nullable": False,
+                },
+            ],
+        },
+    }
+    version = FunctionVersion(**value)
+
+    application = version(body=col("body"), title=col("title")).rename(
+        columns={
+            "normalized_text": "search_text",
+            "token_count": "search_token_count",
+        }
+    )
+
+    assert [value.parameter for value in application.inputs] == ["title", "body"]
+    assert [field.name for field in application.output.fields] == [
+        "normalized_text",
+        "token_count",
+    ]
+    assert dict(application.columns) == {
+        "normalized_text": "search_text",
+        "token_count": "search_token_count",
+    }
+
+
+def test_unknown_fields_and_discriminators_are_forward_decodable():
+    value = job_result("remote_function_job.json")
+    value["future_version_metadata"] = {"retention_class": "catalog"}
+    value["runtime"] = {"kind": "wasm", "module_digest": "sha256:wasm"}
+    value["signature"]["output"]["kind"] = "future_output_shape"
+
+    version = FunctionVersion.from_json(json.dumps(value))
+    assert version.runtime.kind == "wasm"
+    assert version.runtime.python_version is None
+    assert version.runtime.environment is None
+    assert json.loads(version.to_canonical_json())["runtime"] == {"kind": "wasm"}
+    assert version.signature.output.kind == "future_output_shape"
+
+
+def test_function_application_uses_rename_columns_only():
+    application = FunctionApplication.from_json(
+        fixture("remote_function_application.json")
+    )
+    renamed = application.rename(columns={"normalized_text": "body_normalized"})
+
+    assert application.columns["normalized_text"] == "search_text"
+    assert renamed.columns["normalized_text"] == "body_normalized"
+    assert renamed.function == application.function
+    assert not hasattr(application, "rename_outputs")
+    with pytest.raises(TypeError, match="immutable"):
+        renamed.columns["normalized_text"] = "changed"
+    with pytest.raises(TypeError, match="immutable"):
+        application.inputs[0].value["path"] = "changed"
+
+    with pytest.raises(ValueError, match="unknown Function result fields"):
+        application.rename(columns={"missing": "search_text"})
+    with pytest.raises(ValueError, match="destinations must be unique"):
+        application.rename(columns={"normalized_text": "same", "token_count": "same"})
+
+    bare_value = json.loads(fixture("remote_function_application.json"))
+    bare_value.pop("columns")
+    bare = FunctionApplication(**bare_value)
+    with pytest.raises(ValueError, match="destinations must be unique"):
+        bare.rename(columns={"normalized_text": "token_count"})
+
+
+def test_binding_and_refresh_result_keep_stable_remote_fields():
+    binding = FunctionBinding.from_json(fixture("remote_function_binding.json"))
+    assert binding.function.version == "fv_01K3TEXT"
+    assert [output.output_ordinal for output in binding.outputs] == [0, 1]
+    assert binding.input_schema is not None
+    assert binding.output_schema is not None
+
+    result = RefreshColumnResult.from_json(
+        json.dumps(job_result("remote_refresh_job.json"))
+    )
+    assert result.rows_filled == result.rows_assigned
+    assert result.version == result.published_version
+
+    result = RefreshColumnResult.from_json(
+        fixture("remote_refresh_result_without_published_version.json")
+    )
+    assert result.published_version is None
+    assert RefreshColumnResult.from_json(result.to_canonical_json()) == result
+
+
+def test_function_literal_numeric_domain_matches_rust():
+    with pytest.raises(ValueError, match="floating-point Function literals"):
+        FunctionApplication.from_json(fixture("remote_function_application_float.json"))
+
+    value = json.loads(fixture("remote_function_application_float.json"))
+    value["inputs"][0]["value"] = 2**64
+    with pytest.raises(ValueError, match="outside the canonical JSON range"):
+        FunctionApplication.from_json(json.dumps(value))
+
+
+def test_empty_default_maps_have_stable_canonical_bytes():
+    runtime = PythonRuntimeSpec(
+        kind="python", python_version="3.12", environment={"kind": "pip"}
+    )
+    assert runtime.to_canonical_json() == (
+        '{"environment":{"kind":"pip"},"kind":"python","python_version":"3.12"}'
+    )
+
+    value = json.loads(fixture("remote_function_application.json"))
+    value.pop("columns")
+    application = FunctionApplication.from_json(json.dumps(value))
+    assert "columns" not in json.loads(application.to_canonical_json())
+
+
+@pytest.mark.parametrize("field", ["rows_assigned", "source_version"])
+def test_refresh_result_rejects_non_u64_values(field):
+    value = job_result("remote_refresh_job.json")
+    value[field] = -1
+    with pytest.raises(ValueError):
+        RefreshColumnResult.from_json(json.dumps(value))
+
+    value[field] = "1"
+    with pytest.raises(ValueError):
+        RefreshColumnResult.from_json(json.dumps(value))
+
+
+class _FunctionDeclarationInner:
+    def __init__(self):
+        self.calls = []
+
+    async def add_function_columns(self, application_json, output_name):
+        self.calls.append((json.loads(application_json), output_name))
+        return "declared"
+
+
+def known_application() -> FunctionApplication:
+    value = json.loads(fixture("remote_function_application.json"))
+    value.pop("future_application")
+    return FunctionApplication(**value)
+
+
+@pytest.mark.asyncio
+async def test_add_columns_routes_struct_as_one_and_multi_output_binding_atomically():
+    inner = _FunctionDeclarationInner()
+    table = AsyncTable(inner)
+    application = known_application()
+
+    result = await table.add_columns(
+        {"features": application._copy(update={"columns": {}})}
+    )
+    assert result == "declared"
+    assert inner.calls[-1][1] == "features"
+
+    bare = application._copy(update={"columns": {}}).rename(
+        columns={"normalized_text": "search_text"}
+    )
+    result = await table.add_columns(bare)
+    assert result == "declared"
+    assert inner.calls[-1][1] is None
+    assert inner.calls[-1][0]["columns"] == {"normalized_text": "search_text"}
+
+
+@pytest.mark.asyncio
+async def test_add_columns_rejects_multiple_bindings_and_unknown_newer_application():
+    inner = _FunctionDeclarationInner()
+    table = AsyncTable(inner)
+    application = known_application()
+
+    with pytest.raises(ValueError, match="exactly one Function binding"):
+        await table.add_columns({"a": application, "b": application})
+
+    future = json.loads(fixture("remote_function_application.json"))
+    application = FunctionApplication(**future)
+    with pytest.raises(ValueError, match="newer contract"):
+        await table.add_columns(application)
+
+    future.pop("future_application")
+    future["output"]["assignment"] = "cell_flag"
+    application = FunctionApplication(**future)
+    assert "assignment" not in json.loads(application.to_canonical_json())["output"]
+    with pytest.raises(ValueError, match="output.assignment"):
+        await table.add_columns(application)
+    assert inner.calls == []
+
+
+def test_rename_requires_named_struct_and_keeps_partial_mapping_immutable():
+    scalar = FunctionApplication.from_json(
+        json.dumps(
+            {
+                "function": {"name": "embed", "version": "fv_exact"},
+                "inputs": [],
+                "output": {
+                    "kind": "scalar",
+                    "arrow_type": "list<float32>",
+                    "nullable": False,
+                },
+            }
+        )
+    )
+    with pytest.raises(ValueError, match="named-struct"):
+        scalar.rename(columns={"value": "embedding"})
+
+    application = known_application()._copy(update={"columns": {}})
+    renamed = application.rename(columns={"normalized_text": "search_text"})
+    assert dict(application.columns) == {}
+    assert dict(renamed.columns) == {"normalized_text": "search_text"}
diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py
new file mode 100644
index 000000000..55257d322
--- /dev/null
+++ b/python/python/tests/test_first_class_function_slice2.py
@@ -0,0 +1,620 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+from __future__ import annotations
+
+import base64
+import contextlib
+import functools
+import importlib.util
+import types
+from datetime import date
+import http.server
+import json
+from pathlib import Path
+import threading
+from typing import Optional
+
+import pyarrow as pa
+import pytest
+
+import lancedb
+from lancedb.functions import UdfDefinition, udf
+
+THRESHOLD = 20
+_CACHE = None
+
+
+FIXTURES = (
+    Path(__file__).parents[3]
+    / "rust"
+    / "lancedb"
+    / "tests"
+    / "fixtures"
+    / "first_class_functions"
+    / "v1"
+)
+
+
+@udf(
+    pip=["numpy>=2"],
+    env={"MODE": "test"},
+    python_version="3.12",
+)
+def normalize_score(value: float) -> float:
+    return value / 100.0
+
+
+def test_scalar_udf_matches_shared_registration_golden_and_remains_callable():
+    assert isinstance(normalize_score, UdfDefinition)
+    assert normalize_score(25.0) == 0.25
+    assert (
+        normalize_score.registration_request.to_canonical_json()
+        == (FIXTURES / "remote_function_registration_request.canonical.json")
+        .read_text()
+        .strip()
+    )
+    request = json.loads(normalize_score.registration_request.to_canonical_json())
+    assert request["artifact"]["adapter"] == {
+        "kind": "scalar_to_arrow_batch",
+        "version": 1,
+    }
+
+
+def _run_packaged(definition, *args):
+    """Execute the shipped artifact in a fresh namespace, as a worker would."""
+    source = base64.b64decode(definition.registration_request.artifact.content.data)
+    namespace: dict = {}
+    exec(compile(source, "<udf>", "exec"), namespace)
+    return namespace[definition.registration_request.artifact.entrypoint](*args)
+
+
+def test_udf_packages_attribute_access_and_body_imports():
+    @udf
+    def word_norm(body: str) -> float:
+        import numpy as np
+
+        try:
+            words = body.split()
+        except AttributeError as error:
+            raise ValueError(str(error)) from error
+        return float(np.linalg.norm([len(w) for w in words]))
+
+    assert _run_packaged(word_norm, "aa bb") == pytest.approx(8**0.5)
+
+
+def test_udf_packages_module_globals_and_global_caches():
+    @udf
+    def label(value: int) -> str:
+        return "big" if value >= THRESHOLD else "small"
+
+    assert _run_packaged(label, 21) == "big"
+
+    @udf
+    def cached(value: int) -> int:
+        global _CACHE
+        if _CACHE is None:
+            _CACHE = 40
+        return _CACHE + value
+
+    assert _run_packaged(cached, 2) == 42
+
+
+def test_udf_annotations_are_not_runtime_names():
+    @udf
+    def identity(value: date) -> date:
+        return value
+
+    assert _run_packaged(identity, date(2026, 8, 25)) == date(2026, 8, 25)
+
+
+def test_udf_nested_scopes_resolve_lexically():
+    @udf
+    def score(value: int) -> int:
+        offset = 2
+
+        def add_offset() -> int:
+            return value + offset
+
+        return add_offset() + sum(v for v in [0])
+
+    assert _run_packaged(score, 3) == 5
+
+
+def test_udf_resolves_module_globals_before_builtins(tmp_path):
+    module_path = tmp_path / "shadowing_udfs.py"
+    module_path.write_text(
+        "max = 7\n"
+        "len = lambda _: 99\n"
+        "\n"
+        "def uses_literal_shadow(value: int) -> int:\n"
+        "    def nested() -> int:\n"
+        "        return max\n"
+        "    return nested() + value\n"
+        "\n"
+        "def uses_callable_shadow(value: int) -> int:\n"
+        "    def nested() -> int:\n"
+        "        return len([1])\n"
+        "    return nested() + value\n"
+    )
+    spec = importlib.util.spec_from_file_location("shadowing_udfs", module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+
+    # The module's `max = 7` is what the interpreter would use, so it ships.
+    assert _run_packaged(udf(module.uses_literal_shadow), 1) == 8
+    # A callable global cannot ship; it must not be silently swapped for the builtin.
+    with pytest.raises(TypeError, match="unsupported global value of type function"):
+        udf(module.uses_callable_shadow)
+
+
+def test_canonical_arrow_type_is_exactly_the_grammar():
+    from lancedb.functions import _GRAMMAR_PRIMITIVES, _canonical_arrow_type
+
+    golden = json.loads(
+        (
+            Path(__file__).parents[3]
+            / "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json"
+        ).read_text()
+    )
+    primitives = [
+        case["arrow_type"] for case in golden["valid"] if "<" not in case["arrow_type"]
+    ]
+    assert [name for _, name in _GRAMMAR_PRIMITIVES] == primitives
+    for outside in [
+        pa.timestamp("us"),
+        pa.decimal128(10, 2),
+        pa.large_string(),
+        pa.large_binary(),
+        pa.binary(4),
+        pa.duration("s"),
+        pa.struct([pa.field("a", pa.int32())]),
+        pa.list_(pa.float32(), 0),
+        pa.list_(pa.timestamp("us")),
+    ]:
+        with pytest.raises(TypeError, match="unsupported Arrow type"):
+            _canonical_arrow_type(outside)
+
+
+def test_udf_nested_annotations_are_postponed_in_the_artifact():
+    @udf
+    def score(value: int) -> int:
+        def identity(item: date) -> date:
+            return item
+
+        identity(date(2026, 8, 25))
+        return value
+
+    assert _run_packaged(score, 3) == 3
+
+
+def test_udf_ships_globals_the_body_deletes():
+    @udf
+    def clear(value: int) -> int:
+        global _CACHE
+        del _CACHE
+        return value
+
+    assert _run_packaged(clear, 3) == 3
+
+
+def test_udf_rejects_a_module_global_that_does_not_import_as_itself(tmp_path):
+    module_path = tmp_path / "fake_module_udfs.py"
+    module_path.write_text(
+        "import types\n"
+        "np = types.ModuleType('numpy')\n"
+        "np.sqrt = lambda x: 0\n"
+        "\n"
+        "def score(value: int) -> int:\n"
+        "    return int(np.sqrt(value))\n"
+    )
+    spec = importlib.util.spec_from_file_location("fake_module_udfs", module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    with pytest.raises(TypeError, match="does not import as 'numpy'"):
+        udf(module.score)
+
+
+def test_udf_rejects_a_module_level_namespace_alias(tmp_path):
+    module_path = tmp_path / "aliasing_udfs.py"
+    module_path.write_text(
+        "import builtins as b\n"
+        "THRESHOLD = 5\n"
+        "\n"
+        "def score(value: int) -> int:\n"
+        "    return value + b.vars(b.__import__('aliasing_udfs'))['THRESHOLD']\n"
+    )
+    spec = importlib.util.spec_from_file_location("aliasing_udfs", module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    with pytest.raises(ValueError, match="dynamic namespace access"):
+        udf(module.score)
+
+
+@pytest.mark.parametrize(
+    "access",
+    [
+        "globals()['THRESHOLD']",
+        "eval('THRESHOLD')",
+        "(lambda g: g()['THRESHOLD'])(globals)",
+        "__import__('sys').modules[__name__].THRESHOLD",
+        "sys.modules[__name__].THRESHOLD",
+    ],
+)
+def test_udf_rejects_dynamic_namespace_access(access):
+    namespace: dict = {}
+    exec(
+        f"def score(value: int) -> int:\n    return value + {access}\n",
+        {"THRESHOLD": 5},
+        namespace,
+    )
+    with pytest.raises(ValueError, match="dynamic namespace access"):
+        _package_from_text(
+            "def score(value: int) -> int:\n"
+            "    import sys\n"
+            f"    return value + {access}\n"
+        )
+
+
+def _package_from_text(source: str, module_globals: dict | None = None):
+    """Load `source` as a real module file so the packager can inspect it."""
+    import tempfile
+
+    directory = tempfile.mkdtemp()
+    path = Path(directory) / "generated_udf_module.py"
+    path.write_text(source)
+    spec = importlib.util.spec_from_file_location(f"generated_udf_{id(source)}", path)
+    module = importlib.util.module_from_spec(spec)
+    if module_globals:
+        module.__dict__.update(module_globals)
+    spec.loader.exec_module(module)
+    functions = [
+        value
+        for value in vars(module).values()
+        if callable(value) and getattr(value, "__module__", None) == module.__name__
+    ]
+    return udf(functions[0])
+
+
+def test_udf_rejects_a_non_standard_builtins_environment():
+    def score(value: int) -> int:
+        return len([1]) + value
+
+    score.__globals__  # noqa: B018 -- real function, real globals
+    import builtins
+
+    patched = types.FunctionType(
+        score.__code__,
+        {"__builtins__": {**vars(builtins), "len": lambda _: 99}},
+        "score",
+    )
+    patched.__annotations__ = score.__annotations__
+    assert patched(3) == 102
+    with pytest.raises(ValueError, match="non-standard builtins environment"):
+        udf(patched)
+
+    class ReportingDict(dict):  # reports standard entries, resolves differently
+        def __missing__(self, key):
+            return vars(builtins)[key]
+
+    disguised = types.FunctionType(
+        score.__code__, {"__builtins__": ReportingDict(len=lambda _: 99)}, "score"
+    )
+    disguised.__annotations__ = score.__annotations__
+    assert disguised(3) == 102
+    with pytest.raises(ValueError, match="non-standard builtins environment"):
+        udf(disguised)
+
+    hooked = types.FunctionType(
+        score.__code__,
+        {"__builtins__": {**vars(builtins), "__import__": lambda *a, **k: None}},
+        "score",
+    )
+    hooked.__annotations__ = score.__annotations__
+    with pytest.raises(ValueError, match="non-standard builtins environment"):
+        udf(hooked)
+
+
+def test_udf_recursion_versus_a_rebound_module_name(tmp_path):
+    module_path = tmp_path / "rebound_udfs.py"
+    module_path.write_text(
+        "def fact(value: int) -> int:\n"
+        "    return 1 if value <= 1 else value * fact(value - 1)\n"
+        "\n"
+        "def score(value: int) -> int:\n"
+        "    return score + value\n"
+    )
+    spec = importlib.util.spec_from_file_location("rebound_udfs", module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    assert _run_packaged(udf(module.fact), 5) == 120
+    raw = module.score
+    module.score = 10
+    with pytest.raises(ValueError, match="binds that name to another value"):
+        udf(raw)
+    # A wrapper that merely exposes __wrapped__ is not the function.
+    module.score = functools.wraps(raw)(lambda value: 41)
+    with pytest.raises(ValueError, match="binds that name to another value"):
+        udf(raw)
+    # The decorator's own result is; a subclass of it is not.
+    module.fact = udf(module.fact)
+    assert _run_packaged(module.fact, 4) == 24
+
+    class Twisted(UdfDefinition):
+        def __call__(self, *args, **kwargs):
+            return 41
+
+    raw_fact = module.fact._function
+    module.fact = Twisted(
+        raw_fact,
+        name=None,
+        input_schema=None,
+        output_schema=None,
+        pip=(),
+        env={},
+        python_version=None,
+    )
+    with pytest.raises(ValueError, match="binds that name to another value"):
+        udf(raw_fact)
+
+
+def test_canonical_arrow_type_rejects_unrepresentable_list_children():
+    from lancedb.functions import _canonical_arrow_type
+
+    for outside in [
+        pa.list_(pa.float32()),  # pyarrow default: nullable child
+        pa.list_(pa.field("custom", pa.float32(), nullable=False)),
+        pa.list_(pa.field("item", pa.float32(), nullable=False, metadata={"k": "v"})),
+        pa.list_(pa.field("item", pa.float32(), nullable=False), 0),
+    ]:
+        with pytest.raises(TypeError, match="unsupported Arrow type"):
+            _canonical_arrow_type(outside)
+    assert (
+        _canonical_arrow_type(
+            pa.list_(pa.field("item", pa.float32(), nullable=False), 3)
+        )
+        == "fixed_size_list<float32, 3>"
+    )
+
+
+def _calls_missing(value: int) -> int:
+    return missing(value)  # noqa: F821
+
+
+def _shadows_missing_in_a_comprehension(value: int) -> int:
+    return missing(value) + sum(missing for missing in ())  # noqa: F821
+
+
+def _shadows_missing_in_a_lambda(value: int) -> int:
+    return (lambda missing: missing)(value) + missing  # noqa: F821
+
+
+@pytest.mark.parametrize(
+    "function",
+    [_calls_missing, _shadows_missing_in_a_comprehension, _shadows_missing_in_a_lambda],
+)
+def test_udf_rejects_a_truly_unresolved_global(function):
+    with pytest.raises(ValueError, match=r"unresolved global names: \['missing'\]"):
+        udf(function)
+
+
+def _arrow_type_from_golden(spec: dict) -> pa.DataType:
+    kind = spec["type"]
+    if kind in ("list", "large_list", "fixed_size_list"):
+        item = _arrow_type_from_golden(spec["fields"][0]["type"])
+        field = pa.field("item", item, nullable=False)
+        if kind == "list":
+            return pa.list_(field)
+        if kind == "large_list":
+            return pa.large_list(field)
+        return pa.list_(field, spec["length"])
+    return {
+        "null": pa.null(),
+        "bool": pa.bool_(),
+        "utf8": pa.string(),
+        "binary": pa.binary(),
+        "float16": pa.float16(),
+        "float32": pa.float32(),
+        "float64": pa.float64(),
+        "date32": pa.date32(),
+        "date64": pa.date64(),
+    }.get(kind) or getattr(pa, kind)()
+
+
+def test_arrow_type_grammar_matches_the_shared_golden():
+    golden = json.loads(
+        (
+            Path(__file__).parents[3]
+            / "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json"
+        ).read_text()
+    )
+    from lancedb.functions import _canonical_arrow_type
+
+    emitted = {
+        case["arrow_type"]: _canonical_arrow_type(_arrow_type_from_golden(case["json"]))
+        for case in golden["valid"]
+    }
+    assert emitted == {
+        case["arrow_type"]: case["arrow_type"] for case in golden["valid"]
+    }
+    assert not set(emitted) & set(golden["invalid"])
+    for case in golden["server_only"]:
+        with pytest.raises(TypeError, match="unsupported Arrow type"):
+            _canonical_arrow_type(_arrow_type_from_golden(case["json"]))
+
+
+def test_explicit_arrow_schema_is_deterministic():
+    input_schema = pa.schema([pa.field("value", pa.float32(), nullable=True)])
+    output_schema = pa.field(
+        "embedding",
+        pa.list_(pa.field("item", pa.float32(), nullable=False), 3),
+        nullable=False,
+    )
+
+    @udf(input_schema=input_schema, output_schema=output_schema)
+    def explicit(value):
+        return [value, value, value]
+
+    signature = explicit.registration_request.signature
+    assert signature.inputs[0].arrow_type == "float32"
+    assert signature.inputs[0].nullable is True
+    assert signature.output.arrow_type == "fixed_size_list<float32, 3>"
+    assert signature.output.nullable is False
+
+
+def test_annotation_and_explicit_schema_validation_fail_closed():
+    with pytest.raises(TypeError, match="missing Function annotations"):
+
+        @udf
+        def missing(value):
+            return value
+
+    with pytest.raises(TypeError, match="unsupported Function annotation"):
+
+        @udf
+        def unsupported(value: set[str]) -> str:
+            return ""
+
+    with pytest.raises(ValueError, match="output must be non-nullable"):
+
+        @udf
+        def nullable_output(value: int) -> Optional[int]:
+            return value
+
+    with pytest.raises(ValueError, match="provided together"):
+
+        @udf(input_schema=pa.schema([pa.field("value", pa.int64())]))
+        def partial_schema(value):
+            return value
+
+    with pytest.raises(ValueError, match="exactly match callable parameters"):
+
+        @udf(
+            input_schema=pa.schema([pa.field("other", pa.int64())]),
+            output_schema=pa.int64(),
+        )
+        def wrong_name(value):
+            return value
+
+    with pytest.raises(ValueError, match="output must be non-nullable"):
+
+        @udf(
+            input_schema=pa.schema([pa.field("value", pa.int64())]),
+            output_schema=pa.field("result", pa.int64(), nullable=True),
+        )
+        def nullable_explicit(value):
+            return value
+
+
+def test_local_function_catalog_operations_are_not_supported(tmp_path):
+    db = lancedb.connect(tmp_path)
+    message = "Function catalog operations are not supported by this database"
+    with pytest.raises(NotImplementedError, match=message):
+        db.create_function(normalize_score)
+    with pytest.raises(NotImplementedError, match=message):
+        db.create_function_async(normalize_score)
+    with pytest.raises(NotImplementedError, match=message):
+        db.get_function("normalize_score", version="fv_exact")
+
+
+@contextlib.contextmanager
+def _mock_remote_function_catalog():
+    state = {"requests": [], "version": None}
+
+    class Handler(http.server.BaseHTTPRequestHandler):
+        def log_message(self, *args):
+            pass
+
+        def do_POST(self):
+            length = int(self.headers.get("Content-Length", "0"))
+            body = json.loads(self.rfile.read(length) or b"{}")
+            state["requests"].append((self.path, body))
+            status = 200
+            if self.path == "/v1/functions/create":
+                state["version"] = {
+                    "name": body["name"],
+                    "version": "fv_exact",
+                    "artifact": {
+                        key: body["artifact"][key]
+                        for key in ("kind", "digest", "entrypoint")
+                    },
+                    "signature": body["signature"],
+                    "runtime": body["runtime"],
+                    "runtime_digest": "sha256:runtime",
+                    "environment_digest": "sha256:environment",
+                    "created_at": "2026-08-21T00:00:00Z",
+                }
+                response = {"job_id": "job-register"}
+                status = 202
+            elif self.path == "/v1/jobs/describe":
+                assert body == {"job_id": "job-register"}
+                response = {
+                    "job_id": "job-register",
+                    "job_type": "create_function",
+                    "job_state": "DONE",
+                    "result": state["version"],
+                }
+            elif self.path == "/v1/functions/describe":
+                assert body == {
+                    "name": "normalize_score",
+                    "version": "fv_exact",
+                }
+                response = state["version"]
+            else:
+                status = 404
+                response = {"error": "not found"}
+            encoded = json.dumps(response).encode()
+            self.send_response(status)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(encoded)))
+            self.end_headers()
+            self.wfile.write(encoded)
+
+    with http.server.HTTPServer(("localhost", 0), Handler) as server:
+        thread = threading.Thread(target=server.serve_forever)
+        thread.start()
+        try:
+            yield f"http://localhost:{server.server_address[1]}", state
+        finally:
+            server.shutdown()
+            thread.join()
+
+
+def test_remote_registration_job_and_exact_version_reopen_round_trip():
+    with _mock_remote_function_catalog() as (host, state):
+        db = lancedb.connect(
+            "db://dev",
+            api_key="fake",
+            host_override=host,
+            client_config={"retry_config": {"retries": 0}},
+        )
+        registration = db.create_function_async(normalize_score)
+        assert registration.id == "job-register"
+        created = registration.wait()
+        reopened = db.get_function("normalize_score", version=created.version)
+
+    assert created == reopened
+    assert reopened.name == "normalize_score"
+    assert reopened.version == "fv_exact"
+    create_request = state["requests"][0][1]
+    assert create_request == json.loads(
+        normalize_score.registration_request.to_canonical_json()
+    )
+
+
+def test_blocking_remote_registration_returns_function_version():
+    with _mock_remote_function_catalog() as (host, state):
+        db = lancedb.connect(
+            "db://dev",
+            api_key="fake",
+            host_override=host,
+            client_config={"retry_config": {"retries": 0}},
+        )
+        created = db.create_function(normalize_score)
+
+    assert created.name == "normalize_score"
+    assert created.version == "fv_exact"
+    assert [path for path, _ in state["requests"]] == [
+        "/v1/functions/create",
+        "/v1/jobs/describe",
+    ]
diff --git a/python/python/tests/test_fts.py b/python/python/tests/test_fts.py
index f791f9886..625198d92 100644
--- a/python/python/tests/test_fts.py
+++ b/python/python/tests/test_fts.py
@@ -245,6 +245,14 @@ def test_create_inverted_index_rejects_invalid_block_size(table):
         table.create_index("text", config=FTS(block_size=129))
 
 
+def test_create_inverted_index_respects_build_memory_limit(table):
+    with pytest.raises(ValueError, match="exceeds worker memory limit"):
+        table.create_index(
+            "text",
+            config=FTS(memory_limit=0, num_workers=1),
+        )
+
+
 def test_custom_stop_words_list(table):
     table.create_index(
         "text",
diff --git a/python/python/tests/test_hybrid_query.py b/python/python/tests/test_hybrid_query.py
index 65a7890bf..5e9b45ecb 100644
--- a/python/python/tests/test_hybrid_query.py
+++ b/python/python/tests/test_hybrid_query.py
@@ -12,7 +12,7 @@ import pyarrow.compute as pc
 import pytest
 import pytest_asyncio
 
-from lancedb.index import FTS
+from lancedb.index import BTree, FTS, IvfPq
 from lancedb.table import AsyncTable, Table
 
 
@@ -99,6 +99,86 @@ async def test_async_hybrid_query_filters(table: AsyncTable):
     assert result["text"].to_pylist() == ["cat", "b"]
 
 
+@pytest.mark.asyncio
+async def test_hybrid_query_with_stale_fixed_size_binary_prefilter(
+    tmpdir_factory,
+):
+    tmp_path = str(tmpdir_factory.mktemp("stale_scalar_prefilter"))
+    db = await lancedb.connect_async(tmp_path)
+
+    def fixed_size_binary(value: int) -> bytes:
+        return value.to_bytes(16, byteorder="big")
+
+    num_rows = 1000
+    data = pa.table(
+        {
+            "space_id": pa.array(
+                [fixed_size_binary(i) for i in range(num_rows)],
+                type=pa.binary(16),
+            ),
+            "text": ["book"] * num_rows,
+            "vector": pa.array(
+                [[float(i), float(i)] for i in range(num_rows)],
+                type=pa.list_(pa.float32(), 2),
+            ),
+        }
+    )
+    table = await db.create_table("test", data)
+    await table.create_index(
+        "vector", config=IvfPq(num_partitions=4, num_sub_vectors=2)
+    )
+    await table.create_index("space_id", config=BTree())
+    await table.create_index("text", config=FTS(with_position=False))
+
+    # Advance the search indices without advancing the scalar index. This is the
+    # state that previously let hybrid search use an incomplete scalar prefilter.
+    await table.add(data)
+    lance_dataset = await table.to_lance()
+    lance_dataset.optimize.optimize_indices(index_names=["vector_idx", "text_idx"])
+    await table.checkout_latest()
+
+    scalar_stats = await table.index_stats("space_id_idx")
+    assert scalar_stats is not None
+    assert scalar_stats.num_indexed_rows == num_rows
+    assert scalar_stats.num_unindexed_rows == num_rows
+
+    for index_name in ["vector_idx", "text_idx"]:
+        search_stats = await table.index_stats(index_name)
+        assert search_stats is not None
+        assert search_stats.num_indexed_rows == num_rows * 2
+        assert search_stats.num_unindexed_rows == 0
+
+    matching_ids = [5, 10, 15, 20, 25, 30]
+    literals = [
+        f"arrow_cast(0x{fixed_size_binary(i).hex()}, 'FixedSizeBinary(16)')"
+        for i in matching_ids
+    ]
+    predicate = f"space_id IN ({', '.join(literals)})"
+    expected_ids = sorted(fixed_size_binary(i) for i in matching_ids for _ in range(2))
+
+    vector_query = (
+        table.query().where(predicate).nearest_to([5.0, 5.0]).limit(num_rows * 2)
+    )
+    vector_results = await vector_query.to_arrow()
+    assert sorted(vector_results["space_id"].to_pylist()) == expected_ids
+
+    fts_query = (
+        table.query().where(predicate).nearest_to_text("book").limit(num_rows * 2)
+    )
+    fts_results = await fts_query.to_arrow()
+    assert sorted(fts_results["space_id"].to_pylist()) == expected_ids
+
+    hybrid_results = await (
+        table.query()
+        .where(predicate)
+        .nearest_to([5.0, 5.0])
+        .nearest_to_text("book")
+        .limit(num_rows * 2)
+        .to_arrow()
+    )
+    assert sorted(hybrid_results["space_id"].to_pylist()) == expected_ids
+
+
 @pytest.mark.asyncio
 async def test_async_hybrid_query_default_limit(table: AsyncTable):
     # add 10 new rows
@@ -123,6 +203,31 @@ async def test_async_hybrid_query_default_limit(table: AsyncTable):
     assert texts.count("a") == 1
 
 
+def test_hybrid_query_offset(sync_table: Table):
+    # The offset window of a hybrid query must be a suffix of the same query
+    # run without an offset -- it must not be silently ignored.
+    full = (
+        sync_table.search(query_type="hybrid")
+        .vector([0.0, 0.4])
+        .text("dog")
+        .limit(4)
+        .with_row_id(True)
+        .to_arrow()
+    )
+    assert len(full) == 4
+
+    offset_result = (
+        sync_table.search(query_type="hybrid")
+        .vector([0.0, 0.4])
+        .text("dog")
+        .offset(2)
+        .limit(2)
+        .with_row_id(True)
+        .to_arrow()
+    )
+    assert offset_result["_rowid"].to_pylist() == full["_rowid"].to_pylist()[2:]
+
+
 def test_hybrid_query_minimum_nprobes_zero_raises(sync_table: Table):
     # minimum_nprobes(0) must raise the same validation error a plain vector
     # query raises, not silently no-op because 0 is falsy.
diff --git a/python/python/tests/test_import.py b/python/python/tests/test_import.py
new file mode 100644
index 000000000..4b87a0ce8
--- /dev/null
+++ b/python/python/tests/test_import.py
@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import re
+import shutil
+import subprocess
+import sys
+
+import lancedb._lancedb as _lancedb
+import pytest
+
+
+@pytest.mark.skipif(sys.platform != "linux", reason="ldd is Linux-specific")
+def test_native_extension_does_not_link_openssl():
+    """OpenSSL-linked wheels abort when imported on RHEL hosts in FIPS mode."""
+    ldd = shutil.which("ldd")
+    if ldd is None:
+        pytest.skip("ldd is not installed")
+
+    result = subprocess.run(
+        [ldd, _lancedb.__file__],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    openssl_libraries = re.findall(
+        r"^\s*(lib(?:crypto|ssl)\S*)\s+=>", result.stdout, flags=re.MULTILINE
+    )
+
+    assert not openssl_libraries, (
+        "the LanceDB native extension must use rustls instead of linking OpenSSL: "
+        f"{openssl_libraries}"
+    )
diff --git a/python/python/tests/test_index.py b/python/python/tests/test_index.py
index 1cf2c733c..fe6ebe87a 100644
--- a/python/python/tests/test_index.py
+++ b/python/python/tests/test_index.py
@@ -88,7 +88,7 @@ async def binary_table(db_async):
 async def test_create_index_async_returns_done_job(some_table: AsyncTable):
     job = await some_table.create_index_async("id", config=BTree())
     assert job.id is None
-    await job.wait()
+    assert await job.wait() is None
     assert len(await some_table.list_indices()) == 1
     await job.cancel()
 
@@ -372,6 +372,31 @@ async def test_create_vector_index(some_table: AsyncTable):
     assert stats.num_indices == 1
 
 
+@pytest.mark.asyncio
+async def test_create_ivf_index_reports_unsplittable_partitions(db_async):
+    dim = 8
+    num_partitions = 300  # More than 256 selects hierarchical k-means.
+    base_vectors = [[float(row == column) for column in range(dim)] for row in range(5)]
+    vectors = pa.array(base_vectors * 200, pa.list_(pa.float32(), dim))
+    table = await db_async.create_table(
+        "unsplittable_partitions",
+        pa.table({"vector": vectors}),
+    )
+
+    error_pattern = (
+        rf"Cannot create {num_partitions} IVF partitions: k-means could only form"
+    )
+    with pytest.raises(RuntimeError, match=error_pattern):
+        await table.create_index(
+            "vector",
+            config=IvfFlat(
+                distance_type="dot",
+                num_partitions=num_partitions,
+                max_iterations=10,
+            ),
+        )
+
+
 @pytest.mark.asyncio
 async def test_create_4bit_ivfpq_index(some_table: AsyncTable):
     # Can create
diff --git a/python/python/tests/test_lsm_write_spec.py b/python/python/tests/test_lsm_write_spec.py
index d38918f09..d43cb7532 100644
--- a/python/python/tests/test_lsm_write_spec.py
+++ b/python/python/tests/test_lsm_write_spec.py
@@ -21,6 +21,11 @@ SCHEMA = pa.schema(
 )
 
 
+def test_lsm_write_spec_module_metadata():
+    assert lancedb.LsmWriteSpec is LsmWriteSpec
+    assert LsmWriteSpec.__module__ == "lancedb._lancedb"
+
+
 def _batch(ids, vs):
     return pa.RecordBatch.from_arrays(
         [pa.array(ids, type=pa.utf8()), pa.array(vs, type=pa.int32())],
@@ -83,7 +88,9 @@ def test_lsm_write_spec_repr():
     assert s.spec_type == "bucket"
     assert s.column == "id"
     assert s.num_buckets == 4
-    assert s.maintained_indexes == []
+    # A fresh spec defers its maintained set to install time.
+    assert s.maintained_indexes is None
+    assert s.with_maintained_indexes([]).maintained_indexes == []
     assert "bucket" in repr(s)
     assert "id" in repr(s)
     assert "4" in repr(s)
@@ -169,18 +176,23 @@ def test_get_lsm_write_spec(tmp_path):
     table.unset_lsm_write_spec()
     assert table.get_lsm_write_spec() is None
 
-    # Identity round-trips (column recovered from the schema).
+    # Identity round-trips (column recovered from the schema). Leaving the
+    # maintained set to be inferred picks up the index on the table, so the
+    # spec reads back naming it rather than as "infer".
     table.set_lsm_write_spec(LsmWriteSpec.identity("id"))
     spec = table.get_lsm_write_spec()
     assert spec.spec_type == "identity"
     assert spec.column == "id"
+    assert spec.maintained_indexes == [idx_name]
     table.unset_lsm_write_spec()
 
-    # Unsharded round-trips (no routing column).
-    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
+    # Unsharded round-trips (no routing column). Opting out is distinct from
+    # the inferred default.
+    table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([]))
     spec = table.get_lsm_write_spec()
     assert spec.spec_type == "unsharded"
     assert spec.column is None
+    assert spec.maintained_indexes == []
 
 
 @pytest.mark.asyncio
diff --git a/python/python/tests/test_materialized_views.py b/python/python/tests/test_materialized_views.py
new file mode 100644
index 000000000..5fa3aa4fb
--- /dev/null
+++ b/python/python/tests/test_materialized_views.py
@@ -0,0 +1,268 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import lancedb
+import pytest
+from lancedb.materialized_view import MaterializedViewDefinition
+
+
+STABLE_ROW_IDS = {"new_table_enable_stable_row_ids": "true"}
+
+
+def make_db(tmp_path):
+    db = lancedb.connect(tmp_path, storage_options=STABLE_ROW_IDS)
+    db.create_table(
+        "people",
+        [
+            {"name": "ada", "age": 36},
+            {"name": "kid", "age": 7},
+            {"name": "grace", "age": 85},
+        ],
+    )
+    return db
+
+
+def test_create_refresh_and_query(tmp_path):
+    db = make_db(tmp_path)
+    view = db.create_materialized_view(
+        "adults",
+        "people",
+        select=["name", ("shout", "upper(name)")],
+        where="age >= 18",
+    )
+    assert view.name == "adults"
+    assert view.table.count_rows() == 0
+
+    result = view.refresh()
+    assert result.mode == "rebuild"
+    assert result.rows_written == 2
+
+    rows = view.table.search().to_list()
+    assert sorted(row["shout"] for row in rows) == ["ADA", "GRACE"]
+
+
+def test_definition_round_trips(tmp_path):
+    db = make_db(tmp_path)
+    db.create_materialized_view("adults", "people", where="age >= 18")
+
+    view = db.open_materialized_view("adults")
+    assert view.definition == MaterializedViewDefinition(
+        source_table="people",
+        projections=[("name", "`name`"), ("age", "`age`")],
+        filter="age >= 18",
+        inputs=["age", "name"],
+    )
+
+
+def test_incremental_refresh_after_append(tmp_path):
+    db = make_db(tmp_path)
+    view = db.create_materialized_view("copy", "people")
+    view.refresh()
+
+    db.open_table("people").add([{"name": "alan", "age": 41}])
+    result = view.refresh()
+    assert result.mode == "incremental"
+    assert result.rows_written == 1
+    assert view.table.count_rows() == 4
+
+    assert view.refresh().mode == "no_op"
+
+
+def test_incremental_refresh_after_update(tmp_path):
+    db = make_db(tmp_path)
+    view = db.create_materialized_view("copy", "people")
+    view.refresh()
+
+    db.open_table("people").update(where="name = 'kid'", values={"age": 8})
+    result = view.refresh()
+    assert result.mode == "incremental"
+    assert result.rows_written == 1
+    rows = view.table.search().to_list()
+    assert sorted(row["age"] for row in rows) == [8, 36, 85]
+
+
+def test_legacy_storage_source_update_rebuilds(tmp_path):
+    db = lancedb.connect(
+        tmp_path,
+        storage_options={**STABLE_ROW_IDS, "new_table_data_storage_version": "legacy"},
+    )
+    db.create_table("people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}])
+    view = db.create_materialized_view("copy", "people")
+    view.refresh()
+
+    db.open_table("people").update(where="name = 'kid'", values={"age": 8})
+    result = view.refresh()
+    assert result.mode == "rebuild"
+    rows = view.table.search().to_list()
+    assert sorted(row["age"] for row in rows) == [8, 36]
+
+
+def test_list_and_not_a_view(tmp_path):
+    db = make_db(tmp_path)
+    db.create_materialized_view("adults", "people", where="age >= 18")
+
+    assert db.list_materialized_views() == ["adults"]
+    with pytest.raises(ValueError, match="not a materialized view"):
+        db.open_materialized_view("people")
+
+
+def test_invalid_expression_fails_at_create(tmp_path):
+    db = make_db(tmp_path)
+    with pytest.raises(Exception, match="missing"):
+        db.create_materialized_view("bad", "people", select=[("x", "missing + 1")])
+    assert "bad" not in db.list_tables().tables
+
+
+@pytest.mark.asyncio
+async def test_async_create_refresh_and_open(tmp_path):
+    db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS)
+    await db.create_table("people", [{"name": "ada", "age": 36}])
+
+    view = await db.create_materialized_view(
+        "shouts", "people", select=[("shout", "upper(name)")]
+    )
+    result = await view.refresh()
+    assert result.mode == "rebuild"
+    assert result.rows_written == 1
+
+    reopened = await db.open_materialized_view("shouts")
+    definition = await reopened.definition()
+    assert definition.projections == [("shout", "upper(name)")]
+    assert await db.list_materialized_views() == ["shouts"]
+
+
+@pytest.mark.asyncio
+async def test_async_incremental(tmp_path):
+    db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS)
+    await db.create_table("people", [{"name": "ada", "age": 36}])
+    view = await db.create_materialized_view("copy", "people")
+    await view.refresh()
+
+    table = await db.open_table("people")
+    await table.add([{"name": "alan", "age": 41}])
+    result = await view.refresh()
+    assert result.mode == "incremental"
+    assert result.rows_written == 1
+
+
+def test_source_requires_stable_row_ids(tmp_path):
+    db = lancedb.connect(tmp_path)
+    db.create_table("plain", [{"x": 1}])
+    with pytest.raises(Exception, match="stable row ids"):
+        db.create_materialized_view("v", "plain")
+
+
+def test_bare_select_names_are_quoted(tmp_path):
+    db = lancedb.connect(tmp_path, storage_options=STABLE_ROW_IDS)
+    db.create_table("odd_names", [{"order item": "widget", "select": 2}])
+
+    view = db.create_materialized_view(
+        "quoted", "odd_names", select=["order item", "select"]
+    )
+    result = view.refresh()
+    assert result.rows_written == 1
+    rows = view.table.search().to_list()
+    assert rows[0]["order item"] == "widget"
+    assert rows[0]["select"] == 2
+
+
+@pytest.mark.asyncio
+async def test_async_remote_is_refused_without_network():
+    db = await lancedb.connect_async(
+        "db://nowhere", api_key="sk_test", region="us-east-1"
+    )
+    with pytest.raises(NotImplementedError, match="local"):
+        await db.create_materialized_view("v", "src")
+    with pytest.raises(NotImplementedError, match="local"):
+        await db.open_materialized_view("v")
+    with pytest.raises(NotImplementedError, match="local"):
+        await db.list_materialized_views()
+
+
+def test_scalar_select_is_one_column(tmp_path):
+    db = make_db(tmp_path)
+    view = db.create_materialized_view("just_name", "people", select="name")
+    view.refresh()
+    rows = view.table.search().to_list()
+    assert set(rows[0]) - {"__source_row_id"} == {"name"}
+    assert sorted(row["name"] for row in rows) == ["ada", "grace", "kid"]
+
+
+@pytest.mark.asyncio
+async def test_async_scalar_select_is_one_column(tmp_path):
+    db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS)
+    await db.create_table("people", [{"name": "ada", "age": 36}])
+    view = await db.create_materialized_view("just_name", "people", select="name")
+    await view.refresh()
+    rows = await view.table.query().to_list()
+    assert set(rows[0]) - {"__source_row_id"} == {"name"}
+
+
+def test_limit_above_i64_max_is_refused(tmp_path):
+    db = make_db(tmp_path)
+    with pytest.raises(ValueError, match="exceeds the maximum"):
+        db.create_materialized_view("too_big", "people", limit=2**63)
+    # The boundary is fine, and zero still means an empty view.
+    db.create_materialized_view("at_max", "people", limit=2**63 - 1)
+    empty = db.create_materialized_view("none", "people", limit=0)
+    empty.refresh()
+    assert empty.table.count_rows() == 0
+
+
+def _namespace_db(tmp_path):
+    return lancedb.connect_namespace(
+        "dir",
+        {"root": str(tmp_path)},
+        storage_options=STABLE_ROW_IDS,
+    )
+
+
+def test_namespace_connection_materialized_views(tmp_path):
+    db = _namespace_db(tmp_path)
+    db.create_table(
+        "people",
+        [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}],
+        storage_options=STABLE_ROW_IDS,
+    )
+
+    view = db.create_materialized_view("adults", "people", where="age >= 18")
+    view.refresh()
+    assert view.table.count_rows() == 1
+    assert db.list_materialized_views() == ["adults"]
+
+    reopened = db.open_materialized_view("adults")
+    assert reopened.definition.source_table == "people"
+    with pytest.raises(ValueError, match="not a materialized view"):
+        db.open_materialized_view("people")
+
+
+@pytest.mark.asyncio
+async def test_async_namespace_connection_materialized_views(tmp_path):
+    db = lancedb.connect_namespace_async(
+        "dir",
+        {"root": str(tmp_path)},
+        storage_options=STABLE_ROW_IDS,
+    )
+    await db.create_table(
+        "people",
+        [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}],
+        storage_options=STABLE_ROW_IDS,
+    )
+
+    view = await db.create_materialized_view("adults", "people", where="age >= 18")
+    await view.refresh()
+    assert await view.table.count_rows() == 1
+    assert await db.list_materialized_views() == ["adults"]
+
+    reopened = await db.open_materialized_view("adults")
+    assert (await reopened.definition()).source_table == "people"
+
+    # The view's table came through the namespace, not straight from the
+    # inner connection: a bare inner table carries no namespace context, so
+    # its pushdown routing differs from a table the namespace opened.
+    through_namespace = await db.open_table("adults")
+    for handle in (view.table, reopened.table):
+        assert (
+            handle._route_pushdown_to_rust == through_namespace._route_pushdown_to_rust
+        )
+        assert handle._namespace_path == through_namespace._namespace_path
diff --git a/python/python/tests/test_merge_insert_lsm.py b/python/python/tests/test_merge_insert_lsm.py
index 5674a05ab..e74c21589 100644
--- a/python/python/tests/test_merge_insert_lsm.py
+++ b/python/python/tests/test_merge_insert_lsm.py
@@ -544,7 +544,7 @@ def test_lsm_read_fts_unmaintained_index_errors(tmp_path):
     table.create_index("text", config=FTS())
     # No maintained indexes: the active memtable FTS arm cannot serve un-compacted
     # docs, so the search would silently omit them — reject instead.
-    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
+    table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([]))
     with pytest.raises(Exception, match="maintained"):
         table.search("fox", query_type="fts", fts_columns="text").to_arrow()
 
@@ -631,7 +631,7 @@ def test_lsm_read_vector_unmaintained_index_errors(tmp_path):
     )
     # Spec with NO maintained indexes: the base vector index's catch-up is untracked,
     # so the scanner rejects rather than risk dropping compacted-but-unindexed rows.
-    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
+    table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([]))
     with pytest.raises(Exception, match="maintained"):
         table.search([1.0] * VECTOR_DIM).to_arrow()
 
diff --git a/python/python/tests/test_package_metadata.py b/python/python/tests/test_package_metadata.py
new file mode 100644
index 000000000..5792f457b
--- /dev/null
+++ b/python/python/tests/test_package_metadata.py
@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import importlib
+import re
+import sys
+from pathlib import Path
+
+import pytest
+
+
+def test_pyo3_abi_matches_minimum_supported_python():
+    project_dir = Path(__file__).parents[2]
+    pyproject = (project_dir / "pyproject.toml").read_text()
+    cargo_manifest = (project_dir / "Cargo.toml").read_text()
+
+    minimum_python = re.search(
+        r'^requires-python\s*=\s*">=(\d+)\.(\d+)"$', pyproject, re.MULTILINE
+    )
+    assert minimum_python is not None
+
+    major, minor = minimum_python.groups()
+    expected_abi = f"abi3-py{major}{minor}"
+    configured_abis = re.findall(r'"(abi3-py\d+)"', cargo_manifest)
+
+    assert configured_abis == [expected_abi, expected_abi], (
+        "the pyo3 runtime and build ABI features must both match requires-python"
+    )
+
+
+@pytest.mark.skipif(sys.platform != "win32", reason="Windows wheel regression test")
+def test_windows_wheel_tag_and_native_import():
+    project_dir = Path(__file__).parents[2]
+    wheels = list((project_dir.parent / "target" / "wheels").glob("lancedb-*.whl"))
+    if not wheels:
+        pytest.skip("no wheel artifact is available in this development environment")
+
+    assert len(wheels) == 1
+    assert wheels[0].name.endswith("-cp310-abi3-win_amd64.whl")
+
+    native_module = importlib.import_module("lancedb._lancedb")
+    assert Path(native_module.__file__).suffix == ".pyd"
diff --git a/python/python/tests/test_permutation.py b/python/python/tests/test_permutation.py
index 6d8f6f431..142fc84f1 100644
--- a/python/python/tests/test_permutation.py
+++ b/python/python/tests/test_permutation.py
@@ -8,6 +8,11 @@ import pytest
 from lancedb import DBConnection, Table, connect
 from lancedb.background_loop import LOOP
 from lancedb.permutation import Permutation, Permutations, permutation_builder
+from utils import (
+    MockPermutationServer,
+    assert_server_safe_row_id_requests,
+    mock_remote_table,
+)
 
 
 def test_split_random_ratios(mem_db):
@@ -51,6 +56,31 @@ def test_execute_does_not_reenter_background_loop(tmp_path, monkeypatch):
     assert permutation_tbl._conn.read_consistency_interval is None
 
 
+def test_pickled_permutation_reads_pinned_version(tmp_path):
+    """An unpickled copy must still read the pinned version, which also covers the
+    version surviving the ``to_arrow()`` round trip in ``__getstate__``."""
+    import pickle
+
+    db = connect(tmp_path)
+    tbl = db.create_table("base", pa.table({"idx": range(20)}))
+    permutation_tbl = permutation_builder(tbl).execute()
+    perm = Permutation.from_tables(tbl, permutation_tbl)
+
+    payload = pickle.dumps(perm)
+
+    # Compact so the stored row addresses no longer describe these rows at latest.
+    tbl.delete("true")
+    tbl.optimize()
+    assert tbl.count_rows() == 0
+
+    # Unpickle after the mutation: __setstate__ reopens at latest, so this only
+    # passes if the recorded version is applied on reopen.
+    restored = pickle.loads(payload)
+    assert len(restored) == 20
+    rows = restored.__getitems__(list(range(20)))
+    assert sorted(row["idx"] for row in rows) == list(range(20))
+
+
 def test_split_random_counts(mem_db):
     """Test random splitting with absolute counts."""
     tbl = mem_db.create_table(
@@ -1214,3 +1244,57 @@ def test_remove_rowid_after_select(some_permutation: Permutation):
     perm_without_rowid = perm_with_rowid.remove_columns(["_rowid"])
     assert "_rowid" not in perm_without_rowid.column_names
     assert perm_without_rowid.column_names == ["id"]
+
+
+def test_permutation_is_stable_when_remote_scan_order_varies():
+    """Splits are assigned by scan position, and every rank builds its own
+    permutation, so two ranks seeing different scan orders must still agree."""
+    server = MockPermutationServer(num_rows=16, vary_scan_order=True)
+
+    def split_of_each_row(permutation_tbl):
+        # Sequential splits are assigned by position, so a reversed scan would put
+        # the last rows in split 0.  Compare the mapping rather than the table order,
+        # which the split-id sort does not pin down.
+        rows = permutation_tbl.search(None).to_arrow().to_pydict()
+        return dict(zip(rows["row_id"], rows["split_id"]))
+
+    with mock_remote_table(server) as table:
+        first = split_of_each_row(
+            permutation_builder(table).split_sequential(fixed=2).execute()
+        )
+        second = split_of_each_row(
+            permutation_builder(table).split_sequential(fixed=2).execute()
+        )
+
+    assert server.scan_calls == 2, "both builds must have scanned"
+    assert first == second
+    assert first[0] == 0 and first[server.num_rows - 1] == 1, first
+
+
+def test_permutation_over_remote_table():
+    """The permutation API accepts a remote table, addressing rows by `_rowid` just
+    as `take_row_ids` does.  Also pins the request shapes sent to the server.
+    """
+    server = MockPermutationServer()
+
+    with mock_remote_table(server) as table:
+        permutation_tbl = permutation_builder(table).split_sequential(fixed=2).execute()
+        assert permutation_tbl.count_rows() == server.num_rows
+
+        permutation = Permutation.from_tables(table, permutation_tbl, 0)
+        assert permutation.num_rows == server.num_rows // 2
+
+        # Compare against the permutation's own order; the split-id sort is not stable.
+        rows = permutation_tbl.search(None).to_arrow().to_pydict()
+        split0 = [
+            row_id
+            for row_id, split in zip(rows["row_id"], rows["split_id"])
+            if not split
+        ]
+        # The mock table's `id` equals its `_rowid`.
+        assert permutation.take_offsets([2, 0]) == [
+            {"id": split0[2]},
+            {"id": split0[0]},
+        ]
+
+    assert_server_safe_row_id_requests(server)
diff --git a/python/python/tests/test_pydantic.py b/python/python/tests/test_pydantic.py
index e1d533784..ca03940f2 100644
--- a/python/python/tests/test_pydantic.py
+++ b/python/python/tests/test_pydantic.py
@@ -10,11 +10,10 @@ import pyarrow as pa
 import pydantic
 import pytest
 from lancedb.pydantic import (
-    PYDANTIC_VERSION,
     LanceModel,
+    MultiVector,
     Vector,
     pydantic_to_schema,
-    MultiVector,
 )
 from pydantic import BaseModel
 from pydantic import Field
@@ -415,22 +414,27 @@ def test_nullable_vector():
     assert schema == pa.schema([pa.field("vec", pa.list_(pa.float32(), 16), True)])
 
 
+def test_bare_vector_raises_clear_error():
+    namespace = {
+        "__name__": "test_model_without_pyarrow",
+        "LanceModel": LanceModel,
+        "Vector": Vector,
+    }
+
+    with pytest.raises(TypeError, match=r"Vector must be parameterized.*Vector\(128\)"):
+        exec("class TestModel(LanceModel):\n    vector: Vector", namespace)
+
+
 def test_fixed_size_list_field():
     class TestModel(pydantic.BaseModel):
         vec: Vector(16)
         li: List[int]
 
     data = TestModel(vec=list(range(16)), li=[1, 2, 3])
-    if PYDANTIC_VERSION.major >= 2:
-        assert json.loads(data.model_dump_json()) == {
-            "vec": list(range(16)),
-            "li": [1, 2, 3],
-        }
-    else:
-        assert data.dict() == {
-            "vec": list(range(16)),
-            "li": [1, 2, 3],
-        }
+    assert json.loads(data.model_dump_json()) == {
+        "vec": list(range(16)),
+        "li": [1, 2, 3],
+    }
 
     schema = pydantic_to_schema(TestModel)
     assert schema == pa.schema(
@@ -440,10 +444,7 @@ def test_fixed_size_list_field():
         ]
     )
 
-    if PYDANTIC_VERSION.major >= 2:
-        json_schema = TestModel.model_json_schema()
-    else:
-        json_schema = TestModel.schema()
+    json_schema = TestModel.model_json_schema()
 
     assert json_schema == {
         "properties": {
diff --git a/python/python/tests/test_query.py b/python/python/tests/test_query.py
index 6840be052..d2629d1a8 100644
--- a/python/python/tests/test_query.py
+++ b/python/python/tests/test_query.py
@@ -570,6 +570,15 @@ def test_query_builder(table):
     assert all(np.array(rs[0]["vector"]) == [1, 2])
 
 
+def test_query_multiple_vectors(table):
+    results = table.search([np.array([1, 2]), np.array([4, 5])]).limit(1).to_list()
+
+    assert len(results) == 2
+    results_by_query = {result["query_index"]: result for result in results}
+    assert results_by_query[0]["id"] == 1
+    assert results_by_query[1]["id"] == 2
+
+
 def test_with_row_id(table: lancedb.table.Table):
     rs = table.search().with_row_id(True).to_arrow()
     assert "_rowid" in rs.column_names
diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py
index d5d3569d3..2952f00a4 100644
--- a/python/python/tests/test_remote_db.py
+++ b/python/python/tests/test_remote_db.py
@@ -35,6 +35,12 @@ def make_mock_http_handler(handler):
     return MockLanceDBHandler
 
 
+@pytest.mark.parametrize("db_name", ["a" * 64, "invalid..database"])
+def test_connect_rejects_invalid_cloud_dns_hostname(db_name):
+    with pytest.raises(ValueError, match="DNS labels must contain 1 to 63 bytes"):
+        lancedb.connect(f"db://{db_name}", api_key="fake")
+
+
 @contextlib.contextmanager
 def mock_lancedb_connection(handler):
     with http.server.HTTPServer(
@@ -236,8 +242,8 @@ def test_remote_table_branches_sync():
         table.branches.delete("exp")
 
 
-def test_remote_table_branch_merge_defaults_to_execute():
-    merge_bodies = []
+def test_remote_table_cherry_pick_defaults_to_execute():
+    cherry_pick_bodies = []
     diff = {
         "fromBranch": "exp",
         "parentVersion": 1,
@@ -259,8 +265,7 @@ def test_remote_table_branch_merge_defaults_to_execute():
         "changedColumns": [],
         "addedIndexes": [],
         "removedIndexes": [],
-        "mergeable": True,
-        "mergeBlockers": [],
+        "errors": [],
     }
 
     def handler(request):
@@ -270,11 +275,11 @@ def test_remote_table_branch_merge_defaults_to_execute():
         else:
             content_len = int(request.headers.get("Content-Length"))
             request_body = json.loads(request.rfile.read(content_len))
-            merge_bodies.append(request_body)
+            cherry_pick_bodies.append(request_body)
             dry_run = request_body["dry_run"]
             status = 200 if dry_run else 409
             body = {
-                "status": "ready" if dry_run else "rejected",
+                "status": "ready" if dry_run else "failed",
                 "diff": diff,
                 "preview": {"promotedColumns": []},
             }
@@ -286,10 +291,10 @@ def test_remote_table_branch_merge_defaults_to_execute():
 
     with mock_lancedb_connection(handler) as db:
         branches = db.open_table("test").branches
-        assert branches.merge("exp")["status"] == "rejected"
-        assert branches.merge("exp", dry_run=True)["status"] == "ready"
+        assert branches.cherry_pick("exp")["status"] == "failed"
+        assert branches.cherry_pick("exp", dry_run=True)["status"] == "ready"
 
-    assert merge_bodies == [
+    assert cherry_pick_bodies == [
         {"from_branch": "exp", "dry_run": False},
         {"from_branch": "exp", "dry_run": True},
     ]
@@ -870,11 +875,85 @@ def test_remote_create_index_async_returns_job():
         table = db.create_table("test", [{"id": 1}])
         job = table.create_index_async("id", config=BTree())
         assert job.id == "job-1"
-        job.wait(timeout=timedelta(seconds=30))
+        assert job.wait(timeout=timedelta(seconds=30)) is None
         assert len(describe_calls) == 2
         job.cancel()
 
 
+def test_remote_refresh_async_returns_typed_terminal_result():
+    terminal_result = {
+        "rows_assigned": 12,
+        "rows_failed": 0,
+        "rows_remaining": 0,
+        "source_version": 7,
+        "published_version": 8,
+    }
+
+    def handler(request):
+        content_len = int(request.headers.get("Content-Length", 0))
+        body = request.rfile.read(content_len) if content_len > 0 else b""
+        if request.path == "/v1/table/test/backfill_column":
+            assert json.loads(body)["column"] == "derived"
+            request.send_response(202)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(b'{"job_id": "refresh-1"}')
+        elif request.path == "/v1/jobs/describe":
+            assert json.loads(body)["job_id"] == "refresh-1"
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(
+                json.dumps(
+                    {
+                        "job_id": "refresh-1",
+                        "job_type": "function_refresh",
+                        "job_state": "DONE",
+                        "result": terminal_result,
+                    }
+                ).encode()
+            )
+        elif request.path == "/v1/table/test/create/?mode=create":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(b"{}")
+        elif request.path == "/v1/table/test/describe/":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(
+                json.dumps(
+                    {
+                        "version": 1,
+                        "schema": {
+                            "fields": [
+                                {
+                                    "name": "id",
+                                    "type": {"type": "int64"},
+                                    "nullable": False,
+                                }
+                            ]
+                        },
+                    }
+                ).encode()
+            )
+        else:
+            request.send_response(404)
+            request.end_headers()
+
+    with mock_lancedb_connection(handler) as db:
+        table = db.create_table("test", [{"id": 1}])
+        job = table.refresh_column_async("derived")
+        assert job.id == "refresh-1"
+        result = job.wait(timeout=timedelta(seconds=30))
+
+    assert isinstance(result, lancedb.RefreshColumnResult)
+    assert result.model_dump() == terminal_result
+    assert result.rows_filled == 12
+    assert result.version == 8
+
+
 def test_remote_job_wait_raises_on_failure():
     from lancedb.exceptions import JobFailedError
     from lancedb.index import BTree
@@ -1127,6 +1206,131 @@ def test_stats():
         assert res == stats
 
 
+@contextlib.contextmanager
+def lsm_test_table(lsm_handler):
+    """A remote table whose LSM routes are served by ``lsm_handler``.
+
+    ``lsm_handler(request, route)`` is called for ``/v1/table/test/<route>/``
+    where route is one of flush_lsm, compact_lsm, get_lsm_stats, and is
+    responsible for writing the response.
+    """
+    routes = ("flush_lsm", "compact_lsm", "get_lsm_stats")
+
+    def handler(request):
+        match = re.fullmatch(r"/v1/table/test/(\w+)/", request.path)
+        route = match.group(1) if match else None
+        if route in routes:
+            lsm_handler(request, route)
+        elif route == "describe":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(b'{"version": 1, "schema": {"fields": []}}')
+        else:
+            request.send_response(404)
+            request.end_headers()
+
+    with mock_lancedb_connection(handler) as db:
+        yield db.open_table("test")
+
+
+def read_json_body(request):
+    content_len = int(request.headers.get("Content-Length"))
+    return json.loads(request.rfile.read(content_len))
+
+
+def send_json(request, payload, status=200):
+    request.send_response(status)
+    request.send_header("Content-Type", "application/json")
+    request.end_headers()
+    request.wfile.write(json.dumps(payload).encode())
+
+
+def test_get_lsm_stats_sync():
+    """The sync wrapper round-trips the server payload into a dict."""
+    bucket = {
+        "shard_id": "b0",
+        "status": "Active",
+        "writer_epoch": 3,
+        "manifest_version": 12,
+        "current_generation": 6,
+        "replay_after_wal_entry_position": 40,
+        "wal_entry_position_last_seen": 42,
+        "generations": [{"generation": 5, "bytes": 1024, "rows": 7}],
+        "compacting": False,
+        "memtables": [
+            {
+                "generation": 6,
+                "rows": 2,
+                "bytes": 64,
+                "batches": 1,
+                "indexes": ["vec_idx"],
+            }
+        ],
+    }
+    seen_bodies = []
+
+    def lsm_handler(request, route):
+        assert route == "get_lsm_stats"
+        seen_bodies.append(read_json_body(request))
+        send_json(request, {"lsm_stats": {"buckets": [bucket]}})
+
+    with lsm_test_table(lsm_handler) as table:
+        assert table.get_lsm_stats() == {"buckets": [bucket]}
+        # Off by default, and forwarded when asked for.
+        assert seen_bodies == [{"include_generation_rows": False}]
+        table.get_lsm_stats(include_generation_rows=True)
+        assert seen_bodies[-1] == {"include_generation_rows": True}
+
+
+def test_get_lsm_stats_sync_returns_none_when_lsm_disabled():
+    """A null envelope means the LSM write path is not enabled, not an error."""
+
+    def lsm_handler(request, route):
+        send_json(request, {"lsm_stats": None})
+
+    with lsm_test_table(lsm_handler) as table:
+        assert table.get_lsm_stats() is None
+
+
+def test_flush_and_compact_lsm_sync():
+    """Both are one-shot POSTs answered 202 with no body."""
+    called = []
+
+    def lsm_handler(request, route):
+        called.append(route)
+        request.send_response(202)
+        request.end_headers()
+
+    with lsm_test_table(lsm_handler) as table:
+        assert table.flush_lsm() is None
+        assert table.compact_lsm() is None
+        assert called == ["flush_lsm", "compact_lsm"]
+
+
+def test_checkpoint_lsm_sync():
+    """Seal, read the watermark, and return once L0 holds nothing.
+
+    The convergence loop itself is covered in Rust; this pins the sync
+    binding to the endpoints it drives.
+    """
+    called = []
+
+    def lsm_handler(request, route):
+        called.append(route)
+        if route == "get_lsm_stats":
+            # An empty L0 yields no target watermark, so the loop is done
+            # after the seal without ever polling compaction.
+            send_json(request, {"lsm_stats": {"buckets": []}})
+        else:
+            request.send_response(202)
+            request.end_headers()
+
+    with lsm_test_table(lsm_handler) as table:
+        assert table.checkpoint_lsm() is None
+        assert called == ["flush_lsm", "get_lsm_stats"]
+
+
 @contextlib.contextmanager
 def query_test_table(query_handler, *, server_version=Version("0.1.0")):
     def handler(request):
diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py
index 069527b21..56e0eacfd 100644
--- a/python/python/tests/test_table.py
+++ b/python/python/tests/test_table.py
@@ -2,10 +2,13 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 
+import ctypes
+import gc
 import os
 import sys
 import threading
 import warnings
+import weakref
 from concurrent.futures import ThreadPoolExecutor
 from datetime import date, datetime, timedelta
 from time import sleep
@@ -99,6 +102,30 @@ def test_basic(mem_db: DBConnection):
     assert table.to_arrow() == expected_data
 
 
+def test_search_preserves_nulls_from_sliced_arrow_table(mem_db: DBConnection):
+    data = pa.table(
+        {
+            "id": [0, 1, 2, 3, 4],
+            "score_cn": [None, 22, None, 5, 8],
+            "score_mt": [None, 42, None, 5, 8],
+            "vector": [
+                [20, 19, -1, -1],
+                [41, 38, 22, 42],
+                [10, 10, -1, -1],
+                [5, 5, 5, 5],
+                [8, 8, 8, 8],
+            ],
+        }
+    ).slice(1)
+
+    table = mem_db.create_table("sliced_nullable", data=data)
+    result = table.search([41, 38, 22, 42]).limit(1).to_arrow()
+
+    assert result["id"].to_pylist() == [1]
+    assert result["score_cn"].to_pylist() == [22]
+    assert result["score_mt"].to_pylist() == [42]
+
+
 def test_table_to_pandas_default_matches_arrow(tmp_db: DBConnection):
     pd = pytest.importorskip("pandas")
     data = pa.table({"id": [1, 2], "text": ["one", "two"]})
@@ -435,6 +462,38 @@ def test_add(mem_db: DBConnection):
     _add(table, schema)
 
 
+def test_add_releases_arrow_buffers_without_gc(mem_db: DBConnection):
+    """Regression test for https://github.com/lancedb/lancedb/issues/2512."""
+    schema = pa.schema([pa.field("x", pa.int64())])
+    table = mem_db.create_table("test_add_releases_arrow_buffers", schema=schema)
+
+    class BufferOwner:
+        def __init__(self, size: int):
+            self.memory = ctypes.create_string_buffer(size)
+
+    owner_refs = []
+    gc_was_enabled = gc.isenabled()
+    gc.disable()
+    try:
+        for _ in range(3):
+            size = 8 * 1024
+            owner = BufferOwner(size)
+            arrow_buffer = pa.foreign_buffer(
+                ctypes.addressof(owner.memory), size, owner
+            )
+            array = pa.Array.from_buffers(pa.int64(), 1024, [None, arrow_buffer])
+            batch = pa.RecordBatch.from_arrays([array], schema=schema)
+            owner_refs.append(weakref.ref(owner))
+
+            table.add(batch)
+            del batch, array, arrow_buffer, owner
+
+        assert all(owner_ref() is None for owner_ref in owner_refs)
+    finally:
+        if gc_was_enabled:
+            gc.enable()
+
+
 def test_add_write_parallelism(mem_db: DBConnection):
     schema = pa.schema([pa.field("id", pa.int64())])
     table = mem_db.create_table("test", schema=schema)
@@ -870,6 +929,7 @@ def test_polars(mem_db: DBConnection):
 
     # enter table to polars dataframe
     result = table.to_polars()
+    assert isinstance(result, pl.LazyFrame)
     assert np.allclose(result.collect()["vector"].to_list(), data["vector"])
 
     # make sure filtering isn't broken
@@ -1407,7 +1467,7 @@ def test_create_index_async_returns_done_job(mem_db: DBConnection):
     table = mem_db.create_table("job_test", [{"id": i} for i in range(10)])
     job = table.create_index_async("id", config=BTree())
     assert job.id is None
-    job.wait()
+    assert job.wait() is None
     assert len(table.list_indices()) == 1
     job.cancel()
 
@@ -1786,6 +1846,27 @@ def test_add_with_empty_fixed_size_list_drops_bad_rows(mem_db: DBConnection):
     assert np.allclose(data["embedding"].to_pylist()[0], np.array([0.1] * 16))
 
 
+def test_add_nullable_fixed_size_list_with_none(mem_db: DBConnection):
+    """Regression test for issue #2340."""
+    table = mem_db.create_table(
+        "test_nullable_fixed_size_list",
+        schema=pa.schema(
+            [
+                pa.field("id", pa.string()),
+                pa.field("feature", pa.list_(pa.float32(), 256)),
+                pa.field("tags", pa.list_(pa.string())),
+            ]
+        ),
+    )
+
+    table.add([{"id": "1", "feature": None, "tags": ["tag1", "tag2"]}])
+
+    result = table.to_arrow()
+    assert result.to_pylist() == [
+        {"id": "1", "feature": None, "tags": ["tag1", "tag2"]}
+    ]
+
+
 def test_add_nullable_struct_with_none(mem_db: DBConnection):
     """Regression test for issue #2654: a nullable struct column whose
     first batch contains only None values must not crash in
@@ -1825,6 +1906,33 @@ def test_add_nullable_struct_with_none(mem_db: DBConnection):
     assert result.column("data").to_pylist() == [{"x": 1.0}, None]
 
 
+def test_read_mostly_null_list_v2_2_page_boundary(tmp_path):
+    # Regression test for #3194. This row/value count crosses a v2.2 structural
+    # encoding page boundary where Lance 3.0.0 sliced repetition/definition
+    # levels by row offset and decoded child arrays at different lengths.
+    num_rows = 64_885
+    num_values = 217
+    list_type = pa.list_(pa.float32())
+    source = pa.table(
+        {
+            "id": np.arange(num_rows, dtype=np.int64),
+            "coords": pa.array(
+                [[1.0, 2.0, 3.0, 4.0]] * num_values + [None] * (num_rows - num_values),
+                type=list_type,
+            ),
+        }
+    )
+    db = lancedb.connect(
+        tmp_path,
+        storage_options={"new_table_data_storage_version": "2.2"},
+    )
+    table = db.create_table("test_sparse_nullable_list", data=source)
+
+    result = table.search().select(["id", "coords"]).limit(num_rows).to_arrow()
+
+    assert result.equals(source)
+
+
 def test_add_with_integer_embeddings_preserves_casting(mem_db: DBConnection):
     class Schema(LanceModel):
         text: str
@@ -2110,6 +2218,45 @@ def test_merge(tmp_db: DBConnection, tmp_path):
     table.merge(other_dataset, left_on="id")
 
 
+@pytest.mark.parametrize("storage_version", ["legacy", "stable"])
+def test_search_after_merge(tmp_path, storage_version):
+    pytest.importorskip("lance")
+    pd = pytest.importorskip("pandas")
+
+    db = lancedb.connect(
+        tmp_path,
+        storage_options={"new_table_data_storage_version": storage_version},
+    )
+    rng = np.random.default_rng(42)
+    row_count = 512
+    vectors = rng.standard_normal((row_count, 8)).astype(np.float32)
+    table = db.create_table(
+        "search_after_merge",
+        data=pd.DataFrame(
+            {
+                "id": [str(i) for i in range(row_count)],
+                "vector": list(vectors),
+            }
+        ),
+    )
+    table.create_index("vector", config=IvfPq(num_partitions=1, num_sub_vectors=2))
+
+    links = pd.DataFrame(
+        {
+            "id": [str(i) for i in range(row_count // 2)],
+            "link": [f"https://example.com/{i}" for i in range(row_count // 2)],
+        }
+    )
+    table.merge(links, left_on="id")
+
+    query = table.search(vectors[-1]).refine_factor(50).limit(10)
+    assert "ANN" in query.explain_plan(verbose=True)
+
+    result = query.to_arrow()
+    links_by_id = dict(zip(result["id"].to_pylist(), result["link"].to_pylist()))
+    assert links_by_id[str(row_count - 1)] is None
+
+
 def test_delete(mem_db: DBConnection):
     table = mem_db.create_table(
         "my_table",
@@ -2196,6 +2343,20 @@ def test_update(mem_db: DBConnection):
     assert np.allclose(v, np.array([[1.2, 1.9], [1.1, 1.1]]))
 
 
+def test_update_with_arrow_scalar(mem_db: DBConnection):
+    schema = pa.schema({"id": pa.int64(), "vector": pa.list_(pa.float32(), 4)})
+    table = mem_db.create_table("my_table", schema=schema)
+    table.add([{"id": 1, "vector": [1.0, 2.0, 3.0, 4.0]}])
+
+    value = table.search().select(["vector"]).limit(1).to_arrow()["vector"][0]
+    assert isinstance(value, pa.FixedSizeListScalar)
+
+    result = table.update(where="id == 1", values={"vector": value})
+
+    assert result.rows_updated == 1
+    assert table.to_arrow()["vector"].to_pylist() == [[1.0, 2.0, 3.0, 4.0]]
+
+
 def test_update_types(mem_db: DBConnection):
     table = mem_db.create_table(
         "my_table",
@@ -2363,6 +2524,55 @@ def test_merge_insert(mem_db: DBConnection):
         )
 
 
+def test_merge_insert_nullable_pandas_into_pydantic_schema(mem_db: DBConnection):
+    # Regression test for https://github.com/lancedb/lancedb/issues/2366
+    pd = pytest.importorskip("pandas")
+
+    class Document(LanceModel):
+        id: int
+        title: str
+        content: str
+
+    table = mem_db.create_table("documents", schema=Document)
+    table.add(
+        pd.DataFrame(
+            {
+                "title": ["Old title", "Unchanged"],
+                "id": [2, 3],
+                "content": ["Old content", "Keep this"],
+            }
+        )
+    )
+
+    # Pandas produces nullable Arrow fields, in an order that differs from the
+    # non-nullable Pydantic schema. This is valid as long as the data has no nulls.
+    new_data = pd.DataFrame(
+        {
+            "title": ["Inserted", "Updated"],
+            "id": [1, 2],
+            "content": ["New row", "New content"],
+        }
+    )
+    result = (
+        table.merge_insert("id")
+        .when_matched_update_all()
+        .when_not_matched_insert_all()
+        .execute(new_data)
+    )
+
+    assert result.num_inserted_rows == 1
+    assert result.num_updated_rows == 1
+    expected = pa.Table.from_pylist(
+        [
+            {"id": 1, "title": "Inserted", "content": "New row"},
+            {"id": 2, "title": "Updated", "content": "New content"},
+            {"id": 3, "title": "Unchanged", "content": "Keep this"},
+        ],
+        schema=Document.to_arrow_schema(),
+    )
+    assert table.to_arrow().sort_by("id") == expected
+
+
 def test_merge_insert_by_source_delete_expr(mem_db: DBConnection):
     table = mem_db.create_table(
         "my_table",
@@ -2463,6 +2673,36 @@ def test_merge_insert_subschema(mem_db: DBConnection, data_format):
     assert table.to_arrow().sort_by("id") == expected
 
 
+def test_repeated_partial_merge_insert_with_scalar_index(mem_db: DBConnection):
+    def make_batch(start: int) -> pa.Table:
+        return pa.table(
+            {
+                "id": [f"id-{i:04}" for i in range(start, start + 100)],
+                "category": ["A"] * 100,
+                "value_a": [float(i) for i in range(start, start + 100)],
+                "value_b": [float(i) / 10 for i in range(100)],
+            }
+        )
+
+    table = mem_db.create_table("my_table", data=make_batch(0))
+    table.add(make_batch(100))
+    table.add(make_batch(200))
+    table.create_index("id", config=BTree())
+
+    ids = [f"id-{i:04}" for i in range(100, 200)]
+    for value in (999.0, 888.0):
+        result = (
+            table.merge_insert("id")
+            .when_matched_update_all()
+            .execute(pa.table({"id": ids, "value_a": [value] * 100}))
+        )
+        assert result.num_updated_rows == 100
+
+    actual = table.to_arrow().sort_by("id")
+    assert actual.num_rows == 300
+    assert actual["value_a"].to_pylist()[100:200] == [888.0] * 100
+
+
 @pytest.mark.asyncio
 async def test_merge_insert_async(mem_db_async: AsyncConnection):
     data = pa.table({"a": [1, 2, 3], "b": ["a", "b", "c"]})
@@ -2532,6 +2772,56 @@ async def test_merge_insert_async(mem_db_async: AsyncConnection):
     assert (await table.to_arrow()).sort_by("a") == expected
 
 
+@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
+@pytest.mark.asyncio
+async def test_merge_insert_encodes_json(mem_db_async: AsyncConnection):
+    json_type = pa.json_()
+    schema = pa.schema([pa.field("id", pa.string()), pa.field("j", json_type)])
+
+    def json_table(rows):
+        json_values = pa.ExtensionArray.from_storage(
+            json_type,
+            pa.array([value for _, value in rows], type=json_type.storage_type),
+        )
+        return pa.Table.from_arrays(
+            [pa.array([row_id for row_id, _ in rows]), json_values], schema=schema
+        )
+
+    table = await mem_db_async.create_table("json_merge", schema=schema)
+    await table.add(json_table([("a", '{"k": 1}'), ("b", '{"k": 9}')]))
+
+    await (
+        table.merge_insert("id")
+        .when_matched_update_all()
+        .execute(json_table([("a", '{"k": 2}')]))
+    )
+
+    rows = sorted(await table.query().to_list(), key=lambda row: row["id"])
+    assert rows == [
+        {"id": "a", "j": '{"k":2}'},
+        {"id": "b", "j": '{"k":9}'},
+    ]
+    filtered = await table.query().where("json_extract(j, '$.k') = '2'").to_list()
+    assert filtered == [{"id": "a", "j": '{"k":2}'}]
+
+
+@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
+@pytest.mark.asyncio
+async def test_add_sanitization_encodes_json(mem_db_async: AsyncConnection):
+    json_type = pa.json_()
+    schema = pa.schema([pa.field("id", pa.string()), pa.field("j", json_type)])
+    json_values = pa.ExtensionArray.from_storage(
+        json_type, pa.array(['{"k": 3}'], type=json_type.storage_type)
+    )
+    data = pa.Table.from_arrays([pa.array(["c"]), json_values], schema=schema)
+
+    table = await mem_db_async.create_table("json_add", schema=schema)
+    await table.add(data, on_bad_vectors="fill")
+
+    rows = await table.query().where("json_extract(j, '$.k') = '3'").to_list()
+    assert rows == [{"id": "c", "j": '{"k":3}'}]
+
+
 def test_create_with_embedding_function(mem_db: DBConnection):
     class MyTable(LanceModel):
         text: str
@@ -2559,15 +2849,40 @@ def test_create_with_embedding_function(mem_db: DBConnection):
     assert actual == expected
 
 
+def test_create_f16_table_from_arrow_data(mem_db: DBConnection):
+    dimension = 32
+    num_rows = 512
+    values = pa.array(
+        np.random.default_rng(42)
+        .standard_normal(num_rows * dimension)
+        .astype(np.float16)
+    )
+    df = pa.table(
+        {
+            "text": [f"s-{i}" for i in range(num_rows)],
+            "vector": pa.FixedSizeListArray.from_arrays(values, dimension),
+        }
+    )
+    table = mem_db.create_table("f16_tbl", data=df)
+    assert table.schema.field("vector").type == pa.list_(pa.float16(), dimension)
+    table.create_index(num_partitions=2, num_sub_vectors=2)
+
+    query = df["vector"][2].as_py()
+    expected = table.search(query).limit(2).to_arrow()
+
+    assert "s-2" in expected["text"].to_pylist()
+
+
 def test_create_f16_table(mem_db: DBConnection):
     class MyTable(LanceModel):
         text: str
         vector: Vector(32, value_type=pa.float16())
 
+    rng = np.random.default_rng(42)
     df = pa.table(
         {
             "text": [f"s-{i}" for i in range(512)],
-            "vector": [np.random.randn(32).astype(np.float16) for _ in range(512)],
+            "vector": [rng.standard_normal(32).astype(np.float16) for _ in range(512)],
         }
     )
     table = mem_db.create_table(
@@ -3448,7 +3763,8 @@ def test_stats(mem_db: DBConnection):
     stats = table.stats()
     print(f"{stats=}")
     assert stats == {
-        "total_bytes": 60,
+        # Full on-disk size of the data file, footer and metadata included.
+        "total_bytes": 633,
         "num_rows": 2,
         "num_indices": 0,
         "fragment_stats": {
@@ -3466,6 +3782,13 @@ def test_stats(mem_db: DBConnection):
         },
     }
 
+    # Index files count toward total_bytes too (only deletion files and
+    # manifests are excluded).
+    table.create_index("id", config=BTree())
+    stats_with_index = table.stats()
+    assert stats_with_index["num_indices"] == 1
+    assert stats_with_index["total_bytes"] > stats["total_bytes"]
+
 
 def test_create_table_empty_list_with_schema(mem_db: DBConnection):
     """Test creating table with empty list data and schema
@@ -3489,8 +3812,8 @@ def test_create_table_empty_list_no_schema_error(mem_db: DBConnection):
         mem_db.create_table("test_empty_no_schema", data=[])
 
 
-def test_add_table_with_empty_embeddings(tmp_path):
-    """Test exact scenario from issue #1968
+def test_create_table_without_data_with_vector_schema(tmp_path):
+    """Test exact scenario from issue #1968.
 
     Regression test for issue #1968:
     https://github.com/lancedb/lancedb/issues/1968
@@ -3502,6 +3825,9 @@ def test_add_table_with_empty_embeddings(tmp_path):
         embedding: Vector(16)
 
     table = db.create_table("test", schema=MySchema)
+    assert table.count_rows() == 0
+    assert table.schema == MySchema.to_arrow_schema()
+
     table.add(
         [{"text": "bar", "embedding": [0.1] * 16}],
         on_bad_vectors="drop",
@@ -3578,3 +3904,80 @@ async def test_async_search_runs_embedding_on_dedicated_executor(
     assert all(name.startswith("lancedb-embedding") for name in captured_threads), (
         f"embedding ran off the dedicated executor: {captured_threads}"
     )
+
+
+def test_computed_column_declare_and_refresh(tmp_path):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("computed", [{"x": 1}, {"x": 2}])
+
+    table.add_columns(computed={"doubled": "x * 2"})
+    assert table.to_arrow()["doubled"].to_pylist() == [None, None]
+
+    result = table.refresh_column("doubled")
+    assert result.rows_filled == 2
+    assert sorted(table.to_arrow()["doubled"].to_pylist()) == [2, 4]
+
+    table.add([{"x": 5}])
+    assert table.refresh_column("doubled").rows_filled == 1
+    assert sorted(table.to_arrow()["doubled"].to_pylist()) == [2, 4, 10]
+
+
+def test_computed_column_rejects_transforms_and_computed_together(tmp_path):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("computed_mixed", [{"x": 1}])
+    with pytest.raises(ValueError):
+        table.add_columns({"a": "x + 1"}, computed={"b": "x * 2"})
+
+
+@pytest.mark.asyncio
+async def test_computed_column_async(tmp_path):
+    db = await lancedb.connect_async(tmp_path)
+    table = await db.create_table("computed_async", [{"x": 3}])
+
+    await table.add_columns(computed={"tripled": "x * 3"})
+    await table.refresh_column("tripled")
+
+    assert (await table.to_arrow())["tripled"].to_pylist() == [9]
+
+
+def test_refresh_column_async_returns_job(tmp_path):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("computed_job", [{"x": 1}, {"x": 2}])
+    table.add_columns(computed={"doubled": "x * 2"})
+
+    job = table.refresh_column_async("doubled")
+    assert job.id is None  # in-process jobs have no server id
+    result = job.wait()
+    assert isinstance(result, lancedb.RefreshColumnResult)
+    assert result.rows_assigned == 2
+    assert result.rows_failed == 0
+    assert result.rows_remaining == 0
+    assert result.source_version == 2
+    assert result.published_version == 3
+    assert job.status() == "finished"
+    assert sorted(table.to_arrow()["doubled"].to_pylist()) == [2, 4]
+
+    no_op = table.refresh_column_async("doubled").wait()
+    assert no_op.rows_assigned == 0
+    assert no_op.source_version == 3
+    assert no_op.published_version is None
+
+    # Bad input raises at the call, not through the job.
+    with pytest.raises(Exception, match="not a computed column"):
+        table.refresh_column_async("x")
+
+
+@pytest.mark.asyncio
+async def test_refresh_column_async_job_async_table(tmp_path):
+    db = await lancedb.connect_async(tmp_path)
+    table = await db.create_table("computed_job_async", [{"x": 3}])
+    await table.add_columns(computed={"tripled": "x * 3"})
+
+    job = await table.refresh_column_async("tripled")
+    result = await job.wait()
+    assert isinstance(result, lancedb.RefreshColumnResult)
+    assert result.rows_assigned == 1
+    assert result.source_version == 2
+    assert result.published_version == 3
+    assert await job.status() == "finished"
+    assert (await table.to_arrow())["tripled"].to_pylist() == [9]
diff --git a/python/python/tests/test_voyageai_embeddings.py b/python/python/tests/test_voyageai_embeddings.py
index ac1554cad..040cade1f 100644
--- a/python/python/tests/test_voyageai_embeddings.py
+++ b/python/python/tests/test_voyageai_embeddings.py
@@ -75,6 +75,22 @@ class TestVoyageAIModelRegistration:
         with pytest.raises(ValueError, match="not supported"):
             func.ndims()
 
+    def test_voyage3_source_embeddings_use_text_api(self, mock_voyageai_client):
+        """Regression test for text table data being sent to the multimodal API."""
+        mock_voyageai_client.tokenize.return_value = [["hello", "world"]]
+        mock_voyageai_client.embed.return_value.embeddings = [[0.1] * 1024]
+
+        registry = get_registry()
+        func = registry.get("voyageai").create(name="voyage-3")
+
+        embeddings = func.compute_source_embeddings("hello world")
+
+        assert embeddings == [[0.1] * 1024]
+        mock_voyageai_client.embed.assert_called_once_with(
+            texts=["hello world"], model="voyage-3", input_type="document"
+        )
+        mock_voyageai_client.multimodal_embed.assert_not_called()
+
     @pytest.mark.parametrize(
         "model_name",
         [
diff --git a/python/python/tests/utils.py b/python/python/tests/utils.py
index 62ec74497..4882f0c28 100644
--- a/python/python/tests/utils.py
+++ b/python/python/tests/utils.py
@@ -1,7 +1,17 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import contextlib
+import http.server
+import json
+import re
+import threading
+
+import lancedb
+import pyarrow as pa
 import pytest
 
+ARROW_FILE_CONTENT_TYPE = "application/vnd.apache.arrow.file"
+
 
 def exception_output(e_info: pytest.ExceptionInfo):
     import traceback
@@ -9,3 +19,199 @@ def exception_output(e_info: pytest.ExceptionInfo):
     # skip traceback part, since it's not worth checking in tests
     lines = traceback.format_exception_only(e_info.type, e_info.value)
     return "".join(lines).strip()
+
+
+def parse_in_list(filter_sql: str) -> list[int]:
+    """Pull the integers out of a `<col> IN (a, b, c)` predicate.
+
+    Scoped to the parenthesised list so a cast in the SQL adds no phantom values.
+    """
+    match = re.search(r"\bIN\s*\(([^)]*)\)", filter_sql, re.IGNORECASE)
+    assert match is not None, f"expected an IN list, got: {filter_sql}"
+    return [int(m) for m in re.findall(r"-?\d+", match.group(1))]
+
+
+def is_row_id_take(body) -> bool:
+    """True when a query body fetches specific rows by row id."""
+    return "_rowid" in (body.get("filter") or "")
+
+
+def arrow_file_bytes(table: pa.Table) -> bytes:
+    """Serialize to the Arrow IPC *file* framing the /query/ route answers with."""
+    sink = pa.BufferOutputStream()
+    with pa.ipc.new_file(sink, table.schema) as writer:
+        writer.write_table(table)
+    return sink.getvalue().to_pybytes()
+
+
+class MockPermutationServer:
+    """A stand-in LanceDB server hosting one table whose ``id`` equals its ``_rowid``.
+
+    Records every ``/query/`` body so tests can assert on the request shapes sent to
+    the server, which is the part that has to stay compatible.
+    """
+
+    def __init__(self, name="remote_data", num_rows=8, vary_scan_order=False):
+        self.name = name
+        self.num_rows = num_rows
+        self.query_bodies = []
+        # Stand in for a distributed scan that answers in no fixed order.
+        self.vary_scan_order = vary_scan_order
+        self.scan_calls = 0
+
+    def __call__(self, request):
+        path = request.path
+        if path == f"/v1/table/{self.name}/describe/":
+            return self._json(
+                request,
+                {
+                    "version": 1,
+                    "schema": {
+                        "fields": [
+                            {"name": "id", "type": {"type": "int64"}, "nullable": False}
+                        ]
+                    },
+                },
+            )
+        if path == f"/v1/table/{self.name}/get_lsm_write_spec/":
+            self._read_body(request)
+            # Null spec: this table has no LSM write path.
+            return self._json(request, {"lsm_write_spec": None})
+        if path == f"/v1/table/{self.name}/count_rows/":
+            self._read_body(request)
+            return self._json(request, self.num_rows)
+        if path == f"/v1/table/{self.name}/query/":
+            return self._query(request, self._read_body(request))
+
+        # Drain first, so an unexpected route cannot desync a keep-alive connection.
+        self._read_body(request)
+        request.send_response(404)
+        request.end_headers()
+
+    @property
+    def scans(self):
+        """Bodies of the permutation build scan: the row id column, nothing else."""
+        return [b for b in self.query_bodies if b.get("columns") == ["_rowid"]]
+
+    @property
+    def takes(self):
+        """Bodies of the row-id takes the loader fetches batches with.
+
+        Keyed on `_rowid`, not "has a filter": the schema probe also has a predicate.
+        """
+        return [b for b in self.query_bodies if is_row_id_take(b)]
+
+    @staticmethod
+    def _read_body(request):
+        content_len = int(request.headers.get("Content-Length") or 0)
+        return json.loads(request.rfile.read(content_len)) if content_len else {}
+
+    @staticmethod
+    def _json(request, payload):
+        request.send_response(200)
+        request.send_header("Content-Type", "application/json")
+        request.end_headers()
+        request.wfile.write(json.dumps(payload).encode())
+
+    @staticmethod
+    def _arrow(request, table):
+        body = arrow_file_bytes(table)
+        request.send_response(200)
+        request.send_header("Content-Type", ARROW_FILE_CONTENT_TYPE)
+        request.send_header("Content-Length", str(len(body)))
+        request.end_headers()
+        request.wfile.write(body)
+
+    def _query(self, request, body):
+        self.query_bodies.append(body)
+
+        if is_row_id_take(body):
+            # A row-id take. Answer ascending, so tests prove the client reorders.
+            row_ids = sorted(parse_in_list(body["filter"]))
+            return self._arrow(
+                request,
+                pa.table(
+                    {
+                        "id": pa.array(row_ids, pa.int64()),
+                        "_rowid": pa.array(row_ids, pa.uint64()),
+                    }
+                ),
+            )
+
+        if body.get("columns") == ["_rowid"]:
+            # The permutation build scan: row ids and nothing else.
+            row_ids = list(range(self.num_rows))
+            if self.vary_scan_order and self.scan_calls % 2:
+                row_ids.reverse()
+            self.scan_calls += 1
+            return self._arrow(
+                request,
+                pa.table({"_rowid": pa.array(row_ids, pa.uint64())}),
+            )
+
+        # The schema probe: filtered to nothing, so it carries schema and no rows.
+        return self._arrow(request, pa.table({"id": pa.array([], pa.int64())}))
+
+
+def _make_handler(serve):
+    class MockLanceDBHandler(http.server.BaseHTTPRequestHandler):
+        def do_GET(self):
+            serve(self)
+
+        def do_POST(self):
+            serve(self)
+
+        def log_message(self, *args):
+            pass  # keep pytest output readable
+
+    return MockLanceDBHandler
+
+
+@contextlib.contextmanager
+def mock_remote_table(server):
+    """Run ``server`` on a local port and yield an open remote table against it.
+
+    Threading: the loader fans out fetch threads a single-threaded server would
+    serialize, hiding the prefetch overlap under test.
+    """
+    with http.server.ThreadingHTTPServer(
+        ("localhost", 0), _make_handler(server)
+    ) as srv:
+        thread = threading.Thread(target=srv.serve_forever)
+        thread.start()
+        try:
+            db = lancedb.connect(
+                "db://dev",
+                api_key="fake",
+                host_override=f"http://localhost:{srv.server_address[1]}",
+                client_config={"timeout_config": {"connect_timeout": 5}},
+            )
+            yield db.open_table(server.name)
+        finally:
+            srv.shutdown()
+            thread.join()
+
+
+def assert_server_safe_row_id_requests(server):
+    """Assert the loader fetched rows by row id and bounded everything else.
+
+    `.get`, not `[...]`, so a dropped field reads as the assertion, not a KeyError.
+    """
+    for body in server.takes:
+        # The fetch needs the row id back to restore the requested order.
+        assert body.get("with_row_id") is True, body
+        assert "_rowid" in body["filter"], body
+
+    # Only the one-off permutation scan may scan the whole table; the schema probe is
+    # built once per split per epoch. `k == 0` counts as unbounded: lance reads a zero
+    # limit as "no limit".
+    def is_unbounded(body):
+        if is_row_id_take(body):
+            return False
+        k = body.get("k")
+        return k is None or k == 0 or k > server.num_rows
+
+    unbounded = [b for b in server.query_bodies if is_unbounded(b)]
+    assert unbounded == server.scans, (
+        f"only the permutation scan may be unbounded, got {unbounded}"
+    )
diff --git a/python/python/type_tests/connect.py b/python/python/type_tests/connect.py
new file mode 100644
index 000000000..eb2cba37c
--- /dev/null
+++ b/python/python/type_tests/connect.py
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+from typing import assert_type
+
+import lancedb
+from lancedb import AsyncConnection, DBConnection
+
+
+def check_connect_type() -> None:
+    assert_type(lancedb.connect("memory://"), DBConnection)
+
+
+async def check_connect_async_type() -> None:
+    assert_type(await lancedb.connect_async("memory://"), AsyncConnection)
diff --git a/python/src/connection.rs b/python/src/connection.rs
index b97d48ad8..902489f4f 100644
--- a/python/src/connection.rs
+++ b/python/src/connection.rs
@@ -333,6 +333,40 @@ impl Connection {
         })
     }
 
+    #[pyo3(signature = (name, source, projections=None, filter=None, limit=None))]
+    pub fn create_materialized_view(
+        self_: PyRef<'_, Self>,
+        name: String,
+        source: String,
+        projections: Option<Vec<(String, String)>>,
+        filter: Option<String>,
+        limit: Option<u64>,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        future_into_py(self_.py(), async move {
+            let mut builder = inner.create_materialized_view(name, source);
+            if let Some(projections) = projections {
+                builder = builder.select(projections);
+            }
+            if let Some(filter) = filter {
+                builder = builder.only_if(filter);
+            }
+            if let Some(limit) = limit {
+                builder = builder.limit(limit);
+            }
+            let view = builder.execute().await.infer_error()?;
+            Ok(Table::new(view.table().clone()))
+        })
+    }
+
+    pub fn list_materialized_views(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        future_into_py(self_.py(), async move {
+            let views = inner.list_materialized_views().await.infer_error()?;
+            Ok(views.into_iter().map(|view| view.name).collect::<Vec<_>>())
+        })
+    }
+
     #[pyo3(signature = (name, namespace_path=None))]
     pub fn drop_table(
         self_: PyRef<'_, Self>,
@@ -346,6 +380,23 @@ impl Connection {
         })
     }
 
+    #[pyo3(signature = (name, namespace_path=None))]
+    pub fn drop_table_async(
+        self_: PyRef<'_, Self>,
+        name: String,
+        namespace_path: Option<Vec<String>>,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        let ns_path = namespace_path.unwrap_or_default();
+        future_into_py(self_.py(), async move {
+            inner
+                .drop_table_async(name, &ns_path)
+                .await
+                .infer_error()
+                .map(crate::job::Job::new)
+        })
+    }
+
     #[pyo3(signature = (namespace_path=None,))]
     pub fn drop_all_tables(
         self_: PyRef<'_, Self>,
@@ -546,6 +597,38 @@ impl Connection {
         Ok(crate::job::Job::new(inner.job(job_id).infer_error()?))
     }
 
+    pub fn create_function_async(
+        self_: PyRef<'_, Self>,
+        request_json: String,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        let request = lancedb::function::FunctionRegistrationRequest::from_json(&request_json)
+            .infer_error()?;
+        future_into_py(self_.py(), async move {
+            inner
+                .create_function_async(request)
+                .await
+                .infer_error()
+                .map(crate::job::Job::new_typed)
+        })
+    }
+
+    pub fn get_function(
+        self_: PyRef<'_, Self>,
+        name: String,
+        version: String,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        future_into_py(self_.py(), async move {
+            inner
+                .get_function(name, version)
+                .await
+                .infer_error()?
+                .to_canonical_json()
+                .infer_error()
+        })
+    }
+
     pub fn list_jobs(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.get_inner()?.clone();
         future_into_py(self_.py(), async move {
diff --git a/python/src/expr.rs b/python/src/expr.rs
index 242e88b05..eae1d96ec 100644
--- a/python/src/expr.rs
+++ b/python/src/expr.rs
@@ -191,8 +191,27 @@ pub fn expr_lit(value: Bound<'_, PyAny>) -> PyResult<PyExpr> {
     }
 
     // datetime.datetime is a subclass of datetime.date, so it must be checked first.
+    //
+    // Python's datetime.timestamp() treats *naive* datetimes as local wall time.
+    // PyArrow (and therefore Lance table storage) encodes naive timestamps as
+    // UTC wall-clock microseconds. Using .timestamp() for naive values therefore
+    // shifts the literal by the local UTC offset on non-UTC machines, so
+    // `col("ts") == lit(naive_dt)` fails against a table that holds the same
+    // naive value. Fix: treat naive datetimes as UTC wall clock (match Arrow);
+    // keep aware datetimes on the real .timestamp() path (correct epoch).
     if let Ok(dt) = value.cast::<PyDateTime>() {
-        let ts: f64 = dt.call_method0("timestamp")?.extract()?;
+        let ts: f64 = if dt.getattr("tzinfo")?.is_none() {
+            // Force UTC interpretation of the naive wall clock.
+            let utc = pyo3::types::PyModule::import(value.py(), "datetime")?
+                .getattr("timezone")?
+                .getattr("utc")?;
+            let kwargs = pyo3::types::PyDict::new(value.py());
+            kwargs.set_item("tzinfo", utc)?;
+            let aware = dt.call_method("replace", (), Some(&kwargs))?;
+            aware.call_method0("timestamp")?.extract()?
+        } else {
+            dt.call_method0("timestamp")?.extract()?
+        };
         let micros = (ts * 1_000_000.0).round() as i64;
         return Ok(PyExpr(df_lit(ScalarValue::TimestampMicrosecond(
             Some(micros),
diff --git a/python/src/index.rs b/python/src/index.rs
index 8c81dcecf..a5ca63c68 100644
--- a/python/src/index.rs
+++ b/python/src/index.rs
@@ -42,7 +42,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
             "Fm" => Ok(LanceDbIndex::Fm(FmIndexBuilder::default())),
             "FTS" => {
                 let params = source.extract::<FtsParams>()?;
-                let inner_opts = FtsIndexBuilder::default()
+                let mut inner_opts = FtsIndexBuilder::default()
                     .base_tokenizer(params.base_tokenizer)
                     .language(&params.language)
                     .map_err(|_| {
@@ -61,6 +61,12 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                     .ngram_max_length(params.ngram_max_length)
                     .ngram_prefix_only(params.prefix_only)
                     .custom_stop_words(params.custom_stop_words);
+                if let Some(memory_limit) = params.memory_limit {
+                    inner_opts = inner_opts.memory_limit_mb(memory_limit);
+                }
+                if let Some(num_workers) = params.num_workers {
+                    inner_opts = inner_opts.num_workers(num_workers);
+                }
                 let inner_opts = inner_opts
                     .block_size(params.block_size)
                     .map_err(|err| PyValueError::new_err(err.to_string()))?;
@@ -213,6 +219,8 @@ struct FtsParams {
     ngram_max_length: u32,
     prefix_only: bool,
     block_size: usize,
+    memory_limit: Option<u64>,
+    num_workers: Option<usize>,
 }
 
 #[derive(FromPyObject)]
@@ -289,7 +297,7 @@ struct IvfHnswFlatParams {
     target_partition_size: Option<u32>,
 }
 
-#[pyclass(get_all)]
+#[pyclass(module = "lancedb._lancedb", get_all)]
 /// A description of an index currently configured on a column
 pub struct IndexConfig {
     /// The type of the index
@@ -444,3 +452,51 @@ impl IndexConfig {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use pyo3::types::{PyDict, PyDictMethods};
+    use serde_json::json;
+
+    #[test]
+    fn fts_build_controls_are_forwarded() {
+        Python::initialize();
+        Python::attach(|py| {
+            let locals = PyDict::new(py);
+            py.run(
+                c"class FTS:
+    with_position = True
+    base_tokenizer = 'simple'
+    language = 'English'
+    max_token_length = None
+    lower_case = True
+    stem = False
+    remove_stop_words = False
+    custom_stop_words = None
+    ascii_folding = False
+    ngram_min_length = 3
+    ngram_max_length = 3
+    prefix_only = False
+    block_size = 128
+    memory_limit = 2048
+    num_workers = 7
+
+config = FTS()",
+                None,
+                Some(&locals),
+            )
+            .unwrap();
+
+            let config = locals.get_item("config").unwrap().unwrap();
+            let index = extract_index_params(&Some(config)).unwrap();
+            let LanceDbIndex::FTS(params) = index else {
+                panic!("expected FTS index parameters");
+            };
+            let training_json = params.to_training_json().unwrap();
+
+            assert_eq!(training_json.get("memory_limit"), Some(&json!(2048)));
+            assert_eq!(training_json.get("num_workers"), Some(&json!(7)));
+        });
+    }
+}
diff --git a/python/src/job.rs b/python/src/job.rs
index 56ee211f4..688cba7f9 100644
--- a/python/src/job.rs
+++ b/python/src/job.rs
@@ -5,18 +5,32 @@ use std::sync::Arc;
 
 use crate::runtime::future_into_py;
 use pyo3::{Bound, PyAny, PyRef, PyResult, pyclass, pymethods};
+use serde::Serialize;
 
 use crate::error::PythonErrorExt;
 
 #[pyclass]
 pub struct Job {
-    inner: Arc<lancedb::Job>,
+    inner: Arc<lancedb::Job<std::result::Result<Option<String>, String>>>,
 }
 
 impl Job {
     pub(crate) fn new(inner: lancedb::Job) -> Self {
         Self {
-            inner: Arc::new(inner),
+            inner: Arc::new(inner.map(|()| Ok(None))),
+        }
+    }
+
+    pub(crate) fn new_typed<T>(inner: lancedb::Job<T>) -> Self
+    where
+        T: Clone + Serialize + Send + Sync + 'static,
+    {
+        Self {
+            inner: Arc::new(inner.map(|result| {
+                serde_json::to_string(&result)
+                    .map(Some)
+                    .map_err(|error| format!("failed to serialize typed job result: {error}"))
+            })),
         }
     }
 }
@@ -39,8 +53,10 @@ impl Job {
     pub fn wait(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner.clone();
         future_into_py(self_.py(), async move {
-            inner.wait().await.infer_error()?;
-            Ok(())
+            let result = inner.wait().await.infer_error()?;
+            result
+                .map_err(|message| lancedb::Error::Runtime { message })
+                .infer_error()
         })
     }
 
diff --git a/python/src/lib.rs b/python/src/lib.rs
index 6b0c0cf97..8d3eab787 100644
--- a/python/src/lib.rs
+++ b/python/src/lib.rs
@@ -16,7 +16,8 @@ use query::{FTSQuery, HybridQuery, Query, VectorQuery};
 use session::Session;
 use table::{
     AddColumnsResult, AddResult, AlterColumnsResult, DeleteResult, DropColumnsResult, FtsToken,
-    LsmWriteSpec, MergeResult, PyBlobFile, Table, UpdateFieldMetadataResult, UpdateResult,
+    LsmWriteSpec, MergeResult, PyBlobFile, RefreshColumnResult, RefreshMaterializedViewResult,
+    Table, UpdateFieldMetadataResult, UpdateResult,
 };
 
 pub mod arrow;
@@ -57,6 +58,8 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<VectorQuery>()?;
     m.add_class::<RecordBatchStream>()?;
     m.add_class::<AddColumnsResult>()?;
+    m.add_class::<RefreshColumnResult>()?;
+    m.add_class::<RefreshMaterializedViewResult>()?;
     m.add_class::<AlterColumnsResult>()?;
     m.add_class::<UpdateFieldMetadataResult>()?;
     m.add_class::<AddResult>()?;
diff --git a/python/src/permutation.rs b/python/src/permutation.rs
index 4dc49cfd3..24ca2b5a3 100644
--- a/python/src/permutation.rs
+++ b/python/src/permutation.rs
@@ -268,7 +268,9 @@ impl PyPermutationReader {
                     .await
                     .infer_error()?
             } else {
-                PermutationReader::identity(base_table).await
+                PermutationReader::identity(base_table)
+                    .await
+                    .infer_error()?
             };
             Ok(Self::from_reader(reader))
         })
diff --git a/python/src/session.rs b/python/src/session.rs
index 891e61e44..4d58dd269 100644
--- a/python/src/session.rs
+++ b/python/src/session.rs
@@ -11,7 +11,7 @@ use pyo3::{PyResult, pyclass, pymethods};
 /// Sessions allow you to configure cache sizes for index and metadata caches,
 /// which can significantly impact memory use and performance. They can
 /// also be re-used across multiple connections to share the same cache state.
-#[pyclass(from_py_object)]
+#[pyclass(module = "lancedb._lancedb", from_py_object)]
 #[derive(Clone)]
 pub struct Session {
     pub(crate) inner: Arc<LanceSession>,
diff --git a/python/src/table.rs b/python/src/table.rs
index 5b5d6596a..784d29136 100644
--- a/python/src/table.rs
+++ b/python/src/table.rs
@@ -28,11 +28,72 @@ use pyo3::{
     Bound, FromPyObject, Py, PyAny, PyRef, PyResult, Python,
     exceptions::{PyRuntimeError, PyValueError},
     pyclass, pyfunction, pymethods,
-    types::{IntoPyDict, PyAnyMethods, PyBytes, PyDict, PyDictMethods},
+    types::{IntoPyDict, PyAnyMethods, PyBytes, PyDict, PyDictMethods, PyList, PyListMethods},
 };
 
 mod scannable;
 
+/// Convert `LsmStats` to a Python dict, preserving the per-bucket list.
+///
+/// Deliberately not flattened to a table-level summary: a table is N
+/// buckets on one node, and the per-bucket detail is the reason the
+/// endpoint exists — flattening hides the single hot bucket someone opened
+/// it to find.
+fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult<Py<PyDict>> {
+    let out = PyDict::new(py);
+    let buckets = PyList::empty(py);
+    for b in &stats.buckets {
+        let e = PyDict::new(py);
+        e.set_item("shard_id", &b.shard_id)?;
+        e.set_item("status", &b.status)?;
+        e.set_item("writer_epoch", b.writer_epoch)?;
+        e.set_item("manifest_version", b.manifest_version)?;
+        e.set_item("current_generation", b.current_generation)?;
+        e.set_item(
+            "replay_after_wal_entry_position",
+            b.replay_after_wal_entry_position,
+        )?;
+        e.set_item(
+            "wal_entry_position_last_seen",
+            b.wal_entry_position_last_seen,
+        )?;
+
+        let generations = PyList::empty(py);
+        for g in &b.generations {
+            let ge = PyDict::new(py);
+            ge.set_item("generation", g.generation)?;
+            ge.set_item("bytes", g.bytes)?;
+            ge.set_item("rows", g.rows)?;
+            generations.append(ge)?;
+        }
+        e.set_item("generations", generations)?;
+        e.set_item("compacting", b.compacting)?;
+
+        e.set_item(
+            "memtables",
+            b.memtables
+                .as_ref()
+                .map(|ms| {
+                    let l = PyList::empty(py);
+                    for m in ms {
+                        let d = PyDict::new(py);
+                        d.set_item("generation", m.generation)?;
+                        d.set_item("rows", m.rows)?;
+                        d.set_item("bytes", m.bytes)?;
+                        d.set_item("batches", m.batches)?;
+                        d.set_item("indexes", m.indexes.clone())?;
+                        l.append(d)?;
+                    }
+                    PyResult::Ok(l.unbind())
+                })
+                .transpose()?,
+        )?;
+        buckets.append(e)?;
+    }
+    out.set_item("buckets", buckets)?;
+    Ok(out.unbind())
+}
+
 #[derive(FromPyObject)]
 enum PredicateArg {
     Expr(PyExpr),
@@ -185,13 +246,23 @@ impl From<lancedb::table::MergeResult> for MergeResult {
     }
 }
 
+/// Render for `__repr__`, so the default reads as Python's `None` rather than
+/// Rust's `Some([..])`.
+fn fmt_maintained(maintained: &Option<Vec<String>>) -> String {
+    match maintained {
+        Some(names) => format!("{:?}", names),
+        None => "None".to_string(),
+    }
+}
+
 /// Specification selecting Lance's MemWAL LSM-style write path for
 /// `merge_insert`.
 ///
 /// Constructed via the `bucket(...)`, `identity(...)`, or `unsharded()`
 /// classmethods, then optionally chain `with_maintained_indexes(...)` and
-/// `with_writer_config_defaults(...)`.
-#[pyclass(from_py_object)]
+/// `with_writer_config_defaults(...)`. A fresh spec maintains every index the
+/// MemWAL supports, resolved on install.
+#[pyclass(module = "lancedb._lancedb", from_py_object)]
 #[derive(Clone, Debug)]
 pub struct LsmWriteSpec {
     inner: lancedb::table::LsmWriteSpec,
@@ -230,11 +301,11 @@ impl LsmWriteSpec {
         }
     }
 
-    /// Replace the list of indexes the MemWAL should keep up to date as
-    /// rows are appended. Each name must reference an index that
-    /// already exists on the table at the time `set_lsm_write_spec`
-    /// is called.
-    pub fn with_maintained_indexes(&self, indexes: Vec<String>) -> Self {
+    /// Set which indexes the MemWAL maintains. `None` (the default)
+    /// resolves every supported index on install; a list is verbatim,
+    /// and an empty list maintains nothing.
+    #[pyo3(signature = (indexes))]
+    pub fn with_maintained_indexes(&self, indexes: Option<Vec<String>>) -> Self {
         Self {
             inner: self.inner.clone().with_maintained_indexes(indexes),
         }
@@ -256,23 +327,29 @@ impl LsmWriteSpec {
                 maintained_indexes,
                 writer_config_defaults,
             } => format!(
-                "LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={:?}, writer_config_defaults={:?})",
-                column, num_buckets, maintained_indexes, writer_config_defaults,
+                "LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})",
+                column,
+                num_buckets,
+                fmt_maintained(maintained_indexes),
+                writer_config_defaults,
             ),
             lancedb::table::LsmWriteSpec::Identity {
                 column,
                 maintained_indexes,
                 writer_config_defaults,
             } => format!(
-                "LsmWriteSpec.identity(column={:?}, maintained_indexes={:?}, writer_config_defaults={:?})",
-                column, maintained_indexes, writer_config_defaults,
+                "LsmWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})",
+                column,
+                fmt_maintained(maintained_indexes),
+                writer_config_defaults,
             ),
             lancedb::table::LsmWriteSpec::Unsharded {
                 maintained_indexes,
                 writer_config_defaults,
             } => format!(
-                "LsmWriteSpec.unsharded(maintained_indexes={:?}, writer_config_defaults={:?})",
-                maintained_indexes, writer_config_defaults,
+                "LsmWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})",
+                fmt_maintained(maintained_indexes),
+                writer_config_defaults,
             ),
         }
     }
@@ -307,10 +384,10 @@ impl LsmWriteSpec {
         }
     }
 
-    /// Names of indexes the MemWAL should keep up to date during writes.
+    /// Indexes the MemWAL keeps up to date, or `None` for every supported one.
     #[getter]
-    pub fn maintained_indexes(&self) -> Vec<String> {
-        self.inner.maintained_indexes().to_vec()
+    pub fn maintained_indexes(&self) -> Option<Vec<String>> {
+        self.inner.maintained_indexes().map(<[String]>::to_vec)
     }
 
     /// Default `ShardWriter` configuration recorded by this spec.
@@ -338,6 +415,67 @@ pub struct AddColumnsResult {
     pub version: u64,
 }
 
+#[pyclass(get_all, from_py_object)]
+#[derive(Clone, Debug)]
+pub struct RefreshColumnResult {
+    pub rows_filled: u64,
+    pub version: u64,
+}
+
+#[pymethods]
+impl RefreshColumnResult {
+    pub fn __repr__(&self) -> String {
+        format!(
+            "RefreshColumnResult(rows_filled={}, version={})",
+            self.rows_filled, self.version
+        )
+    }
+}
+
+impl From<lancedb::table::RefreshColumnResult> for RefreshColumnResult {
+    fn from(result: lancedb::table::RefreshColumnResult) -> Self {
+        Self {
+            rows_filled: result.rows_filled,
+            version: result.version,
+        }
+    }
+}
+
+#[pyclass(get_all, from_py_object)]
+#[derive(Clone, Debug)]
+pub struct RefreshMaterializedViewResult {
+    pub mode: String,
+    pub rows_written: u64,
+    pub source_version: u64,
+    pub version: u64,
+}
+
+#[pymethods]
+impl RefreshMaterializedViewResult {
+    pub fn __repr__(&self) -> String {
+        format!(
+            "RefreshMaterializedViewResult(mode={}, rows_written={}, source_version={}, version={})",
+            self.mode, self.rows_written, self.source_version, self.version
+        )
+    }
+}
+
+impl From<lancedb::RefreshMaterializedViewResult> for RefreshMaterializedViewResult {
+    fn from(result: lancedb::RefreshMaterializedViewResult) -> Self {
+        let mode = match result.mode {
+            lancedb::RefreshMode::Rebuild => "rebuild",
+            lancedb::RefreshMode::Incremental => "incremental",
+            lancedb::RefreshMode::NoOp => "no_op",
+        };
+        Self {
+            mode: mode.to_string(),
+            rows_written: result.rows_written,
+            source_version: result.source_version,
+            version: result.version,
+        }
+    }
+}
+
 #[pymethods]
 impl AddColumnsResult {
     pub fn __repr__(&self) -> String {
@@ -502,7 +640,7 @@ impl PyBlobFile {
     }
 }
 
-#[pyclass(get_all, from_py_object)]
+#[pyclass(module = "lancedb._lancedb", get_all, from_py_object)]
 #[derive(Clone, Debug)]
 pub struct FtsToken {
     pub text: String,
@@ -642,15 +780,19 @@ impl Table {
         })
     }
 
-    #[pyo3(signature = (data, mode, progress=None, write_parallelism=None))]
+    #[pyo3(signature = (data, mode, progress=None, write_parallelism=None, allow_external_blob_outside_bases=false))]
     pub fn add<'a>(
         self_: PyRef<'a, Self>,
         data: PyScannable,
         mode: String,
         progress: Option<Py<PyAny>>,
         write_parallelism: Option<usize>,
+        allow_external_blob_outside_bases: bool,
     ) -> PyResult<Bound<'a, PyAny>> {
-        let mut op = self_.inner_ref()?.add(data);
+        let mut op = self_
+            .inner_ref()?
+            .add(data)
+            .allow_external_blob_outside_bases(allow_external_blob_outside_bases);
         if mode == "append" {
             op = op.mode(AddDataMode::Append);
         } else if mode == "overwrite" {
@@ -1339,6 +1481,51 @@ impl Table {
         })
     }
 
+    /// Converge the table's LSM write path into its base table.
+    ///
+    /// Best-effort: with writes flowing, new rows may land after the last
+    /// pass. Errors if the table stops making progress.
+    pub fn checkpoint_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            inner.checkpoint_lsm().await.infer_error()
+        })
+    }
+
+    /// Seal every bucket's active memtable into L0.
+    pub fn flush_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(
+            self_.py(),
+            async move { inner.flush_lsm().await.infer_error() },
+        )
+    }
+
+    /// Trigger a background L0 → base pass per bucket. Returns once the
+    /// passes are dispatched, not once they finish — watch `get_lsm_stats`.
+    pub fn compact_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            inner.compact_lsm().await.infer_error()
+        })
+    }
+
+    /// Live LSM state, or `None` when the LSM write path is not enabled.
+    #[pyo3(signature = (include_generation_rows=false))]
+    pub fn get_lsm_stats(
+        self_: PyRef<'_, Self>,
+        include_generation_rows: bool,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            let stats = inner
+                .get_lsm_stats(include_generation_rows)
+                .await
+                .infer_error()?;
+            Python::attach(|py| stats.map(|s| lsm_stats_to_py(py, &s)).transpose())
+        })
+    }
+
     pub fn close_lsm_writers(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
         future_into_py(self_.py(), async move {
@@ -1388,6 +1575,78 @@ impl Table {
         })
     }
 
+    pub fn add_computed_columns(
+        self_: PyRef<'_, Self>,
+        columns: Vec<(String, String)>,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            let mut builder = inner.add_columns();
+            for (name, expression) in columns {
+                builder = builder.computed(name, expression);
+            }
+            let result = builder.execute().await.infer_error()?;
+            Ok(AddColumnsResult::from(result))
+        })
+    }
+
+    pub fn add_function_columns(
+        self_: PyRef<'_, Self>,
+        application_json: String,
+        output_name: Option<String>,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let application =
+            lancedb::function::FunctionApplication::from_json(&application_json).infer_error()?;
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            let builder = match output_name {
+                Some(name) => inner.add_columns().function_as(name, application),
+                None => inner.add_columns().function(application),
+            };
+            let result = builder.execute().await.infer_error()?;
+            Ok(AddColumnsResult::from(result))
+        })
+    }
+
+    pub fn refresh_column(self_: PyRef<'_, Self>, column: String) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            let result = inner.refresh_column(column).await.infer_error()?;
+            Ok(RefreshColumnResult::from(result))
+        })
+    }
+
+    pub fn refresh_column_async(
+        self_: PyRef<'_, Self>,
+        column: String,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            let job = inner.refresh_column_async(column).await.infer_error()?;
+            Ok(crate::job::Job::new_typed(job))
+        })
+    }
+
+    #[pyo3(signature = (full=false, source_version=None))]
+    pub fn refresh_materialized_view(
+        self_: PyRef<'_, Self>,
+        full: bool,
+        source_version: Option<u64>,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            let view = lancedb::MaterializedView::from_table(inner)
+                .await
+                .infer_error()?;
+            let mut builder = view.refresh().full(full);
+            if let Some(version) = source_version {
+                builder = builder.source_version(version);
+            }
+            let result = builder.execute().await.infer_error()?;
+            Ok(RefreshMaterializedViewResult::from(result))
+        })
+    }
+
     pub fn add_columns_with_schema(
         self_: PyRef<'_, Self>,
         schema: PyArrowType<Schema>,
@@ -1685,7 +1944,7 @@ impl Branches {
     }
 
     #[pyo3(signature = (from_branch, dry_run=false))]
-    pub fn merge(
+    pub fn cherry_pick(
         self_: PyRef<'_, Self>,
         from_branch: String,
         dry_run: bool,
@@ -1693,7 +1952,7 @@ impl Branches {
         let inner = self_.inner.clone();
         future_into_py(self_.py(), async move {
             let result = inner
-                .merge_branch(&from_branch, dry_run)
+                .cherry_pick(&from_branch, dry_run)
                 .await
                 .infer_error()?;
             Python::attach(|py| struct_to_wire_py(py, &result))
diff --git a/python/uv.lock b/python/uv.lock
index 551dc3f68..c957a4c06 100644
--- a/python/uv.lock
+++ b/python/uv.lock
@@ -1998,12 +1998,12 @@ requires-dist = [
     { name = "pillow", marker = "extra == 'clip'", specifier = ">=12.1.1" },
     { name = "pillow", marker = "extra == 'embeddings'", specifier = ">=12.1.1" },
     { name = "pillow", marker = "extra == 'siglip'", specifier = ">=12.1.1" },
-    { name = "polars", marker = "extra == 'tests'", specifier = ">=0.19,<=1.3.0" },
+    { name = "polars", marker = "extra == 'tests'", specifier = ">=0.19,<=1.32.3" },
     { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.5.0" },
     { name = "pyarrow", specifier = ">=16" },
     { name = "pyarrow", marker = "extra == 'tests'", specifier = "<25" },
     { name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" },
-    { name = "pydantic", specifier = ">=1.10" },
+    { name = "pydantic", specifier = ">=2.7.4,<3" },
     { name = "pylance", marker = "extra == 'pylance'", specifier = ">=5.0.0b5" },
     { name = "pylance", marker = "extra == 'tests'", specifier = "==9.0.0rc1" },
     { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" },
diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml
index 66db3cf12..8276e5bb3 100644
--- a/rust/lancedb/Cargo.toml
+++ b/rust/lancedb/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.37.1-beta.0"
+version = "0.38.0-beta.10"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
@@ -49,24 +49,22 @@ lance-namespace = { workspace = true }
 lance-namespace-impls = { workspace = true }
 metrics = { workspace = true, optional = true }
 metrics-util = { workspace = true, optional = true }
-# Pin the transitive GooseFS SDK until the 0.1.6 compile break is fixed upstream.
-goosefs-sdk = { version = "=0.1.5", optional = true }
 moka = { workspace = true }
 pin-project = { workspace = true }
-tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
+tokio = { workspace = true }
 log.workspace = true
-async-trait = "0"
-bytes = "1"
+async-trait = { workspace = true }
+bytes = { workspace = true }
 futures.workspace = true
 num-traits.workspace = true
 url.workspace = true
 rand.workspace = true
 regex.workspace = true
-serde = { version = "^1" }
-serde_json = { version = "1" }
+serde = { workspace = true }
+serde_json = { workspace = true }
 async-openai = { version = "0.20.0", optional = true }
 serde_with = { version = "3.8.1" }
-tempfile = "3.5.0"
+tempfile = { workspace = true }
 aws-sdk-bedrockruntime = { version = "1.27.0", optional = true }
 # For remote feature
 reqwest = { version = "0.12.0", default-features = false, features = [
@@ -75,11 +73,13 @@ reqwest = { version = "0.12.0", default-features = false, features = [
     "http2",
     "json",
     "macos-system-configuration",
+    # Avoid linking OpenSSL into Python wheels, which breaks on FIPS hosts.
+    "rustls-tls-native-roots",
     "stream",
 ], optional = true }
 http = { version = "1", optional = true } # Matching what is in reqwest
 urlencoding = { version = "2", optional = true }
-uuid = { version = "1.7.0", features = ["v4", "v5"] }
+uuid = { workspace = true, features = ["v5"] }
 polars-arrow = { version = ">=0.37,<0.40.0", optional = true }
 polars = { version = ">=0.37,<0.40.0", optional = true }
 hf-hub = { version = "0.4.1", optional = true, default-features = false, features = [
@@ -96,10 +96,11 @@ semver = { workspace = true }
 [dev-dependencies]
 anyhow = "1"
 lance-testing = { workspace = true }
-tempfile = "3.5.0"
+tempfile = { workspace = true }
 random_word = { version = "0.4.3", features = ["en"] }
-tokio = { version = "1.23", features = ["io-util", "macros", "net", "rt-multi-thread", "sync"] }
-uuid = { version = "1.7.0", features = ["v4"] }
+roaring = "0.11.4"
+tokio = { workspace = true, features = ["io-util", "macros", "net", "test-util"] }
+uuid = { workspace = true }
 walkdir = "2"
 aws-sdk-dynamodb = { version = "1.55.0" }
 aws-sdk-s3 = { version = "1.55.0" }
@@ -133,7 +134,6 @@ azure = [
 ]
 cos = ["lance/tencent", "lance-io/tencent"]
 goosefs = [
-    "dep:goosefs-sdk",
     "lance/goosefs",
     "lance-io/goosefs",
     "lance-namespace-impls/dir-goosefs",
@@ -188,6 +188,9 @@ required-features = ["bedrock"]
 [[example]]
 name = "bench_streaming_dataloader"
 
+[[example]]
+name = "bench_open_missing_table"
+
 [[example]]
 name = "simple"
 
diff --git a/rust/lancedb/examples/bench_open_missing_table.rs b/rust/lancedb/examples/bench_open_missing_table.rs
new file mode 100644
index 000000000..fbfddf86c
--- /dev/null
+++ b/rust/lancedb/examples/bench_open_missing_table.rs
@@ -0,0 +1,150 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+// Benchmark for opening a missing table as sibling-table cardinality grows.
+//
+// The fixture uses real `.lance` directories and marker files. Fixture creation is
+// outside the timed section. Defaults intentionally cover 1k, 10k, and 100k siblings
+// with 10 warmups and 100 distinct missing-table opens per scale:
+//
+// ```text
+// cargo run --profile release-no-lto -p lancedb --example bench_open_missing_table
+// ```
+//
+// `BENCH_SIBLINGS`, `BENCH_WARMUPS`, and `BENCH_TRIALS` override those defaults.
+// Reduced settings are useful only as a smoke test. Performance comparisons require
+// the same machine, filesystem, fixture sizes, settings, lockfile, and alternating
+// baseline/candidate execution order.
+
+use std::time::{Duration, Instant};
+
+use anyhow::{Context, Result, bail};
+use lancedb::connection::Connection;
+use lancedb::{Error, connect};
+use object_store::ObjectStoreExt as _;
+use object_store::path::Path;
+
+const MAX_SIBLINGS: usize = 1_000_000;
+const MAX_WARMUPS: usize = 10_000;
+const MAX_TRIALS: usize = 100_000;
+
+fn env_usize(key: &str, default: usize, max: usize) -> Result<usize> {
+    let value = match std::env::var(key) {
+        Ok(value) => value
+            .parse()
+            .with_context(|| format!("invalid {key} value: {value}"))?,
+        Err(std::env::VarError::NotPresent) => default,
+        Err(error) => return Err(error).with_context(|| format!("reading {key}")),
+    };
+    if value == 0 || value > max {
+        bail!("{key} must be between 1 and {max}");
+    }
+    Ok(value)
+}
+
+fn sibling_counts() -> Result<Vec<usize>> {
+    let raw = std::env::var("BENCH_SIBLINGS").unwrap_or_else(|_| "1000,10000,100000".into());
+    let mut counts = raw
+        .split(',')
+        .map(|value| {
+            value
+                .trim()
+                .parse::<usize>()
+                .with_context(|| format!("invalid BENCH_SIBLINGS value: {value}"))
+        })
+        .collect::<Result<Vec<_>>>()?;
+    counts.sort_unstable();
+    counts.dedup();
+    if counts.is_empty() || counts[0] == 0 || counts[counts.len() - 1] > MAX_SIBLINGS {
+        bail!("BENCH_SIBLINGS values must be between 1 and {MAX_SIBLINGS}");
+    }
+    Ok(counts)
+}
+
+async fn add_siblings(
+    store: &object_store::local::LocalFileSystem,
+    start: usize,
+    end: usize,
+) -> Result<()> {
+    for index in start..end {
+        let marker = Path::from(format!("sibling_{index:06}.lance/_marker"));
+        store
+            .put(&marker, bytes::Bytes::new().into())
+            .await
+            .with_context(|| format!("creating benchmark marker {marker}"))?;
+    }
+    Ok(())
+}
+
+async fn time_missing_open(db: &Connection, name: &str) -> Result<Duration> {
+    let started = Instant::now();
+    let result = db.open_table(name).execute().await;
+    let elapsed = started.elapsed();
+    match result {
+        Err(Error::TableNotFound { .. }) => Ok(elapsed),
+        Err(error) => bail!("expected TableNotFound for {name}, got {error:?}"),
+        Ok(_) => bail!("benchmark missing-table name unexpectedly exists: {name}"),
+    }
+}
+
+fn percentile(sorted: &[Duration], percentile: usize) -> Duration {
+    let rank = (sorted.len() * percentile).div_ceil(100).saturating_sub(1);
+    sorted[rank]
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let counts = sibling_counts()?;
+    let warmups = env_usize("BENCH_WARMUPS", 10, MAX_WARMUPS)?;
+    let trials = env_usize("BENCH_TRIALS", 100, MAX_TRIALS)?;
+
+    let fixture = tempfile::tempdir().context("creating benchmark fixture")?;
+    let database_path = fixture.path();
+    let fixture_store = object_store::local::LocalFileSystem::new_with_prefix(database_path)
+        .context("creating benchmark object store")?;
+    let db = connect(database_path.to_str().context("non-UTF-8 fixture path")?)
+        .execute()
+        .await?;
+
+    println!(
+        "config: siblings={counts:?} warmups={warmups} trials={trials} profile={} os={} arch={}",
+        if cfg!(debug_assertions) {
+            "debug"
+        } else {
+            "release"
+        },
+        std::env::consts::OS,
+        std::env::consts::ARCH,
+    );
+    println!("lower is better; fixture setup and teardown are excluded");
+    println!("| siblings | samples | p50 | p95 | max |");
+    println!("| ---: | ---: | ---: | ---: | ---: |");
+
+    let mut created = 0;
+    for sibling_count in counts {
+        add_siblings(&fixture_store, created, sibling_count).await?;
+        created = sibling_count;
+
+        for index in 0..warmups {
+            let name = format!("__missing_warmup_{sibling_count}_{index}");
+            let _ = time_missing_open(&db, &name).await?;
+        }
+
+        let mut samples = Vec::with_capacity(trials);
+        for index in 0..trials {
+            let name = format!("__missing_trial_{sibling_count}_{index}");
+            samples.push(time_missing_open(&db, &name).await?);
+        }
+        samples.sort_unstable();
+
+        println!(
+            "| {sibling_count} | {} | {:?} | {:?} | {:?} |",
+            samples.len(),
+            percentile(&samples, 50),
+            percentile(&samples, 95),
+            samples[samples.len() - 1],
+        );
+    }
+
+    Ok(())
+}
diff --git a/rust/lancedb/examples/bench_streaming_dataloader.rs b/rust/lancedb/examples/bench_streaming_dataloader.rs
index 087268ff8..a46d924d1 100644
--- a/rust/lancedb/examples/bench_streaming_dataloader.rs
+++ b/rust/lancedb/examples/bench_streaming_dataloader.rs
@@ -5,10 +5,10 @@
 //! streaming dataloader.
 //!
 //! Normal sweep:
-//!   cargo run --release --example bench_streaming_dataloader
+//!   cargo run --profile release-with-debug --example bench_streaming_dataloader
 //!
 //! Flamegraph (self-contained, no perf/dtrace needed):
-//!   BENCH_PROFILE=1 BENCH_CHUNK=64 cargo run --release \
+//!   BENCH_PROFILE=1 BENCH_CHUNK=64 cargo run --profile release-with-debug \
 //!       --example bench_streaming_dataloader
 //!   # writes flamegraph.svg in the current directory
 //!
diff --git a/rust/lancedb/src/blob.rs b/rust/lancedb/src/blob.rs
index 3448257ab..d59123ec3 100644
--- a/rust/lancedb/src/blob.rs
+++ b/rust/lancedb/src/blob.rs
@@ -17,7 +17,7 @@ use arrow_array::builder::LargeBinaryBuilder;
 use arrow_schema::{DataType, Field, Schema};
 use lance::dataset::{BlobRangeRequest as LanceBlobRangeRequest, Dataset, WriteParams};
 use lance_arrow::FieldExt;
-use lance_encoding::version::LanceFileVersion;
+use lance_file::version::{ConcreteFileVersion, LanceFileVersion};
 use lance_io::object_store::ObjectStore;
 use object_store::path::Path;
 
@@ -333,7 +333,10 @@ pub(crate) fn ensure_blob_storage_version(schema: &Schema, params: &mut WritePar
         .data_storage_version
         .unwrap_or(LanceFileVersion::Stable)
         .resolve();
-    if resolved < LanceFileVersion::V2_2 {
+    if matches!(
+        resolved,
+        ConcreteFileVersion::V1 | ConcreteFileVersion::V2_0 | ConcreteFileVersion::V2_1
+    ) {
         params.data_storage_version = Some(LanceFileVersion::V2_2);
     }
 }
@@ -499,7 +502,7 @@ mod tests {
         ensure_blob_storage_version(&blob_schema(), &mut params);
         assert_eq!(
             params.data_storage_version.unwrap().resolve(),
-            LanceFileVersion::V2_2
+            ConcreteFileVersion::V2_2
         );
     }
 
@@ -512,7 +515,7 @@ mod tests {
         ensure_blob_storage_version(&blob_schema(), &mut params);
         assert_eq!(
             params.data_storage_version.unwrap().resolve(),
-            LanceFileVersion::V2_2
+            ConcreteFileVersion::V2_2
         );
     }
 
diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs
index 89e59e12e..5f66d9dee 100644
--- a/rust/lancedb/src/connection.rs
+++ b/rust/lancedb/src/connection.rs
@@ -34,14 +34,14 @@ use crate::remote::{
     db::{OPT_REMOTE_API_KEY, OPT_REMOTE_HOST_OVERRIDE, OPT_REMOTE_REGION},
 };
 use lance::io::ObjectStoreParams;
-pub use lance_encoding::version::LanceFileVersion;
+pub use lance_file::version::LanceFileVersion;
 #[cfg(feature = "remote")]
 use lance_io::object_store::StorageOptions;
 use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
 
 mod create_table;
 
-fn merge_storage_options(
+pub(crate) fn merge_storage_options(
     store_params: &mut ObjectStoreParams,
     pairs: impl IntoIterator<Item = (String, String)>,
 ) {
@@ -409,6 +409,11 @@ impl Connection {
     ///
     /// The names will be returned in lexicographical order (ascending)
     ///
+    /// Listing databases discover physical `*.lance` entries without opening every
+    /// dataset. The result is a point-in-time discovery snapshot: an entry may still be
+    /// under creation, may contain only uncommitted storage, or may be concurrently
+    /// dropped before it is opened.
+    ///
     /// The parameters `page_token` and `limit` can be used to paginate the results
     pub fn table_names(&self) -> TableNamesBuilder {
         TableNamesBuilder::new(self.internal.clone())
@@ -456,10 +461,9 @@ impl Connection {
     ///
     /// # Returns
     /// Created [`TableRef`], or [`Error::TableNotFound`] if the table does not exist.
-    /// If the table's storage is present but holds no readable dataset (for example a
-    /// `<name>.lance` directory left behind by an interrupted drop and re-create, which
-    /// [`Self::table_names`] still lists) this returns [`Error::TableCorrupted`]
-    /// instead.
+    /// On listing databases, a committed Lance manifest is authoritative for table
+    /// existence. Uncommitted files or a physical `<name>.lance` directory alone do not
+    /// make a table openable.
     pub fn open_table(&self, name: impl Into<String>) -> OpenTableBuilder {
         OpenTableBuilder::new(
             self.internal.clone(),
@@ -492,6 +496,33 @@ impl Connection {
         )
     }
 
+    /// Register a Python callable as a new immutable Function version.
+    ///
+    /// Registration is remote-only and always asynchronous. Waiting on the
+    /// returned typed job yields the durable [`crate::function::FunctionVersion`].
+    /// Local databases return [`Error::NotSupported`].
+    pub async fn create_function_async(
+        &self,
+        request: crate::function::FunctionRegistrationRequest,
+    ) -> Result<crate::job::Job<crate::function::FunctionVersion>> {
+        self.internal.create_function_async(request).await
+    }
+
+    /// Look up one exact immutable Function version in the remote catalog.
+    ///
+    /// Both the logical name and server-assigned version id are required;
+    /// mutable aliases and latest-version lookup are intentionally absent.
+    /// Local databases return [`Error::NotSupported`].
+    pub async fn get_function(
+        &self,
+        name: impl AsRef<str>,
+        version: impl AsRef<str>,
+    ) -> Result<crate::function::FunctionVersion> {
+        self.internal
+            .get_function(name.as_ref(), version.as_ref())
+            .await
+    }
+
     /// Rename a table in the database.
     ///
     /// This is only supported in LanceDB Cloud.
@@ -561,6 +592,21 @@ impl Connection {
             .await
     }
 
+    /// Start dropping a table and return a handle to the cleanup job.
+    ///
+    /// The table may become unavailable before its physical data is removed.
+    /// Call [`crate::job::Job::wait`] to wait for cleanup to finish. Local
+    /// backends may complete the drop before returning the handle.
+    pub async fn drop_table_async(
+        &self,
+        name: impl AsRef<str>,
+        namespace_path: &[String],
+    ) -> Result<crate::job::Job> {
+        self.internal
+            .drop_table_async(name.as_ref(), namespace_path)
+            .await
+    }
+
     /// Drop the database
     ///
     /// This is the same as dropping all of the tables
@@ -1633,6 +1679,50 @@ mod tests {
         assert_eq!(tables, names[..7]);
     }
 
+    #[tokio::test]
+    async fn test_list_tables_walks_page_boundaries() {
+        let tc = new_test_connection().await.unwrap();
+        if tc.is_remote {
+            // What resumes a page is the server's to decide, and asserting it here would be
+            // asserting the server's contract rather than this one.
+            return;
+        }
+        let db = tc.connection;
+        let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)]));
+        let mut names = Vec::with_capacity(5);
+        for _ in 0..5 {
+            let name = uuid::Uuid::new_v4().to_string();
+            names.push(name.clone());
+            db.create_empty_table(name, schema.clone())
+                .execute()
+                .await
+                .unwrap();
+        }
+        names.sort();
+
+        // Walking in pages has to reach every table exactly once, with nothing lost at a
+        // page boundary.
+        let mut seen = Vec::with_capacity(names.len());
+        let mut page_token = None;
+        loop {
+            let page = db
+                .list_tables(ListTablesRequest {
+                    id: Some(Vec::new()),
+                    limit: Some(2),
+                    page_token,
+                    ..Default::default()
+                })
+                .await
+                .unwrap();
+            seen.extend(page.tables);
+            page_token = page.page_token.filter(|token| !token.is_empty());
+            if page_token.is_none() {
+                break;
+            }
+        }
+        assert_eq!(seen, names);
+    }
+
     #[tokio::test]
     async fn test_open_table() {
         let tc = new_test_connection().await.unwrap();
diff --git a/rust/lancedb/src/connection/create_table.rs b/rust/lancedb/src/connection/create_table.rs
index 66f6dfa8d..39cc82ec0 100644
--- a/rust/lancedb/src/connection/create_table.rs
+++ b/rust/lancedb/src/connection/create_table.rs
@@ -202,6 +202,17 @@ mod tests {
         assert_eq!(table.count_rows(None).await.unwrap(), 0);
     }
 
+    #[tokio::test]
+    async fn create_table_in_named_memory_database() {
+        let db = connect("memory://foo").execute().await.unwrap();
+        let batch = record_batch!(("id", Int64, [1, 2, 3])).unwrap();
+
+        let table = db.create_table("my_table", batch).execute().await.unwrap();
+
+        assert_eq!(table.uri().await.unwrap(), "memory://foo/my_table.lance");
+        assert_eq!(table.count_rows(None).await.unwrap(), 3);
+    }
+
     async fn test_create_table_with_data<T>(data: T)
     where
         T: Scannable + 'static,
@@ -427,10 +438,9 @@ mod tests {
             .await
             .unwrap()
             .data_storage_format
-            .lance_file_version()
-            .unwrap();
+            .lance_file_format();
         // Compare resolved versions since Stable/Next are aliases that resolve at storage time
-        assert_eq!(storage_format.resolve(), data_storage_version.resolve());
+        assert_eq!(storage_format, data_storage_version.resolve());
     }
 
     #[tokio::test]
diff --git a/rust/lancedb/src/database.rs b/rust/lancedb/src/database.rs
index f99f6e12a..6c4537972 100644
--- a/rust/lancedb/src/database.rs
+++ b/rust/lancedb/src/database.rs
@@ -241,6 +241,12 @@ fn job_op_not_supported<T>(what: &str) -> Result<T> {
     })
 }
 
+fn function_catalog_not_supported<T>() -> Result<T> {
+    Err(crate::error::Error::NotSupported {
+        message: "Function catalog operations are not supported by this database".to_string(),
+    })
+}
+
 /// The `Database` trait defines the interface for database implementations.
 ///
 /// A database is responsible for managing tables and their metadata.
@@ -286,6 +292,21 @@ pub trait Database:
     ///
     /// See [`CloneTableRequest`] for detailed documentation and examples.
     async fn clone_table(&self, request: CloneTableRequest) -> Result<Arc<dyn BaseTable>>;
+    /// Register an immutable Function version through the remote catalog.
+    async fn create_function_async(
+        &self,
+        _request: crate::function::FunctionRegistrationRequest,
+    ) -> Result<crate::job::Job<crate::function::FunctionVersion>> {
+        function_catalog_not_supported()
+    }
+    /// Look up one exact immutable Function version.
+    async fn get_function(
+        &self,
+        _name: &str,
+        _version: &str,
+    ) -> Result<crate::function::FunctionVersion> {
+        function_catalog_not_supported()
+    }
     /// A [`crate::job::Job`] handle for a server-side job by id, suitable for
     /// waiting on or cancelling the job. The handle is constructed without a
     /// server round trip; an unknown id surfaces when the handle is used.
@@ -323,6 +344,18 @@ pub trait Database:
     ) -> Result<()>;
     /// Drop a table in the database
     async fn drop_table(&self, name: &str, namespace_path: &[String]) -> Result<()>;
+    /// Start dropping a table and return a handle to the cleanup job.
+    ///
+    /// Backends without asynchronous cleanup complete the drop before
+    /// returning an already-finished job.
+    async fn drop_table_async(
+        &self,
+        name: &str,
+        namespace_path: &[String],
+    ) -> Result<crate::job::Job> {
+        self.drop_table(name, namespace_path).await?;
+        Ok(crate::job::Job::new_done())
+    }
     /// Drop all tables in the database
     async fn drop_all_tables(&self, namespace_path: &[String]) -> Result<()>;
     fn as_any(&self) -> &dyn std::any::Any;
diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs
index 454498d54..17dc82756 100644
--- a/rust/lancedb/src/database/listing.rs
+++ b/rust/lancedb/src/database/listing.rs
@@ -12,7 +12,7 @@ use lance::dataset::refs::Ref;
 use lance::dataset::{ReadParams, WriteMode, builder::DatasetBuilder};
 use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore};
 use lance_datafusion::utils::StreamingWriteSource;
-use lance_encoding::version::LanceFileVersion;
+use lance_file::version::LanceFileVersion;
 use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
 use lance_table::io::commit::commit_handler_from_url;
 use object_store::local::LocalFileSystem;
@@ -765,60 +765,13 @@ impl ListingDatabase {
         }
     }
 
-    /// Extract storage option overrides from the request
-    fn extract_storage_overrides(
-        &self,
-        request: &CreateTableRequest,
-    ) -> Result<(Option<LanceFileVersion>, Option<bool>, Option<bool>)> {
-        let storage_options = request
-            .write_options
-            .lance_write_params
-            .as_ref()
-            .and_then(|p| p.store_params.as_ref())
-            .and_then(|sp| sp.storage_options());
-
-        let storage_version_override = storage_options
-            .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION))
-            .map(|s| s.parse::<LanceFileVersion>())
-            .transpose()?;
-
-        let v2_manifest_override = storage_options
-            .and_then(|opts| opts.get(OPT_NEW_TABLE_V2_MANIFEST_PATHS))
-            .map(|s| s.parse::<bool>())
-            .transpose()
-            .map_err(|_| Error::InvalidInput {
-                message: "enable_v2_manifest_paths must be a boolean".to_string(),
-            })?;
-
-        let stable_row_ids_override = storage_options
-            .and_then(|opts| opts.get(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS))
-            .map(|s| s.parse::<bool>())
-            .transpose()
-            .map_err(|_| Error::InvalidInput {
-                message: "enable_stable_row_ids must be a boolean".to_string(),
-            })?;
-
-        Ok((
-            storage_version_override,
-            v2_manifest_override,
-            stable_row_ids_override,
-        ))
-    }
-
     /// Prepare write parameters for table creation
     fn prepare_write_params(
         &self,
         request: &CreateTableRequest,
-        storage_version_override: Option<LanceFileVersion>,
-        v2_manifest_override: Option<bool>,
-        stable_row_ids_override: Option<bool>,
+        mut write_params: lance::dataset::WriteParams,
+        overrides: NewTableConfig,
     ) -> lance::dataset::WriteParams {
-        let mut write_params = request
-            .write_options
-            .lance_write_params
-            .clone()
-            .unwrap_or_default();
-
         // Only modify the storage options if we actually have something to
         // inherit. There is a difference between storage_options=None and
         // storage_options=Some({}). Using storage_options=None will cause the
@@ -842,18 +795,21 @@ impl ListingDatabase {
             store_params.storage_options_accessor = Some(Arc::new(accessor));
         }
 
-        write_params.data_storage_version = storage_version_override
+        write_params.data_storage_version = overrides
+            .data_storage_version
             .or(write_params.data_storage_version)
             .or(self.new_table_config.data_storage_version);
 
-        if let Some(enable_v2_manifest_paths) =
-            v2_manifest_override.or(self.new_table_config.enable_v2_manifest_paths)
+        if let Some(enable_v2_manifest_paths) = overrides
+            .enable_v2_manifest_paths
+            .or(self.new_table_config.enable_v2_manifest_paths)
         {
             write_params.enable_v2_manifest_paths = enable_v2_manifest_paths;
         }
 
         let data_schema = request.data.arrow_schema();
-        if let Some(enable_stable_row_ids) = stable_row_ids_override
+        if let Some(enable_stable_row_ids) = overrides
+            .enable_stable_row_ids
             .or(self.new_table_config.enable_stable_row_ids)
             .or(has_blob_columns(&data_schema).then_some(true))
         {
@@ -1018,20 +974,19 @@ impl Database for ListingDatabase {
             f.drain(0..index);
         }
 
-        // Determine if there's a next page
-        let next_page_token = if let Some(limit) = request.limit {
-            if f.len() > limit as usize {
-                let token = f[limit as usize].clone();
+        // Determine if there's a next page. The token is the last name of this page,
+        // not the first of the next one: the next page resumes strictly after the
+        // token, so naming the next page's first entry would skip it.
+        let next_page_token = match request.limit {
+            Some(limit) if f.len() > limit as usize => {
                 f.truncate(limit as usize);
-                Some(token)
-            } else {
-                None
+                f.last().cloned()
             }
-        } else {
-            None
+            _ => None,
         };
 
         Ok(ListTablesResponse {
+            context: None,
             tables: f,
             page_token: next_page_token,
         })
@@ -1047,15 +1002,13 @@ impl Database for ListingDatabase {
             .clone()
             .unwrap_or_else(|| self.table_uri(&request.name).unwrap());
 
-        let (storage_version_override, v2_manifest_override, stable_row_ids_override) =
-            self.extract_storage_overrides(&request)?;
-
-        let write_params = self.prepare_write_params(
-            &request,
-            storage_version_override,
-            v2_manifest_override,
-            stable_row_ids_override,
-        );
+        let mut write_params = request
+            .write_options
+            .lance_write_params
+            .clone()
+            .unwrap_or_default();
+        let overrides = take_request_creation_overrides(&mut write_params)?;
+        let write_params = self.prepare_write_params(&request, write_params, overrides);
 
         let data_schema = request.data.arrow_schema();
 
@@ -1287,18 +1240,249 @@ impl Database for ListingDatabase {
     }
 }
 
+/// Parse the request-level `new_table_*` creation keys into overrides and
+/// strip them from the store options in one step: every create path that
+/// honors them must also keep them out of the object store.
+pub(crate) fn take_request_creation_overrides(
+    params: &mut lance::dataset::WriteParams,
+) -> Result<NewTableConfig> {
+    let storage_options = params
+        .store_params
+        .as_ref()
+        .and_then(|sp| sp.storage_options());
+    let overrides = NewTableConfig {
+        data_storage_version: storage_options
+            .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION))
+            .map(|s| s.parse::<LanceFileVersion>())
+            .transpose()?,
+        enable_v2_manifest_paths: storage_options
+            .and_then(|opts| opts.get(OPT_NEW_TABLE_V2_MANIFEST_PATHS))
+            .map(|s| s.parse::<bool>())
+            .transpose()
+            .map_err(|_| Error::InvalidInput {
+                message: "enable_v2_manifest_paths must be a boolean".to_string(),
+            })?,
+        enable_stable_row_ids: storage_options
+            .and_then(|opts| opts.get(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS))
+            .map(|s| s.parse::<bool>())
+            .transpose()
+            .map_err(|_| Error::InvalidInput {
+                message: "enable_stable_row_ids must be a boolean".to_string(),
+            })?,
+    };
+    if let Some(store_params) = params.store_params.as_mut() {
+        strip_new_table_creation_keys(store_params);
+    }
+    Ok(overrides)
+}
+
+/// Strip the `new_table_*` creation keys from request store options: they are
+/// creation config, not credentials, and left in place they fork a fresh
+/// store connection for the request.
+fn strip_new_table_creation_keys(store_params: &mut ObjectStoreParams) {
+    let mut options = store_params.storage_options().cloned().unwrap_or_default();
+    let mut removed = false;
+    for key in [
+        OPT_NEW_TABLE_STORAGE_VERSION,
+        OPT_NEW_TABLE_V2_MANIFEST_PATHS,
+        OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS,
+    ] {
+        removed |= options.remove(key).is_some();
+    }
+    if !removed {
+        return;
+    }
+    let provider = store_params
+        .storage_options_accessor
+        .as_ref()
+        .and_then(|accessor| accessor.provider().cloned());
+    store_params.storage_options_accessor = match (options.is_empty(), provider) {
+        (true, None) => None,
+        (true, Some(provider)) => Some(Arc::new(StorageOptionsAccessor::with_provider(provider))),
+        (false, Some(provider)) => Some(Arc::new(
+            StorageOptionsAccessor::with_initial_and_provider(options, provider),
+        )),
+        (false, None) => Some(Arc::new(StorageOptionsAccessor::with_static_options(
+            options,
+        ))),
+    };
+}
+
 #[cfg(test)]
 mod tests {
+    #[tokio::test]
+    async fn request_level_creation_keys_do_not_fork_the_store() {
+        use crate::query::ExecutableQuery;
+        use futures::TryStreamExt;
+
+        let db = crate::connect("memory://").execute().await.unwrap();
+        let batch = arrow_array::record_batch!(("x", Int32, [1, 2])).unwrap();
+        let store_params = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([(
+                    OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
+                    "true".to_string(),
+                )]),
+            ))),
+            ..Default::default()
+        };
+        db.create_table("t", batch)
+            .write_options(crate::table::WriteOptions {
+                lance_write_params: Some(lance::dataset::WriteParams {
+                    store_params: Some(store_params),
+                    ..Default::default()
+                }),
+            })
+            .execute()
+            .await
+            .unwrap();
+
+        let table = db.open_table("t").execute().await.unwrap();
+        let rows: usize = table
+            .query()
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap()
+            .iter()
+            .map(|b| b.num_rows())
+            .sum();
+        assert_eq!(rows, 2, "the table must live in the session's store");
+    }
+
+    mod strip_new_table_creation_keys {
+        use super::super::*;
+
+        #[derive(Debug)]
+        struct EmptyProvider;
+
+        #[async_trait::async_trait]
+        impl StorageOptionsProvider for EmptyProvider {
+            async fn fetch_storage_options(
+                &self,
+            ) -> lance_core::Result<Option<HashMap<String, String>>> {
+                Ok(Some(HashMap::new()))
+            }
+
+            fn provider_id(&self) -> String {
+                "empty-test-provider".into()
+            }
+        }
+
+        fn params_with_static(options: &[(&str, &str)]) -> ObjectStoreParams {
+            ObjectStoreParams {
+                storage_options_accessor: Some(Arc::new(
+                    StorageOptionsAccessor::with_static_options(
+                        options
+                            .iter()
+                            .map(|(k, v)| (k.to_string(), v.to_string()))
+                            .collect(),
+                    ),
+                )),
+                ..Default::default()
+            }
+        }
+
+        #[test]
+        fn creation_keys_are_removed_and_store_keys_kept() {
+            let mut params = params_with_static(&[
+                ("region", "us-west-2"),
+                (OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, "true"),
+            ]);
+            strip_new_table_creation_keys(&mut params);
+            let options = params.storage_options().cloned().unwrap();
+            assert_eq!(options.get("region").map(String::as_str), Some("us-west-2"));
+            assert!(!options.contains_key(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS));
+
+            // Creation keys alone: no accessor survives to fork a store.
+            let mut params = params_with_static(&[(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, "true")]);
+            strip_new_table_creation_keys(&mut params);
+            assert!(params.storage_options_accessor.is_none());
+        }
+
+        /// A provider must survive every shape of strip: untouched accessors
+        /// keep their identity, emptied ones still fetch, and residual
+        /// statics ride along.
+        #[test]
+        fn provider_accessors_survive_the_strip() {
+            let accessor = Arc::new(StorageOptionsAccessor::with_provider(Arc::new(
+                EmptyProvider,
+            )));
+            let mut params = ObjectStoreParams {
+                storage_options_accessor: Some(accessor.clone()),
+                ..Default::default()
+            };
+            strip_new_table_creation_keys(&mut params);
+            assert!(Arc::ptr_eq(
+                params.storage_options_accessor.as_ref().unwrap(),
+                &accessor
+            ));
+
+            let mut params = ObjectStoreParams {
+                storage_options_accessor: Some(Arc::new(
+                    StorageOptionsAccessor::with_initial_and_provider(
+                        HashMap::from([
+                            ("region".to_string(), "us-west-2".to_string()),
+                            (
+                                OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
+                                "true".to_string(),
+                            ),
+                        ]),
+                        Arc::new(EmptyProvider),
+                    ),
+                )),
+                ..Default::default()
+            };
+            strip_new_table_creation_keys(&mut params);
+            let accessor = params.storage_options_accessor.unwrap();
+            assert!(accessor.has_provider());
+            assert_eq!(
+                accessor
+                    .initial_storage_options()
+                    .and_then(|o| o.get("region").cloned())
+                    .as_deref(),
+                Some("us-west-2")
+            );
+
+            // Emptied entirely: a first-fetch accessor, not one caching {}.
+            let mut params = ObjectStoreParams {
+                storage_options_accessor: Some(Arc::new(
+                    StorageOptionsAccessor::with_initial_and_provider(
+                        HashMap::from([(
+                            OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
+                            "true".to_string(),
+                        )]),
+                        Arc::new(EmptyProvider),
+                    ),
+                )),
+                ..Default::default()
+            };
+            strip_new_table_creation_keys(&mut params);
+            let accessor = params.storage_options_accessor.unwrap();
+            assert!(accessor.has_provider());
+            assert!(accessor.initial_storage_options().is_none());
+        }
+    }
+
     use super::*;
     use crate::Table;
+    use crate::arrow::{SendableRecordBatchStream, SimpleRecordBatchStream};
     use crate::connection::ConnectRequest;
     use crate::data::scannable::Scannable;
     use crate::database::{CreateTableMode, CreateTableRequest};
-    use crate::table::WriteOptions;
+    use crate::query::QueryRequest;
+    use crate::table::{AnyQuery, WriteOptions};
     use arrow_array::{Int32Array, RecordBatch, StringArray};
-    use arrow_schema::{DataType, Field, Schema};
+    use arrow_schema::{DataType, Field, Schema, SchemaRef};
+    use futures::{TryStreamExt, stream::once};
     use std::path::PathBuf;
+    use std::sync::Arc;
+    use std::time::Duration;
     use tempfile::tempdir;
+    use tokio::sync::Barrier;
+    use tokio::time::timeout;
 
     async fn setup_database() -> (tempfile::TempDir, ListingDatabase) {
         let tempdir = tempdir().unwrap();
@@ -1322,6 +1506,114 @@ mod tests {
         (tempdir, db)
     }
 
+    struct BarrierScannable {
+        batch: RecordBatch,
+        barrier: Arc<Barrier>,
+    }
+
+    impl Scannable for BarrierScannable {
+        fn schema(&self) -> SchemaRef {
+            self.batch.schema()
+        }
+
+        fn scan_as_stream(&mut self) -> SendableRecordBatchStream {
+            let batch = self.batch.clone();
+            let schema = batch.schema();
+            let barrier = self.barrier.clone();
+            Box::pin(SimpleRecordBatchStream {
+                schema,
+                stream: once(async move {
+                    barrier.wait().await;
+                    Ok(batch)
+                }),
+            })
+        }
+    }
+
+    fn create_request(name: &str, data: Box<dyn Scannable>) -> CreateTableRequest {
+        CreateTableRequest {
+            name: name.to_string(),
+            namespace_path: vec![],
+            data,
+            mode: CreateTableMode::Create,
+            write_options: Default::default(),
+            location: None,
+            namespace_client: None,
+        }
+    }
+
+    #[tokio::test]
+    async fn test_create_ignores_uncommitted_storage_without_manifest() {
+        let (tmp_dir, db) = setup_database().await;
+        let data_dir = tmp_dir.path().join("test.lance/data");
+        std::fs::create_dir_all(&data_dir).unwrap();
+        std::fs::write(data_dir.join("orphan.lance"), b"uncommitted").unwrap();
+
+        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
+        let batch =
+            RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1]))]).unwrap();
+
+        let table = db
+            .create_table(create_request("test", Box::new(batch)))
+            .await
+            .unwrap();
+        assert_eq!(table.count_rows(None).await.unwrap(), 1);
+    }
+
+    #[tokio::test]
+    async fn test_concurrent_create_is_arbitrated_by_manifest_commit() {
+        let uri = format!("memory:///concurrent-create-{}", uuid::Uuid::new_v4());
+        let db = crate::connect(&uri).execute().await.unwrap();
+        let store: Arc<dyn object_store::ObjectStore> =
+            Arc::new(object_store::memory::InMemory::new());
+        let table_url = url::Url::parse("memory:///database/test.lance").unwrap();
+        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
+        let batch =
+            RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1]))]).unwrap();
+        let barrier = Arc::new(Barrier::new(2));
+
+        #[allow(deprecated)]
+        let request = |batch, barrier| {
+            let mut request = create_request("test", Box::new(BarrierScannable { batch, barrier }));
+            request.write_options = WriteOptions {
+                lance_write_params: Some(lance::dataset::WriteParams {
+                    store_params: Some(ObjectStoreParams {
+                        object_store: Some((store.clone(), table_url.clone())),
+                        ..Default::default()
+                    }),
+                    commit_handler: Some(Arc::new(
+                        lance_table::io::commit::ConditionalPutCommitHandler,
+                    )),
+                    ..Default::default()
+                }),
+            };
+            request
+        };
+
+        let left = db
+            .database()
+            .create_table(request(batch.clone(), barrier.clone()));
+        let right = db.database().create_table(request(batch, barrier));
+        let (left, right) = timeout(Duration::from_secs(30), async { tokio::join!(left, right) })
+            .await
+            .expect("concurrent creates deadlocked");
+
+        let results = [left, right];
+        assert_eq!(
+            results.iter().filter(|result| result.is_ok()).count(),
+            1,
+            "expected one successful create, got {results:?}"
+        );
+        assert_eq!(
+            results
+                .iter()
+                .filter(|result| matches!(result, Err(Error::TableAlreadyExists { .. })))
+                .count(),
+            1,
+            "expected one manifest conflict, got {results:?}"
+        );
+    }
+
     #[tokio::test]
     async fn test_listing_database_root_ops_do_not_create_manifest() {
         let tempdir = tempdir().unwrap();
@@ -1376,6 +1668,156 @@ mod tests {
         assert!(!tempdir.path().join("__manifest").exists());
     }
 
+    /// Regression test for https://github.com/lancedb/lancedb/issues/1600.
+    ///
+    /// Opening a table used to create a separate object-store client instead of
+    /// reusing the one that successfully connected to the database.  Repeating
+    /// credential discovery made S3 table opens intermittent, especially in AWS
+    /// Lambda, and the failed open was reported as `TableNotFound`.
+    #[tokio::test]
+    async fn test_open_table_reuses_connection_object_store() {
+        let tempdir = tempdir().unwrap();
+        let uri = tempdir.path().to_str().unwrap();
+        let registry = Arc::new(lance_io::object_store::ObjectStoreRegistry::default());
+        let session = Arc::new(lance::session::Session::new(16, 16, registry.clone()));
+
+        let request = ConnectRequest {
+            uri: uri.to_string(),
+            #[cfg(feature = "remote")]
+            client_config: Default::default(),
+            options: Default::default(),
+            namespace_client_properties: Default::default(),
+            manifest_enabled: false,
+            read_consistency_interval: None,
+            session: Some(session),
+        };
+        let db = ListingDatabase::connect_with_options(&request)
+            .await
+            .unwrap();
+
+        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
+        db.create_table(CreateTableRequest {
+            name: "test".to_string(),
+            namespace_path: vec![],
+            data: Box::new(RecordBatch::new_empty(schema)) as Box<dyn Scannable>,
+            mode: CreateTableMode::Create,
+            write_options: Default::default(),
+            location: None,
+            namespace_client: None,
+        })
+        .await
+        .unwrap();
+
+        let before_open = registry.stats();
+        for _ in 0..3 {
+            let table = db
+                .open_table(OpenTableRequest {
+                    name: "test".to_string(),
+                    namespace_path: vec![],
+                    index_cache_size: None,
+                    lance_read_params: None,
+                    location: None,
+                    namespace_client: None,
+                    managed_versioning: None,
+                })
+                .await
+                .unwrap();
+            assert_eq!(table.count_rows(None).await.unwrap(), 0);
+        }
+
+        let after_open = registry.stats();
+        assert_eq!(after_open.misses, before_open.misses);
+        assert!(after_open.hits >= before_open.hits + 3);
+    }
+
+    /// Regression test for https://github.com/lancedb/lancedb/issues/3197.
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn test_open_table_follows_hugging_face_symlinks() {
+        let (tempdir, db) = setup_database().await;
+        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
+        db.create_table(CreateTableRequest {
+            name: "test".to_string(),
+            namespace_path: vec![],
+            data: Box::new(
+                RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))])
+                    .unwrap(),
+            ) as Box<dyn Scannable>,
+            mode: CreateTableMode::Create,
+            write_options: Default::default(),
+            location: None,
+            namespace_client: None,
+        })
+        .await
+        .unwrap();
+
+        let table_dir = tempdir.path().join("test.lance");
+        let versions_dir = table_dir.join("_versions");
+        let manifest_path = std::fs::read_dir(&versions_dir)
+            .unwrap()
+            .map(|entry| entry.unwrap().path())
+            .find(|path| path.extension().is_some_and(|ext| ext == "manifest"))
+            .unwrap();
+        let data_path = std::fs::read_dir(table_dir.join("data"))
+            .unwrap()
+            .map(|entry| entry.unwrap().path())
+            .find(|path| path.extension().is_some_and(|ext| ext == "lance"))
+            .unwrap();
+
+        // Hugging Face snapshots keep dataset objects in a separate blob directory and
+        // expose them through relative symlinks.
+        let blobs_dir = tempdir.path().join("blobs");
+        std::fs::create_dir(&blobs_dir).unwrap();
+        let manifest_blob = "9b603c63d0e692e05d58be25605f2f2064cc781e5ff94fe983a405059547b816";
+        let data_blob = "be64f20e5723bd0a27cfdbdb41cf7d6fad94cd572a71973b717fb8340f4310c5";
+        std::fs::rename(&manifest_path, blobs_dir.join(manifest_blob)).unwrap();
+        std::fs::rename(&data_path, blobs_dir.join(data_blob)).unwrap();
+        std::os::unix::fs::symlink(Path::new("../../blobs").join(manifest_blob), &manifest_path)
+            .unwrap();
+        std::os::unix::fs::symlink(Path::new("../../blobs").join(data_blob), &data_path).unwrap();
+        let symlink_len = std::fs::symlink_metadata(&manifest_path).unwrap().len();
+        let target_len = std::fs::metadata(&manifest_path).unwrap().len();
+        assert_ne!(symlink_len, target_len);
+
+        drop(db);
+        let db = ListingDatabase::connect_with_options(&ConnectRequest {
+            uri: tempdir.path().to_str().unwrap().to_string(),
+            #[cfg(feature = "remote")]
+            client_config: Default::default(),
+            options: Default::default(),
+            namespace_client_properties: Default::default(),
+            manifest_enabled: false,
+            read_consistency_interval: None,
+            session: None,
+        })
+        .await
+        .unwrap();
+
+        let table = db
+            .open_table(OpenTableRequest {
+                name: "test".to_string(),
+                namespace_path: vec![],
+                index_cache_size: None,
+                lance_read_params: None,
+                location: None,
+                namespace_client: None,
+                managed_versioning: None,
+            })
+            .await
+            .unwrap();
+        let batches = table
+            .query(
+                &AnyQuery::Query(QueryRequest::default()),
+                Default::default(),
+            )
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 3);
+    }
+
     #[tokio::test]
     async fn test_clone_table_basic() {
         let (_tempdir, db) = setup_database().await;
@@ -2280,7 +2722,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_table_uri() {
-        let (_tempdir, db) = setup_database().await;
+        let (_tempdir, mut db) = setup_database().await;
 
         let mut pb = PathBuf::new();
         pb.push(db.uri.clone());
@@ -2289,6 +2731,33 @@ mod tests {
         let expected = pb.to_str().unwrap();
         let uri = db.table_uri("test").ok().unwrap();
         assert_eq!(uri, expected);
+
+        // URI paths always use forward slashes, even on Windows. Using
+        // `Path::join` here used to produce `az://container/prefix\\test.lance`,
+        // which Azure treated as a different object from the table returned by
+        // `table_names` (https://github.com/lancedb/lancedb/issues/1072).
+        for base_uri in ["az://container/prefix", "az://container/prefix/"] {
+            db.uri = base_uri.to_string();
+            assert_eq!(
+                db.table_uri("test").unwrap(),
+                "az://container/prefix/test.lance"
+            );
+        }
+    }
+
+    /// Regression test for https://github.com/lancedb/lancedb/issues/2283.
+    ///
+    /// Object-store URIs must use `/` on every platform. In particular, joining
+    /// with `std::path::Path` used to insert a `\\` into Azure blob keys on
+    /// Windows.
+    #[tokio::test]
+    async fn test_table_uri_uses_forward_slashes_for_azure() {
+        let (_tempdir, mut db) = setup_database().await;
+        db.uri = "az://test/db/test".to_string();
+
+        let uri = db.table_uri("test").unwrap();
+
+        assert_eq!(uri, "az://test/db/test/test.lance");
     }
 
     /// Regression: connecting via a URL-style URI (which goes through
diff --git a/rust/lancedb/src/database/namespace.rs b/rust/lancedb/src/database/namespace.rs
index d18c78682..250d933f6 100644
--- a/rust/lancedb/src/database/namespace.rs
+++ b/rust/lancedb/src/database/namespace.rs
@@ -26,10 +26,7 @@ use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler;
 use crate::blob::{ensure_blob_storage_version, has_blob_columns};
 use crate::connection::NamespaceClientPushdownOperation;
 use crate::database::ReadConsistency;
-use crate::database::listing::{
-    NewTableConfig, OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, OPT_NEW_TABLE_STORAGE_VERSION,
-    OPT_NEW_TABLE_V2_MANIFEST_PATHS,
-};
+use crate::database::listing::{NewTableConfig, take_request_creation_overrides};
 use crate::database::read_freshness::{
     FreshnessBaselines, ReadFreshnessContextProvider, TableFreshness,
 };
@@ -197,69 +194,28 @@ impl LanceNamespaceDatabase {
         TableFreshness::new(self.freshness_baselines.clone(), key)
     }
 
-    fn extract_storage_overrides(
-        &self,
-        request: &DbCreateTableRequest,
-    ) -> Result<(
-        Option<lance_encoding::version::LanceFileVersion>,
-        Option<bool>,
-        Option<bool>,
-    )> {
-        let storage_options = request
-            .write_options
-            .lance_write_params
-            .as_ref()
-            .and_then(|p| p.store_params.as_ref())
-            .and_then(|sp| sp.storage_options());
-
-        let storage_version_override = storage_options
-            .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION))
-            .map(|s| s.parse::<lance_encoding::version::LanceFileVersion>())
-            .transpose()?;
-
-        let v2_manifest_override = storage_options
-            .and_then(|opts| opts.get(OPT_NEW_TABLE_V2_MANIFEST_PATHS))
-            .map(|s| s.parse::<bool>())
-            .transpose()
-            .map_err(|_| Error::InvalidInput {
-                message: "enable_v2_manifest_paths must be a boolean".to_string(),
-            })?;
-
-        let stable_row_ids_override = storage_options
-            .and_then(|opts| opts.get(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS))
-            .map(|s| s.parse::<bool>())
-            .transpose()
-            .map_err(|_| Error::InvalidInput {
-                message: "enable_stable_row_ids must be a boolean".to_string(),
-            })?;
-
-        Ok((
-            storage_version_override,
-            v2_manifest_override,
-            stable_row_ids_override,
-        ))
-    }
-
     fn apply_new_table_config(
         &self,
         params: &mut lance::dataset::WriteParams,
         request: &DbCreateTableRequest,
     ) -> Result<()> {
-        let (storage_version_override, v2_manifest_override, stable_row_ids_override) =
-            self.extract_storage_overrides(request)?;
+        let overrides = take_request_creation_overrides(params)?;
 
-        params.data_storage_version = storage_version_override
+        params.data_storage_version = overrides
+            .data_storage_version
             .or(params.data_storage_version)
             .or(self.new_table_config.data_storage_version);
 
-        if let Some(enable_v2_manifest_paths) =
-            v2_manifest_override.or(self.new_table_config.enable_v2_manifest_paths)
+        if let Some(enable_v2_manifest_paths) = overrides
+            .enable_v2_manifest_paths
+            .or(self.new_table_config.enable_v2_manifest_paths)
         {
             params.enable_v2_manifest_paths = enable_v2_manifest_paths;
         }
 
         let data_schema = request.data.schema();
-        if let Some(enable_stable_row_ids) = stable_row_ids_override
+        if let Some(enable_stable_row_ids) = overrides
+            .enable_stable_row_ids
             .or(self.new_table_config.enable_stable_row_ids)
             .or(has_blob_columns(data_schema.as_ref()).then_some(true))
         {
@@ -644,6 +600,146 @@ mod tests {
         RecordBatch::try_new(schema, vec![Arc::new(id_array), Arc::new(name_array)]).unwrap()
     }
 
+    /// The shared parse-and-sanitize boundary is wired into this path: the
+    /// request-level creation key must act as an override (the strip itself
+    /// is covered by the listing tests).
+    #[tokio::test]
+    async fn request_level_creation_keys_are_taken_as_overrides() {
+        use crate::database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS;
+
+        let tmp_dir = tempdir().unwrap();
+        let mut properties = HashMap::new();
+        properties.insert(
+            "root".to_string(),
+            tmp_dir.path().to_str().unwrap().to_string(),
+        );
+        let db = connect_namespace("dir", properties)
+            .execute()
+            .await
+            .unwrap();
+
+        let store_params = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([(
+                    OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
+                    "true".to_string(),
+                )]),
+            ))),
+            ..Default::default()
+        };
+        let table = db
+            .create_table("t", create_test_data())
+            .write_options(crate::table::WriteOptions {
+                lance_write_params: Some(lance::dataset::WriteParams {
+                    store_params: Some(store_params),
+                    ..Default::default()
+                }),
+            })
+            .execute()
+            .await
+            .unwrap();
+        let native = table.as_native().unwrap();
+        assert!(
+            native
+                .dataset
+                .get()
+                .await
+                .unwrap()
+                .manifest
+                .uses_stable_row_ids(),
+            "the creation key must be honored as an override"
+        );
+
+        let table = db.open_table("t").execute().await.unwrap();
+        let rows: usize = table
+            .query()
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap()
+            .iter()
+            .map(|b| b.num_rows())
+            .sum();
+        assert_eq!(rows, 5);
+    }
+
+    /// Sanitation on this path: apply must strip the creation keys from the
+    /// store options while genuine options and the provider survive.
+    #[tokio::test]
+    async fn apply_new_table_config_sanitizes_request_store_options() {
+        use crate::database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS;
+        use lance_io::object_store::StorageOptionsProvider;
+
+        #[derive(Debug)]
+        struct EmptyProvider;
+
+        #[async_trait::async_trait]
+        impl StorageOptionsProvider for EmptyProvider {
+            async fn fetch_storage_options(
+                &self,
+            ) -> lance_core::Result<Option<HashMap<String, String>>> {
+                Ok(Some(HashMap::new()))
+            }
+
+            fn provider_id(&self) -> String {
+                "empty-test-provider".into()
+            }
+        }
+
+        let tmp_dir = tempdir().unwrap();
+        let mut properties = HashMap::new();
+        properties.insert(
+            "root".to_string(),
+            tmp_dir.path().to_str().unwrap().to_string(),
+        );
+        let db = LanceNamespaceDatabase::connect_with_new_table_config(
+            "dir",
+            properties,
+            HashMap::new(),
+            None,
+            None,
+            HashSet::new(),
+            NewTableConfig::default(),
+        )
+        .await
+        .unwrap();
+
+        let request = DbCreateTableRequest::new("t".to_string(), Box::new(create_test_data()));
+        let mut params = lance::dataset::WriteParams {
+            store_params: Some(ObjectStoreParams {
+                storage_options_accessor: Some(Arc::new(
+                    StorageOptionsAccessor::with_initial_and_provider(
+                        HashMap::from([
+                            ("region".to_string(), "us-west-2".to_string()),
+                            (
+                                OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
+                                "true".to_string(),
+                            ),
+                        ]),
+                        Arc::new(EmptyProvider),
+                    ),
+                )),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+        db.apply_new_table_config(&mut params, &request).unwrap();
+
+        assert!(params.enable_stable_row_ids);
+        let store_params = params.store_params.unwrap();
+        let options = store_params.storage_options().cloned().unwrap();
+        assert!(!options.contains_key(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS));
+        assert_eq!(options.get("region").map(String::as_str), Some("us-west-2"));
+        assert!(
+            store_params
+                .storage_options_accessor
+                .unwrap()
+                .has_provider()
+        );
+    }
+
     #[tokio::test]
     async fn test_namespace_connection_simple() {
         // Test that namespace connections work with simple connect_namespace(impl_type, properties)
diff --git a/rust/lancedb/src/dataloader/permutation/builder.rs b/rust/lancedb/src/dataloader/permutation/builder.rs
index 6b4ae2303..ba0d0e180 100644
--- a/rust/lancedb/src/dataloader/permutation/builder.rs
+++ b/rust/lancedb/src/dataloader/permutation/builder.rs
@@ -27,6 +27,12 @@ pub const SRC_ROW_ID_COL: &str = "row_id";
 
 pub const SPLIT_NAMES_CONFIG_KEY: &str = "split_names";
 
+/// Base table version the permutation was built against.
+pub const BASE_VERSION_CONFIG_KEY: &str = "base_version";
+
+/// Base table branch the permutation was built against.  Absent means main.
+pub const BASE_BRANCH_CONFIG_KEY: &str = "base_branch";
+
 pub const DEFAULT_MEMORY_LIMIT: usize = 100 * 1024 * 1024;
 
 /// Where to store the permutation table
@@ -160,9 +166,10 @@ impl PermutationBuilder {
         self
     }
 
-    async fn sort_by_split_id(
+    async fn sort_by_column(
         &self,
         data: SendableRecordBatchStream,
+        column: &str,
     ) -> Result<SendableRecordBatchStream> {
         let memory_limit = std::env::var("LANCEDB_PERM_BUILDER_MEMORY_LIMIT")
             .unwrap_or_else(|_| DEFAULT_MEMORY_LIMIT.to_string())
@@ -188,45 +195,36 @@ impl PermutationBuilder {
         let df = ctx
             .read_one_shot(data.into_df_stream())
             .map_err(|e| Error::Other {
-                message: format!("Failed to setup sort by split id: {}", e),
+                message: format!("Failed to setup sort by {}: {}", column, e),
                 source: Some(e.into()),
             })?;
         let df_stream = df
-            .sort_by(vec![col(SPLIT_ID_COLUMN)])
+            .sort_by(vec![col(column)])
             .map_err(|e| Error::Other {
-                message: format!("Failed to plan sort by split id: {}", e),
+                message: format!("Failed to plan sort by {}: {}", column, e),
                 source: Some(e.into()),
             })?
             .execute_stream()
             .await
             .map_err(|e| Error::Other {
-                message: format!("Failed to sort by split id: {}", e),
+                message: format!("Failed to sort by {}: {}", column, e),
                 source: Some(e.into()),
             })?;
 
+        let column = column.to_string();
         let schema = df_stream.schema();
-        let stream = df_stream.map_err(|e| Error::Other {
-            message: format!("Failed to execute sort by split id: {}", e),
+        let stream = df_stream.map_err(move |e| Error::Other {
+            message: format!("Failed to execute sort by {}: {}", column, e),
             source: Some(e.into()),
         });
         Ok(Box::pin(SimpleRecordBatchStream { schema, stream }))
     }
 
-    fn add_split_names(
+    fn add_config_metadata(
         data: SendableRecordBatchStream,
-        split_names: &[String],
+        metadata: HashMap<String, String>,
     ) -> Result<SendableRecordBatchStream> {
-        let schema = data
-            .schema()
-            .as_ref()
-            .clone()
-            .with_metadata(HashMap::from([(
-                SPLIT_NAMES_CONFIG_KEY.to_string(),
-                serde_json::to_string(split_names).map_err(|e| Error::Other {
-                    message: format!("Failed to serialize split names: {}", e),
-                    source: Some(e.into()),
-                })?,
-            )]));
+        let schema = data.schema().as_ref().clone().with_metadata(metadata);
         let schema = Arc::new(schema);
         let schema_clone = schema.clone();
         let stream = data.map_ok(move |batch| batch.with_schema(schema.clone()).unwrap());
@@ -237,8 +235,44 @@ impl PermutationBuilder {
     }
 
     /// Builds the permutation table and stores it in the given database.
-    pub async fn build(self) -> Result<Table> {
-        // First pass, apply filter and load row ids
+    pub async fn build(mut self) -> Result<Table> {
+        // Remote tables resolve latest independently for each request. Use a
+        // separate pinned handle so count, projection, and scan all refer to one
+        // snapshot without changing the caller's table checkout state. Native
+        // tables return `None` here and retain their existing behavior.
+        if let Some(snapshot) = self
+            .base_table
+            .base_table()
+            .snapshot_at_current_version()
+            .await?
+        {
+            self.base_table = Table::from(snapshot);
+        }
+
+        // Unflushed rows have no row id, so a permutation cannot address them.
+        match self.base_table.base_table().get_lsm_write_spec().await {
+            Ok(Some(_)) => {
+                return Err(Error::NotSupported {
+                    message: "the data loader does not support tables with an LSM write \
+                              spec: rows that have not been flushed to the base table \
+                              have no row id, so a permutation cannot reference them"
+                        .to_string(),
+                });
+            }
+            Ok(None) => {}
+            // No LSM write path means no spec.
+            Err(Error::NotSupported { .. }) => {}
+            Err(err) => return Err(err),
+        }
+
+        // The handle above is already pinned to one version.  Record which one, so a
+        // reader -- in a DataLoader worker, against a table that has since moved --
+        // resolves these row addresses against the same snapshot.
+        let base_version = self.base_table.version().await?;
+        let base_branch = self.base_table.current_branch();
+
+        // First pass, apply filter and load row ids.  `Shuffler` permutes positions, so
+        // every rank must scan the rows in the same order to build the same permutation.
         let mut rows = self.base_table.query().select(Select::columns(&[ROW_ID]));
 
         if let Some(filter) = &self.config.filter {
@@ -263,6 +297,12 @@ impl PermutationBuilder {
 
         // Apply splits
         let rows = rows.execute().await?;
+        // Splits are assigned by position, so the scan has to arrive in a fixed order.
+        let rows = if self.base_table.base_table().scan_order_is_deterministic() {
+            rows
+        } else {
+            self.sort_by_column(rows, ROW_ID).await?
+        };
         let split_data = splitter.apply(rows, num_rows).await?;
 
         // Shuffle data if requested
@@ -284,7 +324,7 @@ impl PermutationBuilder {
         needs_sort |= !matches!(self.config.shuffle_strategy, ShuffleStrategy::None);
 
         let sorted = if needs_sort {
-            self.sort_by_split_id(shuffled).await?
+            self.sort_by_column(shuffled, SPLIT_ID_COLUMN).await?
         } else {
             shuffled
         };
@@ -292,11 +332,24 @@ impl PermutationBuilder {
         // Rename _rowid to row_id
         let renamed = rename_column(sorted, ROW_ID, SRC_ROW_ID_COL)?;
 
-        let streaming_data = if let Some(split_names) = &self.config.split_names {
-            Self::add_split_names(renamed, split_names)?
-        } else {
-            renamed
-        };
+        let mut metadata = HashMap::from([(
+            BASE_VERSION_CONFIG_KEY.to_string(),
+            base_version.to_string(),
+        )]);
+        // Version numbers are per-branch, so the branch is part of the coordinate.
+        if let Some(branch) = &base_branch {
+            metadata.insert(BASE_BRANCH_CONFIG_KEY.to_string(), branch.clone());
+        }
+        if let Some(split_names) = &self.config.split_names {
+            metadata.insert(
+                SPLIT_NAMES_CONFIG_KEY.to_string(),
+                serde_json::to_string(split_names).map_err(|e| Error::Other {
+                    message: format!("Failed to serialize split names: {}", e),
+                    source: Some(e.into()),
+                })?,
+            );
+        }
+        let streaming_data = Self::add_config_metadata(renamed, metadata)?;
 
         let (name, database) = match &self.config.destination {
             PermutationDestination::Permanent(database, table_name) => {
@@ -367,6 +420,269 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_native_scan_order_is_deterministic() {
+        let temp_dir = tempfile::tempdir().unwrap();
+        let db = connect(temp_dir.path().to_str().unwrap())
+            .execute()
+            .await
+            .unwrap();
+        let data = lance_datagen::gen_batch()
+            .col("col_a", lance_datagen::array::step::<Int32Type>())
+            .into_ldb_stream(RowCount::from(10), BatchCount::from(1));
+        let table = db.create_table("t", data).execute().await.unwrap();
+
+        // Native tables skip the canonicalizing sort; remote does not.
+        assert!(table.base_table().scan_order_is_deterministic());
+    }
+
+    #[cfg(feature = "remote")]
+    #[tokio::test]
+    async fn test_remote_permutation_builder_pins_snapshot() {
+        use std::sync::{
+            Mutex,
+            atomic::{AtomicU64, Ordering},
+        };
+
+        use arrow_array::{RecordBatch, UInt64Array};
+        use arrow_schema::{DataType, Field, Schema};
+
+        let row_ids = RecordBatch::try_new(
+            Arc::new(Schema::new(vec![Field::new(
+                ROW_ID,
+                DataType::UInt64,
+                false,
+            )])),
+            vec![Arc::new(UInt64Array::from(vec![100]))],
+        )
+        .unwrap();
+        let mut query_body = Vec::new();
+        {
+            let mut writer =
+                arrow_ipc::writer::FileWriter::try_new(&mut query_body, &row_ids.schema()).unwrap();
+            writer.write(&row_ids).unwrap();
+            writer.finish().unwrap();
+        }
+
+        let latest = Arc::new(AtomicU64::new(7));
+        let expected_snapshot = Arc::new(AtomicU64::new(7));
+        let planning_versions = Arc::new(Mutex::new(Vec::new()));
+        let latest_ref = latest.clone();
+        let expected_snapshot_ref = expected_snapshot.clone();
+        let planning_versions_ref = planning_versions.clone();
+        let table = Table::new_with_handler("remote_base", move |request| {
+            let path = request.url().path();
+            let body = request
+                .body()
+                .and_then(|body| body.as_bytes())
+                .map(|body| serde_json::from_slice::<serde_json::Value>(body).unwrap());
+
+            match path {
+                "/v1/table/remote_base/describe/" => {
+                    let requested = body.as_ref().and_then(|body| body["version"].as_u64());
+                    let version = requested.unwrap_or_else(|| latest_ref.load(Ordering::SeqCst));
+                    http::Response::builder()
+                        .status(200)
+                        .body(
+                            format!(r#"{{"version":{version},"schema":{{"fields":[]}}}}"#)
+                                .into_bytes(),
+                        )
+                        .unwrap()
+                }
+                "/v1/table/remote_base/get_lsm_write_spec/" => http::Response::builder()
+                    .status(200)
+                    .body(br#"{"lsm_write_spec":null}"#.to_vec())
+                    .unwrap(),
+                "/v1/table/remote_base/count_rows/" => {
+                    let body = body.unwrap();
+                    let version = body["version"].as_u64().unwrap();
+                    assert_eq!(version, expected_snapshot_ref.load(Ordering::SeqCst));
+                    assert_eq!(body["predicate"], "value > 0");
+                    planning_versions_ref.lock().unwrap().push(version);
+
+                    // Simulate a concurrent append after count_rows. An unpinned
+                    // scan would now resolve version 8 and include different rows.
+                    latest_ref.store(8, Ordering::SeqCst);
+                    http::Response::builder()
+                        .status(200)
+                        .body(b"1".to_vec())
+                        .unwrap()
+                }
+                "/v1/table/remote_base/query/" => {
+                    let body = body.unwrap();
+                    let version = body["version"].as_u64().unwrap();
+                    assert_eq!(version, expected_snapshot_ref.load(Ordering::SeqCst));
+                    assert_eq!(body["filter"], "value > 0");
+                    assert_eq!(body["columns"], serde_json::json!([ROW_ID]));
+                    planning_versions_ref.lock().unwrap().push(version);
+                    http::Response::builder()
+                        .status(200)
+                        .header("content-type", "application/vnd.apache.arrow.file")
+                        .body(query_body.clone())
+                        .unwrap()
+                }
+                _ => panic!("unexpected request: {path}"),
+            }
+        });
+
+        let permutation = PermutationBuilder::new(table.clone())
+            .with_filter("value > 0".to_string())
+            .build()
+            .await
+            .unwrap();
+        assert_eq!(permutation.count_rows(None).await.unwrap(), 1);
+
+        // Building uses a separate handle and must not pin the caller's table.
+        assert_eq!(table.version().await.unwrap(), 8);
+
+        // An explicit checkout is copied as-is and remains checked out afterward.
+        expected_snapshot.store(6, Ordering::SeqCst);
+        table.checkout(6).await.unwrap();
+        let permutation = PermutationBuilder::new(table.clone())
+            .with_filter("value > 0".to_string())
+            .build()
+            .await
+            .unwrap();
+        assert_eq!(permutation.count_rows(None).await.unwrap(), 1);
+        assert_eq!(table.version().await.unwrap(), 6);
+        assert_eq!(*planning_versions.lock().unwrap(), vec![7, 7, 6, 6]);
+    }
+
+    #[tokio::test]
+    async fn test_permutation_records_base_version() {
+        let temp_dir = tempfile::tempdir().unwrap();
+
+        let db = connect(temp_dir.path().to_str().unwrap())
+            .execute()
+            .await
+            .unwrap();
+
+        let initial_data = lance_datagen::gen_batch()
+            .col("col_a", lance_datagen::array::step::<Int32Type>())
+            .into_ldb_stream(RowCount::from(100), BatchCount::from(2));
+        let data_table = db
+            .create_table("base_tbl", initial_data)
+            .execute()
+            .await
+            .unwrap();
+
+        let build_version = data_table.version().await.unwrap();
+        let permutation_table = PermutationBuilder::new(data_table.clone())
+            .build()
+            .await
+            .unwrap();
+
+        let recorded = permutation_table
+            .schema()
+            .await
+            .unwrap()
+            .metadata
+            .get(BASE_VERSION_CONFIG_KEY)
+            .expect("permutation should record the base version")
+            .parse::<u64>()
+            .unwrap();
+        assert_eq!(recorded, build_version);
+
+        // Advancing the base table must not move the recorded version.
+        let more_data = lance_datagen::gen_batch()
+            .col("col_a", lance_datagen::array::step::<Int32Type>())
+            .into_ldb_stream(RowCount::from(50), BatchCount::from(1));
+        data_table.add(more_data).execute().await.unwrap();
+        assert!(data_table.version().await.unwrap() > recorded);
+        assert_eq!(
+            permutation_table
+                .schema()
+                .await
+                .unwrap()
+                .metadata
+                .get(BASE_VERSION_CONFIG_KEY)
+                .unwrap()
+                .parse::<u64>()
+                .unwrap(),
+            recorded,
+        );
+    }
+
+    /// Version numbers are per-branch, so a permutation built on a branch must record
+    /// it -- a worker reopens by name and lands on main at the same number.
+    #[tokio::test]
+    async fn test_permutation_records_base_branch() {
+        let temp_dir = tempfile::tempdir().unwrap();
+        let db = connect(temp_dir.path().to_str().unwrap())
+            .execute()
+            .await
+            .unwrap();
+
+        let initial_data = lance_datagen::gen_batch()
+            .col("col_a", lance_datagen::array::step::<Int32Type>())
+            .into_ldb_stream(RowCount::from(10), BatchCount::from(1));
+        let data_table = db
+            .create_table("base_tbl", initial_data)
+            .execute()
+            .await
+            .unwrap();
+
+        let branch = data_table
+            .create_branch("exp", lance::dataset::refs::Ref::from(("main", 1)))
+            .await
+            .unwrap();
+        let permutation_table = PermutationBuilder::new(branch.clone())
+            .build()
+            .await
+            .unwrap();
+
+        let metadata = permutation_table.schema().await.unwrap().metadata.clone();
+        assert_eq!(
+            metadata.get(BASE_BRANCH_CONFIG_KEY).map(String::as_str),
+            Some("exp")
+        );
+
+        // Main records nothing, so an absent key keeps meaning main.
+        let main_permutation = PermutationBuilder::new(data_table.clone())
+            .build()
+            .await
+            .unwrap();
+        assert!(
+            !main_permutation
+                .schema()
+                .await
+                .unwrap()
+                .metadata
+                .contains_key(BASE_BRANCH_CONFIG_KEY)
+        );
+    }
+
+    #[tokio::test]
+    async fn test_build_does_not_pin_the_callers_table() {
+        let temp_dir = tempfile::tempdir().unwrap();
+
+        let db = connect(temp_dir.path().to_str().unwrap())
+            .execute()
+            .await
+            .unwrap();
+
+        let initial_data = lance_datagen::gen_batch()
+            .col("col_a", lance_datagen::array::step::<Int32Type>())
+            .into_ldb_stream(RowCount::from(100), BatchCount::from(1));
+        let data_table = db
+            .create_table("base_tbl", initial_data)
+            .execute()
+            .await
+            .unwrap();
+
+        PermutationBuilder::new(data_table.clone())
+            .build()
+            .await
+            .unwrap();
+
+        // The builder pins its own handle; the caller's must still track latest.
+        let more_data = lance_datagen::gen_batch()
+            .col("col_a", lance_datagen::array::step::<Int32Type>())
+            .into_ldb_stream(RowCount::from(50), BatchCount::from(1));
+        data_table.add(more_data).execute().await.unwrap();
+        assert_eq!(data_table.count_rows(None).await.unwrap(), 150);
+    }
+
     #[tokio::test]
     async fn test_permutation_builder() {
         let temp_dir = tempfile::tempdir().unwrap();
@@ -416,4 +732,48 @@ mod tests {
             283
         );
     }
+
+    /// Rows that have not been flushed to the base table have no row id, so a
+    /// permutation cannot reference them.  Reading the base table alone would drop
+    /// them from training without saying so, so the table is refused instead.
+    #[tokio::test]
+    async fn test_permutation_rejects_lsm_write_spec() {
+        use crate::table::LsmWriteSpec;
+        use arrow_array::{Int32Array, RecordBatchIterator};
+        use arrow_schema::{DataType, Field, Schema};
+
+        // MemWAL needs a real dataset directory and a non-nullable primary key.
+        let temp_dir = tempfile::tempdir().unwrap();
+        let db = connect(temp_dir.path().to_str().unwrap())
+            .execute()
+            .await
+            .unwrap();
+        let schema = Arc::new(Schema::new(vec![Field::new("idx", DataType::Int32, false)]));
+        let batch = arrow_array::RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int32Array::from(vec![0, 1, 2, 3]))],
+        )
+        .unwrap();
+        let reader: Box<dyn arrow_array::RecordBatchReader + Send> =
+            Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema.clone()));
+        let table = db.create_table("tbl", reader).execute().await.unwrap();
+
+        // Without a spec the build succeeds.
+        PermutationBuilder::new(table.clone())
+            .build()
+            .await
+            .unwrap();
+
+        table.set_unenforced_primary_key(["idx"]).await.unwrap();
+        table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded())
+            .await
+            .unwrap();
+
+        let err = PermutationBuilder::new(table).build().await.unwrap_err();
+        assert!(
+            err.to_string().contains("LSM write spec"),
+            "expected the pre-check to refuse the table, got: {err}"
+        );
+    }
 }
diff --git a/rust/lancedb/src/dataloader/permutation/reader.rs b/rust/lancedb/src/dataloader/permutation/reader.rs
index afe79b0ad..9757dc552 100644
--- a/rust/lancedb/src/dataloader/permutation/reader.rs
+++ b/rust/lancedb/src/dataloader/permutation/reader.rs
@@ -8,7 +8,9 @@
 //! the rows from a source table that correspond to row IDs stored in a separate table.
 
 use crate::arrow::{SendableRecordBatchStream, SimpleRecordBatchStream};
-use crate::dataloader::permutation::builder::SRC_ROW_ID_COL;
+use crate::dataloader::permutation::builder::{
+    BASE_BRANCH_CONFIG_KEY, BASE_VERSION_CONFIG_KEY, SRC_ROW_ID_COL,
+};
 use crate::dataloader::permutation::split::SPLIT_ID_COLUMN;
 use crate::error::Error;
 use crate::query::{
@@ -23,6 +25,7 @@ use arrow_array::{RecordBatch, UInt64Array};
 use arrow_schema::SchemaRef;
 use datafusion_expr::{Expr, col, lit};
 use futures::{StreamExt, TryStreamExt};
+use lance::dataset::refs::MAIN_BRANCH;
 use lance::dataset::scanner::DatasetRecordBatchStream;
 use lance::io::RecordBatchStream;
 use lance_arrow::RecordBatchExt;
@@ -69,6 +72,10 @@ impl PermutationReader {
         permutation_table: Option<Arc<dyn BaseTable>>,
         split: u64,
     ) -> Result<Self> {
+        let base_table = match &permutation_table {
+            Some(permutation_table) => Self::pin_base_table(base_table, permutation_table).await?,
+            None => base_table,
+        };
         let mut slf = Self {
             base_table,
             permutation_table,
@@ -89,6 +96,34 @@ impl PermutationReader {
         Ok(slf)
     }
 
+    /// Pins the base table to the version the permutation was built against.
+    /// Permutations written before that was recorded carry no key and stay unpinned.
+    async fn pin_base_table(
+        base_table: Arc<dyn BaseTable>,
+        permutation_table: &Arc<dyn BaseTable>,
+    ) -> Result<Arc<dyn BaseTable>> {
+        let schema = permutation_table.schema().await?;
+        let Some(raw) = schema.metadata.get(BASE_VERSION_CONFIG_KEY) else {
+            return Ok(base_table);
+        };
+        let version = raw.parse::<u64>().map_err(|e| Error::InvalidInput {
+            message: format!(
+                "Permutation table has an unreadable {} of {:?}: {}",
+                BASE_VERSION_CONFIG_KEY, raw, e
+            ),
+        })?;
+        // The recorded branch, not the handle's: a worker reopens by name and lands
+        // on main, and version numbers are per-branch.
+        let branch = schema
+            .metadata
+            .get(BASE_BRANCH_CONFIG_KEY)
+            .map(String::as_str)
+            .unwrap_or(MAIN_BRANCH);
+        base_table
+            .checkout_branch_version(branch, Some(version))
+            .await
+    }
+
     pub async fn try_from_tables(
         base_table: Arc<dyn BaseTable>,
         permutation_table: Arc<dyn BaseTable>,
@@ -97,8 +132,10 @@ impl PermutationReader {
         Self::inner_new(base_table, Some(permutation_table), split).await
     }
 
-    pub async fn identity(base_table: Arc<dyn BaseTable>) -> Self {
-        Self::inner_new(base_table, None, 0).await.unwrap()
+    /// A reader over the base table in storage order, with no permutation.
+    /// Fallible because construction counts the base table.
+    pub async fn identity(base_table: Arc<dyn BaseTable>) -> Result<Self> {
+        Self::inner_new(base_table, None, 0).await
     }
 
     /// Validates the limit and offset and returns the number of rows that will be read
@@ -487,7 +524,13 @@ impl PermutationReader {
 
     pub async fn output_schema(&self, selection: Select) -> Result<SchemaRef> {
         let table = Table::from(self.base_table.clone());
-        table.query().select(selection).output_schema().await
+        // limit(1) because some table types execute the query to get its schema
+        table
+            .query()
+            .select(selection)
+            .limit(1)
+            .output_schema()
+            .await
     }
 
     pub fn count_rows(&self) -> u64 {
@@ -503,9 +546,13 @@ mod tests {
     use lance_datagen::{BatchCount, RowCount};
     use rand::seq::SliceRandom;
 
+    // Aliased: `test_utils::datagen` exports a trait of the same name.
+    use crate::arrow::LanceDbDatagenExt as _;
     use crate::{
         Table,
         arrow::SendableRecordBatchStream,
+        connect,
+        dataloader::permutation::builder::PermutationBuilder,
         query::{ExecutableQuery, QueryBase},
         test_utils::datagen::{LanceDbDatagenExt, virtual_table},
     };
@@ -537,6 +584,58 @@ mod tests {
         .await
     }
 
+    /// Compaction moves row addresses, so the reader must read the pinned version.
+    #[tokio::test]
+    async fn test_reader_pins_base_version() {
+        let temp_dir = tempfile::tempdir().unwrap();
+        let db = connect(temp_dir.path().to_str().unwrap())
+            .execute()
+            .await
+            .unwrap();
+
+        let data = lance_datagen::gen_batch()
+            .col("idx", lance_datagen::array::step::<Int32Type>())
+            .into_ldb_stream(RowCount::from(20), BatchCount::from(1));
+        let base_table = db.create_table("base_tbl", data).execute().await.unwrap();
+
+        let permutation_table = PermutationBuilder::new(base_table.clone())
+            .build()
+            .await
+            .unwrap();
+
+        base_table.delete("true").await.unwrap();
+        base_table
+            .optimize(crate::table::OptimizeAction::All)
+            .await
+            .unwrap();
+        assert_eq!(base_table.count_rows(None).await.unwrap(), 0);
+
+        let reader = PermutationReader::try_from_tables(
+            base_table.base_table().clone(),
+            permutation_table.base_table().clone(),
+            0,
+        )
+        .await
+        .unwrap();
+
+        let values = collect_from_stream::<Int32Type>(
+            reader
+                .read(
+                    Select::Columns(vec!["idx".to_string()]),
+                    QueryExecutionOptions::default(),
+                )
+                .await
+                .unwrap(),
+            "idx",
+        )
+        .await;
+        assert_eq!(
+            values.len(),
+            20,
+            "reader should still see the pinned version"
+        );
+    }
+
     #[tokio::test]
     async fn test_permutation_reader() {
         let base_table = lance_datagen::gen_batch()
@@ -779,7 +878,9 @@ mod tests {
             .into_mem_table("tbl", RowCount::from(10), BatchCount::from(1))
             .await;
 
-        let reader = PermutationReader::identity(base_table.base_table().clone()).await;
+        let reader = PermutationReader::identity(base_table.base_table().clone())
+            .await
+            .unwrap();
 
         // With no permutation table, take_offsets uses the base table directly
         let offsets = vec![0, 2, 4, 6];
@@ -961,7 +1062,9 @@ mod tests {
             .into_mem_table("tbl", RowCount::from(10), BatchCount::from(1))
             .await;
 
-        let reader = PermutationReader::identity(base_table.base_table().clone()).await;
+        let reader = PermutationReader::identity(base_table.base_table().clone())
+            .await
+            .unwrap();
 
         let batch = reader.take_offsets(&[], Select::All).await.unwrap();
 
diff --git a/rust/lancedb/src/error.rs b/rust/lancedb/src/error.rs
index f6f596f3d..be4641388 100644
--- a/rust/lancedb/src/error.rs
+++ b/rust/lancedb/src/error.rs
@@ -71,6 +71,16 @@ pub enum Error {
     IndexNotFound { name: String },
     #[snafu(display("Embedding function '{name}' was not found. : {reason}"))]
     EmbeddingFunctionNotFound { name: String, reason: String },
+    #[snafu(display("Column '{name}' was not found"))]
+    ColumnNotFound { name: String },
+    #[snafu(display("Column '{name}' already exists"))]
+    ColumnAlreadyExists { name: String },
+    #[snafu(display("Column '{name}' is not a computed column"))]
+    NotAComputedColumn { name: String },
+    #[snafu(display("Table '{name}' is not a materialized view"))]
+    NotAMaterializedView { name: String },
+    #[snafu(display("Invalid expression for column '{column}': {message}"))]
+    InvalidExpression { column: String, message: String },
 
     #[snafu(display("Table '{name}' already exists"))]
     TableAlreadyExists { name: String },
@@ -169,6 +179,12 @@ impl From<DataFusionError> for Error {
 
 impl From<lance::Error> for Error {
     fn from(source: lance::Error) -> Self {
+        if has_unsupported_local_filesystem_source(&source) {
+            return Self::NotSupported {
+                message: "the filesystem does not support an operation required for safe Lance commits (such as atomic rename). Object-storage mounts such as Mountpoint for Amazon S3 are not supported; use the native object-store URI (for example, s3://bucket/path) instead".to_string(),
+            };
+        }
+
         // Try to unwrap external errors that were wrapped by lance
         match source {
             lance::Error::Wrapped { error, .. } => Self::from_box_error(error),
@@ -181,6 +197,27 @@ impl From<lance::Error> for Error {
     }
 }
 
+fn has_unsupported_local_filesystem_source(error: &(dyn std::error::Error + 'static)) -> bool {
+    let mut current = Some(error);
+    let mut is_local_filesystem = false;
+    let mut is_unsupported = false;
+    while let Some(error) = current {
+        is_local_filesystem |= error
+            .downcast_ref::<object_store::Error>()
+            .is_some_and(|error| {
+                matches!(error, object_store::Error::Generic { store, .. } if *store == "LocalFileSystem")
+            });
+        is_unsupported |= error
+            .downcast_ref::<std::io::Error>()
+            .is_some_and(|error| error.kind() == std::io::ErrorKind::Unsupported);
+        if is_local_filesystem && is_unsupported {
+            return true;
+        }
+        current = error.source();
+    }
+    false
+}
+
 impl Error {
     fn from_box_error(mut source: Box<dyn std::error::Error + Send + Sync>) -> Self {
         source = match source.downcast::<Self>() {
@@ -270,3 +307,46 @@ impl From<candle_core::Error> for Error {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn unsupported_filesystem_operations_have_actionable_error() {
+        let object_store_error = object_store::Error::Generic {
+            store: "LocalFileSystem",
+            source: Box::new(std::io::Error::from(std::io::ErrorKind::Unsupported)),
+        };
+        let lance_error = lance::Error::io_source(Box::new(object_store_error));
+
+        let error = Error::from(lance_error);
+
+        assert!(matches!(
+            error,
+            Error::NotSupported { message }
+                if message.contains("Mountpoint for Amazon S3")
+                    && message.contains("s3://bucket/path")
+        ));
+    }
+
+    #[test]
+    fn other_io_errors_remain_lance_errors() {
+        let object_store_error = object_store::Error::Generic {
+            store: "LocalFileSystem",
+            source: Box::new(std::io::Error::from(std::io::ErrorKind::PermissionDenied)),
+        };
+        let lance_error = lance::Error::io_source(Box::new(object_store_error));
+
+        assert!(matches!(Error::from(lance_error), Error::Lance { .. }));
+    }
+
+    #[test]
+    fn unsupported_non_filesystem_errors_remain_lance_errors() {
+        let lance_error = lance::Error::io_source(Box::new(std::io::Error::from(
+            std::io::ErrorKind::Unsupported,
+        )));
+
+        assert!(matches!(Error::from(lance_error), Error::Lance { .. }));
+    }
+}
diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs
new file mode 100644
index 000000000..52c70a4b1
--- /dev/null
+++ b/rust/lancedb/src/function.rs
@@ -0,0 +1,585 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+//! Canonical Function values exchanged with the Enterprise service, plus the
+//! backend-neutral terminal result of a computed-column refresh.
+//!
+//! This module contains client/wire values only. Catalog persistence,
+//! environment bake, and execution are owned by Sophon.
+
+use std::collections::BTreeMap;
+
+use serde::de::{self, DeserializeOwned};
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use serde_json::Value;
+
+use crate::{Error, Result};
+
+fn invalid_json(error: impl std::fmt::Display) -> Error {
+    Error::InvalidInput {
+        message: format!("invalid remote Function JSON: {error}"),
+    }
+}
+
+fn write_canonical_json(value: &Value, output: &mut String) -> serde_json::Result<()> {
+    match value {
+        Value::Object(map) => {
+            output.push('{');
+            let mut entries = map.iter().collect::<Vec<_>>();
+            entries.sort_unstable_by_key(|(key, _)| *key);
+            for (index, (key, value)) in entries.into_iter().enumerate() {
+                if index != 0 {
+                    output.push(',');
+                }
+                output.push_str(&serde_json::to_string(key)?);
+                output.push(':');
+                write_canonical_json(value, output)?;
+            }
+            output.push('}');
+        }
+        Value::Array(values) => {
+            output.push('[');
+            for (index, value) in values.iter().enumerate() {
+                if index != 0 {
+                    output.push(',');
+                }
+                write_canonical_json(value, output)?;
+            }
+            output.push(']');
+        }
+        other => output.push_str(&serde_json::to_string(other)?),
+    }
+    Ok(())
+}
+
+fn canonical_json<T: Serialize>(value: &T) -> Result<String> {
+    let value = serde_json::to_value(value).map_err(invalid_json)?;
+    let mut output = String::new();
+    write_canonical_json(&value, &mut output).map_err(invalid_json)?;
+    Ok(output)
+}
+
+fn from_json<T: DeserializeOwned>(json: &str) -> Result<T> {
+    serde_json::from_str(json).map_err(invalid_json)
+}
+
+fn validate_literal(value: &Value) -> Result<()> {
+    match value {
+        Value::Number(number) if number.is_f64() => Err(Error::InvalidInput {
+            message: "floating-point Function literals are not part of the Slice 1 canonical wire contract"
+                .to_string(),
+        }),
+        Value::Array(values) => values.iter().try_for_each(validate_literal),
+        Value::Object(values) => values.values().try_for_each(validate_literal),
+        _ => Ok(()),
+    }
+}
+
+fn has_unknown_keys(value: &Value, allowed: &[&str]) -> bool {
+    value
+        .as_object()
+        .is_some_and(|object| object.keys().any(|key| !allowed.contains(&key.as_str())))
+}
+
+fn application_has_unknown_nested_fields(value: &Value) -> bool {
+    let Some(application) = value.as_object() else {
+        return false;
+    };
+    if application
+        .get("function")
+        .is_some_and(|value| has_unknown_keys(value, &["name", "version"]))
+    {
+        return true;
+    }
+    if application
+        .get("inputs")
+        .and_then(Value::as_array)
+        .is_some_and(|inputs| {
+            inputs
+                .iter()
+                .any(|input| has_unknown_keys(input, &["parameter", "kind", "value"]))
+        })
+    {
+        return true;
+    }
+    application.get("output").is_some_and(|output| {
+        has_unknown_keys(output, &["kind", "arrow_type", "nullable", "fields"])
+            || output
+                .get("fields")
+                .and_then(Value::as_array)
+                .is_some_and(|fields| {
+                    fields
+                        .iter()
+                        .any(|field| has_unknown_keys(field, &["name", "arrow_type", "nullable"]))
+                })
+    })
+}
+
+macro_rules! impl_json {
+    ($type:ty) => {
+        impl $type {
+            /// Decode a remote value. Unknown fields and discriminator values
+            /// are accepted so newer servers remain readable.
+            pub fn from_json(json: &str) -> Result<Self> {
+                from_json(json)
+            }
+
+            /// Encode the known client contract with bytewise-sorted JSON keys.
+            pub fn to_canonical_json(&self) -> Result<String> {
+                canonical_json(self)
+            }
+        }
+    };
+}
+
+/// Packaged Python artifact identity. Source bytes are never part of this value.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct FunctionArtifact {
+    pub kind: String,
+    pub digest: String,
+    pub entrypoint: String,
+}
+
+/// One ordered Arrow input parameter.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct FunctionParameter {
+    pub name: String,
+    pub arrow_type: String,
+    pub nullable: bool,
+}
+
+/// One field of an ordered named-struct result.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct FunctionResultField {
+    pub name: String,
+    pub arrow_type: String,
+    pub nullable: bool,
+}
+
+/// Scalar or named-struct Function output.
+///
+/// `kind` remains a string so unknown future result shapes can be decoded.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct FunctionOutput {
+    pub kind: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub arrow_type: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub nullable: Option<bool>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub fields: Vec<FunctionResultField>,
+}
+
+/// Ordered language-neutral Function signature.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct FunctionSignature {
+    pub inputs: Vec<FunctionParameter>,
+    pub output: FunctionOutput,
+}
+
+/// One Python environment source.
+///
+/// The selected source is interpreted by Sophon. `kind` is open for forward
+/// compatible decoding.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct PythonEnvironmentSpec {
+    pub kind: String,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub packages: Vec<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub path: Option<String>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub modules: Vec<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub image: Option<String>,
+}
+
+/// Reproducible Python runtime definition understood by Sophon.
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum PythonRuntimeSpec {
+    /// The V1 Sophon-managed Python runtime.
+    Python {
+        python_version: String,
+        environment: PythonEnvironmentSpec,
+        env: BTreeMap<String, String>,
+    },
+    /// A runtime kind introduced by a newer server.
+    ///
+    /// Unknown payload fields are intentionally not retained because the
+    /// client does not proxy catalog values.
+    Unrecognized { kind: String },
+}
+
+impl PythonRuntimeSpec {
+    /// The wire discriminator reported by Sophon.
+    pub fn kind(&self) -> &str {
+        match self {
+            Self::Python { .. } => "python",
+            Self::Unrecognized { kind } => kind,
+        }
+    }
+
+    /// The Python version for the V1 runtime, or `None` for an unknown kind.
+    pub fn python_version(&self) -> Option<&str> {
+        match self {
+            Self::Python { python_version, .. } => Some(python_version),
+            Self::Unrecognized { .. } => None,
+        }
+    }
+
+    /// The Python environment for the V1 runtime, or `None` for an unknown kind.
+    pub fn environment(&self) -> Option<&PythonEnvironmentSpec> {
+        match self {
+            Self::Python { environment, .. } => Some(environment),
+            Self::Unrecognized { .. } => None,
+        }
+    }
+
+    /// Environment variables, or `None` for an unknown kind.
+    pub fn env(&self) -> Option<&BTreeMap<String, String>> {
+        match self {
+            Self::Python { env, .. } => Some(env),
+            Self::Unrecognized { .. } => None,
+        }
+    }
+}
+
+#[derive(Deserialize)]
+struct PythonRuntimeWire {
+    kind: String,
+    #[serde(default)]
+    python_version: Option<String>,
+    #[serde(default)]
+    environment: Option<PythonEnvironmentSpec>,
+    #[serde(default)]
+    env: BTreeMap<String, String>,
+}
+
+impl<'de> Deserialize<'de> for PythonRuntimeSpec {
+    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
+        let wire = PythonRuntimeWire::deserialize(deserializer)?;
+        if wire.kind == "python" {
+            Ok(Self::Python {
+                python_version: wire
+                    .python_version
+                    .ok_or_else(|| de::Error::missing_field("python_version"))?,
+                environment: wire
+                    .environment
+                    .ok_or_else(|| de::Error::missing_field("environment"))?,
+                env: wire.env,
+            })
+        } else {
+            Ok(Self::Unrecognized { kind: wire.kind })
+        }
+    }
+}
+
+impl Serialize for PythonRuntimeSpec {
+    fn serialize<S: Serializer>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error> {
+        #[derive(Serialize)]
+        struct PythonRuntimeRef<'a> {
+            kind: &'static str,
+            python_version: &'a str,
+            environment: &'a PythonEnvironmentSpec,
+            #[serde(skip_serializing_if = "BTreeMap::is_empty")]
+            env: &'a BTreeMap<String, String>,
+        }
+
+        #[derive(Serialize)]
+        struct UnrecognizedRuntimeRef<'a> {
+            kind: &'a str,
+        }
+
+        match self {
+            Self::Python {
+                python_version,
+                environment,
+                env,
+            } => PythonRuntimeRef {
+                kind: "python",
+                python_version,
+                environment,
+                env,
+            }
+            .serialize(serializer),
+            Self::Unrecognized { kind } => UnrecognizedRuntimeRef { kind }.serialize(serializer),
+        }
+    }
+}
+
+/// Immutable Function version returned by the Enterprise catalog.
+///
+/// Scheduling resources, priority, concurrency, and retry policy belong to
+/// the submitting Job and are not part of this identity.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct FunctionVersion {
+    name: String,
+    version: String,
+    artifact: FunctionArtifact,
+    signature: FunctionSignature,
+    runtime: PythonRuntimeSpec,
+    runtime_digest: String,
+    environment_digest: String,
+    created_at: String,
+}
+
+impl FunctionVersion {
+    pub fn name(&self) -> &str {
+        &self.name
+    }
+
+    pub fn version(&self) -> &str {
+        &self.version
+    }
+
+    pub fn artifact(&self) -> &FunctionArtifact {
+        &self.artifact
+    }
+
+    pub fn signature(&self) -> &FunctionSignature {
+        &self.signature
+    }
+
+    pub fn runtime(&self) -> &PythonRuntimeSpec {
+        &self.runtime
+    }
+
+    pub fn runtime_digest(&self) -> &str {
+        &self.runtime_digest
+    }
+
+    pub fn environment_digest(&self) -> &str {
+        &self.environment_digest
+    }
+
+    pub fn created_at(&self) -> &str {
+        &self.created_at
+    }
+}
+
+impl_json!(FunctionVersion);
+
+/// Encoded artifact bytes uploaded with a Function registration request.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct FunctionArtifactContent {
+    /// Encoding of `data`. V1 Python authoring uses `base64`.
+    pub encoding: String,
+    pub data: String,
+}
+
+/// Internal execution adapter selected for a Python callable artifact.
+///
+/// The adapter converts the public scalar callable to the Arrow batch ABI
+/// used by the remote executor. It is part of the request envelope, not a
+/// public batch-UDF authoring mode.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct PythonAdapterSpec {
+    pub kind: String,
+    pub version: u32,
+}
+
+/// Python artifact uploaded while registering a Function.
+///
+/// Unlike [`FunctionArtifact`], which is the durable artifact identity
+/// returned by the catalog, this request value contains the encoded source
+/// bytes that Sophon must durably bake before publishing a FunctionVersion.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct FunctionArtifactRequest {
+    pub kind: String,
+    pub digest: String,
+    pub entrypoint: String,
+    pub content: FunctionArtifactContent,
+    pub adapter: PythonAdapterSpec,
+}
+
+/// Stable request envelope for remote immutable Function registration.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct FunctionRegistrationRequest {
+    pub name: String,
+    pub artifact: FunctionArtifactRequest,
+    pub signature: FunctionSignature,
+    pub runtime: PythonRuntimeSpec,
+}
+
+impl_json!(FunctionRegistrationRequest);
+
+/// Exact FunctionVersion reference embedded in applications and bindings.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct FunctionVersionRef {
+    pub name: String,
+    pub version: String,
+}
+
+/// Parameter binding in a FunctionApplication.
+///
+/// `kind` remains open until Python authoring is added in Slice 2. Slice 1
+/// freezes JSON integers, strings, booleans, nulls, arrays, and objects as
+/// canonical literal values. Floating-point literals are rejected until a
+/// language-neutral numeric representation is defined.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct ApplicationInput {
+    pub parameter: String,
+    pub kind: String,
+    pub value: Value,
+}
+
+/// Pre-declaration application of an exact FunctionVersion.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct FunctionApplication {
+    function: FunctionVersionRef,
+    inputs: Vec<ApplicationInput>,
+    output: FunctionOutput,
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
+    columns: BTreeMap<String, String>,
+    #[serde(default, flatten, skip_serializing)]
+    unknown_fields: BTreeMap<String, Value>,
+    #[serde(default, skip)]
+    unknown_nested_fields: bool,
+}
+
+impl FunctionApplication {
+    pub fn function(&self) -> &FunctionVersionRef {
+        &self.function
+    }
+
+    pub fn inputs(&self) -> &[ApplicationInput] {
+        &self.inputs
+    }
+
+    pub fn output(&self) -> &FunctionOutput {
+        &self.output
+    }
+
+    pub fn columns(&self) -> &BTreeMap<String, String> {
+        &self.columns
+    }
+    /// Whether a newer writer attached application fields this client cannot
+    /// validate. Such applications remain readable but must not be declared.
+    pub fn has_unknown_fields(&self) -> bool {
+        !self.unknown_fields.is_empty() || self.unknown_nested_fields
+    }
+
+    /// Decode a remote application after validating the Slice 1 literal domain.
+    pub fn from_json(json: &str) -> Result<Self> {
+        let value: Value = from_json(json)?;
+        let has_unknown_nested_fields = application_has_unknown_nested_fields(&value);
+        let mut application: Self = serde_json::from_value(value).map_err(invalid_json)?;
+        application.unknown_nested_fields = has_unknown_nested_fields;
+        application
+            .inputs
+            .iter()
+            .try_for_each(|input| validate_literal(&input.value))?;
+        Ok(application)
+    }
+
+    /// Encode the application with bytewise-sorted JSON keys.
+    pub fn to_canonical_json(&self) -> Result<String> {
+        self.inputs
+            .iter()
+            .try_for_each(|input| validate_literal(&input.value))?;
+        canonical_json(self)
+    }
+}
+
+/// Stable table input bound to a registered parameter.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct InputBinding {
+    pub parameter: String,
+    pub field_id: i32,
+    pub field_path: String,
+    pub arrow_type: String,
+    pub nullable: bool,
+}
+
+/// Ordered result-field to table-field mapping for a Function binding.
+///
+/// Assignment state is not part of the Slice 1 client contract. During the
+/// NULL transition there is no public Lance cell-flag identifier to persist.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct OutputMapping {
+    pub result_field: String,
+    pub output_name: String,
+    pub output_field_id: i32,
+    pub output_ordinal: u32,
+    pub arrow_type: String,
+    pub nullable: bool,
+}
+
+/// Immutable Function binding persisted by the Enterprise table service.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct FunctionBinding {
+    binding_id: String,
+    function: FunctionVersionRef,
+    inputs: Vec<InputBinding>,
+    outputs: Vec<OutputMapping>,
+    /// Exact Arrow schema presented to the Function, encoded with the Lance
+    /// Namespace Arrow JSON representation.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    input_schema: Option<Value>,
+    /// Exact physical Arrow schema of the binding's table outputs.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    output_schema: Option<Value>,
+}
+
+impl FunctionBinding {
+    pub fn binding_id(&self) -> &str {
+        &self.binding_id
+    }
+
+    pub fn function(&self) -> &FunctionVersionRef {
+        &self.function
+    }
+
+    pub fn inputs(&self) -> &[InputBinding] {
+        &self.inputs
+    }
+
+    pub fn outputs(&self) -> &[OutputMapping] {
+        &self.outputs
+    }
+
+    pub fn input_schema(&self) -> Option<&Value> {
+        self.input_schema.as_ref()
+    }
+
+    pub fn output_schema(&self) -> Option<&Value> {
+        self.output_schema.as_ref()
+    }
+}
+
+impl_json!(FunctionBinding);
+
+/// Stable terminal result of an expression-backed or Function-backed column
+/// refresh [`crate::Job`].
+///
+/// Local refresh jobs produce this value in process. LanceDB Cloud and
+/// Enterprise decode the same value from the durable job's terminal payload.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct RefreshColumnResult {
+    /// Rows assigned a value by this refresh.
+    pub rows_assigned: u64,
+    /// Rows whose computation failed.
+    pub rows_failed: u64,
+    /// Rows that still need a value when the job completes.
+    pub rows_remaining: u64,
+    /// Exact table version the refresh read.
+    pub source_version: u64,
+    /// Table version made visible by the refresh, when one was published.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub published_version: Option<u64>,
+}
+
+impl RefreshColumnResult {
+    /// Deprecated compatibility alias for `rows_assigned`.
+    pub fn rows_filled(&self) -> u64 {
+        self.rows_assigned
+    }
+
+    /// Deprecated compatibility alias for `published_version`.
+    pub fn version(&self) -> Option<u64> {
+        self.published_version
+    }
+}
+
+impl_json!(RefreshColumnResult);
diff --git a/rust/lancedb/src/io/object_store.rs b/rust/lancedb/src/io/object_store.rs
index d27357b82..d594bd857 100644
--- a/rust/lancedb/src/io/object_store.rs
+++ b/rust/lancedb/src/io/object_store.rs
@@ -132,9 +132,14 @@ impl ObjectStore for MirroringObjectStore {
         if to.primary_only() {
             self.primary.copy_opts(from, to, options).await
         } else {
-            self.secondary.copy_opts(from, to, options.clone()).await?;
-            self.primary.copy_opts(from, to, options).await?;
-            Ok(())
+            // The secondary store can be process-local and less durable than the
+            // primary, so a source written by another process may not exist here
+            // or may be evicted before the copy begins.
+            match self.secondary.copy_opts(from, to, options.clone()).await {
+                Ok(()) | Err(Error::NotFound { .. }) => {}
+                Err(err) => return Err(err),
+            }
+            self.primary.copy_opts(from, to, options).await
         }
     }
 }
@@ -192,7 +197,8 @@ mod test {
     use futures::TryStreamExt;
     use lance::{dataset::WriteParams, io::ObjectStoreParams};
     use lance_testing::datagen::{BatchGenerator, IncrementingInt32, RandomVector};
-    use object_store::local::LocalFileSystem;
+    use object_store::{local::LocalFileSystem, memory::InMemory};
+    use std::time::Duration;
     use tempfile;
 
     use crate::{
@@ -201,6 +207,139 @@ mod test {
         table::WriteOptions,
     };
 
+    #[derive(Debug)]
+    struct EvictBeforeCopyStore {
+        inner: Arc<dyn ObjectStore>,
+    }
+
+    impl std::fmt::Display for EvictBeforeCopyStore {
+        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+            write!(f, "EvictBeforeCopyStore")
+        }
+    }
+
+    #[async_trait]
+    impl ObjectStore for EvictBeforeCopyStore {
+        async fn put_opts(
+            &self,
+            location: &Path,
+            payload: PutPayload,
+            options: PutOptions,
+        ) -> Result<PutResult> {
+            self.inner.put_opts(location, payload, options).await
+        }
+
+        async fn put_multipart_opts(
+            &self,
+            location: &Path,
+            options: PutMultipartOptions,
+        ) -> Result<Box<dyn MultipartUpload>> {
+            self.inner.put_multipart_opts(location, options).await
+        }
+
+        async fn get_opts(&self, location: &Path, options: GetOptions) -> Result<GetResult> {
+            self.inner.get_opts(location, options).await
+        }
+
+        fn delete_stream(
+            &self,
+            locations: BoxStream<'static, Result<Path>>,
+        ) -> BoxStream<'static, Result<Path>> {
+            self.inner.delete_stream(locations)
+        }
+
+        fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
+            self.inner.list(prefix)
+        }
+
+        async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {
+            self.inner.list_with_delimiter(prefix).await
+        }
+
+        async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
+            self.inner.delete(from).await?;
+            self.inner.copy_opts(from, to, options).await
+        }
+    }
+
+    #[tokio::test]
+    async fn test_copy_when_source_is_missing_from_secondary() {
+        let primary_dir = tempfile::tempdir().unwrap();
+        let secondary_dir = tempfile::tempdir().unwrap();
+        let primary: Arc<dyn ObjectStore> =
+            Arc::new(LocalFileSystem::new_with_prefix(primary_dir.path()).unwrap());
+        let secondary: Arc<dyn ObjectStore> =
+            Arc::new(LocalFileSystem::new_with_prefix(secondary_dir.path()).unwrap());
+        let store = MirroringObjectStore {
+            primary: primary.clone(),
+            secondary: secondary.clone(),
+        };
+        let staging = Path::from("_versions/1.manifest-staging");
+        let finalized = Path::from("_versions/1.manifest");
+
+        primary
+            .put(&staging, "manifest contents".into())
+            .await
+            .unwrap();
+
+        tokio::time::timeout(Duration::from_secs(5), store.copy(&staging, &finalized))
+            .await
+            .expect("copy should not hang when the secondary source is missing")
+            .unwrap();
+
+        let copied = primary
+            .get(&finalized)
+            .await
+            .unwrap()
+            .bytes()
+            .await
+            .unwrap();
+        assert_eq!(copied, "manifest contents");
+        assert!(matches!(
+            secondary.head(&finalized).await,
+            Err(Error::NotFound { .. })
+        ));
+    }
+
+    #[tokio::test]
+    async fn test_copy_when_secondary_source_disappears_after_head() {
+        let primary: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+        let secondary_inner: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+        let secondary: Arc<dyn ObjectStore> = Arc::new(EvictBeforeCopyStore {
+            inner: secondary_inner.clone(),
+        });
+        let store = MirroringObjectStore {
+            primary: primary.clone(),
+            secondary,
+        };
+        let staging = Path::from("_versions/1.manifest-staging");
+        let finalized = Path::from("_versions/1.manifest");
+
+        primary
+            .put(&staging, "manifest contents".into())
+            .await
+            .unwrap();
+        secondary_inner
+            .put(&staging, "manifest contents".into())
+            .await
+            .unwrap();
+
+        store.copy(&staging, &finalized).await.unwrap();
+
+        let copied = primary
+            .get(&finalized)
+            .await
+            .unwrap()
+            .bytes()
+            .await
+            .unwrap();
+        assert_eq!(copied, "manifest contents");
+        assert!(matches!(
+            secondary_inner.head(&finalized).await,
+            Err(Error::NotFound { .. })
+        ));
+    }
+
     // This test is ignored because lance 3.0 introduced LocalWriter optimization
     // that bypasses the object store wrapper for local writes. The mirroring feature
     // still works for remote/cloud storage, but can't be tested with local storage.
diff --git a/rust/lancedb/src/job.rs b/rust/lancedb/src/job.rs
index 789ce8312..94d1ba2b6 100644
--- a/rust/lancedb/src/job.rs
+++ b/rust/lancedb/src/job.rs
@@ -6,6 +6,8 @@
 use std::sync::Arc;
 
 use async_trait::async_trait;
+use serde::{Serialize, de::DeserializeOwned};
+use serde_json::Value;
 use tokio::sync::watch;
 use tokio::task::{AbortHandle, JoinHandle};
 
@@ -19,43 +21,137 @@ pub(crate) trait JobHandle: Send + Sync {
         None
     }
     async fn status(&self) -> Result<String>;
-    async fn wait(&self) -> Result<()>;
+    async fn wait(&self) -> Result<TerminalResult>;
     async fn cancel(&self) -> Result<()>;
 }
 
+/// A backend-neutral successful terminal result.
+#[derive(Clone)]
+pub(crate) struct TerminalResult {
+    value: Option<Value>,
+    request_id: Option<String>,
+}
+
+impl TerminalResult {
+    fn local(value: Value) -> Self {
+        Self {
+            value: Some(value),
+            request_id: None,
+        }
+    }
+
+    pub(crate) fn remote(value: Option<Value>, request_id: String) -> Self {
+        Self {
+            value,
+            request_id: Some(request_id),
+        }
+    }
+
+    fn decode<T: DeserializeOwned>(self) -> Result<T> {
+        let value = self.value.ok_or_else(|| match &self.request_id {
+            Some(request_id) => Error::Http {
+                source: "successful typed job response did not contain a result".into(),
+                request_id: request_id.clone(),
+                status_code: None,
+            },
+            None => Error::Runtime {
+                message: "successful typed job did not contain a result".to_string(),
+            },
+        })?;
+        serde_json::from_value(value).map_err(|error| match self.request_id {
+            Some(request_id) => Error::Http {
+                source: format!("failed to parse typed job result: {error}").into(),
+                request_id,
+                status_code: None,
+            },
+            None => Error::Runtime {
+                message: format!("failed to parse typed job result: {error}"),
+            },
+        })
+    }
+}
+
+type ResultDecoder<T> = Arc<dyn Fn(TerminalResult) -> Result<T> + Send + Sync>;
+
+enum JobInner<T> {
+    Handle {
+        handle: Box<dyn JobHandle>,
+        decode: ResultDecoder<T>,
+    },
+    Completed(T),
+}
+
 /// A handle to an operation that may still be running.
 ///
-/// The operation may already be complete when the handle is created.
-pub struct Job {
-    handle: Option<Box<dyn JobHandle>>,
+/// The operation may already be complete when the handle is created. `T` is
+/// the endpoint's successful terminal result; unit-result operations use the
+/// default `Job<()>`.
+pub struct Job<T = ()>
+where
+    T: Clone + Send + Sync + 'static,
+{
+    inner: JobInner<T>,
 }
 
-impl std::fmt::Debug for Job {
+impl<T> std::fmt::Debug for Job<T>
+where
+    T: Clone + Send + Sync + 'static,
+{
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("Job")
             .field("id", &self.id())
-            .field("done", &self.handle.is_none())
+            .field("done", &matches!(self.inner, JobInner::Completed(_)))
             .finish()
     }
 }
 
-impl Job {
+impl Job<()> {
     /// A job whose operation finished before the handle was created.
     pub(crate) fn new_done() -> Self {
-        Self { handle: None }
+        Self {
+            inner: JobInner::Completed(()),
+        }
     }
 
     pub(crate) fn new(handle: Box<dyn JobHandle>) -> Self {
         Self {
-            handle: Some(handle),
+            inner: JobInner::Handle {
+                handle,
+                decode: Arc::new(|_| Ok(())),
+            },
         }
     }
+}
 
-    /// A job running as a task in this process.
-    pub(crate) fn spawned(task: JoinHandle<Result<()>>) -> Self {
-        Self::new(Box::new(SpawnedJob::new(task)))
+impl<T> Job<T>
+where
+    T: Clone + DeserializeOwned + Send + Sync + 'static,
+{
+    /// Construct a typed remote Job for a result-specific submit API.
+    pub(crate) fn new_typed(handle: Box<dyn JobHandle>) -> Self {
+        Self {
+            inner: JobInner::Handle {
+                handle,
+                decode: Arc::new(TerminalResult::decode::<T>),
+            },
+        }
     }
+}
 
+impl<T> Job<T>
+where
+    T: Clone + Serialize + DeserializeOwned + Send + Sync + 'static,
+{
+    /// A typed job running as a task in this process.
+    pub(crate) fn spawned(task: JoinHandle<Result<T>>) -> Self {
+        Self::new_typed(Box::new(SpawnedJob::new(task)))
+    }
+}
+
+impl<T> Job<T>
+where
+    T: Clone + Send + Sync + 'static,
+{
     /// Identifies the operation on the server that is running it.
     ///
     /// Returned for correlating with server logs or the jobs API. Operations
@@ -63,7 +159,10 @@ impl Job {
     /// value is opaque: parsing it or storing it to resume the job later is
     /// not supported.
     pub fn id(&self) -> Option<&str> {
-        self.handle.as_ref().and_then(|handle| handle.id())
+        match &self.inner {
+            JobInner::Handle { handle, .. } => handle.id(),
+            JobInner::Completed(_) => None,
+        }
     }
 
     /// The operation's current lifecycle state: "running", "finished",
@@ -73,20 +172,22 @@ impl Job {
     /// terminal failure state, or retry. States a newer server reports that
     /// this client version does not know pass through as-is.
     pub async fn status(&self) -> Result<String> {
-        match &self.handle {
-            None => Ok("finished".to_string()),
-            Some(handle) => handle.status().await,
+        match &self.inner {
+            JobInner::Handle { handle, .. } => handle.status().await,
+            JobInner::Completed(_) => Ok("finished".to_string()),
         }
     }
 
     /// Waits until the operation reaches a terminal state.
     ///
+    /// Returns the endpoint's typed result. Unit-result jobs return `()`.
+    ///
     /// Returns [`crate::Error::JobFailed`] if the operation failed and
     /// [`crate::Error::JobCancelled`] if it was cancelled.
-    pub async fn wait(&self) -> Result<()> {
-        match &self.handle {
-            None => Ok(()),
-            Some(handle) => handle.wait().await,
+    pub async fn wait(&self) -> Result<T> {
+        match &self.inner {
+            JobInner::Handle { handle, decode } => (decode)(handle.wait().await?),
+            JobInner::Completed(result) => Ok(result.clone()),
         }
     }
 
@@ -94,9 +195,41 @@ impl Job {
     ///
     /// Cancelling an operation that already finished is a no-op.
     pub async fn cancel(&self) -> Result<()> {
-        match &self.handle {
-            None => Ok(()),
-            Some(handle) => handle.cancel().await,
+        match &self.inner {
+            JobInner::Handle { handle, .. } => handle.cancel().await,
+            JobInner::Completed(_) => Ok(()),
+        }
+    }
+
+    /// Maps a successful terminal result without changing the job lifecycle.
+    /// The mapping may run once for each call to [`Job::wait`], so it should
+    /// be deterministic and free of externally visible side effects.
+    ///
+    /// ```
+    /// use lancedb::{Job, function::RefreshColumnResult};
+    ///
+    /// # async fn rows_assigned(
+    /// #     job: Job<RefreshColumnResult>,
+    /// # ) -> lancedb::Result<u64> {
+    /// let job = job.map(|result| result.rows_assigned);
+    /// job.wait().await
+    /// # }
+    /// ```
+    pub fn map<U, F>(self, map: F) -> Job<U>
+    where
+        U: Clone + Send + Sync + 'static,
+        F: Fn(T) -> U + Send + Sync + 'static,
+    {
+        match self.inner {
+            JobInner::Handle { handle, decode } => Job {
+                inner: JobInner::Handle {
+                    handle,
+                    decode: Arc::new(move |result| Ok(map((decode)(result)?))),
+                },
+            },
+            JobInner::Completed(result) => Job {
+                inner: JobInner::Completed(map(result)),
+            },
         }
     }
 }
@@ -105,15 +238,15 @@ impl Job {
 /// the outcome; [`Error`] is not, so failures share one behind an [`Arc`].
 #[derive(Clone)]
 enum Outcome {
-    Succeeded,
+    Succeeded(TerminalResult),
     Failed(Arc<Error>),
     Cancelled,
 }
 
 impl Outcome {
-    fn into_result(self) -> Result<()> {
+    fn into_result(self) -> Result<TerminalResult> {
         match self {
-            Self::Succeeded => Ok(()),
+            Self::Succeeded(result) => Ok(result),
             Self::Failed(source) => Err(Error::JobFailed {
                 job_id: None,
                 failure: JobFailure::from_source(source),
@@ -132,16 +265,24 @@ struct SpawnedJob {
 }
 
 impl SpawnedJob {
-    fn new(task: JoinHandle<Result<()>>) -> Self {
+    fn new<T>(task: JoinHandle<Result<T>>) -> Self
+    where
+        T: Serialize + Send + 'static,
+    {
         let abort = task.abort_handle();
         let (tx, outcome) = watch::channel(None);
         tokio::spawn(async move {
             let outcome = match task.await {
-                Ok(Ok(())) => Outcome::Succeeded,
+                Ok(Ok(result)) => match serde_json::to_value(result) {
+                    Ok(value) => Outcome::Succeeded(TerminalResult::local(value)),
+                    Err(err) => Outcome::Failed(Arc::new(Error::Runtime {
+                        message: format!("failed to serialize job result: {err}"),
+                    })),
+                },
                 Ok(Err(err)) => Outcome::Failed(Arc::new(err)),
                 Err(err) if err.is_cancelled() => Outcome::Cancelled,
                 Err(err) => Outcome::Failed(Arc::new(Error::Runtime {
-                    message: format!("index job task failed: {err}"),
+                    message: format!("job task failed: {err}"),
                 })),
             };
             let _ = tx.send(Some(outcome));
@@ -155,20 +296,20 @@ impl JobHandle for SpawnedJob {
     async fn status(&self) -> Result<String> {
         let label = match &*self.outcome.borrow() {
             None => "running",
-            Some(Outcome::Succeeded) => "finished",
+            Some(Outcome::Succeeded(_)) => "finished",
             Some(Outcome::Failed(_)) => "failed",
             Some(Outcome::Cancelled) => "cancelled",
         };
         Ok(label.to_string())
     }
 
-    async fn wait(&self) -> Result<()> {
+    async fn wait(&self) -> Result<TerminalResult> {
         let mut outcome = self.outcome.clone();
         let settled = outcome
             .wait_for(|outcome| outcome.is_some())
             .await
             .map_err(|_| Error::Runtime {
-                message: "index job outcome was dropped before it completed".to_string(),
+                message: "job outcome was dropped before it completed".to_string(),
             })?
             .clone()
             .expect("wait_for returns once an outcome is set");
@@ -180,3 +321,29 @@ impl JobHandle for SpawnedJob {
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::future::pending;
+
+    use super::*;
+
+    #[tokio::test]
+    async fn mapped_spawned_job_reuses_outcome() {
+        let job = Job::spawned(tokio::spawn(async { Ok(41_u64) })).map(|value| value + 1);
+
+        assert_eq!(job.wait().await.unwrap(), 42);
+        assert_eq!(job.wait().await.unwrap(), 42);
+        assert_eq!(job.status().await.unwrap(), "finished");
+    }
+
+    #[tokio::test]
+    async fn mapped_spawned_job_preserves_cancellation() {
+        let job = Job::spawned(tokio::spawn(async { pending::<Result<u64>>().await }))
+            .map(|value| value.to_string());
+
+        job.cancel().await.unwrap();
+        assert!(matches!(job.wait().await, Err(Error::JobCancelled { .. })));
+        assert_eq!(job.status().await.unwrap(), "cancelled");
+    }
+}
diff --git a/rust/lancedb/src/lib.rs b/rust/lancedb/src/lib.rs
index 70d023ccc..9c3c199ff 100644
--- a/rust/lancedb/src/lib.rs
+++ b/rust/lancedb/src/lib.rs
@@ -181,10 +181,12 @@ pub mod dataloader;
 pub mod embeddings;
 pub mod error;
 pub mod expr;
+pub mod function;
 pub mod index;
 pub mod io;
 pub mod ipc;
 pub mod job;
+pub mod materialized_view;
 #[cfg(feature = "metrics-otel")]
 pub mod metrics_otel;
 #[cfg(feature = "polars")]
@@ -205,9 +207,13 @@ use serde::{Deserialize, Serialize};
 pub use blob::{BlobRangeRequest, blob, is_blob};
 pub use connection::{ConnectNamespaceBuilder, Connection};
 pub use error::{Error, JobFailure, Result};
+pub use function::FunctionVersion;
 pub use job::Job;
 use lance_index::vector::ApproxMode as LanceApproxMode;
 use lance_linalg::distance::DistanceType as LanceDistanceType;
+pub use materialized_view::{
+    MaterializedView, MaterializedViewDefinition, RefreshMaterializedViewResult, RefreshMode,
+};
 /// Re-export of the [`metrics`](https://docs.rs/metrics) crate facade. Enable
 /// the `metrics` feature to publish LanceDB's internal metrics; install any
 /// `metrics`-compatible recorder to collect them. See also [`metrics_otel`] for
diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs
new file mode 100644
index 000000000..b28d52931
--- /dev/null
+++ b/rust/lancedb/src/materialized_view.rs
@@ -0,0 +1,2106 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+//! Materialized views.
+//!
+//! A materialized view is a table whose contents are defined by a query over
+//! one source table and maintained by refresh rather than by writes. Creation
+//! commits an empty table carrying the kind-tagged definition in schema
+//! metadata; a kind added later reads back as unrefreshable, not as a plain
+//! table. Queries, indexes and search work on the view unchanged.
+
+pub mod refresh;
+
+#[cfg(test)]
+mod differential;
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use arrow_schema::{DataType, Field as ArrowField, FieldRef, Schema as ArrowSchema, SchemaRef};
+use datafusion_common::ScalarValue;
+use lance_core::ROW_ID;
+use lance_datafusion::planner::Planner;
+use serde::{Deserialize, Serialize};
+
+use crate::connection::Connection;
+use crate::database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS;
+use crate::database::{CreateTableRequest, Database, OpenTableRequest};
+use crate::embeddings::EmbeddingDefinition;
+use crate::table::Table;
+use crate::table::refresh::quote_identifier;
+use crate::table::{ColumnDefinition, ColumnKind};
+use crate::{Error, Result};
+
+pub use refresh::{RefreshMaterializedViewResult, RefreshMode};
+
+/// Schema metadata key holding the view definition, as kind-tagged JSON.
+pub const DEFINITION_META_KEY: &str = "mv.definition";
+
+/// Schema metadata key holding the view's incarnation: a token minted at each
+/// physical creation of a view table, so a view dropped and recreated under
+/// the same name and definition is still told apart from the one a caller
+/// captured. A view whose metadata was replaced wholesale, or one declared
+/// before tokens existed, carries none until its next refresh mints one.
+pub const INCARNATION_META_KEY: &str = "mv.incarnation";
+
+/// Schema metadata key holding the source table version the view was last
+/// refreshed to. Absent until the first refresh.
+pub const SOURCE_VERSION_META_KEY: &str = "mv.source_version";
+
+/// Schema metadata key holding the wall-clock time of the last refresh,
+/// in milliseconds since the epoch.
+pub const REFRESHED_AT_MS_META_KEY: &str = "mv.refreshed_at_ms";
+
+/// Column recording which source row produced each view row: the source's
+/// stable `_rowid` at refresh time, which is why sources must keep stable
+/// row ids.
+pub const SOURCE_ROW_ID_COLUMN: &str = "__source_row_id";
+
+/// Field metadata namespace for declarations about schema structure, such as
+/// an unenforced primary key.
+const SCHEMA_DECLARATION_META_PREFIX: &str = "lance-schema:";
+
+/// A field's identity in its own schema, which is not the view's.
+const LANCE_FIELD_ID_KEY: &str = "lance:field_id";
+
+/// Schema metadata key holding embedding-function configuration. It describes
+/// columns rather than storage, so a view carries it through.
+const EMBEDDING_FUNCTIONS_META_KEY: &str = "embedding_functions";
+
+/// Schema metadata key holding lancedb's own column definitions, one per
+/// field in schema order. It marks which columns an embedding function
+/// produces, which is what lets a query embed its own text.
+const COLUMN_DEFINITIONS_META_KEY: &str = "lancedb::column_definitions";
+
+/// Value of the definition's `kind` tag for the projected `select` form.
+pub const SELECT_KIND: &str = "select";
+
+/// Which view outputs each source column is projected to directly. A column
+/// may be projected more than once, so each carries every name the view gives
+/// it, in projection order.
+type Lineage = HashMap<String, Vec<String>>;
+
+/// One projected output column of a view.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ViewProjection {
+    /// Name of the column in the view.
+    pub output: String,
+    /// SQL expression over the source table that computes it.
+    pub expression: String,
+}
+
+/// The query that defines a materialized view.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct MaterializedViewDefinition {
+    /// Name of the source table, in the same database as the view.
+    pub source_table: String,
+    /// The projected output columns, in view schema order.
+    pub projections: Vec<ViewProjection>,
+    /// SQL predicate selecting the source rows the view holds.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub filter: Option<String>,
+    /// Cap on the number of rows the view holds, in materialization order.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub limit: Option<u64>,
+    /// Source columns the projections and filter read, derived at creation.
+    #[serde(default)]
+    pub inputs: Vec<String>,
+}
+
+/// A view definition as read back from schema metadata. Non-exhaustive so a
+/// kind added later is additive.
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum MaterializedViewKind {
+    /// The projected `select` form.
+    Select(MaterializedViewDefinition),
+    /// A kind written by a newer version, reported so a caller can tell an
+    /// unrefreshable view apart from a plain table. Nothing produces this.
+    Unrecognized {
+        /// The kind as it was found in the metadata.
+        kind: String,
+    },
+}
+
+/// Serialize `definition` into the kind-tagged form stored under
+/// [`DEFINITION_META_KEY`].
+pub(crate) fn definition_to_metadata(definition: &MaterializedViewDefinition) -> Result<String> {
+    let mut value = serde_json::to_value(definition).map_err(|e| Error::Runtime {
+        message: format!("failed to serialize view definition: {e}"),
+    })?;
+    value["kind"] = serde_json::Value::String(SELECT_KIND.to_string());
+    Ok(value.to_string())
+}
+
+/// Read a view declaration off a schema metadata map, if it carries one.
+/// `Ok(None)` for a plain table; a declaration that does not parse is an
+/// error, because treating a view as plain would let it be rewritten.
+pub fn materialized_view_kind(
+    metadata: &HashMap<String, String>,
+) -> Result<Option<MaterializedViewKind>> {
+    let Some(raw) = metadata.get(DEFINITION_META_KEY) else {
+        return Ok(None);
+    };
+    let unreadable = |e: &dyn std::fmt::Display| Error::Runtime {
+        message: format!("unreadable materialized view definition: {e}"),
+    };
+    let value: serde_json::Value = serde_json::from_str(raw).map_err(|e| unreadable(&e))?;
+    let kind = value
+        .get("kind")
+        .and_then(|k| k.as_str())
+        .ok_or_else(|| unreadable(&"missing kind tag"))?;
+    if kind != SELECT_KIND {
+        return Ok(Some(MaterializedViewKind::Unrecognized {
+            kind: kind.to_string(),
+        }));
+    }
+    let definition = serde_json::from_value(value).map_err(|e| unreadable(&e))?;
+    Ok(Some(MaterializedViewKind::Select(definition)))
+}
+
+/// Resolve a definition against the source schema into the view's projected
+/// fields, with `inputs` filled in. Everything statically checkable is
+/// checked here rather than at refresh time. Empty `projections` selects
+/// every source column as the schema stands now.
+pub(crate) fn plan(
+    source_schema: SchemaRef,
+    source_table: &str,
+    projections: &[(String, String)],
+    filter: Option<&str>,
+    limit: Option<u64>,
+) -> Result<(MaterializedViewDefinition, Vec<ArrowField>, Lineage)> {
+    let projections: Vec<(String, String)> = if projections.is_empty() {
+        source_schema
+            .fields()
+            .iter()
+            // A source that is itself a view carries its own provenance
+            // column; the new view records its own, not a copy.
+            .filter(|f| f.name() != SOURCE_ROW_ID_COLUMN)
+            .map(|f| (f.name().clone(), quote_identifier(f.name())))
+            .collect()
+    } else {
+        projections.to_vec()
+    };
+
+    // A scan takes the cap as i64. Rejecting it here keeps creation and
+    // refresh from disagreeing about whether a view is valid.
+    if let Some(limit) = limit
+        && i64::try_from(limit).is_err()
+    {
+        return Err(Error::InvalidInput {
+            message: format!("view limit {limit} exceeds the maximum of {}", i64::MAX),
+        });
+    }
+
+    let planner = Planner::new(source_schema.clone());
+    let mut fields = Vec::with_capacity(projections.len());
+    let mut inputs = Vec::new();
+    let mut declared: Vec<&str> = Vec::with_capacity(projections.len());
+    let mut lineage: Lineage = HashMap::new();
+
+    for (output, expression) in &projections {
+        if declared.contains(&output.as_str()) {
+            return Err(Error::ColumnAlreadyExists {
+                name: output.clone(),
+            });
+        }
+        if output == SOURCE_ROW_ID_COLUMN || output == ROW_ID {
+            return Err(Error::InvalidInput {
+                message: format!("view column name '{output}' is reserved"),
+            });
+        }
+
+        let parsed = planner
+            .parse_expr(expression)
+            .map_err(|e| Error::InvalidExpression {
+                column: output.clone(),
+                message: e.to_string(),
+            })?;
+        // Before optimization: the simplifier folds a stable-but-not-immutable
+        // call like now() into a literal, hiding it from the check while the
+        // stored definition keeps the call.
+        ensure_immutable(&parsed, |message| Error::InvalidExpression {
+            column: output.clone(),
+            message,
+        })?;
+        let expr = planner
+            .optimize_expr(parsed)
+            .map_err(|e| Error::InvalidExpression {
+                column: output.clone(),
+                message: e.to_string(),
+            })?;
+        let expr_inputs =
+            resolve_inputs(&source_schema, &expr, |message| Error::InvalidExpression {
+                column: output.clone(),
+                message,
+            })?;
+
+        // Physical expressions address columns by position, so the planner
+        // that types the expression is built on the projected schema.
+        let read_schema = project_schema(&source_schema, &expr_inputs);
+        let physical = Planner::new(read_schema.clone())
+            .create_physical_expr(&expr)
+            .map_err(|e| Error::InvalidExpression {
+                column: output.clone(),
+                message: e.to_string(),
+            })?;
+        let data_type =
+            physical
+                .data_type(read_schema.as_ref())
+                .map_err(|e| Error::InvalidExpression {
+                    column: output.clone(),
+                    message: e.to_string(),
+                })?;
+
+        // Always nullable: what a refresh appends must fit the declared field
+        // whatever nullability the evaluator reports for a given batch.
+        let mut field = ArrowField::new(output, data_type, true);
+        // Identity projections keep descriptive field metadata (blob markers);
+        // computed values carry none. Structural declarations never come along.
+        if let Some(source_field) = projected_field(&expr, &source_schema) {
+            field = field.with_metadata(source_field.metadata().clone());
+        }
+        if let Some(path) = projected_path(&expr)
+            && let [column] = path.as_slice()
+        {
+            lineage
+                .entry(column.clone())
+                .or_default()
+                .push(output.clone());
+        }
+        fields.push(without_declarations(&field));
+        inputs.extend(expr_inputs);
+        declared.push(output);
+    }
+
+    if let Some(filter) = filter {
+        let expr = planner
+            .parse_filter(filter)
+            .map_err(|e| Error::InvalidInput {
+                message: format!("invalid view filter: {e}"),
+            })?;
+        ensure_immutable(&expr, |message| Error::InvalidInput {
+            message: format!("invalid view filter: {message}"),
+        })?;
+        let filter_inputs = resolve_inputs(&source_schema, &expr, |message| Error::InvalidInput {
+            message: format!("invalid view filter: {message}"),
+        })?;
+        // A committed filter has to be usable as a predicate.
+        let read_schema = project_schema(&source_schema, &filter_inputs);
+        let data_type = Planner::new(read_schema.clone())
+            .create_physical_expr(&expr)
+            .map_err(|e| Error::InvalidInput {
+                message: format!("invalid view filter: {e}"),
+            })?
+            .data_type(read_schema.as_ref())
+            .map_err(|e| Error::InvalidInput {
+                message: format!("invalid view filter: {e}"),
+            })?;
+        if data_type != DataType::Boolean {
+            return Err(Error::InvalidInput {
+                message: format!("view filter must be a boolean predicate, not {data_type}"),
+            });
+        }
+        inputs.extend(filter_inputs);
+    }
+
+    inputs.sort();
+    inputs.dedup();
+
+    let definition = MaterializedViewDefinition {
+        source_table: source_table.to_string(),
+        projections: projections
+            .into_iter()
+            .map(|(output, expression)| ViewProjection { output, expression })
+            .collect(),
+        filter: filter.map(String::from),
+        limit,
+        inputs,
+    };
+    Ok((definition, fields, lineage))
+}
+
+/// Reject any function that is not immutable: a view definition has to
+/// evaluate identically across refreshes, or incremental maintenance would
+/// mix rows from different evaluations of the same definition.
+fn ensure_immutable(expr: &datafusion_expr::Expr, error: impl Fn(String) -> Error) -> Result<()> {
+    use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};
+    use datafusion_expr::Volatility;
+
+    // Labeled immutable but not determined by row values alone: version()
+    // depends on the build, the arrow_* introspectors on schema state.
+    const NOT_VALUE_DETERMINED: &[&str] =
+        &["version", "arrow_typeof", "arrow_field", "arrow_metadata"];
+
+    let mut offending: Option<String> = None;
+    expr.apply(|node| {
+        if let datafusion_expr::Expr::ScalarFunction(function) = node {
+            let name = function.func.name();
+            if function.func.signature().volatility != Volatility::Immutable
+                || NOT_VALUE_DETERMINED.contains(&name)
+            {
+                offending = Some(name.to_string());
+                return Ok(TreeNodeRecursion::Stop);
+            }
+        }
+        Ok(TreeNodeRecursion::Continue)
+    })
+    .map_err(|e| error(e.to_string()))?;
+    match offending {
+        Some(name) => Err(error(format!(
+            "function '{name}' is not immutable and would evaluate differently \
+             across refreshes"
+        ))),
+        None => Ok(()),
+    }
+}
+
+/// The root of a possibly-dotted column path: `metadata.age` -> `metadata`.
+fn root(path: &str) -> &str {
+    path.split('.').next().unwrap_or(path)
+}
+
+/// The columns `expr` reads, kept as the planner reports them (a nested
+/// reference stays a dotted path) but resolved by root field.
+/// Embedding configuration rewritten for the view: entries whose columns the
+/// view projects directly are kept under the view's names; the rest describe
+/// a table that does not exist and are dropped.
+fn embedding_config_for_view(raw: &str, lineage: &Lineage) -> Option<String> {
+    // Every representation the writers use: the Python bindings name the
+    // destination `vector_column`, the Rust definition `dest_column`, and the
+    // Node bindings spell both halves in camelCase.
+    const SOURCE_KEYS: [&str; 2] = ["source_column", "sourceColumn"];
+    const DEST_KEYS: [&str; 4] = ["vector_column", "dest_column", "vectorColumn", "destColumn"];
+
+    let entries: Vec<serde_json::Value> = serde_json::from_str(raw).ok()?;
+    let mut kept = Vec::new();
+    for entry in &entries {
+        let Some(object) = entry.as_object() else {
+            continue;
+        };
+        let named = |keys: &[&str]| {
+            let key = keys.iter().find(|key| object.contains_key(**key))?;
+            let outputs = lineage.get(object.get(*key)?.as_str()?)?;
+            Some(((*key).to_string(), outputs))
+        };
+        let (Some((source_key, sources)), Some((dest_key, dests))) =
+            (named(&SOURCE_KEYS), named(&DEST_KEYS))
+        else {
+            continue;
+        };
+        // A projection may give one source column several names, and every
+        // pairing of the two is a real relationship in the view.
+        for source in sources {
+            for dest in dests {
+                let mut object = object.clone();
+                object.insert(source_key.clone(), source.clone().into());
+                object.insert(dest_key.clone(), dest.clone().into());
+                kept.push(serde_json::Value::Object(object));
+            }
+        }
+    }
+    (!kept.is_empty()).then(|| serde_json::Value::Array(kept).to_string())
+}
+
+/// Lancedb's column definitions rewritten for the view: positional, one per
+/// view field. Directly projected embedding columns keep their definition
+/// under the view's names; everything else is physical. `None` = no key.
+fn column_definitions_for_view(
+    raw: &str,
+    source_schema: &ArrowSchema,
+    view_fields: &[ArrowField],
+    lineage: &Lineage,
+) -> Option<String> {
+    let source_definitions: Vec<ColumnDefinition> = serde_json::from_str(raw).ok()?;
+    // The definition sits on the column the function writes, so the source
+    // schema's field name at that position is the embedding's destination.
+    let embeddings: HashMap<&str, &EmbeddingDefinition> = source_schema
+        .fields()
+        .iter()
+        .zip(&source_definitions)
+        .filter_map(|(field, definition)| match &definition.kind {
+            ColumnKind::Embedding(embedding) => Some((field.name().as_str(), embedding)),
+            ColumnKind::Physical => None,
+        })
+        .collect();
+    let sources: HashMap<&str, &str> = lineage
+        .iter()
+        .flat_map(|(source, outputs)| outputs.iter().map(move |o| (o.as_str(), source.as_str())))
+        .collect();
+
+    let mut kept = false;
+    let definitions: Vec<ColumnDefinition> = view_fields
+        .iter()
+        .map(|field| {
+            let kind = embedding_for_output(field.name(), &embeddings, &sources, lineage)
+                .map(|embedding| {
+                    kept = true;
+                    ColumnKind::Embedding(embedding)
+                })
+                .unwrap_or(ColumnKind::Physical);
+            ColumnDefinition { kind }
+        })
+        .collect();
+    kept.then(|| serde_json::to_string(&definitions).ok())?
+}
+
+/// The embedding `output` inherits, renamed to the view's columns. `None`
+/// unless the view projects both the function's input and its output
+/// directly: anything else advertises a column the view cannot recompute.
+fn embedding_for_output(
+    output: &str,
+    embeddings: &HashMap<&str, &EmbeddingDefinition>,
+    sources: &HashMap<&str, &str>,
+    lineage: &Lineage,
+) -> Option<EmbeddingDefinition> {
+    let embedding = embeddings.get(sources.get(output)?)?;
+    // The input may be projected several times; the first name the view gives
+    // it is the one this column is defined against.
+    let input = lineage.get(&embedding.source_column)?.first()?;
+    Some(EmbeddingDefinition {
+        source_column: input.clone(),
+        dest_column: Some(output.to_string()),
+        embedding_name: embedding.embedding_name.clone(),
+    })
+}
+
+/// The source field a projection reads directly, if it reads one: a bare
+/// column, or a path of struct field accesses over one. Anything computed
+/// produces a new value and has no source field.
+fn projected_field<'a>(
+    expr: &datafusion_expr::Expr,
+    schema: &'a ArrowSchema,
+) -> Option<&'a ArrowField> {
+    let path = projected_path(expr)?;
+    let mut segments = path.iter();
+    let mut field = schema.field_with_name(segments.next()?).ok()?;
+    for segment in segments {
+        let DataType::Struct(children) = field.data_type() else {
+            return None;
+        };
+        field = children.iter().find(|c| c.name() == segment)?;
+    }
+    Some(field)
+}
+
+/// The dotted path a projection reads directly, root first.
+fn projected_path(expr: &datafusion_expr::Expr) -> Option<Vec<String>> {
+    let mut path = Vec::new();
+    let mut node = expr;
+    loop {
+        match node {
+            datafusion_expr::Expr::Column(column) => {
+                path.push(column.name.clone());
+                break;
+            }
+            // `a.b` parses to get_field(a, "b"), nested for deeper paths.
+            datafusion_expr::Expr::ScalarFunction(call) if call.func.name() == "get_field" => {
+                let [
+                    inner,
+                    datafusion_expr::Expr::Literal(ScalarValue::Utf8(Some(name)), _),
+                ] = call.args.as_slice()
+                else {
+                    return None;
+                };
+                path.push(name.clone());
+                node = inner;
+            }
+            _ => return None,
+        }
+    }
+
+    path.reverse();
+    Some(path)
+}
+
+/// `field` without the metadata that declares how a column is written, at
+/// every depth; descriptive metadata (blob markers) stays. A view is written
+/// by refresh alone, and its always-nullable fields contradict declarations.
+fn is_declaration(key: &str) -> bool {
+    key.starts_with(SCHEMA_DECLARATION_META_PREFIX)
+        || key == LANCE_FIELD_ID_KEY
+        || crate::table::computed_columns::is_declaration_key(key)
+}
+
+fn without_declarations(field: &ArrowField) -> ArrowField {
+    let metadata: HashMap<String, String> = field
+        .metadata()
+        .iter()
+        .filter(|(key, _)| !is_declaration(key))
+        .map(|(key, value)| (key.clone(), value.clone()))
+        .collect();
+    let strip = |child: &FieldRef| Arc::new(without_declarations(child));
+    // Every Arrow variant that carries a field carries that field's metadata
+    // with it, so all of them are descended.
+    let data_type = match field.data_type() {
+        DataType::Struct(children) => DataType::Struct(children.iter().map(strip).collect()),
+        DataType::List(child) => DataType::List(strip(child)),
+        DataType::ListView(child) => DataType::ListView(strip(child)),
+        DataType::LargeList(child) => DataType::LargeList(strip(child)),
+        DataType::LargeListView(child) => DataType::LargeListView(strip(child)),
+        DataType::Map(entries, sorted) => DataType::Map(strip(entries), *sorted),
+        DataType::FixedSizeList(child, len) => DataType::FixedSizeList(strip(child), *len),
+        DataType::Union(variants, mode) => DataType::Union(
+            variants
+                .iter()
+                .map(|(id, child)| (id, strip(child)))
+                .collect(),
+            *mode,
+        ),
+        DataType::RunEndEncoded(run_ends, values) => {
+            DataType::RunEndEncoded(strip(run_ends), strip(values))
+        }
+        other => other.clone(),
+    };
+    ArrowField::new(field.name(), data_type, field.is_nullable()).with_metadata(metadata)
+}
+
+fn resolve_inputs(
+    schema: &ArrowSchema,
+    expr: &datafusion_expr::Expr,
+    error: impl Fn(String) -> Error,
+) -> Result<Vec<String>> {
+    let mut inputs = Planner::column_names_in_expr(expr);
+    inputs.sort();
+    inputs.dedup();
+    for input in &inputs {
+        if schema.field_with_name(root(input)).is_err() {
+            return Err(error(format!("unknown column '{input}'")));
+        }
+    }
+    Ok(inputs)
+}
+
+/// Project the root fields of `columns`, deduplicated, in schema order.
+fn project_schema(schema: &ArrowSchema, columns: &[String]) -> SchemaRef {
+    let roots: std::collections::HashSet<&str> = columns.iter().map(|c| root(c)).collect();
+    let fields: Vec<ArrowField> = schema
+        .fields()
+        .iter()
+        .filter(|f| roots.contains(f.name().as_str()))
+        .map(|f| f.as_ref().clone())
+        .collect();
+    Arc::new(ArrowSchema::new(fields))
+}
+
+/// A validated view declaration, ready to become a table: the projected
+/// fields plus [`SOURCE_ROW_ID_COLUMN`], definition stamped in metadata.
+/// Produced only by [`prepare_declaration`].
+#[derive(Clone)]
+pub struct PreparedDeclaration {
+    schema: SchemaRef,
+    definition: MaterializedViewDefinition,
+    /// The source's own database: the only place
+    /// [`PreparedDeclaration::create`] will put the view, because refresh
+    /// resolves the recorded source name through the view's database.
+    database: Arc<dyn Database>,
+}
+
+impl std::fmt::Debug for PreparedDeclaration {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("PreparedDeclaration")
+            .field("definition", &self.definition)
+            .finish_non_exhaustive()
+    }
+}
+
+impl PreparedDeclaration {
+    /// The query the declaration records.
+    pub fn definition(&self) -> &MaterializedViewDefinition {
+        &self.definition
+    }
+
+    /// Create the view table and verify it, consuming the declaration.
+    ///
+    /// The view goes in the source's own database, where refresh resolves the
+    /// recorded source name. Stable row ids are requested at both levels and
+    /// verified rather than trusted; nothing is rolled back on failure.
+    pub async fn create(self, name: &str) -> Result<MaterializedView> {
+        let empty: Vec<std::result::Result<arrow_array::RecordBatch, arrow_schema::ArrowError>> =
+            vec![];
+        // Minted here, not at preparation: a declaration can be cloned and
+        // create more than one physical table, and each needs its own token.
+        let incarnation = uuid::Uuid::new_v4().to_string();
+        let mut metadata = self.schema.metadata().clone();
+        metadata.insert(INCARNATION_META_KEY.to_string(), incarnation.clone());
+        let schema = Arc::new(ArrowSchema::new_with_metadata(
+            self.schema.fields().clone(),
+            metadata,
+        ));
+        let reader: Box<dyn arrow_array::RecordBatchReader + Send> =
+            Box::new(arrow_array::RecordBatchIterator::new(empty, schema));
+        let mut request = CreateTableRequest::new(name.to_string(), Box::new(reader));
+        let write_params = request
+            .write_options
+            .lance_write_params
+            .get_or_insert_with(Default::default);
+        write_params.enable_stable_row_ids = true;
+        let store_params = write_params
+            .store_params
+            .get_or_insert_with(Default::default);
+        crate::connection::merge_storage_options(
+            store_params,
+            [(
+                OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
+                "true".to_string(),
+            )],
+        );
+        let table = self.database.clone().create_table(request).await?;
+        let table = Table::new(table, self.database);
+        let stable = match table.as_native() {
+            Some(native) => native.dataset.get().await?.manifest.uses_stable_row_ids(),
+            None => false,
+        };
+        if !stable {
+            return Err(Error::Runtime {
+                message: format!(
+                    "view '{name}' was created without stable row ids: the database \
+                     ignored the creation option; the table remains and is not \
+                     usable as a materialized view"
+                ),
+            });
+        }
+        Ok(MaterializedView {
+            table,
+            definition: self.definition,
+            incarnation: Some(incarnation),
+        })
+    }
+}
+
+/// Validate a view declaration against its live source and hold what its
+/// creation needs. The declaration is canonicalized through the coordinate a
+/// refresh will resolve, so a handle that does not resolve back to itself is
+/// rejected, as is a namespaced source. Same creation-time checks as
+/// [`Connection::create_materialized_view`].
+///
+/// ```no_run
+/// # #![recursion_limit = "256"]
+/// # use lancedb::materialized_view::prepare_declaration;
+/// # async fn declare(source: &lancedb::Table) -> Result<(), Box<dyn std::error::Error>> {
+/// let prepared = prepare_declaration(
+///     source,
+///     &[("id".into(), "id".into()), ("double".into(), "value * 2".into())],
+///     Some("value > 0"),
+///     None,
+/// )
+/// .await?;
+/// let view = prepared.create("doubles").await?;
+/// # Ok(())
+/// # }
+/// ```
+pub async fn prepare_declaration(
+    source: &Table,
+    projections: &[(String, String)],
+    filter: Option<&str>,
+    limit: Option<u64>,
+) -> Result<PreparedDeclaration> {
+    let Some(caller_native) = source.as_native() else {
+        return Err(Error::NotSupported {
+            message: "materialized views are supported only on local databases".into(),
+        });
+    };
+    // The definition records the source by bare name; any other source
+    // form would be recorded as a name its refresh cannot resolve.
+    if !source.namespace().is_empty() {
+        return Err(Error::NotSupported {
+            message: format!(
+                "a namespaced source cannot be recorded in a view definition; \
+                 '{}' must be a root-namespace table",
+                source.name()
+            ),
+        });
+    }
+    let database = source
+        .database_opt()
+        .ok_or_else(|| Error::InvalidInput {
+            message: "the source was not opened through a database connection".into(),
+        })?
+        .clone();
+
+    // Canonicalize: resolve the recorded coordinate exactly the way a
+    // refresh will, and plan from what it reaches. A handle that does not
+    // resolve back to itself must not be declared under this name.
+    let resolved = database
+        .open_table(OpenTableRequest {
+            name: source.name().to_string(),
+            namespace_path: vec![],
+            index_cache_size: None,
+            lance_read_params: None,
+            location: None,
+            namespace_client: None,
+            managed_versioning: None,
+        })
+        .await?;
+    let resolved = Table::new(resolved, database.clone());
+    let Some(native) = resolved.as_native() else {
+        return Err(Error::NotSupported {
+            message: "materialized views are supported only on local databases".into(),
+        });
+    };
+    let caller_uri = caller_native.dataset.get().await?.uri().to_string();
+    let resolved_uri = native.dataset.get().await?.uri().to_string();
+    if caller_uri != resolved_uri {
+        return Err(Error::InvalidInput {
+            message: format!(
+                "the source handle does not resolve to itself through its \
+                 database: '{}' resolves to '{resolved_uri}', but the handle \
+                 reads '{caller_uri}'",
+                source.name()
+            ),
+        });
+    }
+    if !native.dataset.get().await?.manifest.uses_stable_row_ids() {
+        return Err(Error::InvalidInput {
+            message: format!(
+                "materialized views require stable row ids on the source table; \
+                 create '{}' with storage option new_table_enable_stable_row_ids=true",
+                source.name()
+            ),
+        });
+    }
+    refresh::ensure_no_mem_wal(
+        native.dataset.get().await?.as_ref(),
+        "source table",
+        resolved.name(),
+    )
+    .await?;
+    let source_schema = resolved.schema().await?;
+    let source_metadata = source_schema.metadata().clone();
+    let (definition, mut fields, lineage) = plan(
+        source_schema.clone(),
+        resolved.name(),
+        projections,
+        filter,
+        limit,
+    )?;
+    fields.push(ArrowField::new(
+        SOURCE_ROW_ID_COLUMN,
+        DataType::UInt64,
+        false,
+    ));
+    // Only column-describing metadata comes along: structural declarations
+    // describe how a table is written, and a view is written by refresh alone.
+    let mut metadata: HashMap<String, String> = HashMap::new();
+    if let Some(raw) = source_metadata.get(EMBEDDING_FUNCTIONS_META_KEY)
+        && let Some(rewritten) = embedding_config_for_view(raw, &lineage)
+    {
+        metadata.insert(EMBEDDING_FUNCTIONS_META_KEY.to_string(), rewritten);
+    }
+    if let Some(raw) = source_metadata.get(COLUMN_DEFINITIONS_META_KEY)
+        && let Some(rewritten) = column_definitions_for_view(raw, &source_schema, &fields, &lineage)
+    {
+        metadata.insert(COLUMN_DEFINITIONS_META_KEY.to_string(), rewritten);
+    }
+    metadata.insert(
+        DEFINITION_META_KEY.to_string(),
+        definition_to_metadata(&definition)?,
+    );
+    Ok(PreparedDeclaration {
+        schema: Arc::new(ArrowSchema::new_with_metadata(fields, metadata)),
+        definition,
+        database,
+    })
+}
+
+/// One row of [`Connection::list_materialized_views`]: a view's name and its
+/// definition kind, which may be one this version cannot refresh.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct MaterializedViewEntry {
+    /// Name of the view's table.
+    pub name: String,
+    /// The view's definition as stored.
+    pub kind: MaterializedViewKind,
+}
+
+/// Materialized views are local-only; refuse a remote connection before any
+/// request is made.
+fn ensure_local(connection: &Connection) -> Result<()> {
+    if connection.uri().starts_with("db://") {
+        return Err(Error::NotSupported {
+            message: "materialized views are supported only on local databases".into(),
+        });
+    }
+    Ok(())
+}
+
+/// Builds a materialized view. Created by
+/// [`Connection::create_materialized_view`].
+pub struct CreateMaterializedViewBuilder {
+    connection: Connection,
+    name: String,
+    source: String,
+    projections: Vec<(String, String)>,
+    filter: Option<String>,
+    limit: Option<u64>,
+}
+
+impl CreateMaterializedViewBuilder {
+    pub(crate) fn new(connection: Connection, name: String, source: String) -> Self {
+        Self {
+            connection,
+            name,
+            source,
+            projections: Vec::new(),
+            filter: None,
+            limit: None,
+        }
+    }
+
+    /// The view's columns, as `(name, SQL expression)` pairs. Not calling
+    /// this selects every source column, expanded at creation time.
+    pub fn select(
+        mut self,
+        columns: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
+    ) -> Self {
+        self.projections = columns
+            .into_iter()
+            .map(|(output, expression)| (output.into(), expression.into()))
+            .collect();
+        self
+    }
+
+    /// Only source rows matching the SQL predicate appear in the view.
+    pub fn only_if(mut self, filter: impl Into<String>) -> Self {
+        self.filter = Some(filter.into());
+        self
+    }
+
+    /// Cap the view at `limit` rows, in materialization order.
+    pub fn limit(mut self, limit: u64) -> Self {
+        self.limit = Some(limit);
+        self
+    }
+
+    /// Create the view: an empty table carrying the definition; refresh
+    /// computes the rows. The source must keep stable row ids -- they hold
+    /// provenance across compaction, and cannot be enabled later.
+    pub async fn execute(self) -> Result<MaterializedView> {
+        ensure_local(&self.connection)?;
+        let source = self.connection.open_table(&self.source).execute().await?;
+        let prepared = prepare_declaration(
+            &source,
+            &self.projections,
+            self.filter.as_deref(),
+            self.limit,
+        )
+        .await?;
+        prepared.create(&self.name).await
+    }
+}
+
+/// A handle on a materialized view: the view table plus its parsed definition.
+#[derive(Debug, Clone)]
+pub struct MaterializedView {
+    table: Table,
+    definition: MaterializedViewDefinition,
+    incarnation: Option<String>,
+}
+
+impl MaterializedView {
+    /// Interpret `table` as a materialized view: [`Error::NotAMaterializedView`]
+    /// for a plain table, [`Error::NotSupported`] for a kind this version
+    /// cannot refresh.
+    pub async fn from_table(table: Table) -> Result<Self> {
+        // Same local-only boundary the connection-level entry points hold,
+        // applied before the schema read so a remote table costs no request.
+        if table.as_native().is_none() {
+            return Err(Error::NotSupported {
+                message: "materialized views are supported only on local databases".into(),
+            });
+        }
+        let schema = table.schema().await?;
+        let incarnation = schema.metadata().get(INCARNATION_META_KEY).cloned();
+        match materialized_view_kind(schema.metadata())? {
+            Some(MaterializedViewKind::Select(definition)) => Ok(Self {
+                table,
+                definition,
+                incarnation,
+            }),
+            Some(MaterializedViewKind::Unrecognized { kind }) => Err(Error::NotSupported {
+                message: format!(
+                    "materialized view '{}' is defined by '{kind}', which this version of \
+                     lancedb cannot refresh",
+                    table.name()
+                ),
+            }),
+            None => Err(Error::NotAMaterializedView {
+                name: table.name().to_string(),
+            }),
+        }
+    }
+
+    /// The view, as the table it is. Queries, indexes and search all apply.
+    pub fn table(&self) -> &Table {
+        &self.table
+    }
+
+    /// The view's table name.
+    pub fn name(&self) -> &str {
+        self.table.name()
+    }
+
+    /// The query that defines the view.
+    pub fn definition(&self) -> &MaterializedViewDefinition {
+        &self.definition
+    }
+
+    /// The view's incarnation token as of when this handle was opened; see
+    /// [`RefreshMaterializedViewBuilder::expect_incarnation`]. `None` for a
+    /// view that has none yet (see [`INCARNATION_META_KEY`]).
+    pub fn incarnation(&self) -> Option<&str> {
+        self.incarnation.as_deref()
+    }
+
+    /// Recompute the view from its source.
+    ///
+    /// By default the refresh is incremental when the source's changes can be
+    /// reconciled into the view, and otherwise rebuilds; see
+    /// [`RefreshMaterializedViewBuilder`].
+    ///
+    /// ```no_run
+    /// # #![recursion_limit = "256"]
+    /// # use lancedb::materialized_view::MaterializedView;
+    /// # async fn refresh(view: &MaterializedView) -> Result<(), Box<dyn std::error::Error>> {
+    /// let result = view.refresh().execute().await?;
+    /// println!("{:?}: {} rows", result.mode, result.rows_written);
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn refresh(&self) -> RefreshMaterializedViewBuilder {
+        RefreshMaterializedViewBuilder {
+            view: self.clone(),
+            full: false,
+            source_version: None,
+            expected_incarnation: None,
+        }
+    }
+}
+
+/// Builds a refresh. Created by [`MaterializedView::refresh`].
+pub struct RefreshMaterializedViewBuilder {
+    view: MaterializedView,
+    full: bool,
+    source_version: Option<u64>,
+    expected_incarnation: Option<String>,
+}
+
+impl RefreshMaterializedViewBuilder {
+    /// Rebuild the view even where an incremental refresh would do.
+    pub fn full(mut self, full: bool) -> Self {
+        self.full = full;
+        self
+    }
+
+    /// Refresh to this source table version instead of the latest.
+    pub fn source_version(mut self, version: u64) -> Self {
+        self.source_version = Some(version);
+        self
+    }
+
+    /// Refresh only if the view is still the incarnation that minted `token`
+    /// (see [`MaterializedView::incarnation`]): a refresh requested against
+    /// one declaration must not land in a view dropped and recreated since,
+    /// even under the same name and definition.
+    ///
+    /// Best effort. The token is read from the latest stored manifest before
+    /// planning and again immediately before every commit, but it is not part
+    /// of the commit's own condition, so a recreation that lands between that
+    /// final read and the commit is not caught.
+    pub fn expect_incarnation(mut self, token: impl Into<String>) -> Self {
+        self.expected_incarnation = Some(token.into());
+        self
+    }
+
+    pub async fn execute(self) -> Result<RefreshMaterializedViewResult> {
+        refresh::execute_refresh(
+            &self.view.table,
+            self.full,
+            self.source_version,
+            self.expected_incarnation.as_deref(),
+        )
+        .await
+    }
+}
+
+impl Connection {
+    /// Define a materialized view named `name` over `source`.
+    ///
+    /// The view is created empty, with the definition recorded in its schema
+    /// metadata; refresh computes the rows. Local databases only.
+    ///
+    /// ```no_run
+    /// # #![recursion_limit = "256"]
+    /// # use lancedb::Connection;
+    /// # async fn create(conn: &Connection) -> Result<(), Box<dyn std::error::Error>> {
+    /// let view = conn
+    ///     .create_materialized_view("loud_adults", "people")
+    ///     .select([("name", "upper(name)"), ("age", "age")])
+    ///     .only_if("age >= 18")
+    ///     .execute()
+    ///     .await?;
+    /// view.refresh().execute().await?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn create_materialized_view(
+        &self,
+        name: impl Into<String>,
+        source: impl Into<String>,
+    ) -> CreateMaterializedViewBuilder {
+        CreateMaterializedViewBuilder::new(self.clone(), name.into(), source.into())
+    }
+
+    /// Open the materialized view named `name`.
+    pub async fn open_materialized_view(
+        &self,
+        name: impl Into<String>,
+    ) -> Result<MaterializedView> {
+        ensure_local(self)?;
+        let table = self.open_table(name).execute().await?;
+        MaterializedView::from_table(table).await
+    }
+
+    /// The materialized views in this database, unrefreshable kinds included.
+    /// Costs a table open per table; one that cannot be opened is skipped
+    /// rather than failing the listing.
+    pub async fn list_materialized_views(&self) -> Result<Vec<MaterializedViewEntry>> {
+        ensure_local(self)?;
+        let names = self.table_names().execute().await?;
+        let mut views = Vec::new();
+        for name in names {
+            let Ok(table) = self.open_table(&name).execute().await else {
+                continue;
+            };
+            let schema = table.schema().await?;
+            if let Some(kind) = materialized_view_kind(schema.metadata())? {
+                views.push(MaterializedViewEntry { name, kind });
+            }
+        }
+        Ok(views)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use arrow_array::record_batch;
+
+    use super::*;
+    use crate::connect;
+    use crate::table::WriteOptions;
+
+    async fn people_db() -> Connection {
+        let conn = connect("memory://").execute().await.unwrap();
+        let batch = record_batch!(
+            ("name", Utf8, ["ada", "grace", "alan"]),
+            ("age", Int32, [36, 85, 41])
+        )
+        .unwrap();
+        conn.create_table("people", batch)
+            .write_options(stable_row_ids())
+            .execute()
+            .await
+            .unwrap();
+        conn
+    }
+
+    /// Sources must keep stable row ids; see the create-time gate.
+    pub(super) fn stable_row_ids() -> WriteOptions {
+        WriteOptions {
+            lance_write_params: Some(lance::dataset::WriteParams {
+                enable_stable_row_ids: true,
+                ..Default::default()
+            }),
+        }
+    }
+
+    /// The error a doomed declaration against `people` produces.
+    async fn declare_err(
+        cfg: impl FnOnce(CreateMaterializedViewBuilder) -> CreateMaterializedViewBuilder,
+    ) -> Error {
+        let conn = people_db().await;
+        cfg(conn.create_materialized_view("bad", "people"))
+            .execute()
+            .await
+            .unwrap_err()
+    }
+
+    #[tokio::test]
+    async fn test_create_records_the_definition() {
+        let conn = people_db().await;
+        let view = conn
+            .create_materialized_view("adults", "people")
+            .select([("name", "name"), ("shout", "upper(name)")])
+            .only_if("age >= 18")
+            .limit(10)
+            .execute()
+            .await
+            .unwrap();
+
+        assert_eq!(view.name(), "adults");
+        assert_eq!(
+            view.definition(),
+            &MaterializedViewDefinition {
+                source_table: "people".into(),
+                projections: vec![
+                    ViewProjection {
+                        output: "name".into(),
+                        expression: "name".into()
+                    },
+                    ViewProjection {
+                        output: "shout".into(),
+                        expression: "upper(name)".into()
+                    },
+                ],
+                filter: Some("age >= 18".into()),
+                limit: Some(10),
+                inputs: vec!["age".into(), "name".into()],
+            }
+        );
+
+        // The definition round-trips off the stored schema, not the handle.
+        let reopened = conn.open_materialized_view("adults").await.unwrap();
+        assert_eq!(reopened.definition(), view.definition());
+    }
+
+    #[tokio::test]
+    async fn test_view_schema_is_derived_from_the_query() {
+        let conn = people_db().await;
+        let view = conn
+            .create_materialized_view("shapes", "people")
+            .select([("shout", "upper(name)"), ("next_age", "age + 1")])
+            .execute()
+            .await
+            .unwrap();
+
+        let schema = view.table().schema().await.unwrap();
+        assert_eq!(
+            schema.field_with_name("shout").unwrap().data_type(),
+            &DataType::Utf8
+        );
+        assert_eq!(
+            schema.field_with_name("next_age").unwrap().data_type(),
+            &DataType::Int32
+        );
+        assert_eq!(
+            schema
+                .field_with_name(SOURCE_ROW_ID_COLUMN)
+                .unwrap()
+                .data_type(),
+            &DataType::UInt64
+        );
+        assert_eq!(view.table().count_rows(None).await.unwrap(), 0);
+    }
+
+    /// No projection selects every source column, expanded now: the schema
+    /// captured at creation is the definition.
+    #[tokio::test]
+    async fn test_default_projection_captures_the_source_schema() {
+        let conn = people_db().await;
+        let view = conn
+            .create_materialized_view("copy", "people")
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(
+            view.definition()
+                .projections
+                .iter()
+                .map(|p| p.output.as_str())
+                .collect::<Vec<_>>(),
+            vec!["name", "age"]
+        );
+        assert_eq!(view.definition().inputs, vec!["age", "name"]);
+    }
+
+    #[tokio::test]
+    async fn test_unknown_column_fails_at_create_time() {
+        let conn = people_db().await;
+        let err = conn
+            .create_materialized_view("bad", "people")
+            .select([("x", "missing + 1")])
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::InvalidExpression { column, .. } if column == "x"));
+        let names = conn.table_names().execute().await.unwrap();
+        assert!(!names.contains(&"bad".to_string()));
+    }
+
+    #[tokio::test]
+    async fn test_unknown_filter_column_fails_at_create_time() {
+        let err = declare_err(|b| b.only_if("missing > 1")).await;
+        assert!(matches!(err, Error::InvalidInput { message } if message.contains("missing")));
+    }
+
+    #[tokio::test]
+    async fn test_duplicate_output_is_rejected() {
+        let err = declare_err(|b| b.select([("dup", "age"), ("dup", "age + 1")])).await;
+        assert!(matches!(err, Error::ColumnAlreadyExists { name } if name == "dup"));
+    }
+
+    #[tokio::test]
+    async fn test_reserved_output_name_is_rejected() {
+        let err = declare_err(|b| b.select([(SOURCE_ROW_ID_COLUMN, "age")])).await;
+        assert!(matches!(err, Error::InvalidInput { message } if message.contains("reserved")));
+    }
+
+    #[tokio::test]
+    async fn test_missing_source_fails() {
+        let conn = connect("memory://").execute().await.unwrap();
+        let err = conn
+            .create_materialized_view("v", "nope")
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::TableNotFound { .. }));
+    }
+
+    /// Provenance has to survive source compactions and updates, and stable
+    /// row ids cannot be enabled after a table exists -- so the requirement
+    /// is checked at the last moment the caller can still act on it.
+    #[tokio::test]
+    async fn test_source_without_stable_row_ids_is_refused() {
+        let conn = connect("memory://").execute().await.unwrap();
+        let batch = record_batch!(("x", Int32, [1, 2])).unwrap();
+        conn.create_table("plain", batch).execute().await.unwrap();
+
+        let err = conn
+            .create_materialized_view("v", "plain")
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(err, Error::InvalidInput { message } if message.contains("stable row ids"))
+        );
+        assert!(
+            !conn
+                .table_names()
+                .execute()
+                .await
+                .unwrap()
+                .contains(&"v".to_string())
+        );
+    }
+
+    #[tokio::test]
+    async fn test_name_collision_fails() {
+        let conn = people_db().await;
+        let err = conn
+            .create_materialized_view("people", "people")
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::TableAlreadyExists { .. }));
+    }
+
+    #[tokio::test]
+    async fn test_a_plain_table_is_not_a_view() {
+        let conn = people_db().await;
+        let table = conn.open_table("people").execute().await.unwrap();
+        let err = MaterializedView::from_table(table).await.unwrap_err();
+        assert!(matches!(err, Error::NotAMaterializedView { name } if name == "people"));
+
+        let err = conn.open_materialized_view("people").await.unwrap_err();
+        assert!(matches!(err, Error::NotAMaterializedView { .. }));
+    }
+
+    /// The reason the kind is tagged: a definition written by a newer version
+    /// reads back as a view this one cannot refresh, not as a plain table.
+    #[tokio::test]
+    async fn test_unrecognized_kind_is_refused_by_name() {
+        let conn = people_db().await;
+        conn.create_materialized_view("v", "people")
+            .execute()
+            .await
+            .unwrap();
+        let table = conn.open_table("v").execute().await.unwrap();
+        table
+            .as_native()
+            .unwrap()
+            .replace_schema_metadata(HashMap::from([(
+                DEFINITION_META_KEY.to_string(),
+                r#"{"kind": "join"}"#.to_string(),
+            )]))
+            .await
+            .unwrap();
+
+        let err = conn.open_materialized_view("v").await.unwrap_err();
+        assert!(matches!(err, Error::NotSupported { message } if message.contains("join")));
+    }
+
+    #[tokio::test]
+    async fn test_list_reports_views_and_only_views() {
+        let conn = people_db().await;
+        conn.create_materialized_view("adults", "people")
+            .only_if("age >= 18")
+            .execute()
+            .await
+            .unwrap();
+
+        let views = conn.list_materialized_views().await.unwrap();
+        assert_eq!(
+            views.iter().map(|v| v.name.as_str()).collect::<Vec<_>>(),
+            vec!["adults"]
+        );
+        let MaterializedViewKind::Select(definition) = &views[0].kind else {
+            panic!("expected a select view");
+        };
+        assert_eq!(definition.filter.as_deref(), Some("age >= 18"));
+    }
+
+    /// The creation option outranks a connection configured to create
+    /// unstable tables: the view still gets stable row ids, on the same
+    /// store (no fork -- the table must be reachable through the
+    /// connection afterwards).
+    #[tokio::test]
+    async fn test_view_is_stable_despite_connection_override() {
+        let conn = connect("memory://")
+            .storage_options([("new_table_enable_stable_row_ids", "false")])
+            .execute()
+            .await
+            .unwrap();
+        let batch = record_batch!(("x", Int32, [1])).unwrap();
+        conn.create_table("src", batch)
+            .storage_option("new_table_enable_stable_row_ids", "true")
+            .execute()
+            .await
+            .unwrap();
+
+        let view = conn
+            .create_materialized_view("v", "src")
+            .execute()
+            .await
+            .unwrap();
+        let stable = view
+            .table()
+            .as_native()
+            .unwrap()
+            .dataset
+            .get()
+            .await
+            .unwrap()
+            .manifest
+            .uses_stable_row_ids();
+        assert!(stable);
+        conn.open_materialized_view("v").await.unwrap();
+    }
+
+    /// A committed filter has to be usable as a predicate.
+    #[tokio::test]
+    async fn test_non_boolean_filter_is_rejected() {
+        let err = declare_err(|b| b.only_if("age + 1")).await;
+        assert!(matches!(err, Error::InvalidInput { message } if message.contains("boolean")));
+    }
+
+    /// Nested references stay dotted paths; resolution is by root field.
+    #[tokio::test]
+    async fn test_struct_columns_can_be_declared() {
+        use arrow_array::{ArrayRef, Int32Array, StructArray};
+
+        let conn = connect("memory://").execute().await.unwrap();
+        let ages = StructArray::from(vec![(
+            Arc::new(ArrowField::new("age", DataType::Int32, false)),
+            Arc::new(Int32Array::from(vec![36, 17])) as ArrayRef,
+        )]);
+        let batch =
+            arrow_array::RecordBatch::try_from_iter(vec![("metadata", Arc::new(ages) as ArrayRef)])
+                .unwrap();
+        conn.create_table("people", batch)
+            .write_options(stable_row_ids())
+            .execute()
+            .await
+            .unwrap();
+
+        let view = conn
+            .create_materialized_view("ages", "people")
+            .select([("age", "metadata.age")])
+            .only_if("metadata.age >= 18")
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(view.definition().inputs, vec!["metadata.age"]);
+        let schema = view.table().schema().await.unwrap();
+        assert_eq!(
+            schema.field_with_name("age").unwrap().data_type(),
+            &DataType::Int32
+        );
+    }
+
+    /// A newer-kind view must not disappear from the listing.
+    #[tokio::test]
+    async fn test_unrecognized_kind_is_listed_with_its_kind() {
+        let conn = people_db().await;
+        conn.create_materialized_view("v", "people")
+            .execute()
+            .await
+            .unwrap();
+        let table = conn.open_table("v").execute().await.unwrap();
+        table
+            .as_native()
+            .unwrap()
+            .replace_schema_metadata(HashMap::from([(
+                DEFINITION_META_KEY.to_string(),
+                r#"{"kind": "join"}"#.to_string(),
+            )]))
+            .await
+            .unwrap();
+
+        let views = conn.list_materialized_views().await.unwrap();
+        assert_eq!(views.len(), 1);
+        assert_eq!(views[0].name, "v");
+        assert_eq!(
+            views[0].kind,
+            MaterializedViewKind::Unrecognized {
+                kind: "join".into()
+            }
+        );
+    }
+
+    /// Remote connections are refused before any request is made.
+    #[cfg(feature = "remote")]
+    #[tokio::test]
+    async fn test_remote_connection_is_refused_up_front() {
+        let conn = connect("db://nowhere")
+            .api_key("sk_test")
+            .region("us-east-1")
+            .execute()
+            .await
+            .unwrap();
+        let err = conn
+            .create_materialized_view("v", "src")
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::NotSupported { .. }));
+        let err = conn.open_materialized_view("v").await.unwrap_err();
+        assert!(matches!(err, Error::NotSupported { .. }));
+        let err = conn.list_materialized_views().await.unwrap_err();
+        assert!(matches!(err, Error::NotSupported { .. }));
+    }
+
+    /// A definition must evaluate identically across refreshes; anything
+    /// less makes incremental maintenance a mix of evaluations.
+    #[tokio::test]
+    async fn test_volatile_and_unstable_expressions_are_rejected() {
+        let conn = people_db().await;
+        for expression in [
+            "random()",
+            "now()",
+            "version()",
+            "arrow_typeof(age)",
+            "arrow_metadata(age, 'k')",
+        ] {
+            let err = conn
+                .create_materialized_view("bad", "people")
+                .select([("x", expression)])
+                .execute()
+                .await
+                .unwrap_err();
+            assert!(
+                matches!(err, Error::InvalidExpression { message, .. }
+                    if message.contains("not immutable")),
+                "{expression} was not rejected"
+            );
+        }
+        for filter in ["age > random() * 100", "age >= 0 and now() is not null"] {
+            let err = conn
+                .create_materialized_view("bad", "people")
+                .only_if(filter)
+                .execute()
+                .await
+                .unwrap_err();
+            assert!(
+                matches!(err, Error::InvalidInput { message } if message.contains("not immutable")),
+                "{filter} was not rejected"
+            );
+        }
+    }
+
+    /// A column projected as itself stays the column it was: blob discovery
+    /// and the blob APIs key off field metadata, which a bare rebuild of the
+    /// field would drop.
+    #[tokio::test]
+    async fn test_identity_projection_keeps_field_metadata() {
+        let conn = connect("memory://").execute().await.unwrap();
+        let schema = Arc::new(ArrowSchema::new_with_metadata(
+            vec![
+                ArrowField::new("id", DataType::Int32, true),
+                crate::blob("payload", true),
+            ],
+            HashMap::new(),
+        ));
+        conn.create_empty_table("src", schema)
+            .write_options(stable_row_ids())
+            .execute()
+            .await
+            .unwrap();
+
+        let view = conn
+            .create_materialized_view("v", "src")
+            .execute()
+            .await
+            .unwrap();
+        let view_schema = view.table().schema().await.unwrap();
+
+        let payload = view_schema.field_with_name("payload").unwrap();
+        assert!(
+            crate::blob::is_blob(payload),
+            "default projection dropped the blob marker: {:?}",
+            payload.metadata()
+        );
+        assert_eq!(
+            view.table().blob_columns().await.unwrap(),
+            vec!["payload".to_string()],
+            "blob discovery no longer finds the projected column"
+        );
+        assert!(view_schema.metadata().contains_key(DEFINITION_META_KEY));
+        // Structural declarations describe how a table is written; a view is
+        // written by refresh, and its fields are always nullable.
+        assert!(!view_schema.metadata().contains_key("lance:primary_key"));
+
+        // A computed column is a new value and carries no source metadata.
+        let computed = conn
+            .create_materialized_view("c", "src")
+            .select([("payload", "payload"), ("n", "id + 1")])
+            .execute()
+            .await
+            .unwrap();
+        let computed_schema = computed.table().schema().await.unwrap();
+        assert!(crate::blob::is_blob(
+            computed_schema.field_with_name("payload").unwrap()
+        ));
+        assert!(
+            computed_schema
+                .field_with_name("n")
+                .unwrap()
+                .metadata()
+                .is_empty()
+        );
+    }
+
+    /// A nested column projected straight through is still that column, and a
+    /// declaration buried in a struct child binds as hard as one on top.
+    #[tokio::test]
+    async fn test_nested_projection_metadata_and_declarations() {
+        let conn = connect("memory://").execute().await.unwrap();
+        let payload = crate::blob("payload", true).with_metadata(HashMap::from([
+            ("lance-encoding:blob".to_string(), "true".to_string()),
+            (
+                "lance-schema:unenforced-primary-key".to_string(),
+                "0".to_string(),
+            ),
+        ]));
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("id", DataType::Int32, true),
+            ArrowField::new("meta", DataType::Struct(vec![payload].into()), true),
+        ]));
+        conn.create_empty_table("src", schema)
+            .write_options(stable_row_ids())
+            .execute()
+            .await
+            .unwrap();
+
+        // A nested path is a direct projection: the leaf's metadata comes with
+        // it, so the blob stays a blob rather than a plain struct.
+        let lifted = conn
+            .create_materialized_view("lifted", "src")
+            .select([("payload", "meta.payload")])
+            .execute()
+            .await
+            .unwrap();
+        let field = lifted.table().schema().await.unwrap();
+        let field = field.field_with_name("payload").unwrap().clone();
+        assert_eq!(
+            field.metadata().get("lance-encoding:blob"),
+            Some(&"true".to_string()),
+            "nested projection lost the leaf's metadata"
+        );
+        assert!(
+            !field
+                .metadata()
+                .contains_key("lance-schema:unenforced-primary-key"),
+            "a structural declaration rode along"
+        );
+
+        // Projecting the struct whole must not carry the child's declaration
+        // out to a view whose fields are nullable.
+        let whole = conn
+            .create_materialized_view("whole", "src")
+            .select([("meta", "meta")])
+            .execute()
+            .await
+            .unwrap();
+        let schema = whole.table().schema().await.unwrap();
+        let DataType::Struct(children) = schema.field_with_name("meta").unwrap().data_type() else {
+            panic!("meta is not a struct");
+        };
+        let child = children.iter().find(|c| c.name() == "payload").unwrap();
+        assert!(
+            !child
+                .metadata()
+                .contains_key("lance-schema:unenforced-primary-key"),
+            "a nested declaration survived: {:?}",
+            child.metadata()
+        );
+        assert_eq!(
+            child.metadata().get("lance-encoding:blob"),
+            Some(&"true".to_string())
+        );
+    }
+
+    /// A map's entries are fields like any other, and a declaration on one
+    /// binds the view's writes just as hard as one on top.
+    #[tokio::test]
+    async fn test_map_declarations_are_stripped() {
+        let conn = connect("memory://").execute().await.unwrap();
+        let value =
+            ArrowField::new("value", DataType::Utf8, false).with_metadata(HashMap::from([(
+                "lance-schema:unenforced-clustering-key:position".to_string(),
+                "1".to_string(),
+            )]));
+        let entries = ArrowField::new(
+            "entries",
+            DataType::Struct(vec![ArrowField::new("key", DataType::Utf8, false), value].into()),
+            false,
+        );
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "props",
+            DataType::Map(Arc::new(entries), false),
+            true,
+        )]));
+        conn.create_empty_table("src", schema)
+            .write_options(stable_row_ids())
+            .execute()
+            .await
+            .unwrap();
+
+        let view = conn
+            .create_materialized_view("view", "src")
+            .execute()
+            .await
+            .unwrap();
+        let schema = view.table().schema().await.unwrap();
+        let DataType::Map(entries, _) = schema.field_with_name("props").unwrap().data_type() else {
+            panic!("props is not a map");
+        };
+        let DataType::Struct(children) = entries.data_type() else {
+            panic!("map entries are not a struct");
+        };
+        let value = children.iter().find(|c| c.name() == "value").unwrap();
+        assert!(
+            !value
+                .metadata()
+                .contains_key("lance-schema:unenforced-clustering-key:position"),
+            "a declaration survived inside a map: {:?}",
+            value.metadata()
+        );
+    }
+
+    /// A computed column is declared by field metadata. Projecting one --
+    /// as itself or under an alias -- must carry its description without its
+    /// declaration, which the target table would reject as foreign.
+    #[tokio::test]
+    async fn test_view_over_a_computed_column() {
+        let conn = connect("memory://").execute().await.unwrap();
+        let batch = record_batch!(("id", Int32, [1, 2])).unwrap();
+        let source = conn
+            .create_table("src", batch)
+            .write_options(stable_row_ids())
+            .execute()
+            .await
+            .unwrap();
+        source
+            .add_columns()
+            .computed("doubled", "id * 2")
+            .execute()
+            .await
+            .unwrap();
+
+        // Default projection reaches the computed column too.
+        let whole = conn
+            .create_materialized_view("whole", "src")
+            .execute()
+            .await
+            .unwrap();
+        let schema = whole.table().schema().await.unwrap();
+        let field = schema.field_with_name("doubled").unwrap();
+        assert!(
+            !field
+                .metadata()
+                .keys()
+                .any(|k| k.starts_with("computed_column")),
+            "a computed-column declaration rode along: {:?}",
+            field.metadata()
+        );
+
+        // And under an alias.
+        conn.create_materialized_view("aliased", "src")
+            .select([("twice", "doubled")])
+            .execute()
+            .await
+            .unwrap();
+    }
+
+    /// Embedding configuration names columns. It comes along only for the
+    /// columns a view actually projects, under the names the view gives them.
+    #[tokio::test]
+    async fn test_embedding_config_follows_the_projection() {
+        let config = r#"[{"name":"f","model":{},"source_column":"text","vector_column":"vec"}]"#;
+        let conn = connect("memory://").execute().await.unwrap();
+        let schema = Arc::new(ArrowSchema::new_with_metadata(
+            vec![
+                ArrowField::new("text", DataType::Utf8, true),
+                ArrowField::new("vec", DataType::Float32, true),
+            ],
+            HashMap::from([("embedding_functions".to_string(), config.to_string())]),
+        ));
+        conn.create_empty_table("src", schema)
+            .write_options(stable_row_ids())
+            .execute()
+            .await
+            .unwrap();
+
+        let carried = |view: &MaterializedView| {
+            let view = view.table().clone();
+            async move {
+                view.schema()
+                    .await
+                    .unwrap()
+                    .metadata()
+                    .get("embedding_functions")
+                    .cloned()
+            }
+        };
+
+        // Both columns projected as themselves: kept as it stands.
+        let whole = conn
+            .create_materialized_view("whole", "src")
+            .execute()
+            .await
+            .unwrap();
+        let kept = carried(&whole).await.expect("config dropped");
+        assert!(kept.contains(r#""source_column":"text""#), "{kept}");
+        assert!(kept.contains(r#""vector_column":"vec""#), "{kept}");
+
+        // Only the source column: the configuration names a vector column the
+        // view does not have, so it describes nothing and goes.
+        let partial = conn
+            .create_materialized_view("partial", "src")
+            .select([("text", "text")])
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(carried(&partial).await, None);
+
+        // Renamed: the configuration follows the names the view uses.
+        let renamed = conn
+            .create_materialized_view("renamed", "src")
+            .select([("body", "text"), ("embedding", "vec")])
+            .execute()
+            .await
+            .unwrap();
+        let remapped = carried(&renamed).await.expect("config dropped");
+        assert!(remapped.contains(r#""source_column":"body""#), "{remapped}");
+        assert!(
+            remapped.contains(r#""vector_column":"embedding""#),
+            "{remapped}"
+        );
+
+        // The Node bindings spell the same configuration in camelCase, and
+        // the Rust definition names the destination `dest_column`.
+        for (config, source_key, dest_key) in [
+            (
+                r#"[{"name":"f","model":{},"sourceColumn":"text","vectorColumn":"vec"}]"#,
+                "sourceColumn",
+                "vectorColumn",
+            ),
+            (
+                r#"[{"name":"f","model":{},"source_column":"text","dest_column":"vec"}]"#,
+                "source_column",
+                "dest_column",
+            ),
+        ] {
+            let schema = Arc::new(ArrowSchema::new_with_metadata(
+                vec![
+                    ArrowField::new("text", DataType::Utf8, true),
+                    ArrowField::new("vec", DataType::Float32, true),
+                ],
+                HashMap::from([("embedding_functions".to_string(), config.to_string())]),
+            ));
+            let name = format!("src_{source_key}");
+            conn.create_empty_table(&name, schema)
+                .write_options(stable_row_ids())
+                .execute()
+                .await
+                .unwrap();
+            let view = conn
+                .create_materialized_view(format!("v_{source_key}"), &name)
+                .select([("body", "text"), ("embedding", "vec")])
+                .execute()
+                .await
+                .unwrap();
+            let carried = carried(&view).await.expect("config dropped");
+            assert!(
+                carried.contains(&format!(r#""{source_key}":"body""#)),
+                "{carried}"
+            );
+            assert!(
+                carried.contains(&format!(r#""{dest_key}":"embedding""#)),
+                "{carried}"
+            );
+        }
+
+        // A computed column is not the source column under another name.
+        let computed = conn
+            .create_materialized_view("computed", "src")
+            .select([("body", "upper(text)"), ("embedding", "vec")])
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(carried(&computed).await, None);
+
+        // One column projected twice is two columns in the view, and the
+        // configuration has to describe both rather than whichever came last.
+        let twice = conn
+            .create_materialized_view("twice", "src")
+            .select([("body", "text"), ("a", "vec"), ("b", "vec")])
+            .execute()
+            .await
+            .unwrap();
+        let carried = carried(&twice).await.expect("config dropped");
+        let entries: Vec<serde_json::Value> = serde_json::from_str(&carried).unwrap();
+        let mut vectors: Vec<&str> = entries
+            .iter()
+            .filter_map(|e| e["vector_column"].as_str())
+            .collect();
+        vectors.sort_unstable();
+        assert_eq!(vectors, ["a", "b"], "{carried}");
+    }
+
+    /// The native Rust producer records embeddings as column definitions
+    /// rather than as `embedding_functions`, and a query embeds its own text
+    /// through them. They are positional, so the view's list covers every one
+    /// of its fields.
+    #[tokio::test]
+    async fn test_native_column_definitions_follow_the_projection() {
+        let conn = connect("memory://").execute().await.unwrap();
+        let rich = crate::table::TableDefinition::new(
+            Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("text", DataType::Utf8, true),
+                ArrowField::new("vector", DataType::Float32, true),
+            ])),
+            vec![
+                ColumnDefinition {
+                    kind: ColumnKind::Physical,
+                },
+                ColumnDefinition {
+                    kind: ColumnKind::Embedding(EmbeddingDefinition::new(
+                        "text",
+                        "model",
+                        Some("vector"),
+                    )),
+                },
+            ],
+        )
+        .into_rich_schema();
+        conn.create_empty_table("src", rich)
+            .write_options(stable_row_ids())
+            .execute()
+            .await
+            .unwrap();
+
+        let view = conn
+            .create_materialized_view("view", "src")
+            .select([("body", "text"), ("embedding", "vector")])
+            .execute()
+            .await
+            .unwrap();
+        let schema = view.table().schema().await.unwrap();
+        let raw = schema
+            .metadata()
+            .get(COLUMN_DEFINITIONS_META_KEY)
+            .expect("the view dropped the native column definitions");
+        let definitions: Vec<ColumnDefinition> = serde_json::from_str(raw).unwrap();
+        assert_eq!(
+            definitions.len(),
+            schema.fields().len(),
+            "column definitions are positional"
+        );
+        let ColumnKind::Embedding(embedding) = &definitions[1].kind else {
+            panic!("the embedding column came back physical: {raw}");
+        };
+        assert_eq!(embedding.source_column, "body");
+        assert_eq!(embedding.dest_column.as_deref(), Some("embedding"));
+        assert_eq!(embedding.embedding_name, "model");
+        assert!(matches!(definitions[0].kind, ColumnKind::Physical));
+        assert!(matches!(definitions[2].kind, ColumnKind::Physical));
+
+        // Without the column the function reads, the view cannot recompute
+        // the embedding, so it carries no definition for it.
+        let partial = conn
+            .create_materialized_view("partial", "src")
+            .select([("embedding", "vector")])
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(
+            partial
+                .table()
+                .schema()
+                .await
+                .unwrap()
+                .metadata()
+                .get(COLUMN_DEFINITIONS_META_KEY),
+            None
+        );
+    }
+
+    /// A scan takes the cap as i64, so a larger one is refused where it is
+    /// declared rather than at the refresh that cannot run it. What a cap of
+    /// zero means is a refresh question, tested there.
+    #[tokio::test]
+    async fn test_limit_above_i64_max_is_refused_at_creation() {
+        let conn = people_db().await;
+        let err = conn
+            .create_materialized_view("too_big", "people")
+            .limit(i64::MAX as u64 + 1)
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(&err, Error::InvalidInput { message } if message.contains("exceeds the maximum")),
+            "got {err:?}"
+        );
+
+        // The boundary itself is accepted.
+        conn.create_materialized_view("at_max", "people")
+            .limit(i64::MAX as u64)
+            .execute()
+            .await
+            .unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_drop_is_drop_table() {
+        let conn = people_db().await;
+        conn.create_materialized_view("v", "people")
+            .execute()
+            .await
+            .unwrap();
+        conn.drop_table("v", &[]).await.unwrap();
+        assert!(conn.list_materialized_views().await.unwrap().is_empty());
+    }
+
+    /// The public declaration contract: prepare validates the source and
+    /// create consumes the declaration into a verified view table; a
+    /// source that cannot anchor refresh and a target outside the
+    /// source's database are both refused.
+    #[tokio::test]
+    async fn prepare_and_create_bind_the_declaration_lifecycle() {
+        let conn = connect("memory://").execute().await.unwrap();
+        let batch = record_batch!(("id", Int32, [1, 2]), ("value", Int32, [3, 4])).unwrap();
+        let source = conn
+            .create_table("src", batch.clone())
+            .write_options(stable_row_ids())
+            .execute()
+            .await
+            .unwrap();
+
+        let projections = [
+            ("id".to_string(), "id".to_string()),
+            ("double".to_string(), "value * 2".to_string()),
+        ];
+        let prepared = prepare_declaration(&source, &projections, Some("value > 0"), None)
+            .await
+            .unwrap();
+        assert_eq!(prepared.definition().source_table, "src");
+
+        let view = prepared.create("v").await.unwrap();
+        let schema = view.table().schema().await.unwrap();
+        let names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect();
+        assert_eq!(names, ["id", "double", SOURCE_ROW_ID_COLUMN]);
+        assert!(schema.metadata().contains_key(DEFINITION_META_KEY));
+
+        // The same call rejects a source without stable row ids, so an
+        // external creation path cannot skip the check.
+        conn.create_table("plain", batch).execute().await.unwrap();
+        let plain = conn.open_table("plain").execute().await.unwrap();
+        let err = prepare_declaration(&plain, &[], None, None)
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("stable row ids"), "{err}");
+
+        // A handle whose location does not resolve back through its name is
+        // refused: the definition would record a name reaching other data.
+        let plain_uri = plain
+            .as_native()
+            .unwrap()
+            .dataset
+            .get()
+            .await
+            .unwrap()
+            .uri()
+            .to_string();
+        let masquerade = conn
+            .open_table("src")
+            .location(plain_uri)
+            .execute()
+            .await
+            .unwrap();
+        let err = prepare_declaration(&masquerade, &[], None, None)
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string().contains("does not resolve to itself"),
+            "{err}"
+        );
+
+        // A table created at a custom location is refused outright: its
+        // recorded name reaches nothing at the database root, so the
+        // canonical reopen fails before any URI comparison.
+        let custom = conn
+            .create_table(
+                "custom_loc",
+                record_batch!(("id", Int32, [1, 2]), ("value", Int32, [3, 4])).unwrap(),
+            )
+            .location("memory://elsewhere/custom_loc")
+            .write_options(stable_row_ids())
+            .execute()
+            .await
+            .unwrap();
+        let err = prepare_declaration(&custom, &[], None, None)
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("custom_loc"), "{err}");
+
+        // A namespaced source cannot be recorded in the definition: the
+        // bare name refresh resolves would reach a different table or none.
+        let namespaced = crate::table::NativeTable::create(
+            "memory://ns_src",
+            "ns_src",
+            vec!["ns".to_string()],
+            Box::new(arrow_array::RecordBatchIterator::new(
+                vec![],
+                std::sync::Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
+                    "id",
+                    arrow_schema::DataType::Int32,
+                    true,
+                )])),
+            )) as Box<dyn arrow_array::RecordBatchReader + Send>,
+            None,
+            None,
+            None,
+            None,
+            std::collections::HashSet::new(),
+        )
+        .await
+        .unwrap();
+        let namespaced = Table::new(std::sync::Arc::new(namespaced), conn.database().clone());
+        let err = prepare_declaration(&namespaced, &[], None, None)
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("namespaced source"), "{err}");
+    }
+}
diff --git a/rust/lancedb/src/materialized_view/differential.rs b/rust/lancedb/src/materialized_view/differential.rs
new file mode 100644
index 000000000..396b4e65b
--- /dev/null
+++ b/rust/lancedb/src/materialized_view/differential.rs
@@ -0,0 +1,730 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+//! Differential refresh testing.
+//!
+//! The refresh contract is a property: after any sequence of source
+//! mutations, a view maintained by default (incremental-where-possible)
+//! refreshes equals the definition evaluated against the source directly,
+//! and so does a forced rebuild. The oracle is an independent read of the
+//! source -- plain column scan, filter applied in Rust -- so it shares
+//! nothing with the refresh path it checks.
+//!
+//! The oracle runs after every step, not just at the end: a later mutation
+//! that forces a rebuild would silently heal an incremental error, and those
+//! transient errors are exactly the bugs this exists to catch.
+
+use arrow_array::{Float32Array, Int32Array, RecordBatch};
+use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
+use futures::{StreamExt, TryStreamExt};
+use lance::dataset::NewColumnTransform;
+use std::sync::Arc;
+
+use super::MaterializedView;
+use super::refresh::RefreshMode;
+use crate::connect;
+use crate::connection::Connection;
+use crate::query::{ExecutableQuery, QueryBase, Select};
+use crate::table::{CompactionOptions, OptimizeAction, Table};
+
+/// One source mutation, one per correctness-relevant class.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum SrcOp {
+    /// Fresh non-colliding ids of both parities, so every other op has
+    /// view-resident rows to act on: the only op that should refresh
+    /// incrementally.
+    AppendNew,
+    /// Deletion in surviving fragments must break the pure-append check.
+    DeleteEven,
+    /// An in-place update; on the filtered shape it crosses the predicate,
+    /// so rows must leave the view.
+    UpdateOddScore,
+    /// Fragment rewrite/renumber must break the pure-append check.
+    Compact,
+    /// A column the view does not read must NOT force a rebuild.
+    AddColumn,
+    /// merge_insert commits an Update whose by-source arm deletes rows, so a
+    /// classifier that reads Update as "changed only" loses those deletions.
+    MergeDropLargest,
+    /// merge_insert that both changes existing rows and inserts new ones in
+    /// one transaction.
+    MergeUpsert,
+}
+
+const ALL_OPS: [SrcOp; 7] = [
+    SrcOp::AppendNew,
+    SrcOp::DeleteEven,
+    SrcOp::UpdateOddScore,
+    SrcOp::Compact,
+    SrcOp::AddColumn,
+    SrcOp::MergeDropLargest,
+    SrcOp::MergeUpsert,
+];
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum Shape {
+    /// SELECT id, score.
+    Identity,
+    /// SELECT id, score WHERE score > 50: additionally sensitive to rows
+    /// crossing the predicate.
+    Filtered,
+    /// SELECT id, score LIMIT 4. Which rows are held depends on the order
+    /// they were first materialized, so the oracle checks containment and
+    /// the cap rather than equality.
+    Limited,
+}
+
+impl Shape {
+    fn filter(&self) -> Option<&'static str> {
+        match self {
+            Self::Identity | Self::Limited => None,
+            Self::Filtered => Some("score > 50"),
+        }
+    }
+
+    fn matches(&self, score: f32) -> bool {
+        match self {
+            Self::Identity | Self::Limited => true,
+            Self::Filtered => score > 50.0,
+        }
+    }
+
+    fn limit(&self) -> Option<usize> {
+        match self {
+            Self::Limited => Some(4),
+            _ => None,
+        }
+    }
+}
+
+struct Case {
+    conn: Connection,
+    source: Table,
+    view: MaterializedView,
+    shape: Shape,
+    next_id: i32,
+    added_columns: u32,
+}
+
+fn rows_batch(ids: &[i32]) -> RecordBatch {
+    let scores: Vec<f32> = ids.iter().map(|id| (*id * 10) as f32).collect();
+    RecordBatch::try_new(
+        Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("id", DataType::Int32, true),
+            ArrowField::new("score", DataType::Float32, true),
+        ])),
+        vec![
+            Arc::new(Int32Array::from(ids.to_vec())),
+            Arc::new(Float32Array::from(scores)),
+        ],
+    )
+    .unwrap()
+}
+
+fn merge_batch(ids: &[i32]) -> RecordBatch {
+    let scores: Vec<f32> = ids.iter().map(|id| (*id * 10 + 5) as f32).collect();
+    RecordBatch::try_new(
+        Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("id", DataType::Int32, true),
+            ArrowField::new("score", DataType::Float32, true),
+        ])),
+        vec![
+            Arc::new(Int32Array::from(ids.to_vec())),
+            Arc::new(Float32Array::from(scores)),
+        ],
+    )
+    .unwrap()
+}
+
+impl Case {
+    async fn new(shape: Shape) -> Self {
+        let conn = connect("memory://").execute().await.unwrap();
+        let source = conn
+            .create_table("src", rows_batch(&[1, 2, 3, 4]))
+            .write_options(crate::materialized_view::tests::stable_row_ids())
+            .execute()
+            .await
+            .unwrap();
+        let mut builder = conn
+            .create_materialized_view("view", "src")
+            .select([("id", "id"), ("score", "score")]);
+        if let Some(filter) = shape.filter() {
+            builder = builder.only_if(filter);
+        }
+        if let Some(limit) = shape.limit() {
+            builder = builder.limit(limit as u64);
+        }
+        let view = builder.execute().await.unwrap();
+        Self {
+            conn,
+            source,
+            view,
+            shape,
+            next_id: 100,
+            added_columns: 0,
+        }
+    }
+
+    async fn apply(&mut self, op: SrcOp) {
+        match op {
+            SrcOp::AppendNew => {
+                // Mixed parity: the middle id is odd, so UpdateOddScore always
+                // has a filter-matching appended row to evict.
+                let ids = vec![self.next_id, self.next_id + 101, self.next_id + 202];
+                self.next_id += 303;
+                self.source.add(rows_batch(&ids)).execute().await.unwrap();
+            }
+            SrcOp::DeleteEven => {
+                self.source.delete("id % 2 = 0").await.unwrap();
+            }
+            SrcOp::UpdateOddScore => {
+                self.source
+                    .update()
+                    .column("score", "-1.0")
+                    .only_if("id % 2 = 1")
+                    .execute()
+                    .await
+                    .unwrap();
+            }
+            SrcOp::Compact => {
+                self.source
+                    .optimize(OptimizeAction::Compact {
+                        options: CompactionOptions::default(),
+                        remap_options: None,
+                    })
+                    .await
+                    .unwrap();
+            }
+            SrcOp::MergeDropLargest => {
+                let mut ids = self.source_ids().await;
+                ids.sort_unstable();
+                ids.pop();
+                if ids.is_empty() {
+                    return;
+                }
+                let batch = rows_batch(&ids);
+                let reader =
+                    arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema());
+                let mut merge = self.source.merge_insert(&["id"]);
+                merge.when_not_matched_by_source_delete(None);
+                merge.execute(Box::new(reader)).await.unwrap();
+            }
+            SrcOp::MergeUpsert => {
+                let mut ids = self.source_ids().await;
+                ids.sort_unstable();
+                // One row that exists (updated in place) and one that does not.
+                let existing = ids.first().copied().unwrap_or(self.next_id);
+                let fresh = self.next_id;
+                self.next_id += 1;
+                let batch = merge_batch(&[existing, fresh]);
+                let reader =
+                    arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema());
+                let mut merge = self.source.merge_insert(&["id"]);
+                merge
+                    .when_matched_update_all(None)
+                    .when_not_matched_insert_all();
+                merge.execute(Box::new(reader)).await.unwrap();
+            }
+            SrcOp::AddColumn => {
+                self.added_columns += 1;
+                let field = ArrowField::new(
+                    format!("extra_{}", self.added_columns),
+                    DataType::Int32,
+                    true,
+                );
+                self.source
+                    .add_columns()
+                    .transform(NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new(
+                        vec![field],
+                    ))))
+                    .execute()
+                    .await
+                    .unwrap();
+            }
+        }
+    }
+
+    async fn source_ids(&self) -> Vec<i32> {
+        read_rows(
+            self.source
+                .query()
+                .select(Select::columns(&["id", "score"])),
+        )
+        .await
+        .into_iter()
+        .map(|(id, _)| id)
+        .collect()
+    }
+
+    /// The definition's result, read independently of the refresh path:
+    /// plain column scan, filter applied here, sorted.
+    async fn oracle(&self) -> Vec<(i32, i32)> {
+        let mut rows = read_rows(
+            self.source
+                .query()
+                .select(Select::columns(&["id", "score"])),
+        )
+        .await
+        .into_iter()
+        .filter(|(_, score)| self.shape.matches(*score as f32))
+        .collect::<Vec<_>>();
+        rows.sort_unstable();
+        rows
+    }
+
+    async fn view_rows(&self) -> Vec<(i32, i32)> {
+        let mut rows = read_rows(
+            self.view
+                .table()
+                .query()
+                .select(Select::columns(&["id", "score"])),
+        )
+        .await;
+        rows.sort_unstable();
+        rows
+    }
+
+    async fn check(&self, label: &str) -> Result<(), String> {
+        let expected = self.oracle().await;
+        let actual = self.view_rows().await;
+        let Some(cap) = self.shape.limit() else {
+            if expected != actual {
+                return Err(format!(
+                    "{label}: view diverged from oracle\n  expected: {expected:?}\n  actual:   {actual:?}"
+                ));
+            }
+            return Ok(());
+        };
+        // A capped view holds some subset of the definition's result, never
+        // more than the cap, and never the same row twice.
+        if actual.len() > cap {
+            return Err(format!(
+                "{label}: view holds {} rows, over its cap of {cap}: {actual:?}",
+                actual.len()
+            ));
+        }
+        let mut unique = actual.clone();
+        unique.dedup();
+        if unique.len() != actual.len() {
+            return Err(format!("{label}: view holds a row twice: {actual:?}"));
+        }
+        if let Some(stray) = actual.iter().find(|row| !expected.contains(row)) {
+            return Err(format!(
+                "{label}: view holds {stray:?}, which the definition does not select: {expected:?}"
+            ));
+        }
+        // Below the cap the view must be complete, or a row was lost.
+        if actual.len() < cap.min(expected.len()) {
+            return Err(format!(
+                "{label}: view holds {} of {} selectable rows under a cap of {cap}: {actual:?}",
+                actual.len(),
+                expected.len()
+            ));
+        }
+        Ok(())
+    }
+}
+
+async fn read_rows(query: impl ExecutableQuery) -> Vec<(i32, i32)> {
+    let batches = query
+        .execute()
+        .await
+        .unwrap()
+        .try_collect::<Vec<_>>()
+        .await
+        .unwrap();
+    batches
+        .iter()
+        .flat_map(|batch| {
+            let ids = batch["id"].as_any().downcast_ref::<Int32Array>().unwrap();
+            let scores = batch["score"]
+                .as_any()
+                .downcast_ref::<Float32Array>()
+                .unwrap();
+            // Scores are integer-valued by construction; compare exactly.
+            (0..batch.num_rows())
+                .map(|i| (ids.value(i), scores.value(i) as i32))
+                .collect::<Vec<_>>()
+        })
+        .collect()
+}
+
+/// Drive one mutation sequence: refresh + oracle-check after every step,
+/// then a forced rebuild checked against the same oracle.
+async fn run_sequence(ops: &[SrcOp], shape: Shape) -> Result<(), String> {
+    let label = format!("{shape:?} {ops:?}");
+    let mut case = Case::new(shape).await;
+    case.view
+        .refresh()
+        .execute()
+        .await
+        .map_err(|e| format!("{label}: initial refresh failed: {e}"))?;
+    case.check(&format!("{label} (initial)")).await?;
+
+    for (step, op) in ops.iter().enumerate() {
+        case.apply(*op).await;
+        case.view
+            .refresh()
+            .execute()
+            .await
+            .map_err(|e| format!("{label}: refresh at step {step} failed: {e}"))?;
+        case.check(&format!("{label} (step {step}, {op:?})"))
+            .await?;
+    }
+
+    case.view
+        .refresh()
+        .full(true)
+        .execute()
+        .await
+        .map_err(|e| format!("{label}: final full refresh failed: {e}"))?;
+    case.check(&format!("{label} (final rebuild)")).await?;
+    // Silence the unused-connection lint without dropping it mid-case.
+    let _ = &case.conn;
+    Ok(())
+}
+
+/// Every op sequence up to `max_len`.
+fn all_sequences(max_len: u32) -> Vec<Vec<SrcOp>> {
+    let mut sequences = Vec::new();
+    for len in 1..=max_len {
+        for mut index in 0..ALL_OPS.len().pow(len) {
+            let mut ops = Vec::with_capacity(len as usize);
+            for _ in 0..len {
+                ops.push(ALL_OPS[index % ALL_OPS.len()]);
+                index /= ALL_OPS.len();
+            }
+            sequences.push(ops);
+        }
+    }
+    sequences
+}
+
+async fn run_exhaustive(max_len: u32) {
+    let mut cases = Vec::new();
+    for shape in [Shape::Identity, Shape::Filtered, Shape::Limited] {
+        for ops in all_sequences(max_len) {
+            cases.push((ops, shape));
+        }
+    }
+    let failures: Vec<String> = futures::stream::iter(cases)
+        .map(|(ops, shape)| async move { run_sequence(&ops, shape).await.err() })
+        .buffer_unordered(8)
+        .filter_map(|failure| async move { failure })
+        .collect()
+        .await;
+    assert!(
+        failures.is_empty(),
+        "{} sequences diverged; first: {}",
+        failures.len(),
+        failures[0]
+    );
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn differential_exhaustive() {
+    run_exhaustive(3).await;
+}
+
+#[tokio::test(flavor = "multi_thread")]
+#[ignore = "longer sweep; run manually"]
+async fn differential_exhaustive_deep() {
+    run_exhaustive(4).await;
+}
+
+/// Named interleavings that double as repro handles. The mode assertions pin
+/// the classifier, which value comparison alone cannot: a wrongly rebuilt
+/// view still matches the oracle.
+#[tokio::test(flavor = "multi_thread")]
+async fn differential_named_regressions() {
+    // An append is the one op that must stay incremental.
+    let mut case = Case::new(Shape::Identity).await;
+    case.view.refresh().execute().await.unwrap();
+    case.apply(SrcOp::AppendNew).await;
+    let result = case.view.refresh().execute().await.unwrap();
+    assert_eq!(result.mode, RefreshMode::Incremental);
+    case.check("append stays incremental").await.unwrap();
+
+    // A column the view does not read must not force a rebuild.
+    let mut case = Case::new(Shape::Identity).await;
+    case.view.refresh().execute().await.unwrap();
+    case.apply(SrcOp::AddColumn).await;
+    let result = case.view.refresh().execute().await.unwrap();
+    assert_eq!(result.mode, RefreshMode::Incremental);
+    assert_eq!(result.rows_written, 0);
+
+    // Compaction rearranges rows without changing them: the watermark
+    // advances and nothing rebuilds.
+    let mut case = Case::new(Shape::Identity).await;
+    case.view.refresh().execute().await.unwrap();
+    case.apply(SrcOp::AppendNew).await;
+    case.view.refresh().execute().await.unwrap();
+    case.apply(SrcOp::Compact).await;
+    let result = case.view.refresh().execute().await.unwrap();
+    assert_eq!(result.mode, RefreshMode::Incremental);
+    assert_eq!(result.rows_written, 0);
+    case.check("compaction alone").await.unwrap();
+
+    // Fragment bookkeeping stays coherent across the compaction: the next
+    // append is separable and computed alone.
+    case.apply(SrcOp::AppendNew).await;
+    let result = case.view.refresh().execute().await.unwrap();
+    assert_eq!(result.mode, RefreshMode::Incremental);
+    assert_eq!(result.rows_written, 3);
+    case.check("compact then append").await.unwrap();
+
+    // A row updated to no longer match the filter must leave the view --
+    // and the fixture must prove the eviction happened, not merely that the
+    // end state matches: an update that never touched a view-resident row
+    // would also "match".
+    let mut case = Case::new(Shape::Filtered).await;
+    case.apply(SrcOp::AppendNew).await;
+    case.view.refresh().execute().await.unwrap();
+    let before = case.view_rows().await.len();
+    case.apply(SrcOp::UpdateOddScore).await;
+    case.view.refresh().execute().await.unwrap();
+    let after = case.view_rows().await.len();
+    assert!(
+        after < before,
+        "no view-resident row was evicted ({before} -> {after}); the fixture \
+         no longer exercises the filtered-update transition"
+    );
+    case.check("update crosses the filter").await.unwrap();
+}
+
+// ---------------------------------------------------------------------------
+// Concurrency
+// ---------------------------------------------------------------------------
+//
+// The sequential cases above cannot observe a cross-process race: the
+// per-view refresh lock is process-local, so a second refresh in this
+// process queues behind the first. What is missing is not more op
+// sequences but a second process. These cases add one, and assert the same
+// property the harness always asserts -- the view holds each row once.
+
+/// Rows the definition selects from the source: every id but the first,
+/// read straight from the source, sharing nothing with the refresh path.
+async fn concurrency_oracle(conn: &Connection) -> Vec<i32> {
+    let batches: Vec<RecordBatch> = conn
+        .open_table("src")
+        .execute()
+        .await
+        .unwrap()
+        .query()
+        .select(Select::columns(&["id"]))
+        .execute()
+        .await
+        .unwrap()
+        .try_collect()
+        .await
+        .unwrap();
+    let mut ids = Vec::new();
+    for batch in &batches {
+        let column = batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap();
+        for i in 0..batch.num_rows() {
+            if column.value(i) > 1 {
+                ids.push(column.value(i));
+            }
+        }
+    }
+    ids.sort_unstable();
+    ids
+}
+
+/// The view's ids, sorted.
+async fn concurrency_view_ids(conn: &Connection) -> Vec<i32> {
+    let batches: Vec<RecordBatch> = conn
+        .open_table("mv")
+        .execute()
+        .await
+        .unwrap()
+        .query()
+        .select(Select::columns(&["id"]))
+        .execute()
+        .await
+        .unwrap()
+        .try_collect()
+        .await
+        .unwrap();
+    let mut ids = Vec::new();
+    for batch in &batches {
+        let column = batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap();
+        for i in 0..batch.num_rows() {
+            ids.push(column.value(i));
+        }
+    }
+    ids.sort_unstable();
+    ids
+}
+
+/// One refresh of the view at `MV_RACE_DIR`, in its own process.
+///
+/// Setup happens before the start barrier so warm-up does not stagger the
+/// two processes. What makes the race certain rather than likely is the
+/// second barrier inside `refresh()` itself, which holds every participant
+/// between staging and commit.
+#[tokio::test]
+#[ignore = "spawned as a child process by the concurrency cases"]
+async fn cross_process_refresh_child() {
+    let Ok(dir) = std::env::var("MV_RACE_DIR") else {
+        return;
+    };
+    let dir = std::path::PathBuf::from(dir);
+    let tag = std::env::var("MV_RACE_TAG").unwrap();
+
+    let conn = connect(dir.to_str().unwrap()).execute().await.unwrap();
+    let table = conn.open_table("mv").execute().await.unwrap();
+    let _ = table.schema().await.unwrap();
+    let _ = table.count_rows(None).await.unwrap();
+    let source = conn.open_table("src").execute().await.unwrap();
+    let _ = source.count_rows(None).await.unwrap();
+    let view = MaterializedView::from_table(table).await.unwrap();
+
+    std::fs::write(dir.join(format!("ready-{tag}")), b"1").unwrap();
+    while !dir.join("START").exists() {
+        std::thread::sleep(std::time::Duration::from_millis(2));
+    }
+
+    let outcome = match view.refresh().execute().await {
+        Ok(result) => format!("committed rows={}", result.rows_written),
+        Err(err) if is_commit_conflict(&err) => "conflicted".to_string(),
+        Err(err) => format!("failed {err}"),
+    };
+    std::fs::write(dir.join(format!("outcome-{tag}")), outcome).unwrap();
+}
+
+/// Whether a refresh lost its commit to a concurrent one, as opposed to
+/// failing for any other reason.
+fn is_commit_conflict(err: &crate::Error) -> bool {
+    let text = err.to_string();
+    text.contains("Retryable commit conflict") || text.contains("preempted by concurrent")
+}
+
+/// Two processes refreshing one view concurrently must leave the view
+/// equal to the oracle: each selected row present exactly once.
+///
+/// Both plan the same incremental delta from one watermark. A refresh is
+/// meant to land on the generation it planned or leave nothing behind, so
+/// at most one of them may write.
+#[tokio::test]
+async fn concurrent_refreshes_hold_each_row_once() {
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().to_str().unwrap().to_string();
+    let conn = connect(&path).execute().await.unwrap();
+    conn.create_table("src", rows_batch(&[1, 2, 3, 4]))
+        .write_options(crate::materialized_view::tests::stable_row_ids())
+        .execute()
+        .await
+        .unwrap();
+    let view = conn
+        .create_materialized_view("mv", "src")
+        .select([("id", "id"), ("score", "score")])
+        .only_if("id > 1")
+        .execute()
+        .await
+        .unwrap();
+    // Seed the watermark so the racing refreshes are both incremental.
+    view.refresh().execute().await.unwrap();
+
+    // Large enough that a refresh is real work rather than a formality.
+    let ids: Vec<i32> = (100..200_100).collect();
+    conn.open_table("src")
+        .execute()
+        .await
+        .unwrap()
+        .add(rows_batch(&ids))
+        .execute()
+        .await
+        .unwrap();
+
+    let tags = ["a", "b"];
+    let exe = std::env::current_exe().unwrap();
+    let children: Vec<std::process::Child> = tags
+        .iter()
+        .map(|tag| {
+            std::process::Command::new(&exe)
+                .args([
+                    "--exact",
+                    "materialized_view::differential::cross_process_refresh_child",
+                    "--ignored",
+                    "--nocapture",
+                ])
+                .env("MV_RACE_DIR", dir.path())
+                .env("MV_RACE_SYNC", dir.path())
+                .env("MV_RACE_PEERS", "2")
+                .env("MV_RACE_TAG", tag)
+                .stdout(std::process::Stdio::null())
+                .stderr(std::process::Stdio::null())
+                .spawn()
+                .unwrap()
+        })
+        .collect();
+
+    let deadline = std::time::Instant::now() + std::time::Duration::from_secs(180);
+    while tags
+        .iter()
+        .any(|tag| !dir.path().join(format!("ready-{tag}")).exists())
+    {
+        assert!(
+            std::time::Instant::now() < deadline,
+            "children never became ready"
+        );
+        std::thread::sleep(std::time::Duration::from_millis(10));
+    }
+    std::fs::write(dir.path().join("START"), b"1").unwrap();
+    for (tag, mut child) in tags.iter().zip(children) {
+        let status = loop {
+            match child.try_wait().unwrap() {
+                Some(status) => break status,
+                None if std::time::Instant::now() >= deadline => {
+                    child.kill().unwrap();
+                    panic!("child {tag} never finished");
+                }
+                None => std::thread::sleep(std::time::Duration::from_millis(10)),
+            }
+        };
+        assert!(status.success(), "child {tag} exited {status}");
+    }
+
+    // Both refreshes reached the commit boundary before either committed --
+    // the in-refresh barrier guarantees it -- so exactly one may win.
+    let outcomes: Vec<String> = tags
+        .iter()
+        .map(|tag| {
+            std::fs::read_to_string(dir.path().join(format!("outcome-{tag}")))
+                .unwrap_or_else(|_| panic!("child {tag} recorded no outcome"))
+        })
+        .collect();
+    for tag in tags {
+        assert!(
+            dir.path().join(format!("planned-{tag}")).exists(),
+            "child {tag} never reached the commit boundary, so nothing was synchronized"
+        );
+    }
+    let committed = outcomes.iter().filter(|o| o.contains("committed")).count();
+    let conflicted = outcomes.iter().filter(|o| o.contains("conflicted")).count();
+    assert_eq!(
+        (committed, conflicted),
+        (1, 1),
+        "exactly one refresh may win the generation both planned: {outcomes:?}"
+    );
+
+    let expected = concurrency_oracle(&conn).await;
+    let actual = concurrency_view_ids(&conn).await;
+    assert_eq!(
+        actual.len(),
+        expected.len(),
+        "the view holds {} rows, the oracle {}: a losing refresh left rows behind",
+        actual.len(),
+        expected.len()
+    );
+    assert_eq!(actual, expected, "the view does not match the oracle");
+}
diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs
new file mode 100644
index 000000000..735751c27
--- /dev/null
+++ b/rust/lancedb/src/materialized_view/refresh.rs
@@ -0,0 +1,2996 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+//! Refreshing materialized views.
+//!
+//! A refresh pins one source version and brings the view to exactly the
+//! definition's result at that version: added rows are computed and appended,
+//! removed or changed rows are evicted by provenance id and recomputed in the
+//! same pass. Compaction outputs cost nothing, which is only sound while
+//! [`SOURCE_ROW_ID_COLUMN`] stays valid across the rewrite. Anything the
+//! classifier cannot prove intact rebuilds; an indexed rebuild swaps all
+//! fragments in one commit that retains index definitions.
+//!
+//! The watermark ([`SOURCE_VERSION_META_KEY`]) lands in a follow-up commit; a
+//! crash or race between the two leaves the view visibly unstamped and the
+//! next refresh rebuilds. In-process refreshes serialize on a per-view lock;
+//! across processes the commit's inserted-rows filter carries a shared token,
+//! so two refreshes of one view conflict and only one lands.
+
+use std::collections::{HashMap, HashSet};
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Mutex as StdMutex, OnceLock};
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use arrow_array::cast::AsArray;
+use arrow_array::types::UInt64Type;
+use arrow_array::{RecordBatch, UInt64Array};
+use arrow_schema::{Schema as ArrowSchema, SchemaRef};
+use datafusion::common::ScalarValue;
+use datafusion::error::DataFusionError;
+use datafusion::physical_plan::SendableRecordBatchStream;
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+use datafusion::prelude::{col, lit};
+use futures::{StreamExt, TryStreamExt};
+use lance::Dataset;
+use lance::dataset::mem_wal::DatasetMemWalExt;
+use lance::dataset::transaction::{Operation, Transaction};
+use lance::dataset::write::delete::DeleteBuilder;
+use lance::dataset::write::merge_insert::inserted_rows::{
+    KeyExistenceFilter, KeyExistenceFilterBuilder, KeyValue,
+};
+use lance::dataset::{CommitBuilder, InsertBuilder, WriteDestination, WriteMode, WriteParams};
+use lance_core::{ROW_CREATED_AT_VERSION, ROW_ID, ROW_LAST_UPDATED_AT_VERSION};
+use lance_file::version::ConcreteFileVersion;
+use lance_table::format::Fragment;
+use serde::{Deserialize, Serialize};
+
+use super::{
+    INCARNATION_META_KEY, MaterializedViewDefinition, REFRESHED_AT_MS_META_KEY,
+    SOURCE_ROW_ID_COLUMN, SOURCE_VERSION_META_KEY,
+};
+use crate::database::OpenTableRequest;
+use crate::table::{NativeTable, NativeTableExt, Table};
+use crate::{Error, Result};
+
+/// How a refresh brought the view up to date.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum RefreshMode {
+    /// The view was recomputed from scratch.
+    Rebuild,
+    /// Rows from source fragments added since the last refresh were appended.
+    Incremental,
+    /// The view was already at the requested source version.
+    NoOp,
+}
+
+/// The result of refreshing a materialized view.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct RefreshMaterializedViewResult {
+    /// How the view was brought up to date.
+    pub mode: RefreshMode,
+    /// Rows written to the view: everything on a rebuild, and on an
+    /// incremental refresh both the rows added and the rows recomputed in
+    /// place of ones the source changed.
+    pub rows_written: u64,
+    /// The source table version the view now reflects.
+    pub source_version: u64,
+    /// The view table version after the refresh.
+    pub version: u64,
+}
+
+/// Schema metadata key holding the view table version a successful refresh
+/// left behind. Any other commit on the view is drift, and refresh rebuilds.
+pub const VIEW_VERSION_META_KEY: &str = "mv.view_version";
+
+/// Schema metadata key holding the commit timestamp of the watermark's source
+/// manifest. A dropped and recreated source reuses version numbers but never
+/// their timestamps, so a mismatch means the watermark describes a different
+/// incarnation and refresh rebuilds.
+pub const SOURCE_VERSION_TS_META_KEY: &str = "mv.source_version_ts";
+
+/// One refresh per view at a time within this process.
+fn refresh_lock(uri: &str) -> Arc<tokio::sync::Mutex<()>> {
+    static LOCKS: OnceLock<StdMutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
+        OnceLock::new();
+    LOCKS
+        .get_or_init(Default::default)
+        .lock()
+        .expect("refresh lock registry poisoned")
+        .entry(uri.to_string())
+        .or_default()
+        .clone()
+}
+
+/// Internal implementation of the refresh logic.
+pub(crate) async fn execute_refresh(
+    view: &Table,
+    full: bool,
+    pinned: Option<u64>,
+    expected_incarnation: Option<&str>,
+) -> Result<RefreshMaterializedViewResult> {
+    let view_native = view.as_native().ok_or_else(|| Error::NotSupported {
+        message: "materialized views are supported only on local tables".into(),
+    })?;
+    view_native.dataset.ensure_mutable()?;
+    let lock = refresh_lock(view_native.dataset.get().await?.uri());
+    let _guard = lock.lock().await;
+    // Force-load the latest view state under the lock: each handle caches
+    // lazily, and a second handle would otherwise plan from a snapshot taken
+    // before another handle's commit -- appending the same rows again or
+    // reporting NoOp over a mutated view.
+    view_native.dataset.reload().await?;
+    let view_ds = view_native.dataset.get().await?.as_ref().clone();
+
+    ensure_incarnation(&view_ds, expected_incarnation, view.name()).await?;
+
+    // The definition a handle cached at open may since have been replaced;
+    // what refresh executes and what it stamps must be one generation.
+    let definition = match super::materialized_view_kind(&view_ds.schema().metadata)? {
+        Some(super::MaterializedViewKind::Select(definition)) => definition,
+        Some(super::MaterializedViewKind::Unrecognized { kind }) => {
+            return Err(Error::NotSupported {
+                message: format!(
+                    "materialized view '{}' is defined by '{kind}', which this \
+                     version of lancedb cannot refresh",
+                    view.name()
+                ),
+            });
+        }
+        None => {
+            return Err(Error::NotAMaterializedView {
+                name: view.name().to_string(),
+            });
+        }
+    };
+    let definition = &definition;
+    ensure_no_mem_wal(&view_ds, "materialized view", view.name()).await?;
+
+    let source_ds = open_source(view, definition).await?;
+    let source_ds = match pinned {
+        Some(version) => source_ds.checkout_version(version).await?,
+        None => source_ds,
+    };
+    ensure_no_mem_wal(&source_ds, "source table", &definition.source_table).await?;
+    let source_version = source_ds.version().version;
+    let source_ts = source_ds.manifest.timestamp_nanos;
+
+    // Re-plan the persisted definition against the current source schema and
+    // require its planned output to be exactly the view's physical schema: a
+    // definition the stored table cannot represent must not be certified.
+    let source_schema = Arc::new(ArrowSchema::from(source_ds.schema()));
+    let projections: Vec<(String, String)> = definition
+        .projections
+        .iter()
+        .map(|p| (p.output.clone(), p.expression.clone()))
+        .collect();
+    validate_inputs(&source_ds, definition)?;
+    let (replanned, mut planned_fields, _renames) = super::plan(
+        source_schema,
+        &definition.source_table,
+        &projections,
+        definition.filter.as_deref(),
+        definition.limit,
+    )?;
+    planned_fields.push(arrow_schema::Field::new(
+        SOURCE_ROW_ID_COLUMN,
+        arrow_schema::DataType::UInt64,
+        false,
+    ));
+    let physical = ArrowSchema::from(view_ds.schema());
+    let planned_shape: Vec<_> = planned_fields
+        .iter()
+        .map(|f| (f.name().clone(), f.data_type().clone(), f.is_nullable()))
+        .collect();
+    let physical_shape: Vec<_> = physical
+        .fields()
+        .iter()
+        .map(|f| (f.name().clone(), f.data_type().clone(), f.is_nullable()))
+        .collect();
+    if planned_shape != physical_shape {
+        return Err(Error::Schema {
+            message: format!(
+                "the stored definition of view '{}' does not produce this \
+                 view's schema; recreate the view",
+                view.name()
+            ),
+        });
+    }
+    let definition = &replanned;
+
+    let metadata = &view_ds.schema().metadata;
+    let watermark: Option<u64> = metadata
+        .get(SOURCE_VERSION_META_KEY)
+        .and_then(|raw| raw.parse().ok());
+    let recorded_ts: Option<u128> = metadata
+        .get(SOURCE_VERSION_TS_META_KEY)
+        .and_then(|raw| raw.parse().ok());
+    // The watermark speaks only for the view state its refresh left behind;
+    // any other commit on the view since then is drift.
+    let view_intact = metadata
+        .get(VIEW_VERSION_META_KEY)
+        .and_then(|raw| raw.parse::<u64>().ok())
+        == Some(view_ds.version().version);
+
+    if !full && watermark == Some(source_version) && view_intact && recorded_ts == Some(source_ts) {
+        return Ok(RefreshMaterializedViewResult {
+            mode: RefreshMode::NoOp,
+            rows_written: 0,
+            source_version,
+            version: view_ds.version().version,
+        });
+    }
+
+    let watermark = watermark.filter(|_| view_intact);
+    match plan_increment(
+        &source_ds,
+        source_version,
+        watermark,
+        recorded_ts,
+        full,
+        definition,
+    )
+    .await
+    {
+        Some(increment) => {
+            let reconciled = incremental(
+                view_native,
+                &view_ds,
+                &source_ds,
+                source_version,
+                source_ts,
+                increment,
+                definition,
+                watermark,
+                expected_incarnation,
+            )
+            .await?;
+            match reconciled {
+                Some(result) => Ok(result),
+                // The delta was too large to reconcile in bounded memory.
+                None => {
+                    rebuild(
+                        view_native,
+                        &view_ds,
+                        &source_ds,
+                        source_version,
+                        source_ts,
+                        definition,
+                        expected_incarnation,
+                    )
+                    .await
+                }
+            }
+        }
+        None => {
+            rebuild(
+                view_native,
+                &view_ds,
+                &source_ds,
+                source_version,
+                source_ts,
+                definition,
+                expected_incarnation,
+            )
+            .await
+        }
+    }
+}
+
+/// The source fragments whose rows are new since the watermark, or `None`
+/// where the view has to rebuild. Two tiers: the transaction walk is exact
+/// where it applies; the fragment-signature check is the fallback for deltas
+/// the walk cannot read, and under it any fragment churn rebuilds.
+async fn plan_increment(
+    source_ds: &Dataset,
+    source_version: u64,
+    watermark: Option<u64>,
+    recorded_ts: Option<u128>,
+    full: bool,
+    definition: &MaterializedViewDefinition,
+) -> Option<Increment> {
+    if full {
+        return None;
+    }
+    let watermark = watermark?;
+    if watermark > source_version {
+        return None;
+    }
+    let old = source_ds.checkout_version(watermark).await.ok()?;
+    // A recreated source reuses version numbers, never their timestamps: a
+    // mismatch means the watermark describes a different incarnation.
+    if recorded_ts != Some(old.manifest.timestamp_nanos) {
+        return None;
+    }
+    let old_ids: HashSet<u64> = old.get_fragments().iter().map(|f| f.id() as u64).collect();
+    let live: Vec<Fragment> = source_ds
+        .get_fragments()
+        .iter()
+        .map(|f| f.metadata().clone())
+        .collect();
+
+    if let Some(delta) = appends_and_rewrites(source_ds, watermark, source_version).await {
+        // A rewrite that consumed a fragment neither present at the watermark
+        // nor produced by an earlier rewrite swallowed a mid-delta append;
+        // its rows cannot be told apart from already-materialized ones.
+        let folded = delta
+            .rewritten
+            .iter()
+            .any(|id| !old_ids.contains(id) && !delta.produced.contains(id));
+        if folded {
+            return None;
+        }
+        // An update rewrites a whole fragment. If it touched one the watermark
+        // never saw, that fragment holds rows appended since -- new rows the
+        // update did not change, which the recompute does not cover and which
+        // this fragment's exclusion from the append set would drop.
+        if delta
+            .updated_in_place
+            .iter()
+            .any(|id| !old_ids.contains(id))
+        {
+            return None;
+        }
+        // Rows past the cap left every delta when the watermark advanced, so
+        // a capped view cannot reconcile a removal incrementally. The rebuild
+        // is cheap for the same reason it is capped: the scan stops there.
+        if definition.limit.is_some() && (delta.deleted_rows || delta.updated_rows) {
+            return None;
+        }
+        // Legacy storage cannot serve the row-version columns update
+        // discovery scans; deletes and appends need none of them.
+        if delta.updated_rows
+            && source_ds.manifest.data_storage_format.lance_file_format() == ConcreteFileVersion::V1
+        {
+            return None;
+        }
+        // Every other fragment new at head is an append: appends and rewrites
+        // are the only operations in the delta that add fragments, and the
+        // rewrite outputs are already-materialized rows rearranged.
+        return Some(Increment {
+            appended: live
+                .into_iter()
+                .filter(|f| !old_ids.contains(&f.id) && !delta.produced.contains(&f.id))
+                .collect(),
+            evict_deleted: delta.deleted_rows,
+            replace_updated: delta.updated_rows,
+        });
+    }
+
+    is_pure_append(&old, source_ds, &relevant_field_ids(source_ds, definition)).then(|| Increment {
+        appended: live
+            .into_iter()
+            .filter(|f| !old_ids.contains(&f.id))
+            .collect(),
+        evict_deleted: false,
+        replace_updated: false,
+    })
+}
+
+/// Version gap beyond which per-version transaction reads stop being cheaper
+/// than one fragment scan.
+const MAX_TRANSACTION_WALK: u64 = 512;
+
+/// Fragment ids moved by the `Rewrite` operations of the delta. Only rewrite
+/// ids are real in transaction files (an Append's are placeholders assigned
+/// at commit), so appends are derived as new-at-head minus rewrite outputs.
+/// What an incremental refresh must do to bring the view up to date.
+struct Increment {
+    /// Source fragments whose rows are not in the view yet.
+    appended: Vec<Fragment>,
+    /// Source rows left the range, so the view holds rows to evict.
+    evict_deleted: bool,
+    /// Source rows changed in the range, so the view holds rows to replace.
+    replace_updated: bool,
+}
+
+struct TxnDelta {
+    /// Fragments consumed by `Rewrite` operations.
+    rewritten: HashSet<u64>,
+    /// Fragments produced by `Rewrite` operations.
+    produced: HashSet<u64>,
+    /// The delta removed source rows, so the view holds rows to evict.
+    deleted_rows: bool,
+    /// The delta changed source rows in place, so the view holds rows to
+    /// recompute.
+    updated_rows: bool,
+    /// Fragments an update modified in place, as opposed to produced.
+    updated_in_place: HashSet<u64>,
+}
+
+/// Read the delta from the transaction log, `None` where it holds anything
+/// but appends and rewrites or cannot be read; `None` only sends the caller
+/// to a slower check. `ReserveFragments` moves no rows and rides along.
+async fn appends_and_rewrites(cur: &Dataset, from: u64, to: u64) -> Option<TxnDelta> {
+    if to <= from || to - from > MAX_TRANSACTION_WALK {
+        return None;
+    }
+    let mut delta = TxnDelta {
+        rewritten: HashSet::new(),
+        produced: HashSet::new(),
+        deleted_rows: false,
+        updated_rows: false,
+        updated_in_place: HashSet::new(),
+    };
+    for version in (from + 1)..=to {
+        let Ok(Some(txn)) = cur.read_transaction_by_version(version).await else {
+            return None;
+        };
+        match txn.operation {
+            Operation::Append { .. } | Operation::ReserveFragments { .. } => {}
+            // A delete removes source rows without changing the ones that
+            // remain, so the view's other rows stay valid: the refresh
+            // evicts exactly the ids that left.
+            Operation::Delete { .. } => delta.deleted_rows = true,
+            // Update outputs carry no new rows, so they are excluded from
+            // the append set like rewrite outputs; the changed rows are
+            // replaced individually below.
+            Operation::Update {
+                removed_fragment_ids,
+                new_fragments,
+                updated_fragments,
+                ..
+            } => {
+                delta.updated_rows = true;
+                // merge_insert reaches here too, and its by-source arm deletes
+                // rows rather than changing them.
+                delta.deleted_rows = true;
+                delta.rewritten.extend(removed_fragment_ids.iter().copied());
+                // Only pre-existing fragment ids are real here; created ones
+                // are placeholders. Rewritten rows are excluded by creation
+                // version below, not by fragment identity.
+                delta
+                    .produced
+                    .extend(updated_fragments.iter().map(|f| f.id));
+                delta
+                    .updated_in_place
+                    .extend(updated_fragments.iter().map(|f| f.id));
+                let _ = new_fragments;
+            }
+            Operation::Rewrite { groups, .. } => {
+                for group in groups {
+                    delta
+                        .rewritten
+                        .extend(group.old_fragments.iter().map(|f| f.id));
+                    delta
+                        .produced
+                        .extend(group.new_fragments.iter().map(|f| f.id));
+                }
+            }
+            _ => return None,
+        }
+    }
+    Some(delta)
+}
+
+/// Fallback pure-append check: every old fragment still present with an
+/// identical signature over the columns the view reads. Compaction, deletes
+/// and updates each break it and force a rebuild; a change to a column the
+/// view does not read leaves it alone, which is what lets this tier pass
+/// deltas the transaction walk cannot.
+fn is_pure_append(old: &Dataset, cur: &Dataset, relevant: &HashSet<i32>) -> bool {
+    let signature = |fragment: &lance::dataset::fragment::FileFragment| {
+        fragment_signature(fragment.metadata(), relevant)
+    };
+    let current: HashSet<(u64, String)> = cur.get_fragments().iter().map(signature).collect();
+    old.get_fragments()
+        .iter()
+        .all(|fragment| current.contains(&signature(fragment)))
+}
+
+/// A fragment's identity as the view observes it: data files and overlays
+/// touching the columns it reads, plus the deletion file. Overlays change no
+/// file path, so they must be part of the signature.
+fn fragment_signature(metadata: &Fragment, relevant: &HashSet<i32>) -> (u64, String) {
+    let touches_relevant =
+        |fields: &[i32]| relevant.is_empty() || fields.iter().any(|id| relevant.contains(id));
+    let mut files: Vec<&str> = metadata
+        .files
+        .iter()
+        .filter(|file| touches_relevant(&file.fields))
+        .map(|file| file.path.as_str())
+        .collect();
+    files.sort_unstable();
+    let mut overlays: Vec<String> = metadata
+        .overlays
+        .iter()
+        .filter(|overlay| touches_relevant(&overlay.data_file.fields))
+        .map(|overlay| format!("{}@{}", overlay.data_file.path, overlay.committed_version))
+        .collect();
+    overlays.sort_unstable();
+    (
+        metadata.id,
+        format!(
+            "{}|{}|{:?}",
+            files.join(","),
+            overlays.join(","),
+            metadata.deletion_file
+        ),
+    )
+}
+
+/// Field ids (with struct descendants) of the source columns the view reads.
+fn relevant_field_ids(source: &Dataset, definition: &MaterializedViewDefinition) -> HashSet<i32> {
+    fn collect(field: &lance_core::datatypes::Field, ids: &mut HashSet<i32>) {
+        ids.insert(field.id);
+        for child in &field.children {
+            collect(child, ids);
+        }
+    }
+    let mut ids = HashSet::new();
+    for input in &definition.inputs {
+        if let Some(field) = source.schema().field(input) {
+            collect(field, &mut ids);
+        }
+    }
+    ids
+}
+
+/// Error if a column the view reads no longer exists in the source.
+fn validate_inputs(source: &Dataset, definition: &MaterializedViewDefinition) -> Result<()> {
+    for input in &definition.inputs {
+        if source.schema().field(input).is_none() {
+            return Err(Error::Schema {
+                message: format!(
+                    "source column '{input}' read by the view no longer exists \
+                     (dropped or renamed in '{}')",
+                    definition.source_table
+                ),
+            });
+        }
+    }
+    Ok(())
+}
+
+/// Reject MemWAL/LSM state on a refresh participant: un-compacted tiers are
+/// invisible to the fragment-planned refresh scan. An active write spec and
+/// retained rows both disqualify; shard directories on storage are the
+/// durable evidence of the latter.
+pub(crate) async fn ensure_no_mem_wal(dataset: &Dataset, role: &str, name: &str) -> Result<()> {
+    let retained = !dataset.list_mem_wal_latest_shard_ids().await?.is_empty();
+    if retained || dataset.mem_wal_index_details().await?.is_some() {
+        return Err(Error::NotSupported {
+            message: format!(
+                "{role} '{name}' has an LSM write spec or retained un-compacted \
+                 rows: rows in un-compacted tiers are invisible to refresh"
+            ),
+        });
+    }
+    Ok(())
+}
+
+async fn open_source(view: &Table, definition: &MaterializedViewDefinition) -> Result<Dataset> {
+    let database = view.database_opt().ok_or_else(|| Error::InvalidInput {
+        message: "the view was not opened through a database connection".into(),
+    })?;
+    let source = database
+        .open_table(OpenTableRequest {
+            name: definition.source_table.clone(),
+            namespace_path: Vec::new(),
+            index_cache_size: None,
+            lance_read_params: None,
+            location: None,
+            namespace_client: None,
+            managed_versioning: None,
+        })
+        .await?;
+    let native = source.as_native().ok_or_else(|| Error::NotSupported {
+        message: "materialized views are supported only on local tables".into(),
+    })?;
+    let dataset = native.dataset.get().await?.as_ref().clone();
+    if !dataset.manifest.uses_stable_row_ids() {
+        return Err(Error::InvalidInput {
+            message: format!(
+                "source table '{}' does not have stable row ids; it is not the \
+                 table this view was declared over",
+                definition.source_table
+            ),
+        });
+    }
+    Ok(dataset)
+}
+
+#[allow(clippy::too_many_arguments)]
+async fn incremental(
+    view_native: &NativeTable,
+    view_ds: &Dataset,
+    source_ds: &Dataset,
+    source_version: u64,
+    source_ts: u128,
+    increment: Increment,
+    definition: &MaterializedViewDefinition,
+    watermark: Option<u64>,
+    expected_incarnation: Option<&str>,
+) -> Result<Option<RefreshMaterializedViewResult>> {
+    let new_fragments = increment.appended;
+    let watermark_version = watermark.unwrap_or(0);
+    // Provenance ids this refresh removes: dropped rows, plus changed rows
+    // recomputed in the same commit. One view row per source row makes the
+    // eviction exact. Staged, not committed: removals ride with the rows
+    // that replace them, so a reader never sees the view without either.
+    let mut eviction = Eviction::new(view_ds, EVICTION_CHUNK);
+    let mut updated_rows = false;
+    if (increment.evict_deleted || increment.replace_updated)
+        && let Some(watermark) = watermark
+    {
+        let delta = source_ds
+            .delta()
+            .with_begin_version(watermark)
+            .with_end_version(source_version)
+            .build()?;
+        // Reconciling holds the delta's provenance ids in staged deletion
+        // vectors; past the cap, the streamed rebuild is the bounded path.
+        let cap = eviction_rebuild_cap();
+        let mut evicted = 0usize;
+        if increment.evict_deleted {
+            let mut stream = delta.get_deleted_row_ids().await?;
+            while let Some(batch) = stream.try_next().await? {
+                let ids = row_ids_of(&batch)?;
+                evicted += ids.len();
+                if evicted > cap {
+                    return Ok(None);
+                }
+                eviction.push(ids).await?;
+            }
+        }
+        if increment.replace_updated {
+            // Ids only: `get_updated_rows` carries every column of every
+            // updated row, and discovery needs none of them.
+            let mut scanner = source_ds.scan();
+            scanner.with_row_id().project(&[ROW_CREATED_AT_VERSION])?;
+            // A fixed bound: the configured default could make one discovery
+            // batch arbitrarily large before the fallback cap is consulted.
+            scanner.batch_size(8192);
+            scanner.filter(&format!(
+                "{ROW_CREATED_AT_VERSION} <= {watermark} 
+                 AND {ROW_LAST_UPDATED_AT_VERSION} > {watermark} 
+                 AND {ROW_LAST_UPDATED_AT_VERSION} <= {source_version}"
+            ))?;
+            let mut stream = scanner.try_into_stream().await?;
+            while let Some(batch) = stream.try_next().await? {
+                let ids = row_ids_of(&batch)?;
+                evicted += ids.len();
+                if evicted > cap {
+                    return Ok(None);
+                }
+                updated_rows |= !ids.is_empty();
+                eviction.push(ids).await?;
+            }
+        }
+    }
+    let eviction = eviction.finish().await?;
+
+    // The cap counts rows already materialized, in first-materialized order.
+    let remaining = match definition.limit {
+        Some(limit) => {
+            let held = view_ds.count_rows(None).await? as u64;
+            Some(limit.saturating_sub(held))
+        }
+        None => None,
+    };
+
+    let mut result = RefreshMaterializedViewResult {
+        mode: RefreshMode::Incremental,
+        rows_written: 0,
+        source_version,
+        version: view_ds.version().version,
+    };
+    let nothing_to_add = (new_fragments.is_empty() && !updated_rows) || remaining == Some(0);
+    if nothing_to_add && eviction.is_none() {
+        result.version = stamp_watermark(
+            view_native,
+            view_ds.clone(),
+            source_version,
+            source_ts,
+            expected_incarnation,
+        )
+        .await?;
+        return Ok(Some(result));
+    }
+    // Rows left but none arrive: the removals still have to be published.
+    if nothing_to_add {
+        let filter = refresh_filter(&empty_keys(view_ds)?)?;
+        let published = publish(
+            view_ds,
+            eviction,
+            Vec::new(),
+            Some(filter),
+            expected_incarnation,
+        )
+        .await?;
+        result.version = stamp_watermark(
+            view_native,
+            published,
+            source_version,
+            source_ts,
+            expected_incarnation,
+        )
+        .await?;
+        return Ok(Some(result));
+    }
+
+    // Appends carry the view's schema as it stands; the watermark moves in a
+    // follow-up commit (see the module docs for the crash window).
+    let schema = Arc::new(ArrowSchema::from(view_ds.schema()));
+    let rows_written = Arc::new(AtomicU64::new(0));
+    // compute_stream counts what it produces; the truncation below can drop
+    // some of that, so the written count comes from the tee instead.
+    let computed = Arc::new(AtomicU64::new(0));
+    let mut stream = compute_stream(
+        source_ds,
+        definition,
+        RowScope {
+            fragments: Some(new_fragments),
+            // An update rewrites whole fragments, so a fragment new at head
+            // can hold rows the view already has. Their creation version
+            // does not change, so it -- not fragment identity -- says which
+            // rows are new.
+            created_after: increment.replace_updated.then_some(watermark_version),
+            limit: remaining,
+            ..Default::default()
+        },
+        schema.clone(),
+        computed.clone(),
+    )
+    .await?;
+
+    // The updated rows' current values, computed the same way and appended
+    // in the same commit as the new fragments' rows.
+    if updated_rows {
+        let recomputed = compute_stream(
+            source_ds,
+            definition,
+            RowScope {
+                updated_between: Some((watermark_version, source_version)),
+                ..Default::default()
+            },
+            schema.clone(),
+            computed.clone(),
+        )
+        .await?;
+        stream = Box::pin(RecordBatchStreamAdapter::new(
+            schema.clone(),
+            recomputed.chain(stream),
+        ));
+    }
+
+    // Nothing survived the filter: the watermark still has to advance or the
+    // same fragments would be rescanned forever, but any removals still do.
+    let Some(first) = stream.try_next().await? else {
+        let published = if eviction.is_some() {
+            publish(
+                view_ds,
+                eviction,
+                Vec::new(),
+                Some(refresh_filter(&empty_keys(view_ds)?)?),
+                expected_incarnation,
+            )
+            .await?
+        } else {
+            view_ds.clone()
+        };
+        result.version = stamp_watermark(
+            view_native,
+            published,
+            source_version,
+            source_ts,
+            expected_incarnation,
+        )
+        .await?;
+        return Ok(Some(result));
+    };
+    let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(
+        schema,
+        futures::stream::iter([Ok(first)]).chain(stream),
+    ));
+
+    // Any two refreshes of one view must not both commit: the filter's
+    // shared token makes lance reject the loser on key overlap however
+    // their planned rows relate.
+    let keys = Arc::new(StdMutex::new(KeyExistenceFilterBuilder::new(vec![
+        source_row_id_field_id(view_ds)?,
+    ])));
+    let stream = collect_source_row_ids(stream, keys.clone(), rows_written.clone());
+
+    let ds = Arc::new(view_ds.clone());
+    let write_txn = InsertBuilder::new(WriteDestination::Dataset(ds.clone()))
+        .with_params(&WriteParams {
+            mode: WriteMode::Append,
+            ..Default::default()
+        })
+        .execute_uncommitted_stream(stream)
+        .await?;
+    let Operation::Append {
+        fragments: new_fragments,
+    } = write_txn.operation
+    else {
+        return Err(Error::Runtime {
+            message: "expected an append when staging the view's new rows".into(),
+        });
+    };
+    let filter = refresh_filter(&keys)?;
+    let appended = publish(
+        view_ds,
+        eviction,
+        new_fragments,
+        Some(filter),
+        expected_incarnation,
+    )
+    .await?;
+    result.rows_written = rows_written.load(Ordering::Relaxed);
+    result.version = stamp_watermark(
+        view_native,
+        appended,
+        source_version,
+        source_ts,
+        expected_incarnation,
+    )
+    .await?;
+    Ok(Some(result))
+}
+
+async fn rebuild(
+    view_native: &NativeTable,
+    view_ds: &Dataset,
+    source_ds: &Dataset,
+    source_version: u64,
+    source_ts: u128,
+    definition: &MaterializedViewDefinition,
+    expected_incarnation: Option<&str>,
+) -> Result<RefreshMaterializedViewResult> {
+    let rows_written = Arc::new(AtomicU64::new(0));
+    let schema = Arc::new(ArrowSchema::from(view_ds.schema()));
+    let stream = compute_stream(
+        source_ds,
+        definition,
+        RowScope {
+            limit: definition.limit,
+            ..Default::default()
+        },
+        schema,
+        rows_written.clone(),
+    )
+    .await?;
+    let keys = Arc::new(StdMutex::new(KeyExistenceFilterBuilder::new(vec![
+        source_row_id_field_id(view_ds)?,
+    ])));
+    let stream = collect_source_row_ids(stream, keys.clone(), Arc::new(AtomicU64::new(0)));
+    // Every rebuild is one fragment swap, indexed or not: an Update commit
+    // carries no schema metadata, so it cannot erase a definition update
+    // that raced in the way an overwrite (which adopts its stream's schema)
+    // durably would -- and it must land on the planned generation or abort.
+    let replaced =
+        replace_retaining_indices(view_ds.clone(), stream, keys, expected_incarnation).await?;
+    let version = stamp_watermark(
+        view_native,
+        replaced,
+        source_version,
+        source_ts,
+        expected_incarnation,
+    )
+    .await?;
+    Ok(RefreshMaterializedViewResult {
+        mode: RefreshMode::Rebuild,
+        rows_written: rows_written.load(Ordering::Relaxed),
+        source_version,
+        version,
+    })
+}
+
+/// Replace all of the view's data in one commit that retains its index
+/// definitions: new fragments staged uncommitted, one `Update` removing every
+/// old fragment. `Update` prunes index bitmaps only for modified fields and
+/// none are modified here, so readers never see the view unindexed or empty.
+async fn replace_retaining_indices(
+    view_ds: Dataset,
+    stream: SendableRecordBatchStream,
+    keys: Arc<StdMutex<KeyExistenceFilterBuilder>>,
+    expected_incarnation: Option<&str>,
+) -> Result<Dataset> {
+    let ds = Arc::new(view_ds);
+    let read_version = ds.version().version;
+    #[cfg(test)]
+    tests::hold_before_publish(ds.uri()).await;
+    ensure_incarnation(&ds, expected_incarnation, ds.uri()).await?;
+    let removed_fragment_ids: Vec<u64> = ds.get_fragments().iter().map(|f| f.id() as u64).collect();
+
+    let write_txn = InsertBuilder::new(WriteDestination::Dataset(ds.clone()))
+        .with_params(&WriteParams {
+            mode: WriteMode::Append,
+            ..Default::default()
+        })
+        .execute_uncommitted_stream(stream)
+        .await?;
+    let Operation::Append {
+        fragments: new_fragments,
+    } = write_txn.operation
+    else {
+        return Err(Error::Runtime {
+            message: "expected an append when staging the view's replacement rows".into(),
+        });
+    };
+
+    // Built only now: the tee fills as the staging drains the stream.
+    let filter = refresh_filter(&keys)?;
+    let transaction = Transaction::new(
+        read_version,
+        Operation::Update {
+            removed_fragment_ids,
+            updated_fragments: Vec::new(),
+            new_fragments,
+            fields_modified: Vec::new(),
+            compacted_sstables: Vec::new(),
+            fields_for_preserving_frag_bitmap: Vec::new(),
+            update_mode: None,
+            // Two refreshes that materialized the same source rows must not
+            // both land -- a raced first rebuild would double the view.
+            inserted_rows_filter: Some(filter),
+            updated_fragment_offsets: None,
+        },
+        None,
+    );
+    let committed = CommitBuilder::new(WriteDestination::Dataset(ds))
+        .execute(transaction)
+        .await?;
+    if committed.version().version != read_version + 1 {
+        return Err(Error::Runtime {
+            message: format!(
+                "a concurrent commit raced this refresh (view version {}); the 
+                 refresh is unrecorded and the next one will rebuild",
+                committed.version().version
+            ),
+        });
+    }
+    Ok(committed)
+}
+
+/// Record that the view now reflects `source_version`, including the view
+/// Refuse to act on a view that is not `expected`'s incarnation, judged from
+/// the latest stored manifest. Not a commit condition; see
+/// `RefreshMaterializedViewBuilder::expect_incarnation`.
+async fn ensure_incarnation(view_ds: &Dataset, expected: Option<&str>, what: &str) -> Result<()> {
+    let Some(expected) = expected else {
+        return Ok(());
+    };
+    let mut latest = view_ds.clone();
+    latest.checkout_latest().await?;
+    match latest.schema().metadata.get(INCARNATION_META_KEY) {
+        Some(actual) if actual == expected => Ok(()),
+        Some(_) => Err(Error::Runtime {
+            message: format!(
+                "materialized view '{what}' is not the incarnation this refresh was \
+                 requested for: it was dropped and recreated"
+            ),
+        }),
+        None => Err(Error::Runtime {
+            message: format!(
+                "materialized view '{what}' carries no incarnation token: its schema \
+                 metadata was replaced since the token was captured"
+            ),
+        }),
+    }
+}
+
+/// version this very commit produces. The version is predicted and then
+/// verified; on a mismatch another commit raced in between, and the stamp
+/// ABORTS rather than certify that commit as the refresh's own generation.
+/// The view is left visibly unstamped, so the next refresh rebuilds.
+async fn stamp_watermark(
+    view_native: &NativeTable,
+    mut dataset: Dataset,
+    source_version: u64,
+    source_ts: u128,
+    expected_incarnation: Option<&str>,
+) -> Result<u64> {
+    ensure_incarnation(&dataset, expected_incarnation, dataset.uri()).await?;
+    let predicted = dataset.version().version + 1;
+    // A view with no token (declared before tokens existed, or its metadata
+    // replaced wholesale) starts a new incarnation here.
+    let incarnation = dataset
+        .schema()
+        .metadata
+        .get(INCARNATION_META_KEY)
+        .cloned()
+        .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
+    dataset
+        .update_schema_metadata([
+            (INCARNATION_META_KEY.to_string(), Some(incarnation)),
+            (
+                SOURCE_VERSION_META_KEY.to_string(),
+                Some(source_version.to_string()),
+            ),
+            (
+                SOURCE_VERSION_TS_META_KEY.to_string(),
+                Some(source_ts.to_string()),
+            ),
+            (
+                REFRESHED_AT_MS_META_KEY.to_string(),
+                Some(now_ms().to_string()),
+            ),
+            (
+                VIEW_VERSION_META_KEY.to_string(),
+                Some(predicted.to_string()),
+            ),
+        ])
+        .await?;
+    let actual = dataset.version().version;
+    if actual != predicted {
+        return Err(Error::Runtime {
+            message: format!(
+                "a concurrent commit raced this refresh (view version {actual}, \
+                 expected {predicted}); the refresh is unrecorded and the next \
+                 one will rebuild"
+            ),
+        });
+    }
+    view_native.dataset.update(dataset);
+    Ok(predicted)
+}
+
+fn now_ms() -> u128 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_millis()
+}
+
+/// Evaluate the definition over `source`, restricted to `fragments` when
+/// given, as batches in the view's schema. The filter is pushed into the
+/// scan; projections are `(name, expression)` pairs, never spliced into SQL;
+/// [`SOURCE_ROW_ID_COLUMN`] is filled by the scan's row id.
+/// Which source rows a compute pass reads.
+#[derive(Default)]
+struct RowScope {
+    /// Read only these fragments.
+    fragments: Option<Vec<Fragment>>,
+    /// Read only rows created after this version.
+    created_after: Option<u64>,
+    /// Read only rows changed in place after the first version and no later
+    /// than the second.
+    updated_between: Option<(u64, u64)>,
+    /// Stop after this many rows.
+    limit: Option<u64>,
+}
+
+async fn compute_stream(
+    source: &Dataset,
+    definition: &MaterializedViewDefinition,
+    scope: RowScope,
+    schema: SchemaRef,
+    rows_written: Arc<AtomicU64>,
+) -> Result<SendableRecordBatchStream> {
+    let RowScope {
+        fragments,
+        created_after,
+        updated_between,
+        limit,
+    } = scope;
+    let mut scanner = source.scan();
+    if let Some(fragments) = fragments {
+        scanner.with_fragments(fragments);
+    }
+    scanner.with_row_id();
+    // Narrowing keeps the definition's filter, so a row updated out of the
+    // view simply does not come back. Changed rows are named by the predicate
+    // `DatasetDelta::get_updated_rows` uses, not its streamed ids: an id list
+    // grows with the delta, this does not.
+    let updated_filter = updated_between.map(|(from, to)| {
+        format!(
+            "{ROW_CREATED_AT_VERSION} <= {from} \
+             AND {ROW_LAST_UPDATED_AT_VERSION} > {from} \
+             AND {ROW_LAST_UPDATED_AT_VERSION} <= {to}"
+        )
+    });
+    let created_filter =
+        created_after.map(|version| format!("{ROW_CREATED_AT_VERSION} > {version}"));
+    let clauses: Vec<String> = definition
+        .filter
+        .clone()
+        .map(|f| format!("({f})"))
+        .into_iter()
+        .chain(updated_filter)
+        .chain(created_filter)
+        .collect();
+    if !clauses.is_empty() {
+        scanner.filter(&clauses.join(" AND "))?;
+    }
+    let transforms: Vec<(&str, &str)> = definition
+        .projections
+        .iter()
+        .map(|p| (p.output.as_str(), p.expression.as_str()))
+        .collect();
+    scanner.project_with_transform(&transforms)?;
+    // A scan reads a limit of zero as no limit at all, so a view capped at
+    // nothing is answered without one.
+    if limit == Some(0) {
+        return Ok(Box::pin(RecordBatchStreamAdapter::new(
+            schema.clone(),
+            futures::stream::empty(),
+        )));
+    }
+    if let Some(limit) = limit {
+        let limit = i64::try_from(limit).map_err(|_| Error::InvalidInput {
+            message: format!("view limit {limit} exceeds the maximum of {}", i64::MAX),
+        })?;
+        scanner.limit(Some(limit), None)?;
+    }
+
+    let out_schema = schema.clone();
+    let mapped = scanner.try_into_stream().await?.map(move |batch| {
+        let batch = batch.map_err(|e| DataFusionError::External(Box::new(e)))?;
+        let mut columns = Vec::with_capacity(out_schema.fields().len());
+        for field in out_schema.fields() {
+            let name = if field.name() == SOURCE_ROW_ID_COLUMN {
+                ROW_ID
+            } else {
+                field.name()
+            };
+            let column = batch.column_by_name(name).ok_or_else(|| {
+                DataFusionError::Internal(format!(
+                    "view column '{}' is not produced by the view's definition",
+                    field.name()
+                ))
+            })?;
+            columns.push(column.clone());
+        }
+        rows_written.fetch_add(batch.num_rows() as u64, Ordering::Relaxed);
+        Ok(RecordBatch::try_new(out_schema.clone(), columns)?)
+    });
+    Ok(Box::pin(RecordBatchStreamAdapter::new(schema, mapped)))
+}
+
+/// Commit the view's removals and additions as one change, on the exact
+/// generation the refresh planned from. Lance rejects an overlapping
+/// provenance key, but an unrelated write to the view is not a key conflict,
+/// so the generation is checked here too.
+async fn publish(
+    view_ds: &Dataset,
+    eviction: Option<(Vec<Fragment>, Vec<u64>)>,
+    new_fragments: Vec<Fragment>,
+    keys: Option<KeyExistenceFilter>,
+    expected_incarnation: Option<&str>,
+) -> Result<Dataset> {
+    let planned = view_ds.version().version;
+    #[cfg(test)]
+    tests::hold_before_publish(view_ds.uri()).await;
+    #[cfg(test)]
+    tests::hold_until_peers_planned();
+    ensure_incarnation(view_ds, expected_incarnation, view_ds.uri()).await?;
+    let (updated_fragments, removed_fragment_ids) = eviction.unwrap_or_default();
+    let committed = CommitBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone())))
+        .execute(Transaction::new(
+            planned,
+            Operation::Update {
+                removed_fragment_ids,
+                updated_fragments,
+                new_fragments,
+                fields_modified: Vec::new(),
+                compacted_sstables: Vec::new(),
+                fields_for_preserving_frag_bitmap: Vec::new(),
+                update_mode: None,
+                inserted_rows_filter: keys,
+                updated_fragment_offsets: None,
+            },
+            None,
+        ))
+        .await?;
+    if committed.version().version != planned + 1 {
+        return Err(Error::Runtime {
+            message: format!(
+                "a concurrent commit raced this refresh (view version {}); 
+                 the refresh is unrecorded and the next one will rebuild",
+                committed.version().version
+            ),
+        });
+    }
+    Ok(committed)
+}
+
+/// A delta batch's row id column, borrowed: a delta batch is as large as one
+/// fragment's deletions, so copying it out would be the unbounded step the
+/// chunking exists to avoid.
+fn row_ids_of(batch: &RecordBatch) -> Result<&UInt64Array> {
+    let column = batch.column_by_name(ROW_ID).ok_or_else(|| Error::Runtime {
+        message: format!("'{ROW_ID}' is missing from a delta batch"),
+    })?;
+    column
+        .as_primitive_opt::<UInt64Type>()
+        .ok_or_else(|| Error::Runtime {
+            message: "row ids are not UInt64".into(),
+        })
+}
+
+/// A provenance id no source row can hold, carried in every refresh's
+/// inserted-rows filter: any two refreshes of one view overlap on it, so
+/// the loser of a race conflicts at commit whatever rows each planned.
+const REFRESH_TOKEN_ID: u64 = u64::MAX;
+
+/// A builder with no collected keys, for publishes that only remove rows.
+fn empty_keys(view_ds: &Dataset) -> Result<Arc<StdMutex<KeyExistenceFilterBuilder>>> {
+    Ok(Arc::new(StdMutex::new(KeyExistenceFilterBuilder::new(
+        vec![source_row_id_field_id(view_ds)?],
+    ))))
+}
+
+/// An inserted-rows filter holding the refresh token plus whatever the
+/// tee collected.
+fn refresh_filter(keys: &Arc<StdMutex<KeyExistenceFilterBuilder>>) -> Result<KeyExistenceFilter> {
+    let mut keys = keys.lock().map_err(|_| Error::Runtime {
+        message: "the provenance key filter was poisoned mid-refresh".into(),
+    })?;
+    keys.insert(KeyValue::UInt64(REFRESH_TOKEN_ID))
+        .map_err(|e| Error::Runtime {
+            message: format!("failed to mark the refresh's filter: {e}"),
+        })?;
+    Ok(keys.build())
+}
+
+/// Reconciled ids past this fall back to the streamed rebuild, whose memory
+/// does not grow with the delta.
+fn eviction_rebuild_cap() -> usize {
+    #[cfg(test)]
+    if let Some(cap) = tests::eviction_cap_override() {
+        return cap;
+    }
+    4 * 1024 * 1024
+}
+
+/// Provenance ids per staged eviction: the delete predicate carries one
+/// literal per id, so a whole delta at once is unbounded. Each chunk costs a
+/// pass over the view's provenance column, which is why it is not smaller.
+const EVICTION_CHUNK: usize = 64 * 1024;
+
+/// Accumulates the view's removals from bounded chunks of provenance ids into
+/// the one set of fragment changes the refresh publishes.
+struct Eviction {
+    /// The view as the chunks staged so far leave it. A chunk's delete has to
+    /// see the deletion vectors the earlier ones wrote, or it stages a
+    /// fragment that drops them.
+    snapshot: Dataset,
+    chunk: usize,
+    updated: HashMap<u64, Fragment>,
+    removed: Vec<u64>,
+    pending: Vec<u64>,
+    staged: bool,
+    /// Largest the buffer ever got, which is the bound under test.
+    #[cfg(test)]
+    peak: usize,
+}
+
+impl Eviction {
+    fn new(view_ds: &Dataset, chunk: usize) -> Self {
+        Self {
+            snapshot: view_ds.clone(),
+            chunk,
+            updated: HashMap::new(),
+            removed: Vec::new(),
+            pending: Vec::with_capacity(chunk),
+            staged: false,
+            #[cfg(test)]
+            peak: 0,
+        }
+    }
+
+    /// Fill a chunk at a time. Taking the batch whole and splitting it would
+    /// hold a fragment's worth of ids and recopy the tail per chunk.
+    async fn push(&mut self, ids: &UInt64Array) -> Result<()> {
+        for id in ids.values() {
+            self.pending.push(*id);
+            #[cfg(test)]
+            {
+                self.peak = self.peak.max(self.pending.len());
+            }
+            if self.pending.len() == self.chunk {
+                self.flush().await?;
+            }
+        }
+        Ok(())
+    }
+
+    /// Stage what is buffered, keeping the buffer's allocation.
+    async fn flush(&mut self) -> Result<()> {
+        let mut chunk = std::mem::take(&mut self.pending);
+        let staged = self.stage(&chunk).await;
+        chunk.clear();
+        self.pending = chunk;
+        staged
+    }
+
+    /// The staged fragment changes, or `None` where nothing was evicted.
+    async fn finish(mut self) -> Result<Option<(Vec<Fragment>, Vec<u64>)>> {
+        if !self.pending.is_empty() {
+            self.flush().await?;
+        }
+        if !self.staged {
+            return Ok(None);
+        }
+        let mut updated: Vec<Fragment> = self.updated.into_values().collect();
+        updated.sort_unstable_by_key(|f| f.id);
+        Ok(Some((updated, self.removed)))
+    }
+
+    async fn stage(&mut self, ids: &[u64]) -> Result<()> {
+        let (updated, removed) = stage_eviction(&self.snapshot, ids).await?;
+        self.snapshot = advance(&self.snapshot, &updated);
+        for fragment in updated {
+            self.updated.insert(fragment.id, fragment);
+        }
+        self.removed.extend(removed);
+        self.staged = true;
+        Ok(())
+    }
+}
+
+/// The view as staged removals leave it, without committing them. Fragments
+/// are replaced in place so the fragment bitmap stays in step; an emptied one
+/// is left alone, since the delta names each provenance id only once.
+fn advance(view_ds: &Dataset, updated: &[Fragment]) -> Dataset {
+    if updated.is_empty() {
+        return view_ds.clone();
+    }
+    let by_id: HashMap<u64, &Fragment> = updated.iter().map(|f| (f.id, f)).collect();
+    let fragments = view_ds
+        .manifest
+        .fragments
+        .iter()
+        .map(|f| by_id.get(&f.id).map_or_else(|| f.clone(), |u| (*u).clone()))
+        .collect();
+    let mut manifest = view_ds.manifest.as_ref().clone();
+    manifest.fragments = Arc::new(fragments);
+    let mut snapshot = view_ds.clone();
+    snapshot.manifest = Arc::new(manifest);
+    snapshot
+}
+
+/// The fragment changes that remove the view's rows for `ids`, staged rather
+/// than committed so they can ride in the refresh's single data commit.
+async fn stage_eviction(view_ds: &Dataset, ids: &[u64]) -> Result<(Vec<Fragment>, Vec<u64>)> {
+    // An expression rather than SQL text: the id list is a value here, not a
+    // predicate string that grows with the delta and has to be parsed.
+    let predicate = col(SOURCE_ROW_ID_COLUMN).in_list(
+        ids.iter()
+            .map(|id| lit(ScalarValue::UInt64(Some(*id))))
+            .collect(),
+        false,
+    );
+    let staged = DeleteBuilder::from_expr(Arc::new(view_ds.clone()), predicate)
+        .execute_uncommitted()
+        .await?;
+    let Operation::Delete {
+        updated_fragments,
+        deleted_fragment_ids,
+        ..
+    } = staged.transaction.operation
+    else {
+        return Err(Error::Runtime {
+            message: "expected a delete when staging the view's evictions".into(),
+        });
+    };
+    Ok((updated_fragments, deleted_fragment_ids))
+}
+
+fn source_row_id_field_id(view_ds: &Dataset) -> Result<i32> {
+    view_ds
+        .schema()
+        .field(SOURCE_ROW_ID_COLUMN)
+        .map(|f| f.id)
+        .ok_or_else(|| Error::Runtime {
+            message: format!("the view has no '{SOURCE_ROW_ID_COLUMN}' column"),
+        })
+}
+
+/// Tee the provenance ids of everything written into `keys`.
+fn collect_source_row_ids(
+    stream: SendableRecordBatchStream,
+    keys: Arc<StdMutex<KeyExistenceFilterBuilder>>,
+    written: Arc<AtomicU64>,
+) -> SendableRecordBatchStream {
+    let schema = stream.schema();
+    let mapped = stream.map(move |batch| {
+        let batch = batch?;
+        let column = batch
+            .column_by_name(SOURCE_ROW_ID_COLUMN)
+            .ok_or_else(|| {
+                DataFusionError::Internal(format!(
+                    "'{SOURCE_ROW_ID_COLUMN}' is missing from the rows being written"
+                ))
+            })?
+            .as_primitive_opt::<UInt64Type>()
+            .ok_or_else(|| {
+                DataFusionError::Internal(format!("'{SOURCE_ROW_ID_COLUMN}' is not a uint64"))
+            })?;
+        let mut keys = keys
+            .lock()
+            .map_err(|_| DataFusionError::Internal("provenance key filter poisoned".into()))?;
+        for id in column.values() {
+            keys.insert(KeyValue::UInt64(*id))
+                .map_err(|e| DataFusionError::Internal(e.to_string()))?;
+        }
+        written.fetch_add(batch.num_rows() as u64, Ordering::Relaxed);
+        Ok(batch)
+    });
+    Box::pin(RecordBatchStreamAdapter::new(schema, mapped))
+}
+
+#[cfg(test)]
+mod tests {
+
+    /// Park a refresh between planning and publication so a test can move
+    /// the view underneath it. Inert unless [`DRIFT_TARGET`] names this view.
+    pub(super) async fn hold_before_publish(uri: &str) {
+        {
+            let mut target = DRIFT_TARGET.lock().unwrap();
+            if target.as_deref() != Some(uri) {
+                return;
+            }
+            // Take it: memory:// uris are relative and repeat across tests, so
+            // leaving it armed would park an unrelated refresh forever.
+            *target = None;
+        }
+        DRIFT_PLANNED.notify_one();
+        DRIFT_RELEASED.notified().await;
+    }
+
+    /// The rendezvous below is one global pair, so the cases that use it run
+    /// one at a time rather than trading each other's signals.
+    pub(super) static EVICTION_CAP: StdMutex<Option<usize>> = StdMutex::new(None);
+    pub(super) fn eviction_cap_override() -> Option<usize> {
+        *EVICTION_CAP.lock().unwrap()
+    }
+
+    pub(super) static DRIFT_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
+    pub(super) static DRIFT_TARGET: StdMutex<Option<String>> = StdMutex::new(None);
+    pub(super) static DRIFT_PLANNED: tokio::sync::Notify = tokio::sync::Notify::const_new();
+    pub(super) static DRIFT_RELEASED: tokio::sync::Notify = tokio::sync::Notify::const_new();
+
+    /// Block until every participant in a cross-process race has planned and
+    /// staged its write, so the commits they then attempt genuinely contend
+    /// rather than depending on the scheduler to overlap them. Inert unless
+    /// `MV_RACE_SYNC` names a directory shared by the participants.
+    pub(super) fn hold_until_peers_planned() {
+        let (Ok(dir), Ok(tag), Ok(peers)) = (
+            std::env::var("MV_RACE_SYNC"),
+            std::env::var("MV_RACE_TAG"),
+            std::env::var("MV_RACE_PEERS"),
+        ) else {
+            return;
+        };
+        let dir = std::path::PathBuf::from(dir);
+        let peers: usize = peers.parse().unwrap();
+        std::fs::write(dir.join(format!("planned-{tag}")), b"1").unwrap();
+        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(120);
+        while planned_count(&dir) < peers {
+            assert!(
+                std::time::Instant::now() < deadline,
+                "peers never reached the commit boundary"
+            );
+            std::thread::sleep(std::time::Duration::from_millis(2));
+        }
+    }
+
+    fn planned_count(dir: &std::path::Path) -> usize {
+        std::fs::read_dir(dir)
+            .map(|entries| {
+                entries
+                    .filter_map(|e| e.ok())
+                    .filter(|e| e.file_name().to_string_lossy().starts_with("planned-"))
+                    .count()
+            })
+            .unwrap_or(0)
+    }
+    use arrow_array::{Int32Array, record_batch};
+    use futures::TryStreamExt;
+    use lance::dataset::NewColumnTransform;
+    use lance_file::version::LanceFileVersion;
+
+    use super::*;
+    use crate::connect;
+    use crate::connection::Connection;
+    use crate::index::Index;
+    use crate::index::scalar::BTreeIndexBuilder;
+    use crate::materialized_view::MaterializedView;
+    use crate::query::{ExecutableQuery, QueryBase, Select};
+    use crate::table::{CompactionOptions, OptimizeAction};
+
+    async fn db_with_source(values: Vec<i32>) -> (Connection, Table) {
+        let conn = connect("memory://").execute().await.unwrap();
+        let batch = record_batch!(("x", Int32, values)).unwrap();
+        let table = conn
+            .create_table("src", batch)
+            .write_options(crate::materialized_view::tests::stable_row_ids())
+            .execute()
+            .await
+            .unwrap();
+        (conn, table)
+    }
+
+    async fn doubled_view(conn: &Connection) -> MaterializedView {
+        conn.create_materialized_view("doubled", "src")
+            .select([("x", "x"), ("twice", "x * 2")])
+            .execute()
+            .await
+            .unwrap()
+    }
+
+    /// A refreshed doubled view over a source holding `values`.
+    async fn refreshed_doubled(values: Vec<i32>) -> (Connection, Table, MaterializedView) {
+        let (conn, source) = db_with_source(values).await;
+        let view = doubled_view(&conn).await;
+        view.refresh().execute().await.unwrap();
+        (conn, source, view)
+    }
+
+    async fn read(table: &Table, column: &str) -> Vec<i32> {
+        let batches = table
+            .query()
+            .select(Select::columns(&[column]))
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        let mut values: Vec<i32> = batches
+            .iter()
+            .flat_map(|batch| {
+                batch[column]
+                    .as_any()
+                    .downcast_ref::<Int32Array>()
+                    .unwrap()
+                    .iter()
+                    .flatten()
+                    .collect::<Vec<_>>()
+            })
+            .collect();
+        values.sort();
+        values
+    }
+
+    async fn append(table: &Table, values: Vec<i32>) {
+        let batch = record_batch!(("x", Int32, values)).unwrap();
+        table.add(batch).execute().await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_first_refresh_materializes_the_view() {
+        let (conn, _) = db_with_source(vec![1, 2, 3]).await;
+        let view = doubled_view(&conn).await;
+
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Rebuild);
+        assert_eq!(result.rows_written, 3);
+        assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6]);
+
+        // The watermark survives on the stored schema, not just the handle.
+        let reopened = conn.open_materialized_view("doubled").await.unwrap();
+        let again = reopened.refresh().execute().await.unwrap();
+        assert_eq!(again.mode, RefreshMode::NoOp);
+        assert_eq!(again.rows_written, 0);
+    }
+
+    #[tokio::test]
+    async fn test_filter_selects_the_source_rows() {
+        let (conn, _) = db_with_source(vec![1, 20, 3, 40]).await;
+        let view = conn
+            .create_materialized_view("big", "src")
+            .select([("x", "x")])
+            .only_if("x > 10")
+            .execute()
+            .await
+            .unwrap();
+
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.rows_written, 2);
+        assert_eq!(read(view.table(), "x").await, vec![20, 40]);
+    }
+
+    #[tokio::test]
+    async fn test_append_refreshes_incrementally() {
+        let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await;
+
+        append(&source, vec![5]).await;
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Incremental);
+        assert_eq!(result.rows_written, 1);
+        assert_eq!(read(view.table(), "twice").await, vec![2, 4, 10]);
+    }
+
+    #[tokio::test]
+    async fn test_incremental_applies_the_filter() {
+        let (conn, source) = db_with_source(vec![1, 20]).await;
+        let view = conn
+            .create_materialized_view("big", "src")
+            .select([("x", "x")])
+            .only_if("x > 10")
+            .execute()
+            .await
+            .unwrap();
+        view.refresh().execute().await.unwrap();
+
+        append(&source, vec![3, 30]).await;
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Incremental);
+        assert_eq!(result.rows_written, 1);
+        assert_eq!(read(view.table(), "x").await, vec![20, 30]);
+    }
+
+    /// The watermark has to advance even when no appended row matches, or the
+    /// same fragments would be rescanned by every later refresh.
+    #[tokio::test]
+    async fn test_incremental_with_nothing_matching_advances_the_watermark() {
+        let (conn, source) = db_with_source(vec![20]).await;
+        let view = conn
+            .create_materialized_view("big", "src")
+            .select([("x", "x")])
+            .only_if("x > 10")
+            .execute()
+            .await
+            .unwrap();
+        view.refresh().execute().await.unwrap();
+
+        append(&source, vec![1, 2]).await;
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Incremental);
+        assert_eq!(result.rows_written, 0);
+
+        let again = view.refresh().execute().await.unwrap();
+        assert_eq!(again.mode, RefreshMode::NoOp);
+    }
+
+    /// Unlike a computed column, a view reflects source mutation: an update
+    /// rebuilds rather than going stale.
+    #[tokio::test]
+    async fn test_update_replaces_the_rows_it_changed() {
+        let (_conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await;
+
+        // An update changes rows in place, so the view replaces exactly
+        // those rows and leaves the rest of what it holds alone.
+        source
+            .update()
+            .column("x", "20")
+            .only_if("x = 2")
+            .execute()
+            .await
+            .unwrap();
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Incremental);
+        assert_eq!(result.rows_written, 1, "only the changed row is recomputed");
+        assert_eq!(read(view.table(), "twice").await, vec![2, 6, 40]);
+    }
+
+    /// Legacy storage cannot serve the row-version columns update discovery
+    /// scans; appends stay incremental, updates rebuild rather than fail.
+    #[tokio::test]
+    async fn test_a_legacy_storage_source_is_reconciled() {
+        let conn = connect("memory://").execute().await.unwrap();
+        let batch = record_batch!(("x", Int32, [1, 2, 3])).unwrap();
+        let source = conn
+            .create_table("legacy_src", batch)
+            .write_options(crate::table::WriteOptions {
+                lance_write_params: Some(lance::dataset::WriteParams {
+                    enable_stable_row_ids: true,
+                    data_storage_version: Some(LanceFileVersion::Legacy),
+                    ..Default::default()
+                }),
+            })
+            .execute()
+            .await
+            .unwrap();
+        let view = conn
+            .create_materialized_view("legacy_doubled", "legacy_src")
+            .select([("x", "x"), ("twice", "x * 2")])
+            .execute()
+            .await
+            .unwrap();
+        view.refresh().execute().await.unwrap();
+
+        source
+            .add(record_batch!(("x", Int32, [4])).unwrap())
+            .execute()
+            .await
+            .unwrap();
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Incremental);
+        assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6, 8]);
+
+        source
+            .update()
+            .column("x", "20")
+            .only_if("x = 2")
+            .execute()
+            .await
+            .unwrap();
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Rebuild);
+        assert_eq!(read(view.table(), "twice").await, vec![2, 6, 8, 40]);
+    }
+
+    #[tokio::test]
+    async fn test_delete_evicts_the_view_rows_it_removed() {
+        let (_conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await;
+
+        // A delete removes source rows without changing the ones that
+        // remain, so the view evicts exactly those rows and keeps the rest
+        // rather than recomputing every row it already held.
+        source.delete("x = 2").await.unwrap();
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Incremental);
+        assert_eq!(read(view.table(), "twice").await, vec![2, 6]);
+
+        // A delete and an append in one span: both are applied.
+        source.delete("x = 1").await.unwrap();
+        source
+            .add(record_batch!(("x", Int32, vec![4])).unwrap())
+            .execute()
+            .await
+            .unwrap();
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Incremental);
+        assert_eq!(read(view.table(), "twice").await, vec![6, 8]);
+    }
+
+    /// merge_insert commits an `Update`, and its by-source arm removes rows
+    /// rather than changing them, so the classifier must treat that
+    /// transaction form as a source of deletions.
+    #[tokio::test]
+    async fn test_merge_insert_by_source_delete_evicts_the_view_rows() {
+        let (_conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await;
+
+        let batch = record_batch!(("x", Int32, vec![1, 3])).unwrap();
+        let reader = arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema());
+        let mut merge = source.merge_insert(&["x"]);
+        merge.when_not_matched_by_source_delete(None);
+        merge.execute(Box::new(reader)).await.unwrap();
+
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Incremental);
+        assert_eq!(read(view.table(), "twice").await, vec![2, 6]);
+    }
+
+    /// A refresh may only certify the generation it planned from. A write
+    /// that lands between planning and publication is drift the refresh did
+    /// not account for, so it aborts rather than stamp it as materialized.
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_refresh_aborts_rather_than_certify_view_drift() {
+        let _serial = DRIFT_LOCK.lock().await;
+        let (conn, source) = db_with_source(vec![1, 2, 3]).await;
+        let view = conn
+            .create_materialized_view("drifting_view", "src")
+            .select([("x", "x"), ("twice", "x * 2")])
+            .execute()
+            .await
+            .unwrap();
+        view.refresh().execute().await.unwrap();
+        let certified = source_version_of(view.table()).await;
+
+        // The source loses a row, so the next refresh plans an eviction.
+        source.delete("x = 2").await.unwrap();
+
+        let uri = view
+            .table()
+            .as_native()
+            .unwrap()
+            .dataset
+            .get()
+            .await
+            .unwrap()
+            .uri()
+            .to_string();
+        *DRIFT_TARGET.lock().unwrap() = Some(uri);
+        let refreshing = tokio::spawn(async move { view.refresh().execute().await });
+
+        // Move the view once the refresh has planned against it.
+        tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified())
+            .await
+            .expect("the refresh never reached the publication boundary");
+        let drifted = conn.open_table("drifting_view").execute().await.unwrap();
+        drifted.delete("twice = 6").await.unwrap();
+        DRIFT_RELEASED.notify_one();
+
+        // Publishing removals and additions as one change makes the drift a
+        // conflict lance itself rejects; a pure append, which touches no
+        // existing fragment, still relies on the generation check.
+        let err = refreshing.await.unwrap().unwrap_err();
+        let message = err.to_string();
+        assert!(
+            message.contains("raced this refresh") || message.contains("preempted by concurrent"),
+            "got {err:?}"
+        );
+        // The watermark still names the generation that was actually proven.
+        assert_eq!(source_version_of(&drifted).await, certified);
+    }
+
+    async fn source_version_of(table: &Table) -> Option<String> {
+        table
+            .schema()
+            .await
+            .unwrap()
+            .metadata()
+            .get(SOURCE_VERSION_META_KEY)
+            .cloned()
+    }
+
+    /// A transaction file carries placeholder ids for the fragments it
+    /// creates. Into an empty source those collide with the ids the commit
+    /// assigns, so treating them as already-materialized drops the very
+    /// first rows while the watermark still advances past them.
+    #[tokio::test]
+    async fn test_merge_into_an_empty_source_is_materialized() {
+        let (_conn, source, view) = refreshed_doubled(vec![]).await;
+        assert_eq!(read(view.table(), "twice").await, Vec::<i32>::new());
+
+        let batch = record_batch!(("x", Int32, vec![1, 2])).unwrap();
+        let reader = arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema());
+        let mut merge = source.merge_insert(&["x"]);
+        merge
+            .when_matched_update_all(None)
+            .when_not_matched_insert_all();
+        merge.execute(Box::new(reader)).await.unwrap();
+
+        view.refresh().execute().await.unwrap();
+        assert_eq!(read(view.table(), "twice").await, vec![2, 4]);
+        // A second refresh must not double them either.
+        view.refresh().execute().await.unwrap();
+        assert_eq!(read(view.table(), "twice").await, vec![2, 4]);
+    }
+
+    /// A refresh publishes what it removes and what it adds as one change,
+    /// so an update never exposes the view without the rows it is replacing.
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_an_update_is_never_visible_as_a_gap() {
+        let _serial = DRIFT_LOCK.lock().await;
+        let (conn, source) = db_with_source(vec![1, 2, 3]).await;
+        let view = conn
+            .create_materialized_view("atomic_view", "src")
+            .select([("x", "x"), ("twice", "x * 2")])
+            .execute()
+            .await
+            .unwrap();
+        view.refresh().execute().await.unwrap();
+
+        source
+            .update()
+            .column("x", "x + 10")
+            .execute()
+            .await
+            .unwrap();
+
+        let uri = view
+            .table()
+            .as_native()
+            .unwrap()
+            .dataset
+            .get()
+            .await
+            .unwrap()
+            .uri()
+            .to_string();
+        *DRIFT_TARGET.lock().unwrap() = Some(uri);
+        let refreshing = tokio::spawn(async move { view.refresh().execute().await });
+
+        // Read the view while the refresh is staged but not yet published.
+        tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified())
+            .await
+            .expect("the refresh never reached the publication boundary");
+        let midway = conn.open_table("atomic_view").execute().await.unwrap();
+        assert_eq!(
+            read(&midway, "twice").await,
+            vec![2, 4, 6],
+            "the pre-refresh rows must still be there in full"
+        );
+        DRIFT_RELEASED.notify_one();
+
+        refreshing.await.unwrap().unwrap();
+        let after = conn.open_table("atomic_view").execute().await.unwrap();
+        assert_eq!(read(&after, "twice").await, vec![22, 24, 26]);
+    }
+
+    async fn provenance_by_x(table: &Table) -> HashMap<i32, u64> {
+        let batches = table
+            .query()
+            .select(Select::columns(&["x", SOURCE_ROW_ID_COLUMN]))
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        batches
+            .iter()
+            .flat_map(|batch| {
+                let xs = batch["x"].as_any().downcast_ref::<Int32Array>().unwrap();
+                let ids = batch[SOURCE_ROW_ID_COLUMN].as_primitive::<UInt64Type>();
+                (0..batch.num_rows())
+                    .map(|i| (xs.value(i), ids.value(i)))
+                    .collect::<Vec<_>>()
+            })
+            .collect()
+    }
+
+    /// One delta batch is as large as a fragment's deletions, so it arrives
+    /// well past a chunk: it has to be drained a chunk at a time without ever
+    /// holding the batch, and the passes' deletion vectors have to accumulate
+    /// rather than replace each other.
+    #[tokio::test]
+    async fn test_a_chunked_eviction_removes_every_row_it_names() {
+        let (_conn, _, view) = refreshed_doubled(vec![1, 2, 3, 4, 5, 6]).await;
+
+        let native = view.table().as_native().unwrap();
+        let view_ds = native.dataset.get().await.unwrap().as_ref().clone();
+        let provenance = provenance_by_x(view.table()).await;
+
+        let mut eviction = Eviction::new(&view_ds, 2);
+        let batch = UInt64Array::from_iter_values([1, 2, 3, 4].map(|x| provenance[&x]));
+        eviction.push(&batch).await.unwrap();
+        assert_eq!(
+            eviction.peak, 2,
+            "a batch past the chunk was buffered whole"
+        );
+        let staged = eviction.finish().await.unwrap();
+        assert!(staged.is_some(), "four ids over a chunk of two stage twice");
+        publish(&view_ds, staged, Vec::new(), None, None)
+            .await
+            .unwrap();
+        native.dataset.reload().await.unwrap();
+
+        assert_eq!(read(view.table(), "x").await, vec![5, 6]);
+    }
+
+    /// Two first rebuilds materialize the same source rows; the loser must
+    /// key-conflict at commit and land nothing, or the view doubles.
+    #[tokio::test]
+    async fn test_a_raced_first_rebuild_lands_nothing() {
+        let _guard = DRIFT_LOCK.lock().await;
+        let (conn, _) = db_with_source(vec![1, 2, 3]).await;
+        let view = conn
+            .create_materialized_view("raced_rebuild", "src")
+            .select([("x", "x"), ("twice", "x * 2")])
+            .execute()
+            .await
+            .unwrap();
+        let native = view.table().as_native().unwrap();
+        let view_ds = native.dataset.get().await.unwrap().as_ref().clone();
+        let uri = view_ds.uri().to_string();
+        *DRIFT_TARGET.lock().unwrap() = Some(uri);
+        let racing = tokio::spawn(async move { view.refresh().execute().await });
+        tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified())
+            .await
+            .expect("the rebuild never reached the publication boundary");
+
+        // The other process's first rebuild, reduced to its commit. Its
+        // source rows are DISJOINT from ours -- the sentinel alone must make
+        // the two rebuilds conflict.
+        let batch = record_batch!(
+            ("x", Int32, [8, 9]),
+            ("twice", Int32, [16, 18]),
+            ("__source_row_id", UInt64, [7u64, 8])
+        )
+        .unwrap();
+        let schema = Arc::new(ArrowSchema::from(view_ds.schema()));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            schema
+                .fields()
+                .iter()
+                .map(|f| batch.column_by_name(f.name()).unwrap().clone())
+                .collect(),
+        )
+        .unwrap();
+        let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(
+            schema.clone(),
+            futures::stream::iter([Ok(batch)]),
+        ));
+        let keys = Arc::new(StdMutex::new(KeyExistenceFilterBuilder::new(vec![
+            source_row_id_field_id(&view_ds).unwrap(),
+        ])));
+        let stream = collect_source_row_ids(stream, keys.clone(), Arc::new(AtomicU64::new(0)));
+        let write_txn = InsertBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone())))
+            .with_params(&WriteParams {
+                mode: WriteMode::Append,
+                ..Default::default()
+            })
+            .execute_uncommitted_stream(stream)
+            .await
+            .unwrap();
+        let Operation::Append { fragments } = write_txn.operation else {
+            panic!("expected an append");
+        };
+        let filter = {
+            let mut keys = keys.lock().unwrap();
+            keys.insert(KeyValue::UInt64(super::REFRESH_TOKEN_ID))
+                .unwrap();
+            keys.build()
+        };
+        CommitBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone())))
+            .execute(Transaction::new(
+                view_ds.version().version,
+                Operation::Update {
+                    removed_fragment_ids: Vec::new(),
+                    updated_fragments: Vec::new(),
+                    new_fragments: fragments,
+                    fields_modified: Vec::new(),
+                    compacted_sstables: Vec::new(),
+                    fields_for_preserving_frag_bitmap: Vec::new(),
+                    update_mode: None,
+                    inserted_rows_filter: Some(filter),
+                    updated_fragment_offsets: None,
+                },
+                None,
+            ))
+            .await
+            .unwrap();
+        DRIFT_RELEASED.notify_one();
+
+        let err = racing.await.unwrap().unwrap_err();
+        // The loser lands nothing: only the winner's rows remain.
+        let raced = conn.open_table("raced_rebuild").execute().await.unwrap();
+        assert_eq!(
+            read(&raced, "twice").await,
+            vec![16, 18],
+            "the losing rebuild must not union with the winner ({err})"
+        );
+    }
+
+    /// An incremental refresh racing any other refresh must land nothing,
+    /// even when their planned rows are disjoint: the shared token makes the
+    /// commits conflict.
+    #[tokio::test]
+    async fn test_a_raced_incremental_lands_nothing() {
+        let _guard = DRIFT_LOCK.lock().await;
+        let (conn, source) = db_with_source(vec![1, 2, 3]).await;
+        let view = conn
+            .create_materialized_view("raced_incremental", "src")
+            .select([("x", "x"), ("twice", "x * 2")])
+            .execute()
+            .await
+            .unwrap();
+        view.refresh().execute().await.unwrap();
+        append(&source, vec![4]).await;
+
+        let native = view.table().as_native().unwrap();
+        native.dataset.reload().await.unwrap();
+        let view_ds = native.dataset.get().await.unwrap().as_ref().clone();
+        *DRIFT_TARGET.lock().unwrap() = Some(view_ds.uri().to_string());
+        let racing = tokio::spawn(async move { view.refresh().execute().await });
+        tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified())
+            .await
+            .expect("the refresh never reached the publication boundary");
+
+        // The concurrent refresh, reduced to its commit: disjoint rows, the
+        // shared token alone must collide.
+        let batch = record_batch!(
+            ("x", Int32, [9]),
+            ("twice", Int32, [18]),
+            ("__source_row_id", UInt64, [8u64])
+        )
+        .unwrap();
+        let schema = Arc::new(ArrowSchema::from(view_ds.schema()));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            schema
+                .fields()
+                .iter()
+                .map(|f| batch.column_by_name(f.name()).unwrap().clone())
+                .collect(),
+        )
+        .unwrap();
+        let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(
+            schema.clone(),
+            futures::stream::iter([Ok(batch)]),
+        ));
+        let keys = empty_keys(&view_ds).unwrap();
+        let stream = collect_source_row_ids(stream, keys.clone(), Arc::new(AtomicU64::new(0)));
+        let write_txn = InsertBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone())))
+            .with_params(&WriteParams {
+                mode: WriteMode::Append,
+                ..Default::default()
+            })
+            .execute_uncommitted_stream(stream)
+            .await
+            .unwrap();
+        let Operation::Append { fragments } = write_txn.operation else {
+            panic!("expected an append");
+        };
+        let filter = refresh_filter(&keys).unwrap();
+        CommitBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone())))
+            .execute(Transaction::new(
+                view_ds.version().version,
+                Operation::Update {
+                    removed_fragment_ids: Vec::new(),
+                    updated_fragments: Vec::new(),
+                    new_fragments: fragments,
+                    fields_modified: Vec::new(),
+                    compacted_sstables: Vec::new(),
+                    fields_for_preserving_frag_bitmap: Vec::new(),
+                    update_mode: None,
+                    inserted_rows_filter: Some(filter),
+                    updated_fragment_offsets: None,
+                },
+                None,
+            ))
+            .await
+            .unwrap();
+        DRIFT_RELEASED.notify_one();
+
+        let err = racing.await.unwrap().unwrap_err();
+        let raced = conn
+            .open_table("raced_incremental")
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(
+            read(&raced, "twice").await,
+            vec![2, 4, 6, 18],
+            "the losing increment must not land its rows ({err})"
+        );
+    }
+
+    /// Past the eviction cap the refresh falls back to the streamed rebuild,
+    /// and the result is identical either way.
+    #[tokio::test]
+    async fn test_oversized_delta_falls_back_to_rebuild() {
+        let (conn, source) = db_with_source((1..=20).collect()).await;
+        let view = doubled_view(&conn).await;
+        view.refresh().execute().await.unwrap();
+
+        source.delete("x <= 10").await.unwrap();
+        *tests::EVICTION_CAP.lock().unwrap() = Some(4);
+        let result = view.refresh().execute().await;
+        *tests::EVICTION_CAP.lock().unwrap() = None;
+        let result = result.unwrap();
+        assert_eq!(
+            result.mode,
+            RefreshMode::Rebuild,
+            "ten evictions, cap of four"
+        );
+        assert_eq!(read(view.table(), "x").await, (11..=20).collect::<Vec<_>>());
+
+        // Under the cap the same shape stays incremental.
+        source.delete("x = 11").await.unwrap();
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Incremental);
+        assert_eq!(read(view.table(), "x").await, (12..=20).collect::<Vec<_>>());
+    }
+
+    /// A cap of zero is a view that holds nothing, not a view without a cap.
+    #[tokio::test]
+    async fn test_zero_limit_holds_no_rows() {
+        let (conn, source) = db_with_source(vec![1, 2, 3]).await;
+        let view = conn
+            .create_materialized_view("empty", "src")
+            .select([("x", "x")])
+            .limit(0)
+            .execute()
+            .await
+            .unwrap();
+
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.rows_written, 0);
+        assert_eq!(view.table().count_rows(None).await.unwrap(), 0);
+
+        // Still nothing after the source grows, and the watermark advances.
+        append(&source, vec![4]).await;
+        view.refresh().execute().await.unwrap();
+        assert_eq!(view.table().count_rows(None).await.unwrap(), 0);
+        assert_eq!(
+            view.refresh().execute().await.unwrap().mode,
+            RefreshMode::NoOp
+        );
+    }
+
+    /// An update rewrites a whole fragment. When it lands on one appended
+    /// since the watermark, that fragment also holds rows the update never
+    /// touched -- rows the recompute does not cover and the append set no
+    /// longer reaches.
+    #[tokio::test]
+    async fn test_update_touching_a_new_fragment_keeps_its_untouched_rows() {
+        let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await;
+
+        // One fragment, appended after the watermark, holding both a row the
+        // update will change and a row it will not.
+        append(&source, vec![3, 40]).await;
+        source
+            .update()
+            .column("x", "99")
+            .only_if("x = 3")
+            .execute()
+            .await
+            .unwrap();
+
+        view.refresh().execute().await.unwrap();
+        assert_eq!(
+            read(view.table(), "twice").await,
+            vec![2, 4, 80, 198],
+            "a row appended into the updated fragment went missing"
+        );
+    }
+
+    async fn compact(source: &Table) {
+        source
+            .optimize(OptimizeAction::Compact {
+                options: CompactionOptions::default(),
+                remap_options: None,
+            })
+            .await
+            .unwrap();
+    }
+
+    /// A compaction rearranges rows without changing them, so it costs the
+    /// view nothing: the watermark advances and no row is recomputed.
+    #[tokio::test]
+    async fn test_compaction_alone_refreshes_incrementally() {
+        let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await;
+        append(&source, vec![3]).await;
+        view.refresh().execute().await.unwrap();
+
+        compact(&source).await;
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Incremental);
+        assert_eq!(result.rows_written, 0);
+        assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6]);
+        assert_eq!(
+            view.refresh().execute().await.unwrap().mode,
+            RefreshMode::NoOp
+        );
+    }
+
+    /// Rows appended after a compaction are separable through the transaction
+    /// log: only the appended fragments are computed.
+    #[tokio::test]
+    async fn test_append_after_compaction_stays_incremental() {
+        let (_conn, source, view) = refreshed_doubled(vec![1]).await;
+        append(&source, vec![2]).await;
+        view.refresh().execute().await.unwrap();
+
+        compact(&source).await;
+        append(&source, vec![3]).await;
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Incremental);
+        assert_eq!(result.rows_written, 1);
+        assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6]);
+    }
+
+    /// An append swallowed by a later compaction cannot be told apart from
+    /// the rows the view already holds, so the refresh rebuilds -- once --
+    /// rather than duplicate or drop.
+    #[tokio::test]
+    async fn test_append_folded_into_compaction_rebuilds() {
+        let (_conn, source, view) = refreshed_doubled(vec![1]).await;
+
+        append(&source, vec![2]).await;
+        compact(&source).await;
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Rebuild);
+        assert_eq!(read(view.table(), "twice").await, vec![2, 4]);
+    }
+
+    /// The create-time gate holds across a drop-and-recreate of the source
+    /// under the same name.
+    #[tokio::test]
+    async fn test_refresh_refuses_a_recreated_source_without_stable_row_ids() {
+        let (conn, _, view) = refreshed_doubled(vec![1]).await;
+
+        conn.drop_table("src", &[]).await.unwrap();
+        let batch = record_batch!(("x", Int32, [9])).unwrap();
+        conn.create_table("src", batch).execute().await.unwrap();
+
+        let err = view.refresh().execute().await.unwrap_err();
+        assert!(
+            matches!(err, Error::InvalidInput { message } if message.contains("stable row ids"))
+        );
+    }
+
+    /// A change to a column the view does not read is not a reason to
+    /// rebuild: the exact signature check is scoped to the view's inputs.
+    #[tokio::test]
+    async fn test_unrelated_column_change_does_not_rebuild() {
+        let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await;
+
+        source
+            .add_columns()
+            .transform(NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new(
+                vec![arrow_schema::Field::new(
+                    "unrelated",
+                    arrow_schema::DataType::Int32,
+                    true,
+                )],
+            ))))
+            .execute()
+            .await
+            .unwrap();
+
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Incremental);
+        assert_eq!(result.rows_written, 0);
+        assert_eq!(read(view.table(), "twice").await, vec![2, 4]);
+    }
+
+    #[tokio::test]
+    async fn test_full_forces_a_rebuild() {
+        let (_conn, source, view) = refreshed_doubled(vec![1]).await;
+
+        append(&source, vec![2]).await;
+        let result = view.refresh().full(true).execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Rebuild);
+        assert_eq!(result.rows_written, 2);
+        assert_eq!(read(view.table(), "twice").await, vec![2, 4]);
+    }
+
+    /// A cap and incremental reconciliation do not compose: rows skipped at
+    /// the cap fall behind the watermark, so room freed later cannot be
+    /// refilled from any delta. A capped view rebuilds instead, which its
+    /// own cap keeps cheap.
+    #[tokio::test]
+    async fn test_limited_view_rebuilds_rather_than_reconcile() {
+        let (conn, source) = db_with_source(vec![1, 2]).await;
+        let view = conn
+            .create_materialized_view("capped", "src")
+            .select([("x", "x")])
+            .limit(2)
+            .execute()
+            .await
+            .unwrap();
+        view.refresh().execute().await.unwrap();
+        assert_eq!(read(view.table(), "x").await, vec![1, 2]);
+
+        append(&source, vec![3, 4]).await;
+        source
+            .update()
+            .column("x", "11")
+            .only_if("x = 1")
+            .execute()
+            .await
+            .unwrap();
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Rebuild);
+        let held = read(view.table(), "x").await;
+        assert_eq!(held.len(), 2, "the cap holds: {held:?}");
+        let selectable = read(&source, "x").await;
+        assert!(
+            held.iter().all(|x| selectable.contains(x)),
+            "{held:?} is not a subset of {selectable:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_limit_caps_the_view() {
+        let (conn, source) = db_with_source(vec![1, 2, 3]).await;
+        let view = conn
+            .create_materialized_view("capped", "src")
+            .select([("x", "x")])
+            .limit(4)
+            .execute()
+            .await
+            .unwrap();
+
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.rows_written, 3);
+
+        // The cap counts already-held rows, so only one appended row lands.
+        append(&source, vec![4, 5, 6]).await;
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Incremental);
+        assert_eq!(result.rows_written, 1);
+        assert_eq!(view.table().count_rows(None).await.unwrap(), 4);
+
+        // At the cap, later appends only move the watermark.
+        append(&source, vec![7]).await;
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.rows_written, 0);
+        assert_eq!(
+            view.refresh().execute().await.unwrap().mode,
+            RefreshMode::NoOp
+        );
+    }
+
+    /// The whole point of the fragment-swap commit: a rebuild must never
+    /// leave the view without its index definitions.
+    #[tokio::test]
+    async fn test_rebuild_retains_indexes() {
+        let (_conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await;
+
+        view.table()
+            .create_index(&["twice"], Index::BTree(BTreeIndexBuilder::default()))
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(view.table().list_indices().await.unwrap().len(), 1);
+
+        source
+            .update()
+            .column("x", "x + 10")
+            .execute()
+            .await
+            .unwrap();
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Rebuild);
+        assert_eq!(view.table().list_indices().await.unwrap().len(), 1);
+        assert_eq!(read(view.table(), "twice").await, vec![22, 24, 26]);
+
+        // The swapped-in rows are reachable through an indexed query.
+        let batches = view
+            .table()
+            .query()
+            .only_if("twice = 24")
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 1);
+    }
+
+    #[tokio::test]
+    async fn test_rebuild_of_an_empty_result_is_an_empty_view() {
+        let (conn, _) = db_with_source(vec![1, 2]).await;
+        let view = conn
+            .create_materialized_view("none", "src")
+            .select([("x", "x")])
+            .only_if("x > 100")
+            .execute()
+            .await
+            .unwrap();
+
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Rebuild);
+        assert_eq!(result.rows_written, 0);
+        assert_eq!(view.table().count_rows(None).await.unwrap(), 0);
+        assert_eq!(
+            view.refresh().execute().await.unwrap().mode,
+            RefreshMode::NoOp
+        );
+    }
+
+    /// Provenance: every view row records the source row that produced it.
+    #[tokio::test]
+    async fn test_source_row_ids_are_recorded() {
+        let (_conn, _, view) = refreshed_doubled(vec![1, 2, 3]).await;
+
+        let batches = view
+            .table()
+            .query()
+            .select(Select::columns(&[SOURCE_ROW_ID_COLUMN]))
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        let total: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert_eq!(total, 3);
+        for batch in &batches {
+            assert_eq!(batch[SOURCE_ROW_ID_COLUMN].null_count(), 0);
+        }
+    }
+
+    #[tokio::test]
+    async fn test_dropping_a_source_input_fails_the_refresh() {
+        let (conn, source) = db_with_source(vec![1]).await;
+        let view = conn
+            .create_materialized_view("v", "src")
+            .select([("twice", "x * 2")])
+            .execute()
+            .await
+            .unwrap();
+        view.refresh().execute().await.unwrap();
+
+        append(&source, vec![2]).await;
+        source
+            .add_columns()
+            .transform(NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new(
+                vec![arrow_schema::Field::new(
+                    "y",
+                    arrow_schema::DataType::Int32,
+                    true,
+                )],
+            ))))
+            .execute()
+            .await
+            .unwrap();
+        source.drop_columns(&["x"]).await.unwrap();
+
+        let err = view.refresh().execute().await.unwrap_err();
+        assert!(matches!(err, Error::Schema { message } if message.contains("'x'")));
+    }
+
+    /// A pinned refresh materializes the source as of `version`; catching up
+    /// to the appends beyond it stays incremental.
+    #[tokio::test]
+    async fn test_pinned_refresh_and_catch_up() {
+        let (conn, source) = db_with_source(vec![1]).await;
+        let view = doubled_view(&conn).await;
+        let pinned = source.version().await.unwrap();
+
+        append(&source, vec![2]).await;
+        let result = view
+            .refresh()
+            .source_version(pinned)
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(result.source_version, pinned);
+        assert_eq!(read(view.table(), "twice").await, vec![2]);
+
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Incremental);
+        assert_eq!(read(view.table(), "twice").await, vec![2, 4]);
+    }
+
+    /// Views chain: a view's stable row ids and provenance column make it a
+    /// source like any other, and the default projection takes its declared
+    /// columns without copying its provenance.
+    #[tokio::test]
+    async fn test_a_view_can_source_another_view() {
+        let (conn, source) = db_with_source(vec![1, 2, 30]).await;
+        let first = doubled_view(&conn).await;
+        first.refresh().execute().await.unwrap();
+
+        let second = conn
+            .create_materialized_view("second", "doubled")
+            .only_if("twice > 10")
+            .execute()
+            .await
+            .unwrap();
+        assert!(
+            second
+                .definition()
+                .projections
+                .iter()
+                .all(|p| p.output != SOURCE_ROW_ID_COLUMN)
+        );
+        let result = second.refresh().execute().await.unwrap();
+        assert_eq!(result.rows_written, 1);
+        assert_eq!(read(second.table(), "twice").await, vec![60]);
+
+        append(&source, vec![50]).await;
+        first.refresh().execute().await.unwrap();
+        let result = second.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Incremental);
+        assert_eq!(read(second.table(), "twice").await, vec![60, 100]);
+    }
+
+    /// The watermark speaks only for the state a refresh left behind: a
+    /// direct write to the view is drift, and the next refresh rebuilds
+    /// rather than preserving it as current.
+    #[tokio::test]
+    async fn test_direct_view_mutation_forces_a_rebuild() {
+        let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await;
+
+        view.table().delete("x = 1").await.unwrap();
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Rebuild);
+        assert_eq!(read(view.table(), "twice").await, vec![2, 4]);
+        assert_eq!(
+            view.refresh().execute().await.unwrap().mode,
+            RefreshMode::NoOp
+        );
+    }
+
+    /// A dropped and recreated source reuses version numbers but never their
+    /// timestamps; the watermark must not vouch for the replacement's rows.
+    #[tokio::test]
+    async fn test_source_recreation_forces_a_rebuild() {
+        let (conn, _, view) = refreshed_doubled(vec![1]).await;
+
+        conn.drop_table("src", &[]).await.unwrap();
+        let batch = record_batch!(("x", Int32, [7])).unwrap();
+        conn.create_table("src", batch)
+            .write_options(crate::materialized_view::tests::stable_row_ids())
+            .execute()
+            .await
+            .unwrap();
+
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Rebuild);
+        assert_eq!(read(view.table(), "twice").await, vec![14]);
+    }
+
+    /// A refresh bound to an incarnation refuses a view dropped and recreated
+    /// since, even under the same name and definition; the recreated view's
+    /// own token is accepted, and the token survives a refresh's stamp.
+    #[tokio::test]
+    async fn test_refresh_refuses_a_recreated_view_incarnation() {
+        let (conn, _, view) = refreshed_doubled(vec![1]).await;
+        let token = view.incarnation().unwrap().to_string();
+        view.refresh()
+            .expect_incarnation(&token)
+            .execute()
+            .await
+            .unwrap();
+        let reopened = conn.open_materialized_view("doubled").await.unwrap();
+        assert_eq!(reopened.incarnation(), Some(token.as_str()));
+
+        conn.drop_table("doubled", &[]).await.unwrap();
+        let recreated = doubled_view(&conn).await;
+        assert_ne!(recreated.incarnation(), Some(token.as_str()));
+
+        let err = recreated
+            .refresh()
+            .expect_incarnation(&token)
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("dropped and recreated"), "{err}");
+        assert_eq!(read(recreated.table(), "twice").await, Vec::<i32>::new());
+
+        recreated
+            .refresh()
+            .expect_incarnation(recreated.incarnation().unwrap())
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(read(recreated.table(), "twice").await, vec![2]);
+    }
+
+    /// A cloned declaration creates two physical tables; each gets its own
+    /// token.
+    #[tokio::test]
+    async fn test_cloned_declaration_mints_a_fresh_incarnation_per_create() {
+        let (conn, source) = db_with_source(vec![1]).await;
+        let prepared = crate::materialized_view::prepare_declaration(
+            &source,
+            &[("x".into(), "x".into()), ("twice".into(), "x * 2".into())],
+            None,
+            None,
+        )
+        .await
+        .unwrap();
+        let replacement = prepared.clone();
+        let first = prepared.create("cloned").await.unwrap();
+        let first_token = first.incarnation().unwrap().to_string();
+
+        conn.drop_table("cloned", &[]).await.unwrap();
+        let second = replacement.create("cloned").await.unwrap();
+        assert_ne!(second.incarnation(), Some(first_token.as_str()));
+    }
+
+    /// A recreation that lands after planning but before publication is
+    /// caught by the pre-commit read: the stale refresh fails and the
+    /// replacement stays empty under its own token.
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_bound_refresh_cannot_publish_into_a_raced_recreation() {
+        let _serial = DRIFT_LOCK.lock().await;
+        let (conn, _) = db_with_source(vec![1]).await;
+        let view = doubled_view(&conn).await;
+        let token = view.incarnation().unwrap().to_string();
+        let uri = view
+            .table()
+            .as_native()
+            .unwrap()
+            .dataset
+            .get()
+            .await
+            .unwrap()
+            .uri()
+            .to_string();
+
+        *DRIFT_TARGET.lock().unwrap() = Some(uri);
+        let refreshing =
+            tokio::spawn(async move { view.refresh().expect_incarnation(token).execute().await });
+        tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified())
+            .await
+            .expect("refresh never reached publication");
+
+        conn.drop_table("doubled", &[]).await.unwrap();
+        let replacement = doubled_view(&conn).await;
+        let replacement_token = replacement.incarnation().unwrap().to_string();
+        DRIFT_RELEASED.notify_one();
+
+        let result = refreshing.await.unwrap();
+        assert!(result.is_err(), "the stale refresh unexpectedly succeeded");
+        let reopened = conn.open_materialized_view("doubled").await.unwrap();
+        assert_eq!(reopened.incarnation(), Some(replacement_token.as_str()));
+        assert_eq!(read(reopened.table(), "twice").await, Vec::<i32>::new());
+    }
+
+    /// Replacing the schema metadata wholesale drops the token. A refresh
+    /// bound to the old token is refused for that reason, not as a
+    /// recreation; an unbound refresh mints the view a fresh one.
+    #[tokio::test]
+    async fn test_a_view_whose_metadata_was_replaced_starts_a_new_incarnation() {
+        let (conn, _, view) = refreshed_doubled(vec![1]).await;
+        let token = view.incarnation().unwrap().to_string();
+        let mut metadata = HashMap::new();
+        metadata.insert(
+            crate::materialized_view::DEFINITION_META_KEY.to_string(),
+            crate::materialized_view::definition_to_metadata(view.definition()).unwrap(),
+        );
+        view.table()
+            .as_native()
+            .unwrap()
+            .replace_schema_metadata(metadata)
+            .await
+            .unwrap();
+        assert_eq!(
+            conn.open_materialized_view("doubled")
+                .await
+                .unwrap()
+                .incarnation(),
+            None
+        );
+
+        let err = view
+            .refresh()
+            .expect_incarnation(&token)
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("no incarnation token"), "{err}");
+
+        view.refresh().execute().await.unwrap();
+        let reopened = conn.open_materialized_view("doubled").await.unwrap();
+        assert!(reopened.incarnation().is_some());
+        assert_ne!(reopened.incarnation(), Some(token.as_str()));
+    }
+
+    /// In-process refreshes of one view serialize: the loser of the race
+    /// observes the winner's watermark instead of appending the same rows.
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_concurrent_refreshes_do_not_duplicate() {
+        let (_conn, source, view) = refreshed_doubled(vec![1]).await;
+
+        append(&source, vec![2, 3]).await;
+        let (a, b) = tokio::join!(view.refresh().execute(), view.refresh().execute());
+        let (a, b) = (a.unwrap(), b.unwrap());
+        assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6]);
+        let modes = [a.mode, b.mode];
+        assert!(modes.contains(&RefreshMode::Incremental));
+        assert!(modes.contains(&RefreshMode::NoOp));
+    }
+
+    /// A second handle's lazy cache must not defeat the lock: after another
+    /// handle's refresh commits, the stale handle plans from the reloaded
+    /// state and no-ops instead of appending the same fragments again.
+    #[tokio::test]
+    async fn test_a_second_handle_does_not_double_append() {
+        let (conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await;
+        let stale = conn.open_materialized_view("doubled").await.unwrap();
+
+        append(&source, vec![4]).await;
+        view.refresh().execute().await.unwrap();
+
+        let result = stale.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::NoOp);
+        assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6, 8]);
+    }
+
+    /// A commit racing between a refresh's data commit and its stamp must
+    /// not be certified as the refresh's generation: the stamp aborts, and
+    /// the next refresh rebuilds from the drifted state.
+    #[tokio::test]
+    async fn test_stamp_aborts_on_a_racing_commit() {
+        let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await;
+
+        let view_native = view.table().as_native().unwrap();
+        let stale = view_native.dataset.get().await.unwrap().as_ref().clone();
+        view.table().delete("x = 1").await.unwrap();
+
+        let err = stamp_watermark(view_native, stale, 99, 99, None).await;
+        assert!(err.is_err());
+
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Rebuild);
+        assert_eq!(read(view.table(), "twice").await, vec![2, 4]);
+    }
+
+    /// What refresh executes and what it stamps must be one generation: a
+    /// replaced definition wins over whatever a stale handle cached.
+    #[tokio::test]
+    async fn test_refresh_uses_the_latest_persisted_definition() {
+        let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await;
+
+        let replacement = crate::materialized_view::MaterializedViewDefinition {
+            source_table: "src".into(),
+            projections: vec![
+                crate::materialized_view::ViewProjection {
+                    output: "x".into(),
+                    expression: "x".into(),
+                },
+                crate::materialized_view::ViewProjection {
+                    output: "twice".into(),
+                    expression: "x * 3".into(),
+                },
+            ],
+            filter: None,
+            limit: None,
+            inputs: vec!["x".into()],
+        };
+        let mut metadata = HashMap::new();
+        metadata.insert(
+            crate::materialized_view::DEFINITION_META_KEY.to_string(),
+            crate::materialized_view::definition_to_metadata(&replacement).unwrap(),
+        );
+        view.table()
+            .as_native()
+            .unwrap()
+            .replace_schema_metadata(metadata)
+            .await
+            .unwrap();
+
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Rebuild);
+        assert_eq!(read(view.table(), "twice").await, vec![3, 6]);
+    }
+
+    /// A persisted definition that does not produce this view's schema must
+    /// not refresh at all, let alone be certified.
+    #[tokio::test]
+    async fn test_definition_view_schema_mismatch_is_refused() {
+        let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await;
+
+        let narrower = crate::materialized_view::MaterializedViewDefinition {
+            source_table: "src".into(),
+            projections: vec![crate::materialized_view::ViewProjection {
+                output: "x".into(),
+                expression: "x".into(),
+            }],
+            filter: None,
+            limit: None,
+            inputs: vec!["x".into()],
+        };
+        let mut metadata = HashMap::new();
+        metadata.insert(
+            crate::materialized_view::DEFINITION_META_KEY.to_string(),
+            crate::materialized_view::definition_to_metadata(&narrower).unwrap(),
+        );
+        view.table()
+            .as_native()
+            .unwrap()
+            .replace_schema_metadata(metadata)
+            .await
+            .unwrap();
+
+        let err = view.refresh().execute().await.unwrap_err();
+        assert!(matches!(err, Error::Schema { message } if message.contains("does not produce")),);
+    }
+
+    /// An overlay replaces cell values without changing any file path; the
+    /// signature must see it, scoped to the columns the view reads like
+    /// data files are.
+    #[test]
+    fn test_fragment_signature_sees_overlays() {
+        use lance_file::version::ConcreteFileVersion;
+        use lance_table::format::DataFile;
+        use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
+
+        let base = Fragment::new(7);
+        let mut file = DataFile::new_unstarted("f0.lance", ConcreteFileVersion::V2_1);
+        file.fields = vec![0, 1].into();
+        let mut with_file = base.clone();
+        with_file.files.push(file.clone());
+
+        let overlay = |field: i32| {
+            let mut data_file = DataFile::new_unstarted("o0.lance", ConcreteFileVersion::V2_1);
+            data_file.fields = vec![field].into();
+            DataOverlayFile {
+                data_file,
+                coverage: OverlayCoverage::PerField(Vec::new()),
+                committed_version: 9,
+            }
+        };
+        let relevant: HashSet<i32> = [0].into_iter().collect();
+
+        let mut overlaid_relevant = with_file.clone();
+        overlaid_relevant.overlays.push(overlay(0));
+        assert_ne!(
+            fragment_signature(&with_file, &relevant),
+            fragment_signature(&overlaid_relevant, &relevant),
+        );
+
+        let mut overlaid_unrelated = with_file.clone();
+        overlaid_unrelated.overlays.push(overlay(5));
+        assert_eq!(
+            fragment_signature(&with_file, &relevant),
+            fragment_signature(&overlaid_unrelated, &relevant),
+        );
+    }
+
+    /// An output whose name needs quoting flows through as a projection
+    /// alias, never spliced into SQL text.
+    #[tokio::test]
+    async fn test_output_names_needing_quotes() {
+        let (conn, _) = db_with_source(vec![1, 2]).await;
+        let view = conn
+            .create_materialized_view("v", "src")
+            .select([("double value", "x * 2")])
+            .execute()
+            .await
+            .unwrap();
+
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.rows_written, 2);
+        assert_eq!(read(view.table(), "double value").await, vec![2, 4]);
+    }
+
+    /// MemWAL tiers are visible to reads but not to the refresh scan, so LSM
+    /// state disqualifies every participant: the source at create, either
+    /// side at refresh, and the view can never accept a spec. Retained
+    /// un-compacted rows (the catch-up flag outlives unset) count as state.
+    #[tokio::test]
+    async fn lsm_state_disqualifies_source_and_view() {
+        use crate::table::LsmWriteSpec;
+        use arrow_array::RecordBatchIterator;
+
+        let tmp_dir = tempfile::tempdir().unwrap();
+        let conn = connect(tmp_dir.path().to_str().unwrap())
+            .execute()
+            .await
+            .unwrap();
+        // Hand-rolled: the LSM primary key must be non-nullable, which
+        // record_batch! cannot express.
+        let schema = Arc::new(ArrowSchema::new(vec![
+            arrow_schema::Field::new("id", arrow_schema::DataType::Int64, false),
+            arrow_schema::Field::new("x", arrow_schema::DataType::Int32, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(arrow_array::Int64Array::from(vec![1, 2])) as _,
+                Arc::new(Int32Array::from(vec![1, 2])) as _,
+            ],
+        )
+        .unwrap();
+        let table = conn
+            .create_table("src", batch.clone())
+            .write_options(crate::materialized_view::tests::stable_row_ids())
+            .execute()
+            .await
+            .unwrap();
+        table.set_unenforced_primary_key(["id"]).await.unwrap();
+        table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded())
+            .await
+            .unwrap();
+
+        // An active-LSM source is refused at create.
+        let err = conn
+            .create_materialized_view("v", "src")
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("un-compacted"), "{err}");
+
+        // Unset with nothing written clears the state: the view creates,
+        // and a spec can never be installed over it.
+        table.unset_lsm_write_spec().await.unwrap();
+        let view = conn
+            .create_materialized_view("v", "src")
+            .execute()
+            .await
+            .unwrap();
+        let err = view
+            .table()
+            .set_lsm_write_spec(LsmWriteSpec::unsharded())
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("materialized view"), "{err}");
+
+        // A source that acquires a spec after create fails refresh.
+        table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded())
+            .await
+            .unwrap();
+        let err = view.refresh().execute().await.unwrap_err();
+        assert!(err.to_string().contains("source table 'src'"), "{err}");
+
+        // Retained rows outlive unset: write through the WAL, unset, and
+        // refresh still refuses.
+        let mut merge = table.merge_insert(&["id"]);
+        merge
+            .when_matched_update_all(None)
+            .when_not_matched_insert_all()
+            .use_lsm(true);
+        merge
+            .execute(Box::new(RecordBatchIterator::new(
+                vec![Ok(batch.clone())],
+                batch.schema(),
+            )))
+            .await
+            .unwrap();
+        table.unset_lsm_write_spec().await.unwrap();
+        let err = view.refresh().execute().await.unwrap_err();
+        assert!(err.to_string().contains("source table 'src'"), "{err}");
+    }
+}
diff --git a/rust/lancedb/src/query.rs b/rust/lancedb/src/query.rs
index b76865043..654777adb 100644
--- a/rust/lancedb/src/query.rs
+++ b/rust/lancedb/src/query.rs
@@ -1661,14 +1661,8 @@ mod tests {
 
     #[tokio::test]
     async fn test_setters_getters() {
-        // TODO: Switch back to memory://foo after https://github.com/lancedb/lancedb/issues/1051
-        // is fixed
-        let tmp_dir = tempdir().unwrap();
-        let dataset_path = tmp_dir.path().join("test.lance");
-        let uri = dataset_path.to_str().unwrap();
-
         let batches = make_test_batches();
-        let conn = connect(uri).execute().await.unwrap();
+        let conn = connect("memory://foo").execute().await.unwrap();
         let table = conn
             .create_table("my_table", batches)
             .execute()
@@ -1763,14 +1757,8 @@ mod tests {
 
     #[tokio::test]
     async fn test_execute() {
-        // TODO: Switch back to memory://foo after https://github.com/lancedb/lancedb/issues/1051
-        // is fixed
-        let tmp_dir = tempdir().unwrap();
-        let dataset_path = tmp_dir.path().join("test.lance");
-        let uri = dataset_path.to_str().unwrap();
-
         let batches = make_non_empty_batches();
-        let conn = connect(uri).execute().await.unwrap();
+        let conn = connect("memory://foo").execute().await.unwrap();
         let table = conn
             .create_table("my_table", batches)
             .execute()
@@ -1786,11 +1774,14 @@ mod tests {
             .postfilter();
         let result = query.execute().await;
         let mut stream = result.expect("should have result");
-        // should only have one batch
+        let mut num_rows = 0;
         while let Some(batch) = stream.next().await {
-            // post filter should have removed some rows
-            assert!(batch.expect("should be Ok").num_rows() < 10);
+            let batch = batch.expect("should be Ok");
+            let ids: &Int32Array = batch["id"].as_primitive();
+            assert!(ids.iter().all(|id| id.unwrap() % 2 == 0));
+            num_rows += batch.num_rows();
         }
+        assert!(num_rows <= 10);
 
         let query = table
             .query()
@@ -1889,14 +1880,8 @@ mod tests {
 
     #[tokio::test]
     async fn test_select_with_transform() {
-        // TODO: Switch back to memory://foo after https://github.com/lancedb/lancedb/issues/1051
-        // is fixed
-        let tmp_dir = tempdir().unwrap();
-        let dataset_path = tmp_dir.path().join("test.lance");
-        let uri = dataset_path.to_str().unwrap();
-
         let batches = make_non_empty_batches();
-        let conn = connect(uri).execute().await.unwrap();
+        let conn = connect("memory://foo").execute().await.unwrap();
         let table = conn
             .create_table("my_table", batches)
             .execute()
@@ -1993,15 +1978,9 @@ mod tests {
 
     #[tokio::test]
     async fn test_execute_no_vector() {
-        // TODO: Switch back to memory://foo after https://github.com/lancedb/lancedb/issues/1051
-        // is fixed
-        let tmp_dir = tempdir().unwrap();
-        let dataset_path = tmp_dir.path().join("test.lance");
-        let uri = dataset_path.to_str().unwrap();
-
         // test that it's ok to not specify a query vector (just filter / limit)
         let batches = make_non_empty_batches();
-        let conn = connect(uri).execute().await.unwrap();
+        let conn = connect("memory://foo").execute().await.unwrap();
         let table = conn
             .create_table("my_table", batches)
             .execute()
diff --git a/rust/lancedb/src/remote.rs b/rust/lancedb/src/remote.rs
index 4b5f8832f..be9d0eef6 100644
--- a/rust/lancedb/src/remote.rs
+++ b/rust/lancedb/src/remote.rs
@@ -19,6 +19,15 @@ const ARROW_FILE_CONTENT_TYPE: &str = "application/vnd.apache.arrow.file";
 #[cfg(test)]
 const JSON_CONTENT_TYPE: &str = "application/json";
 
+fn extract_job_id(body: &str) -> Option<String> {
+    serde_json::from_str::<serde_json::Value>(body)
+        .ok()?
+        .get("job_id")?
+        .as_str()
+        .filter(|job_id| !job_id.is_empty())
+        .map(str::to_string)
+}
+
 pub use client::{ClientConfig, HeaderProvider, RetryConfig, TimeoutConfig, TlsConfig};
 pub use db::{RemoteDatabaseOptions, RemoteDatabaseOptionsBuilder};
 pub use oauth::{OAuthConfig, OAuthFlow, OAuthHeaderProvider};
diff --git a/rust/lancedb/src/remote/client.rs b/rust/lancedb/src/remote/client.rs
index 9e34fca9f..57dd89890 100644
--- a/rust/lancedb/src/remote/client.rs
+++ b/rust/lancedb/src/remote/client.rs
@@ -373,6 +373,37 @@ pub fn parse_db_url(db_url: &str) -> Result<ParsedDbUrl> {
     Ok(ParsedDbUrl { db_name, db_prefix })
 }
 
+fn validate_dns_hostname(hostname: &str) -> Result<()> {
+    let ascii_hostname = match url::Host::parse(hostname) {
+        Ok(url::Host::Domain(hostname)) => hostname,
+        Ok(_) => {
+            return Err(Error::InvalidInput {
+                message: "LanceDB Cloud database URI or region produced a non-DNS hostname"
+                    .to_string(),
+            });
+        }
+        Err(err) => {
+            return Err(Error::InvalidInput {
+                message: format!(
+                    "LanceDB Cloud database URI or region produced an invalid hostname: {err}"
+                ),
+            });
+        }
+    };
+
+    if ascii_hostname.len() > 253
+        || ascii_hostname
+            .split('.')
+            .any(|label| label.is_empty() || label.len() > 63)
+    {
+        return Err(Error::InvalidInput {
+            message: "LanceDB Cloud database URI or region produced an invalid hostname: DNS labels must contain 1 to 63 bytes and the full hostname must not exceed 253 bytes".to_string(),
+        });
+    }
+
+    Ok(())
+}
+
 impl RestfulLanceDbClient<Sender> {
     fn get_timeout(passed: Option<Duration>, env_var: &str) -> Result<Option<Duration>> {
         if let Some(passed) = passed {
@@ -480,7 +511,11 @@ impl RestfulLanceDbClient<Sender> {
 
         let host = match host_override {
             Some(host_override) => host_override,
-            None => format!("https://{}.{}.api.lancedb.com", parsed_url.db_name, region),
+            None => {
+                let hostname = format!("{}.{}.api.lancedb.com", parsed_url.db_name, region);
+                validate_dns_hostname(&hostname)?;
+                format!("https://{hostname}")
+            }
         };
         debug!("Created client for host: {}", host);
         let retry_config = client_config.retry_config.clone().try_into()?;
@@ -1157,6 +1192,29 @@ mod tests {
         assert_eq!(headers.get("x-api-key").unwrap(), "api-key");
     }
 
+    #[test]
+    fn test_rejects_invalid_cloud_dns_hostname() {
+        let invalid_database_names = ["a".repeat(64), "invalid..database".to_string()];
+
+        for db_name in invalid_database_names {
+            let parsed_url = parse_db_url(&format!("db://{db_name}")).unwrap();
+            let error = RestfulLanceDbClient::<Sender>::try_new(
+                &parsed_url,
+                "us-east-1",
+                None,
+                HeaderMap::new(),
+                ClientConfig::default(),
+                None,
+            )
+            .unwrap_err();
+
+            assert!(
+                matches!(error, Error::InvalidInput { ref message } if message.contains("DNS labels must contain 1 to 63 bytes")),
+                "unexpected error: {error}"
+            );
+        }
+    }
+
     // Test implementation of HeaderProvider
     #[derive(Debug, Clone)]
     struct TestHeaderProvider {
diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs
index 839cb3797..3f4216bc8 100644
--- a/rust/lancedb/src/remote/db.rs
+++ b/rust/lancedb/src/remote/db.rs
@@ -9,6 +9,7 @@ use http::StatusCode;
 use lance_io::object_store::StorageOptions;
 use lance_namespace_impls::{DynamicContextProvider, OperationInfo};
 use moka::future::Cache;
+use reqwest::Response;
 use reqwest::header::CONTENT_TYPE;
 
 use lance_namespace::models::{
@@ -23,15 +24,18 @@ use crate::database::{
     JobDescription, JobInfo, OpenTableRequest, ReadConsistency, TableNamesRequest,
 };
 use crate::error::Result;
+use crate::function::{FunctionRegistrationRequest, FunctionVersion};
+use crate::job::Job;
+use crate::remote::job::{DescribeJobResponse, RemoteJob, job_state_to_client};
 use crate::remote::util::stream_as_body;
 use crate::table::BaseTable;
 
-use super::ARROW_STREAM_CONTENT_TYPE;
 use super::client::{
     ClientConfig, HeaderProvider, HttpSend, RequestResultExt, RestfulLanceDbClient, Sender,
 };
 use super::table::RemoteTable;
 use super::util::parse_server_version;
+use super::{ARROW_STREAM_CONTENT_TYPE, extract_job_id};
 
 // Request structure for the remote clone table API
 #[derive(serde::Serialize)]
@@ -326,6 +330,78 @@ impl RemoteDatabase {
     }
 }
 
+impl<S: HttpSend> RemoteDatabase<S> {
+    async fn submit_drop_table(
+        &self,
+        name: &str,
+        namespace_path: &[String],
+    ) -> Result<(String, Response)> {
+        let identifier = build_table_identifier(name, namespace_path, &self.client.id_delimiter);
+        let cache_key = build_cache_key(name, namespace_path);
+        let req = self.client.post(&format!("/v1/table/{}/drop/", identifier));
+        let (request_id, resp) = self.client.send(req).await?;
+        let resp = self.client.check_response(&request_id, resp).await?;
+        self.table_cache.remove(&cache_key).await;
+        Ok((request_id, resp))
+    }
+
+    /// Collect the tables of a namespace in name order, for `table_names`.
+    ///
+    /// `table_names` promises name order and resumes after a table name, but the namespace
+    /// route's `page_token` is opaque -- it belongs to the store the listing walks, and a
+    /// token this client invented would resume from the wrong place. So the whole namespace is
+    /// walked by handing each response's token straight back, and the name semantics are
+    /// applied here. Constructing no token is what makes this work against a server on either
+    /// side of the change: it only ever repeats what the server said.
+    ///
+    /// This is the cost `table_names` already paid -- the server used to enumerate and sort the
+    /// namespace on every request -- and it is why `list_tables` replaces it.
+    async fn table_names_in_namespace(
+        &self,
+        request: &TableNamesRequest,
+    ) -> Result<(Vec<String>, ServerVersion)> {
+        let namespace_id =
+            build_namespace_identifier(&request.namespace_path, &self.client.id_delimiter);
+        let path = format!("/v1/namespace/{}/table/list", namespace_id);
+
+        let mut names = Vec::new();
+        // Every page reports the same server, so keep the first page's version.
+        let mut version: Option<ServerVersion> = None;
+        let mut page_token: Option<String> = None;
+        loop {
+            let mut req = self.client.get(&path);
+            if let Some(ref token) = page_token {
+                req = req.query(&[("page_token", token)]);
+            }
+            let (request_id, rsp) = self.client.send_with_retry(req, None, true).await?;
+            let rsp = self.client.check_response(&request_id, rsp).await?;
+            if version.is_none() {
+                version = Some(parse_server_version(&request_id, &rsp)?);
+            }
+            let response: ListTablesResponse = rsp.json().await.err_to_http(request_id)?;
+            names.extend(response.tables);
+            // An empty token is the end of the listing, not a token to send back: a server
+            // that reads an empty token as "start from the beginning" would hand back the
+            // first page again.
+            match response.page_token.filter(|token| !token.is_empty()) {
+                // A server that repeated a token would never finish; treat that as the end
+                // rather than looping on it.
+                Some(token) if Some(&token) != page_token.as_ref() => page_token = Some(token),
+                _ => break,
+            }
+        }
+
+        names.sort();
+        if let Some(ref start_after) = request.start_after {
+            names.retain(|name| name > start_after);
+        }
+        if let Some(limit) = request.limit {
+            names.truncate(limit as usize);
+        }
+        Ok((names, version.unwrap_or_default()))
+    }
+}
+
 #[cfg(all(test, feature = "remote"))]
 mod test_utils {
     use super::*;
@@ -453,48 +529,6 @@ struct RemoteListJobsResponse {
     page_token: Option<String>,
 }
 
-/// The server's account of why a job failed. Absent from older servers,
-/// which report only the terminal state.
-#[derive(serde::Deserialize)]
-struct RemoteReportedFailure {
-    #[serde(default)]
-    phase: Option<String>,
-    #[serde(default)]
-    message: Option<String>,
-    #[serde(default)]
-    retryable: Option<bool>,
-}
-
-#[derive(serde::Deserialize)]
-struct RemoteDescribeJobResponse {
-    job_id: String,
-    #[serde(default)]
-    job_type: String,
-    job_state: String,
-    #[serde(default)]
-    creation_ms: i64,
-    #[serde(default)]
-    spec: serde_json::Value,
-    #[serde(default)]
-    failure: Option<RemoteReportedFailure>,
-}
-
-/// Server job states -> the client vocabulary ("running" / "finished" /
-/// "failed" / "cancelled"). Covers both the describe enum (IN_PROGRESS /
-/// DONE / FAILED / CANCELLED) and the registry's lowercase list-row states
-/// (in_progress / succeeded / failed / canceled / timed_out). States this
-/// client version does not know (e.g. created, queued) pass through as-is.
-fn job_state_to_client(state: &str) -> String {
-    match state {
-        "IN_PROGRESS" | "in_progress" => "running",
-        "DONE" | "done" | "succeeded" => "finished",
-        "FAILED" | "failed" | "TIMED_OUT" | "timed_out" => "failed",
-        "CANCELLED" | "cancelled" | "canceled" => "cancelled",
-        other => other,
-    }
-    .to_string()
-}
-
 /// Bound on `list_jobs` page walking; a warning is logged when the listing
 /// is truncated at this many pages.
 const MAX_LIST_JOBS_PAGES: usize = 100;
@@ -512,6 +546,39 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
         })
     }
 
+    async fn create_function_async(
+        &self,
+        request: FunctionRegistrationRequest,
+    ) -> Result<Job<FunctionVersion>> {
+        let req = self.client.post("/v1/functions/create").json(&request);
+        let (request_id, response) = self.client.send(req).await?;
+        let response = self.client.check_response(&request_id, response).await?;
+        let status = response.status();
+        let body = response.text().await.err_to_http(request_id.clone())?;
+        let job_id = extract_job_id(&body).ok_or_else(|| Error::Http {
+            source: "Function registration response did not contain a valid job_id".into(),
+            request_id,
+            status_code: Some(status),
+        })?;
+        Ok(Job::new_typed(Box::new(RemoteJob::new(
+            self.client.clone(),
+            job_id,
+        ))))
+    }
+
+    async fn get_function(&self, name: &str, version: &str) -> Result<FunctionVersion> {
+        let req = self
+            .client
+            .post("/v1/functions/describe")
+            .json(&serde_json::json!({
+                "name": name,
+                "version": version,
+            }));
+        let (request_id, response) = self.client.send(req).await?;
+        let response = self.client.check_response(&request_id, response).await?;
+        response.json().await.err_to_http(request_id)
+    }
+
     fn job(&self, job_id: &str) -> Result<crate::job::Job> {
         Ok(crate::job::Job::new(Box::new(super::job::RemoteJob::new(
             self.client.clone(),
@@ -567,19 +634,14 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
             }) => return Ok(None),
             Err(err) => return Err(err),
         };
-        let body: RemoteDescribeJobResponse = rsp.json().await.err_to_http(request_id)?;
+        let body: DescribeJobResponse = rsp.json().await.err_to_http(request_id)?;
         Ok(Some(JobDescription {
             job_id: body.job_id,
             job_type: body.job_type,
             state: job_state_to_client(&body.job_state),
             creation_ms: body.creation_ms,
             spec: body.spec,
-            failure: body.failure.map(|reported| crate::error::JobFailure {
-                phase: reported.phase,
-                message: reported.message,
-                retryable: reported.retryable,
-                source: None,
-            }),
+            failure: body.failure.map(|reported| reported.into_job_failure()),
         }))
     }
 
@@ -615,29 +677,29 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
     }
 
     async fn table_names(&self, request: TableNamesRequest) -> Result<Vec<String>> {
-        let mut req = if !request.namespace_path.is_empty() {
-            let namespace_id =
-                build_namespace_identifier(&request.namespace_path, &self.client.id_delimiter);
-            self.client
-                .get(&format!("/v1/namespace/{}/table/list", namespace_id))
+        let (tables, version) = if request.namespace_path.is_empty() {
+            // The flat route resumes after a table name and orders by name, which is exactly
+            // what `start_after` means, so the server does the paging.
+            let mut req = self.client.get("/v1/table/");
+            if let Some(limit) = request.limit {
+                req = req.query(&[("limit", limit)]);
+            }
+            if let Some(ref start_after) = request.start_after {
+                req = req.query(&[("page_token", start_after)]);
+            }
+            let (request_id, rsp) = self.client.send_with_retry(req, None, true).await?;
+            let rsp = self.client.check_response(&request_id, rsp).await?;
+            let version = parse_server_version(&request_id, &rsp)?;
+            let tables = rsp
+                .json::<ListTablesResponse>()
+                .await
+                .err_to_http(request_id)?
+                .tables;
+            (tables, version)
         } else {
-            self.client.get("/v1/table/")
+            self.table_names_in_namespace(&request).await?
         };
 
-        if let Some(limit) = request.limit {
-            req = req.query(&[("limit", limit)]);
-        }
-        if let Some(start_after) = request.start_after {
-            req = req.query(&[("page_token", start_after)]);
-        }
-        let (request_id, rsp) = self.client.send_with_retry(req, None, true).await?;
-        let rsp = self.client.check_response(&request_id, rsp).await?;
-        let version = parse_server_version(&request_id, &rsp)?;
-        let tables = rsp
-            .json::<ListTablesResponse>()
-            .await
-            .err_to_http(request_id)?
-            .tables;
         for table in &tables {
             let table_identifier =
                 build_table_identifier(table, &request.namespace_path, &self.client.id_delimiter);
@@ -894,13 +956,28 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
     }
 
     async fn drop_table(&self, name: &str, namespace_path: &[String]) -> Result<()> {
-        let identifier = build_table_identifier(name, namespace_path, &self.client.id_delimiter);
-        let cache_key = build_cache_key(name, namespace_path);
-        let req = self.client.post(&format!("/v1/table/{}/drop/", identifier));
-        let (request_id, resp) = self.client.send(req).await?;
-        self.client.check_response(&request_id, resp).await?;
-        self.table_cache.remove(&cache_key).await;
-        Ok(())
+        self.submit_drop_table(name, namespace_path)
+            .await
+            .map(|_| ())
+    }
+
+    async fn drop_table_async(&self, name: &str, namespace_path: &[String]) -> Result<Job> {
+        let (request_id, response) = self.submit_drop_table(name, namespace_path).await?;
+        let status = response.status();
+        let body = response.text().await.err_to_http(request_id.clone())?;
+        let job_id = extract_job_id(&body);
+        Ok(match job_id {
+            Some(job_id) => Job::new(Box::new(RemoteJob::new(self.client.clone(), job_id))),
+            None if status == StatusCode::ACCEPTED => {
+                return Err(Error::Http {
+                    source: "asynchronous drop-table response did not contain a valid job_id"
+                        .into(),
+                    request_id,
+                    status_code: Some(status),
+                });
+            }
+            None => Job::new_done(),
+        })
     }
 
     async fn drop_all_tables(&self, namespace_path: &[String]) -> Result<()> {
@@ -1206,6 +1283,101 @@ mod tests {
         assert_eq!(names, vec!["table1", "table2"]);
     }
 
+    #[tokio::test]
+    async fn test_table_names_in_a_namespace_never_invents_a_page_token() {
+        // The namespace route's token belongs to the store, so `table_names` cannot build one
+        // from `start_after`. It walks the namespace on the server's own tokens and applies the
+        // name semantics itself, which is what keeps it working either side of the change.
+        let page = Arc::new(AtomicUsize::new(0));
+        let conn = Connection::new_with_handler(move |request| {
+            assert_eq!(request.url().path(), "/v1/namespace/ns/table/list");
+            let query = request.url().query().unwrap_or("");
+            assert!(
+                !query.contains("page_token=users"),
+                "a table name must never be sent as a page token: {query}"
+            );
+            match page.fetch_add(1, Ordering::SeqCst) {
+                0 => {
+                    assert!(
+                        !query.contains("page_token"),
+                        "the walk starts with no token"
+                    );
+                    http::Response::builder()
+                        .status(200)
+                        .body(r#"{"tables": ["users", "orders"], "page_token": "opaque-1"}"#)
+                        .unwrap()
+                }
+                _ => {
+                    assert!(query.contains("page_token=opaque-1"));
+                    http::Response::builder()
+                        .status(200)
+                        .body(r#"{"tables": ["widgets"]}"#)
+                        .unwrap()
+                }
+            }
+        });
+
+        let names = conn
+            .table_names()
+            .namespace(vec!["ns".to_string()])
+            .start_after("users")
+            .execute()
+            .await
+            .unwrap();
+        // Name order, resumed after "users": "orders" sorts before it and is dropped.
+        assert_eq!(names, vec!["widgets"]);
+    }
+
+    #[tokio::test]
+    async fn test_table_names_in_a_namespace_stops_on_a_repeated_token() {
+        // A server that handed back the token it was given would never finish the walk.
+        let conn = Connection::new_with_handler(|_request| {
+            http::Response::builder()
+                .status(200)
+                .body(r#"{"tables": ["a"], "page_token": "same"}"#)
+                .unwrap()
+        });
+
+        let names = conn
+            .table_names()
+            .namespace(vec!["ns".to_string()])
+            .execute()
+            .await
+            .unwrap();
+        // The guard bounds the walk instead of letting it run forever. The repeat is the
+        // server breaking the token contract and is not papered over here.
+        assert_eq!(names, vec!["a", "a"]);
+    }
+
+    #[tokio::test]
+    async fn test_table_names_in_a_namespace_stops_on_an_empty_token() {
+        // An empty token ends the listing. Sending it back would ask a server that reads it
+        // as "start from the beginning" for the first page a second time, and every name on
+        // that page would be collected twice.
+        let requests = Arc::new(AtomicUsize::new(0));
+        let seen = requests.clone();
+        let conn = Connection::new_with_handler(move |request| {
+            seen.fetch_add(1, Ordering::SeqCst);
+            assert!(
+                !request.url().query().unwrap_or("").contains("page_token"),
+                "an empty token must never be sent back"
+            );
+            http::Response::builder()
+                .status(200)
+                .body(r#"{"tables": ["a"], "page_token": ""}"#)
+                .unwrap()
+        });
+
+        let names = conn
+            .table_names()
+            .namespace(vec!["ns".to_string()])
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(names, vec!["a"]);
+        assert_eq!(requests.load(Ordering::SeqCst), 1);
+    }
+
     #[tokio::test]
     async fn test_table_names_pagination() {
         let conn = Connection::new_with_handler(|request| {
@@ -1492,6 +1664,67 @@ mod tests {
         // NOTE: the API will return 200 even if the table does not exist. So we shouldn't expect 404.
     }
 
+    #[tokio::test]
+    async fn test_drop_table_does_not_read_response_body() {
+        let conn = Connection::new_with_handler(|_| {
+            http::Response::builder()
+                .status(200)
+                .body(vec![0xff])
+                .unwrap()
+        });
+
+        conn.drop_table("table1", &[]).await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_drop_table_async_returns_job() {
+        let conn = Connection::new_with_handler(|request| {
+            assert_eq!(request.method(), &reqwest::Method::POST);
+            assert_eq!(request.url().path(), "/v1/table/table1/drop/");
+            http::Response::builder()
+                .status(202)
+                .body(r#"{"job_id":"drop-job-123"}"#)
+                .unwrap()
+        });
+
+        let job = conn.drop_table_async("table1", &[]).await.unwrap();
+        assert_eq!(job.id(), Some("drop-job-123"));
+    }
+
+    #[tokio::test]
+    async fn test_drop_table_async_old_server_returns_done_job() {
+        let conn = Connection::new_with_handler(|_| {
+            http::Response::builder().status(200).body("").unwrap()
+        });
+
+        let job = conn.drop_table_async("table1", &[]).await.unwrap();
+        assert_eq!(job.id(), None);
+        assert_eq!(job.status().await.unwrap(), "finished");
+    }
+
+    #[tokio::test]
+    async fn test_drop_table_async_rejects_accepted_response_without_job_id() {
+        let conn = Connection::new_with_handler(|_| {
+            http::Response::builder().status(202).body("{}").unwrap()
+        });
+
+        let error = conn.drop_table_async("table1", &[]).await.err().unwrap();
+        assert!(error.to_string().contains("valid job_id"));
+    }
+
+    #[tokio::test]
+    async fn test_drop_table_async_rejects_empty_job_id() {
+        let conn = Connection::new_with_handler(|_| {
+            http::Response::builder()
+                .status(202)
+                .body(r#"{"job_id":""}"#)
+                .unwrap()
+        });
+
+        let error = conn.drop_table_async("table1", &[]).await.err().unwrap();
+        assert!(error.to_string().contains("valid job_id"));
+    }
+
     #[tokio::test]
     async fn test_rename_table() {
         let conn = Connection::new_with_handler(|request| {
@@ -2398,6 +2631,60 @@ mod tests {
         assert_eq!(batches[0].num_rows(), 2);
     }
 
+    #[tokio::test]
+    async fn test_create_function_async_sends_canonical_request_and_decodes_typed_job() {
+        const REQUEST: &str = include_str!(
+            "../../tests/fixtures/first_class_functions/v1/remote_function_registration_request.json"
+        );
+        const FUNCTION_JOB: &str =
+            include_str!("../../tests/fixtures/first_class_functions/v1/remote_function_job.json");
+        let expected: serde_json::Value = serde_json::from_str(REQUEST).unwrap();
+        let conn = Connection::new_with_handler(move |request| match request.url().path() {
+            "/v1/functions/create" => {
+                assert_eq!(request.method(), &reqwest::Method::POST);
+                let body: serde_json::Value =
+                    serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
+                assert_eq!(body, expected);
+                http::Response::builder()
+                    .status(202)
+                    .body(r#"{"job_id":"job-function-1"}"#)
+                    .unwrap()
+            }
+            "/v1/jobs/describe" => http::Response::builder()
+                .status(200)
+                .body(FUNCTION_JOB)
+                .unwrap(),
+            path => panic!("unexpected path: {path}"),
+        });
+        let request = crate::function::FunctionRegistrationRequest::from_json(REQUEST).unwrap();
+        let job = conn.create_function_async(request).await.unwrap();
+        assert_eq!(job.id(), Some("job-function-1"));
+        let version = job.wait().await.unwrap();
+        assert_eq!(version.name(), "embed");
+        assert_eq!(version.version(), "fv_01K3EXACT");
+    }
+
+    #[tokio::test]
+    async fn test_get_function_requires_and_sends_exact_version() {
+        const VERSION: &str = include_str!(
+            "../../tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json"
+        );
+        let conn = Connection::new_with_handler(|request| {
+            assert_eq!(request.method(), &reqwest::Method::POST);
+            assert_eq!(request.url().path(), "/v1/functions/describe");
+            let body: serde_json::Value =
+                serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
+            assert_eq!(
+                body,
+                serde_json::json!({"name": "embed", "version": "fv_01K3EXACT"})
+            );
+            http::Response::builder().status(200).body(VERSION).unwrap()
+        });
+        let version = conn.get_function("embed", "fv_01K3EXACT").await.unwrap();
+        assert_eq!(version.name(), "embed");
+        assert_eq!(version.version(), "fv_01K3EXACT");
+    }
+
     #[tokio::test]
     async fn test_conn_job_waits_to_done() {
         let polls = Arc::new(AtomicUsize::new(0));
@@ -2412,7 +2699,7 @@ mod tests {
             http::Response::builder()
                 .status(200)
                 .body(format!(
-                    r#"{{"job_id": "job-1", "job_type": "create_index", "job_state": "{}", "creation_ms": 1}}"#,
+                    r#"{{"job_id": "job-1", "job_type": "create_function", "job_state": "{}", "creation_ms": 1, "result": {{"name": "embed", "version": "fv_1"}}}}"#,
                     state
                 ))
                 .unwrap()
diff --git a/rust/lancedb/src/remote/job.rs b/rust/lancedb/src/remote/job.rs
index 2fc99da59..0d41dbb35 100644
--- a/rust/lancedb/src/remote/job.rs
+++ b/rust/lancedb/src/remote/job.rs
@@ -8,10 +8,10 @@ use std::time::Duration;
 use async_trait::async_trait;
 use tokio::time::sleep;
 
-use serde::{Deserialize, Deserializer};
+use serde::Deserialize;
 
 use crate::error::{Error, JobFailure, Result};
-use crate::job::JobHandle;
+use crate::job::{JobHandle, TerminalResult};
 use crate::remote::client::{HttpSend, RequestResultExt, RestfulLanceDbClient};
 
 /// Delay before the second job-state poll; doubles up to [`MAX_POLL_INTERVAL`].
@@ -29,12 +29,6 @@ enum JobState {
     Other(String),
 }
 
-impl<'de> Deserialize<'de> for JobState {
-    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
-        Ok(Self::from(String::deserialize(deserializer)?.as_str()))
-    }
-}
-
 impl JobState {
     /// The client vocabulary label for this state.
     fn client_label(&self) -> String {
@@ -51,22 +45,26 @@ impl JobState {
 impl From<&str> for JobState {
     fn from(state: &str) -> Self {
         match state {
-            "IN_PROGRESS" => Self::InProgress,
-            "CANCELLED" => Self::Cancelled,
+            "IN_PROGRESS" | "in_progress" => Self::InProgress,
+            "CANCELLED" | "cancelled" | "canceled" => Self::Cancelled,
             // The server reports a timed-out job as FAILED on describe;
             // accept the raw registry state too in case a future server
             // stops folding it.
-            "FAILED" | "TIMED_OUT" => Self::Failed,
-            "DONE" => Self::Done,
+            "FAILED" | "failed" | "TIMED_OUT" | "timed_out" => Self::Failed,
+            "DONE" | "done" | "succeeded" => Self::Done,
             other => Self::Other(other.to_string()),
         }
     }
 }
 
+pub(super) fn job_state_to_client(state: &str) -> String {
+    JobState::from(state).client_label()
+}
+
 /// The server's account of why a job failed. Absent from older servers, which
 /// report only the terminal state.
 #[derive(Deserialize)]
-struct ReportedFailure {
+pub(super) struct ReportedFailure {
     #[serde(default)]
     phase: Option<String>,
     #[serde(default)]
@@ -75,11 +73,43 @@ struct ReportedFailure {
     retryable: Option<bool>,
 }
 
+/// Forward-compatible `/v1/jobs/describe` wire envelope.
 #[derive(Deserialize)]
-struct DescribeJobResponse {
-    job_state: JobState,
+pub(super) struct DescribeJobResponse {
     #[serde(default)]
-    failure: Option<ReportedFailure>,
+    pub(super) job_id: String,
+    #[serde(default)]
+    pub(super) job_type: String,
+    pub(super) job_state: String,
+    #[serde(default)]
+    pub(super) creation_ms: i64,
+    #[serde(default)]
+    pub(super) spec: serde_json::Value,
+    #[serde(default)]
+    result: Option<serde_json::Value>,
+    #[serde(default)]
+    pub(super) failure: Option<ReportedFailure>,
+}
+
+impl ReportedFailure {
+    pub(super) fn into_job_failure(self) -> JobFailure {
+        JobFailure {
+            phase: self.phase,
+            message: self.message,
+            retryable: self.retryable,
+            source: None,
+        }
+    }
+}
+
+impl DescribeJobResponse {
+    fn state(&self) -> JobState {
+        JobState::from(self.job_state.as_str())
+    }
+
+    fn into_terminal_result(self, request_id: String) -> TerminalResult {
+        TerminalResult::remote(self.result, request_id)
+    }
 }
 
 pub struct RemoteJob<S: HttpSend> {
@@ -93,7 +123,7 @@ impl<S: HttpSend> RemoteJob<S> {
     }
 
     /// One `/v1/jobs/describe` round trip.
-    async fn describe(&self) -> Result<DescribeJobResponse> {
+    async fn describe(&self) -> Result<(String, DescribeJobResponse)> {
         let request = self
             .client
             .post("/v1/jobs/describe")
@@ -104,10 +134,10 @@ impl<S: HttpSend> RemoteJob<S> {
         let description: DescribeJobResponse =
             serde_json::from_str(&body).map_err(|e| Error::Http {
                 source: format!("failed to parse job description: {}", e).into(),
-                request_id,
+                request_id: request_id.clone(),
                 status_code: None,
             })?;
-        Ok(description)
+        Ok((request_id, description))
     }
 }
 
@@ -118,26 +148,21 @@ impl<S: HttpSend> JobHandle for RemoteJob<S> {
     }
 
     async fn status(&self) -> Result<String> {
-        Ok(self.describe().await?.job_state.client_label())
+        Ok(self.describe().await?.1.state().client_label())
     }
 
-    async fn wait(&self) -> Result<()> {
+    async fn wait(&self) -> Result<TerminalResult> {
         let mut interval = INITIAL_POLL_INTERVAL;
         loop {
-            let description = self.describe().await?;
-            match description.job_state {
-                JobState::Done => return Ok(()),
+            let (request_id, description) = self.describe().await?;
+            match description.state() {
+                JobState::Done => return Ok(description.into_terminal_result(request_id)),
                 JobState::Failed => {
                     return Err(Error::JobFailed {
                         job_id: Some(self.job_id.clone()),
                         failure: description
                             .failure
-                            .map(|reported| JobFailure {
-                                phase: reported.phase,
-                                message: reported.message,
-                                retryable: reported.retryable,
-                                source: None,
-                            })
+                            .map(ReportedFailure::into_job_failure)
                             .unwrap_or_default(),
                     });
                 }
@@ -168,3 +193,78 @@ impl<S: HttpSend> JobHandle for RemoteJob<S> {
             .map(|_| ())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use async_trait::async_trait;
+
+    use crate::Result;
+    use crate::function::{FunctionVersion, RefreshColumnResult};
+    use crate::job::{Job, JobHandle, TerminalResult};
+
+    use super::DescribeJobResponse;
+
+    const FUNCTION_JOB: &str =
+        include_str!("../../tests/fixtures/first_class_functions/v1/remote_function_job.json");
+    const REFRESH_JOB: &str =
+        include_str!("../../tests/fixtures/first_class_functions/v1/remote_refresh_job.json");
+    const UNIT_JOB: &str =
+        include_str!("../../tests/fixtures/first_class_functions/v1/remote_unit_job.json");
+    const MISSING_RESULT_JOB: &str = r#"{"job_state":"DONE"}"#;
+
+    struct FixtureRemoteJob(&'static str);
+
+    #[async_trait]
+    impl JobHandle for FixtureRemoteJob {
+        async fn status(&self) -> Result<String> {
+            Ok("finished".to_string())
+        }
+
+        async fn wait(&self) -> Result<TerminalResult> {
+            let description: DescribeJobResponse =
+                serde_json::from_str(self.0).expect("remote job fixture");
+            Ok(description.into_terminal_result("fixture-request".to_string()))
+        }
+
+        async fn cancel(&self) -> Result<()> {
+            Ok(())
+        }
+    }
+
+    #[tokio::test]
+    async fn typed_remote_job_fixtures_decode_terminal_results() {
+        let function = Job::<FunctionVersion>::new_typed(Box::new(FixtureRemoteJob(FUNCTION_JOB)));
+        let result = function.wait().await.expect("typed FunctionVersion result");
+        assert_eq!(result.version(), "fv_01K3EXACT");
+
+        let refresh =
+            Job::<RefreshColumnResult>::new_typed(Box::new(FixtureRemoteJob(REFRESH_JOB)));
+        let result = refresh.wait().await.expect("typed RefreshColumnResult");
+        assert_eq!(result.rows_assigned, 999_998_800);
+        assert_eq!(result.rows_filled(), result.rows_assigned);
+
+        let unit = Job::new(Box::new(FixtureRemoteJob(UNIT_JOB)));
+        unit.wait()
+            .await
+            .expect("unit result ignores additive remote payloads");
+    }
+
+    #[tokio::test]
+    async fn typed_remote_job_requires_a_terminal_result() {
+        let typed =
+            Job::<RefreshColumnResult>::new_typed(Box::new(FixtureRemoteJob(MISSING_RESULT_JOB)));
+        let error = typed.wait().await.unwrap_err();
+        assert!(
+            error
+                .to_string()
+                .contains("successful typed job response did not contain a result")
+        );
+    }
+
+    #[test]
+    fn remote_wire_unknown_fields_are_forward_decodable() {
+        let response: DescribeJobResponse =
+            serde_json::from_str(FUNCTION_JOB).expect("function job fixture");
+        assert_eq!(response.job_state, "DONE");
+    }
+}
diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs
index b1b41cae7..b116083c8 100644
--- a/rust/lancedb/src/remote/table.rs
+++ b/rust/lancedb/src/remote/table.rs
@@ -8,7 +8,7 @@ use self::insert::{RemoteWriteExec, WriteOp};
 use super::client::RequestResultExt;
 use super::client::{HttpSend, RestfulLanceDbClient, Sender};
 use super::db::ServerVersion;
-use super::{ARROW_FILE_CONTENT_TYPE, ARROW_STREAM_CONTENT_TYPE};
+use super::{ARROW_FILE_CONTENT_TYPE, ARROW_STREAM_CONTENT_TYPE, extract_job_id};
 use crate::blob::BlobFile;
 use crate::data::scannable::{PeekedScannable, Scannable, estimate_write_partitions};
 use crate::expr::expr_to_sql_string;
@@ -21,17 +21,21 @@ use crate::remote::job::RemoteJob;
 use crate::table::AddColumnsResult;
 use crate::table::AddResult;
 use crate::table::BranchDiff;
+use crate::table::CherryPickResult;
 use crate::table::DeleteResult;
 use crate::table::DropColumnsResult;
+use crate::table::LsmStats;
 use crate::table::LsmWriteSpec;
-use crate::table::MergeBranchResult;
 use crate::table::MergeResult;
 use crate::table::Tags;
 use crate::table::UpdateResult;
+use crate::table::lsm_stats::GetLsmStatsResponse;
 use crate::table::merge::MergeFilter;
 use crate::table::query::create_multi_vector_plan;
 use crate::table::write_progress::FinishOnDrop;
-use crate::table::{AlterColumnsResult, FieldMetadataUpdate, UpdateFieldMetadataResult};
+use crate::table::{
+    AlterColumnsResult, FieldMetadataUpdate, RefreshColumnResult, UpdateFieldMetadataResult,
+};
 use crate::table::{AnyQuery, Filter, Predicate, PreprocessingOutput, TableStatistics};
 use crate::utils::background_cache::BackgroundCache;
 use crate::utils::{
@@ -138,6 +142,40 @@ impl FreshnessHeaders {
     }
 }
 
+/// A backfill job whose successful wait establishes a read-freshness
+/// baseline on the submitting handle, so a later read cannot be served
+/// from a cache older than the completed fill. A handle pinned by checkout
+/// at completion keeps its time-travel view instead.
+struct FreshnessJob<S: HttpSend> {
+    inner: RemoteJob<S>,
+    freshness: Arc<Mutex<FreshnessState>>,
+    version: Arc<RwLock<Option<u64>>>,
+}
+
+#[async_trait]
+impl<S: HttpSend> crate::job::JobHandle for FreshnessJob<S> {
+    fn id(&self) -> Option<&str> {
+        crate::job::JobHandle::id(&self.inner)
+    }
+
+    async fn status(&self) -> Result<String> {
+        crate::job::JobHandle::status(&self.inner).await
+    }
+
+    async fn wait(&self) -> Result<crate::job::TerminalResult> {
+        let result = crate::job::JobHandle::wait(&self.inner).await?;
+        let version = self.version.read().await;
+        if version.is_none() {
+            self.freshness.lock().unwrap().checkout_baseline = Some(SystemTime::now());
+        }
+        Ok(result)
+    }
+
+    async fn cancel(&self) -> Result<()> {
+        crate::job::JobHandle::cancel(&self.inner).await
+    }
+}
+
 fn compute_min_timestamp(
     state: &FreshnessState,
     interval: Option<Duration>,
@@ -272,10 +310,10 @@ pub struct RemoteTable<S: HttpSend = Sender> {
     identifier: String,
     server_version: ServerVersion,
 
-    version: RwLock<Option<u64>>,
+    version: Arc<RwLock<Option<u64>>>,
     location: RwLock<Option<String>>,
     schema_cache: BackgroundCache<SchemaRef, Error>,
-    freshness: Mutex<FreshnessState>,
+    freshness: Arc<Mutex<FreshnessState>>,
     /// The branch this handle is scoped to, or `None` for the main branch.
     /// Stamped onto every branch-accepting request so reads and writes resolve
     /// on the branch's own version chain rather than main's.
@@ -390,13 +428,7 @@ impl<S: HttpSend> RemoteTable<S> {
             .text()
             .await
             .ok()
-            .and_then(|body| serde_json::from_str::<serde_json::Value>(&body).ok())
-            .and_then(|value| {
-                value
-                    .get("job_id")
-                    .and_then(|id| id.as_str())
-                    .map(str::to_string)
-            });
+            .and_then(|body| extract_job_id(&body));
 
         if let Some(wait_timeout) = index.wait_timeout {
             let index_name = index.name.unwrap_or_else(|| format!("{}_idx", column));
@@ -419,10 +451,10 @@ impl<S: HttpSend> RemoteTable<S> {
             namespace,
             identifier,
             server_version,
-            version: RwLock::new(None),
+            version: Arc::new(RwLock::new(None)),
             location: RwLock::new(None),
             schema_cache: BackgroundCache::new(SCHEMA_CACHE_TTL, SCHEMA_CACHE_REFRESH_WINDOW),
-            freshness: Mutex::new(FreshnessState::default()),
+            freshness: Arc::new(Mutex::new(FreshnessState::default())),
             branch: None,
         }
     }
@@ -451,10 +483,10 @@ impl<S: HttpSend> RemoteTable<S> {
             namespace: self.namespace.clone(),
             identifier: self.identifier.clone(),
             server_version: self.server_version.clone(),
-            version: RwLock::new(None),
+            version: Arc::new(RwLock::new(None)),
             location: RwLock::new(None),
             schema_cache: BackgroundCache::new(SCHEMA_CACHE_TTL, SCHEMA_CACHE_REFRESH_WINDOW),
-            freshness: Mutex::new(FreshnessState::default()),
+            freshness: Arc::new(Mutex::new(FreshnessState::default())),
             branch,
         }
     }
@@ -991,6 +1023,18 @@ impl<S: HttpSend> RemoteTable<S> {
         }
     }
 
+    /// Send an LSM operator request with the transport retry layer **off**.
+    ///
+    /// Retry policy on these routes belongs to the checkpoint loop, which
+    /// reads the status and can tell contention from a lost claim. Leaving the
+    /// transport layer on would re-ask on its own schedule first, and surface
+    /// an `Error::Retry` whose status the loop would then have to unwrap.
+    async fn send_lsm_route(&self, request: RequestBuilder) -> Result<(String, reqwest::Response)> {
+        let (request_id, response) = self.send(request, false).await?;
+        let response = self.check_table_response(&request_id, response).await?;
+        Ok((request_id, response))
+    }
+
     /// Build a POST request and attach the read-freshness headers
     /// (`x-lancedb-min-version`, `x-lancedb-min-timestamp`).
     fn post_read(&self, uri: &str) -> RequestBuilder {
@@ -1260,10 +1304,10 @@ mod test_utils {
                 namespace: vec![],
                 identifier: name,
                 server_version: version.map(ServerVersion).unwrap_or_default(),
-                version: RwLock::new(None),
+                version: Arc::new(RwLock::new(None)),
                 location: RwLock::new(None),
                 schema_cache: BackgroundCache::new(SCHEMA_CACHE_TTL, SCHEMA_CACHE_REFRESH_WINDOW),
-                freshness: Mutex::new(FreshnessState::default()),
+                freshness: Arc::new(Mutex::new(FreshnessState::default())),
                 branch: None,
             }
         }
@@ -1284,10 +1328,10 @@ mod test_utils {
                 namespace: vec![],
                 identifier: name,
                 server_version: ServerVersion::default(),
-                version: RwLock::new(None),
+                version: Arc::new(RwLock::new(None)),
                 location: RwLock::new(None),
                 schema_cache: BackgroundCache::new(SCHEMA_CACHE_TTL, SCHEMA_CACHE_REFRESH_WINDOW),
-                freshness: Mutex::new(FreshnessState::default()),
+                freshness: Arc::new(Mutex::new(FreshnessState::default())),
                 branch: None,
             }
         }
@@ -1317,10 +1361,10 @@ mod test_utils {
                 namespace: vec![],
                 identifier: name,
                 server_version: version.map(ServerVersion).unwrap_or_default(),
-                version: RwLock::new(None),
+                version: Arc::new(RwLock::new(None)),
                 location: RwLock::new(None),
                 schema_cache: BackgroundCache::new(SCHEMA_CACHE_TTL, SCHEMA_CACHE_REFRESH_WINDOW),
-                freshness: Mutex::new(FreshnessState::default()),
+                freshness: Arc::new(Mutex::new(FreshnessState::default())),
                 branch: None,
             }
         }
@@ -1680,15 +1724,37 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
     }
     async fn query_snapshot(&self) -> Result<Arc<dyn BaseTable>> {
         let description = self.describe().await?;
-        let schema: arrow_schema::Schema = description.schema.try_into()?;
+        let TableDescription {
+            version,
+            schema,
+            location,
+        } = description;
+        let schema = Arc::new(arrow_schema::Schema::try_from(schema)?);
         let snapshot = self.with_branch(self.branch.clone());
-        *snapshot.version.write().await = Some(description.version);
-        snapshot.schema_cache.seed(Arc::new(schema));
+        *snapshot.version.write().await = Some(version);
+        *snapshot.location.write().await = location;
+        snapshot.schema_cache.seed(schema);
         Ok(Arc::new(snapshot))
     }
     async fn version(&self) -> Result<u64> {
         self.describe().await.map(|desc| desc.version)
     }
+
+    async fn checkout_current(&self) -> Result<Arc<dyn BaseTable>> {
+        let description = self.describe().await?;
+        let TableDescription {
+            version,
+            schema,
+            location,
+        } = description;
+        let schema = Arc::new(arrow_schema::Schema::try_from(schema)?);
+        let snapshot = self.with_branch(self.branch.clone());
+        *snapshot.version.write().await = Some(version);
+        *snapshot.location.write().await = location;
+        snapshot.schema_cache.seed(schema);
+        Ok(Arc::new(snapshot))
+    }
+
     async fn checkout(&self, version: u64) -> Result<()> {
         // Validate the version exists. The describe is sent without freshness
         // headers so a stale `min_version` from a previous write doesn't ride
@@ -1739,6 +1805,18 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
 
         Ok(())
     }
+    async fn snapshot_at_current_version(&self) -> Result<Option<Arc<dyn BaseTable>>> {
+        // A checked-out handle already names its snapshot. Otherwise resolve
+        // latest exactly once before creating the independent pinned handle.
+        let version = match self.current_version().await {
+            Some(version) => version,
+            None => self.describe().await?.version,
+        };
+
+        let snapshot = self.with_branch(self.branch.clone());
+        *snapshot.version.write().await = Some(version);
+        Ok(Some(Arc::new(snapshot)))
+    }
     async fn restore(&self) -> Result<()> {
         let mut request = self
             .client
@@ -1995,7 +2073,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
     async fn diff_branch(&self, from_branch: &str) -> Result<BranchDiff> {
         if from_branch.trim().is_empty() {
             return Err(Error::InvalidInput {
-                message: "from_branch must be a non-empty string".into(),
+                message: "Branch name cannot be empty.".into(),
             });
         }
         let request = self
@@ -2022,20 +2100,23 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
         })
     }
 
-    async fn merge_branch(&self, from_branch: &str, dry_run: bool) -> Result<MergeBranchResult> {
+    async fn cherry_pick(&self, from_branch: &str, dry_run: bool) -> Result<CherryPickResult> {
         if from_branch.trim().is_empty() {
             return Err(Error::InvalidInput {
-                message: "from_branch must be a non-empty string".into(),
+                message: "Branch name cannot be empty.".into(),
             });
         }
         let request = self
             .client
-            .post(&format!("/v1/table/{}/branches/merge/", self.identifier))
+            .post(&format!(
+                "/v1/table/{}/branches/cherry_pick/",
+                self.identifier
+            ))
             .json(&serde_json::json!({
                 "from_branch": from_branch,
                 "dry_run": dry_run,
             }));
-        // No retry. 409 rejected merge is final and carries a body.
+        // No retry. HTTP 409 is CherryPickStatus::Failed with a body, not a transport error.
         let (request_id, response) = self.send(request, false).await?;
         let status = response.status();
         if status == StatusCode::NOT_FOUND {
@@ -2044,11 +2125,11 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
                 source: format!("branch '{}' does not exist", from_branch).into(),
             });
         }
-        // 200 and 409 both carry MergeBranchResult.
+        // 200 and 409 both carry CherryPickResult.
         if status != StatusCode::OK && status != StatusCode::CONFLICT {
             let body = response.text().await.unwrap_or_default();
             return Err(Error::Http {
-                source: format!("unexpected status {status} from merge_branch: {body}").into(),
+                source: format!("unexpected status {status} from cherry_pick: {body}").into(),
                 request_id,
                 status_code: Some(status),
             });
@@ -2056,7 +2137,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
         let body = response.text().await.err_to_http(request_id.clone())?;
         serde_json::from_str(&body).map_err(|err| Error::Http {
             source: format!(
-                "Failed to parse merge_branch response: {}, body: {}",
+                "Failed to parse cherry_pick response: {}, body: {}",
                 err, body
             )
             .into(),
@@ -2110,7 +2191,17 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
     async fn add(&self, mut add: AddDataBuilder) -> Result<AddResult> {
         self.check_mutable().await?;
 
+        if add.allow_external_blob_outside_bases {
+            return Err(Error::NotSupported {
+                message: "allow_external_blob_outside_bases is only supported on local tables"
+                    .to_string(),
+            });
+        }
+        // String blob values still coerce to the uri child in into_plan.
+        // Remote and local share that input shape.
+
         let table_schema = self.schema().await?;
+        crate::table::computed_columns::ensure_supported_function_metadata(table_schema.as_ref())?;
         let table_def = TableDefinition::try_from_rich_schema(table_schema.clone())?;
 
         let num_partitions = if self.server_version.support_multipart_write() {
@@ -2476,13 +2567,47 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
         })
     }
 
+    async fn flush_lsm(&self) -> Result<()> {
+        let request = self
+            .client
+            .post(&format!("/v1/table/{}/flush_lsm/", self.identifier));
+        self.send_lsm_route(request).await?;
+        Ok(())
+    }
+
+    async fn compact_lsm(&self) -> Result<()> {
+        let request = self
+            .client
+            .post(&format!("/v1/table/{}/compact_lsm/", self.identifier));
+        self.send_lsm_route(request).await?;
+        Ok(())
+    }
+
+    async fn get_lsm_stats(&self, include_generation_rows: bool) -> Result<Option<LsmStats>> {
+        // Read-semantics POST, like `get_lsm_write_spec`.
+        let request = self
+            .post_read(&format!("/v1/table/{}/get_lsm_stats/", self.identifier))
+            .json(&serde_json::json!({
+                "include_generation_rows": include_generation_rows,
+            }));
+        let (request_id, response) = self.send_lsm_route(request).await?;
+        let body = response.text().await.err_to_http(request_id.clone())?;
+        let parsed: GetLsmStatsResponse = serde_json::from_str(&body).map_err(|e| Error::Http {
+            source: format!("Failed to parse get_lsm_stats response: {e}").into(),
+            request_id,
+            status_code: None,
+        })?;
+        // `null` — and only — when the table has no LSM write path.
+        Ok(parsed.lsm_stats)
+    }
+
     async fn set_lsm_write_spec(&self, spec: LsmWriteSpec) -> Result<()> {
         self.check_mutable().await?;
 
         // Map the spec onto the server's request DTO. `sharding` is internally
-        // tagged on `mode` to mirror sophon's `Sharding` enum; `maintained_indexes`
-        // and `writer_config_defaults` are sent verbatim (an empty list means "no
-        // maintained indexes", not "default to all").
+        // tagged on `mode` to mirror sophon's `Sharding` enum. A null
+        // `maintained_indexes` asks the server to resolve every maintainable
+        // index at HEAD; a list is verbatim, an empty one meaning none.
         let sharding = match &spec {
             LsmWriteSpec::Bucket {
                 column,
@@ -2628,6 +2753,10 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
         _read_columns: Option<Vec<String>>,
     ) -> Result<AddColumnsResult> {
         self.check_mutable().await?;
+        crate::table::computed_columns::ensure_no_function_bindings_for_mutation(
+            self.schema().await?.as_ref(),
+            "schema evolution",
+        )?;
         match transforms {
             NewColumnTransform::SqlExpressions(expressions) => {
                 let body = expressions
@@ -2674,6 +2803,149 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
         }
     }
 
+    async fn add_computed_columns(&self, columns: &[(String, String)]) -> Result<AddColumnsResult> {
+        self.check_mutable().await?;
+        crate::table::computed_columns::ensure_no_function_bindings_for_mutation(
+            self.schema().await?.as_ref(),
+            "schema evolution",
+        )?;
+        // The server plans the declaration: expression validation, type
+        // inference and the persisted binding all happen there.
+        let entries = columns
+            .iter()
+            .map(
+                |(name, expression)| lance_namespace::models::AddColumnsEntry {
+                    name: name.clone(),
+                    computed: Some(Some(expression.clone())),
+                    ..Default::default()
+                },
+            )
+            .collect::<Vec<_>>();
+        let mut body = serde_json::json!({ "new_columns": entries });
+        self.apply_branch_body(&mut body);
+        let request = self
+            .client
+            .post(&format!("/v1/table/{}/add_columns/", self.identifier))
+            .json(&body);
+        let (request_id, response) = self.send(request, true).await?;
+        let response = self.check_table_response(&request_id, response).await?;
+        let body = response.text().await.err_to_http(request_id.clone())?;
+
+        if body.trim().is_empty() {
+            // Backward compatible with old servers
+            return Ok(AddColumnsResult { version: 0 });
+        }
+
+        let result: AddColumnsResult = serde_json::from_str(&body).map_err(|e| Error::Http {
+            source: format!("Failed to parse add_columns response: {}", e).into(),
+            request_id,
+            status_code: None,
+        })?;
+
+        self.invalidate_schema_cache();
+        self.track_write_version(result.version);
+
+        Ok(result)
+    }
+
+    async fn add_function_columns(
+        &self,
+        application: &crate::function::FunctionApplication,
+        output_name: Option<&str>,
+    ) -> Result<AddColumnsResult> {
+        self.check_mutable().await?;
+        let schema = self.schema().await?;
+        let plan = crate::table::computed_columns::plan_function_application(
+            schema.as_ref(),
+            application,
+            output_name,
+        )?;
+        let new_columns = plan
+            .outputs
+            .iter()
+            .map(|output| {
+                serde_json::json!({
+                    "name": output.output_name,
+                    "all_null": true,
+                })
+            })
+            .collect::<Vec<_>>();
+        let mut body = serde_json::json!({
+            "new_columns": new_columns,
+            "function": {
+                "application": plan.application,
+                "binding_metadata_version": plan.binding_metadata_version,
+                "input_bindings": plan.input_bindings,
+                "input_schema": plan.input_schema,
+                "output_schema": plan.output_schema,
+                "outputs": plan.outputs,
+            },
+        });
+        self.apply_branch_body(&mut body);
+        let request = self
+            .client
+            .post(&format!("/v1/table/{}/add_columns/", self.identifier))
+            .json(&body);
+        let (request_id, response) = self.send(request, true).await?;
+        let response = self.check_table_response(&request_id, response).await?;
+        let body = response.text().await.err_to_http(request_id.clone())?;
+
+        if body.trim().is_empty() {
+            return Ok(AddColumnsResult { version: 0 });
+        }
+
+        let result: AddColumnsResult = serde_json::from_str(&body).map_err(|e| Error::Http {
+            source: format!("Failed to parse add Function columns response: {e}").into(),
+            request_id,
+            status_code: None,
+        })?;
+
+        self.invalidate_schema_cache();
+        self.track_write_version(result.version);
+        Ok(result)
+    }
+
+    async fn refresh_column(&self, _column: &str) -> Result<RefreshColumnResult> {
+        // The server runs a refresh as a job and does not report a fill
+        // count, so the blocking form has no honest result to return.
+        Err(Error::NotSupported {
+            message: "a remote refresh runs as a server job; use refresh_column_async and \
+                      wait on the returned handle"
+                .into(),
+        })
+    }
+
+    async fn refresh_column_async(
+        &self,
+        column: &str,
+    ) -> Result<Job<crate::function::RefreshColumnResult>> {
+        self.check_mutable().await?;
+        let mut body = serde_json::json!({ "column": column });
+        self.apply_branch_body(&mut body);
+        let request = self
+            .post_read(&format!("/v1/table/{}/backfill_column", self.identifier))
+            .json(&body);
+        let (request_id, response) = self.send(request, true).await?;
+        let response = self.check_table_response(&request_id, response).await?;
+        let body = response.text().await.err_to_http(request_id.clone())?;
+
+        #[derive(serde::Deserialize)]
+        struct BackfillResponse {
+            job_id: String,
+        }
+        let response: BackfillResponse = serde_json::from_str(&body).map_err(|e| Error::Http {
+            source: format!("Failed to parse backfill_column response: {}", e).into(),
+            request_id,
+            status_code: None,
+        })?;
+
+        Ok(Job::new_typed(Box::new(FreshnessJob {
+            inner: RemoteJob::new(self.client.clone(), response.job_id),
+            freshness: self.freshness.clone(),
+            version: self.version.clone(),
+        })))
+    }
+
     async fn alter_columns(&self, alterations: &[ColumnAlteration]) -> Result<AlterColumnsResult> {
         self.check_mutable().await?;
         let body = alterations
@@ -2799,9 +3071,10 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
     }
 
     async fn index_stats(&self, index_name: &str) -> Result<Option<IndexStatistics>> {
+        let encoded_name = urlencoding::encode(index_name);
         let mut request = self.post_read(&format!(
-            "/v1/table/{}/index/{}/stats/",
-            self.identifier, index_name
+            "/v1/table/{}/index/{encoded_name}/stats/",
+            self.identifier
         ));
         let version = self.current_version().await;
         let mut body = serde_json::json!({ "version": version });
@@ -2828,9 +3101,10 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
     }
 
     async fn drop_index(&self, index_name: &str) -> Result<()> {
+        let encoded_name = urlencoding::encode(index_name);
         let request = self.apply_branch_query(self.client.post(&format!(
-            "/v1/table/{}/index/{}/drop/",
-            self.identifier, index_name
+            "/v1/table/{}/index/{encoded_name}/drop/",
+            self.identifier
         )));
         let (request_id, response) = self.send(request, true).await?;
         if response.status() == StatusCode::NOT_FOUND {
@@ -2843,9 +3117,10 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
     }
 
     async fn prewarm_index(&self, index_name: &str) -> Result<()> {
+        let encoded_name = urlencoding::encode(index_name);
         let request = self.client.post(&format!(
-            "/v1/table/{}/index/{}/prewarm/",
-            self.identifier, index_name
+            "/v1/table/{}/index/{encoded_name}/prewarm/",
+            self.identifier
         ));
         let (request_id, response) = self.send(request, true).await?;
         if response.status() == StatusCode::NOT_FOUND {
@@ -2947,7 +3222,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
 }
 
 #[derive(Serialize, Clone, Debug)]
-pub(crate) struct MergeInsertRequest {
+pub struct MergeInsertRequest {
     on: String,
     when_matched_update_all: bool,
     when_matched_update_all_filt: Option<String>,
@@ -3033,7 +3308,10 @@ mod tests {
     use arrow::{array::AsArray, compute::concat_batches, datatypes::Int32Type};
     use arrow_array::Array;
     use arrow_array::builder::LargeBinaryBuilder;
-    use arrow_array::{BinaryArray, Int32Array, RecordBatch, RecordBatchIterator, record_batch};
+    use arrow_array::{
+        BinaryArray, Int32Array, Int64Array, RecordBatch, RecordBatchIterator, StringArray,
+        StructArray, record_batch,
+    };
     use arrow_schema::{DataType, Field, Schema};
     use chrono::{DateTime, Utc};
     use futures::{StreamExt, TryFutureExt, future::BoxFuture};
@@ -3059,6 +3337,21 @@ mod tests {
         },
     };
 
+    fn refresh_done(job_id: &str) -> String {
+        json!({
+            "job_id": job_id,
+            "job_state": "DONE",
+            "result": {
+                "rows_assigned": 12,
+                "rows_failed": 0,
+                "rows_remaining": 0,
+                "source_version": 7,
+                "published_version": 8,
+            }
+        })
+        .to_string()
+    }
+
     #[tokio::test]
     async fn test_not_found() {
         let table = Table::new_with_handler("my_table", |_| {
@@ -3370,6 +3663,88 @@ mod tests {
         assert_eq!(&body, &expected_body);
     }
 
+    #[tokio::test]
+    async fn add_rejects_external_blob_flag_before_any_request() {
+        let table = Table::new_with_handler::<String>("my_table", |request| {
+            panic!("Unexpected request: {}", request.url().path())
+        });
+        let data = RecordBatch::try_new(
+            Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
+            vec![Arc::new(Int32Array::from(vec![1]))],
+        )
+        .unwrap();
+
+        let err = table
+            .add(data)
+            .allow_external_blob_outside_bases(true)
+            .execute()
+            .await
+            .unwrap_err();
+
+        assert!(matches!(err, Error::NotSupported { .. }), "got {err:?}");
+        assert!(err.to_string().contains("local tables"));
+    }
+
+    #[tokio::test]
+    async fn add_string_blob_becomes_uri_struct_without_the_local_flag() {
+        let table_schema = Schema::new(vec![
+            Field::new("id", DataType::Int64, false),
+            crate::blob("image", true),
+        ]);
+        let describe_body = describe_response(&table_schema);
+        let input = RecordBatch::try_new(
+            Arc::new(Schema::new(vec![
+                Field::new("id", DataType::Int64, false),
+                Field::new("image", DataType::Utf8, true),
+            ])),
+            vec![
+                Arc::new(Int64Array::from(vec![1])),
+                Arc::new(StringArray::from(vec![Some("s3://bucket/key")])),
+            ],
+        )
+        .unwrap();
+
+        let (sender, receiver) = std::sync::mpsc::channel();
+        let table =
+            Table::new_with_handler("my_table", move |mut request| match request.url().path() {
+                "/v1/table/my_table/describe/" => http::Response::builder()
+                    .status(200)
+                    .body(describe_body.clone())
+                    .unwrap(),
+                "/v1/table/my_table/insert/" => {
+                    let mut body_out = reqwest::Body::from(Vec::new());
+                    std::mem::swap(request.body_mut().as_mut().unwrap(), &mut body_out);
+                    sender.send(body_out).unwrap();
+                    http::Response::builder()
+                        .status(200)
+                        .body(r#"{"version": 2}"#.to_string())
+                        .unwrap()
+                }
+                path => panic!("Unexpected path: {path}"),
+            });
+
+        table.add(input).execute().await.unwrap();
+
+        let body = collect_body(receiver.recv().unwrap()).await;
+        let mut reader =
+            arrow_ipc::reader::StreamReader::try_new(std::io::Cursor::new(body), None).unwrap();
+        let batch = reader.next().unwrap().unwrap();
+        let image = batch
+            .column_by_name("image")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<StructArray>()
+            .expect("remote add should send the coerced blob struct");
+        let uri: &StringArray = image
+            .column_by_name("uri")
+            .unwrap()
+            .as_any()
+            .downcast_ref()
+            .unwrap();
+        assert_eq!(uri.value(0), "s3://bucket/key");
+        assert!(image.column_by_name("data").unwrap().is_null(0));
+    }
+
     #[rstest]
     #[case(true)]
     #[case(false)]
@@ -3657,11 +4032,14 @@ mod tests {
                 assert_eq!(rename, "y");
 
                 if old_server {
-                    http::Response::builder().status(200).body("{}").unwrap()
+                    http::Response::builder()
+                        .status(200)
+                        .body("{}".to_string())
+                        .unwrap()
                 } else {
                     http::Response::builder()
                         .status(200)
-                        .body(r#"{"version": 43}"#)
+                        .body(r#"{"version": 43}"#.to_string())
                         .unwrap()
                 }
             } else {
@@ -3792,11 +4170,14 @@ mod tests {
                 assert_eq!(predicate, "id in (1, 2, 3)");
 
                 if old_server {
-                    http::Response::builder().status(200).body("{}").unwrap()
+                    http::Response::builder()
+                        .status(200)
+                        .body("{}".to_string())
+                        .unwrap()
                 } else {
                     http::Response::builder()
                         .status(200)
-                        .body(r#"{"version": 43}"#)
+                        .body(r#"{"version": 43}"#.to_string())
                         .unwrap()
                 }
             } else {
@@ -4079,6 +4460,56 @@ mod tests {
         write_ipc_stream_uncompressed(&one_row_blob_batch(column))
     }
 
+    #[tokio::test]
+    async fn test_remote_scan_order_is_not_deterministic() {
+        // A distributed scan answers in no fixed order, so callers that assign meaning
+        // to row position have to sort for themselves.
+        let table = Table::new_with_handler("my_table", |_| {
+            http::Response::builder()
+                .status(200)
+                .body(Vec::new())
+                .unwrap()
+        });
+        assert!(!table.base_table().scan_order_is_deterministic());
+    }
+
+    #[tokio::test]
+    async fn test_checkout_branch_pins_without_touching_the_original() {
+        let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
+        let recorder = seen.clone();
+        let table = Table::new_with_handler_version(
+            "my_table",
+            semver::Version::new(0, 5, 0),
+            move |request| match request.url().path() {
+                "/v1/table/my_table/describe/" => http::Response::builder()
+                    .status(200)
+                    .body(br#"{"version": 42, "schema": {"fields": []}}"#.to_vec())
+                    .unwrap(),
+                "/v1/table/my_table/count_rows/" => {
+                    let body = request_body_json(&request);
+                    recorder.lock().unwrap().push(body["version"].clone());
+                    http::Response::builder()
+                        .status(200)
+                        .body(b"0".to_vec())
+                        .unwrap()
+                }
+                path => panic!("unexpected request path: {path}"),
+            },
+        );
+
+        let pinned = table.checkout_branch("main", Some(42)).await.unwrap();
+        pinned.count_rows(None).await.unwrap();
+        table.count_rows(None).await.unwrap();
+
+        let seen = seen.lock().unwrap();
+        assert_eq!(seen[0], 42, "the pinned handle must send its version");
+        assert!(
+            seen[1].is_null(),
+            "the original handle must still track latest, got {:?}",
+            seen[1]
+        );
+    }
+
     #[tokio::test]
     async fn test_fetch_blobs_sends_the_checked_out_version() {
         let ipc = one_row_blob_ipc_stream("image");
@@ -5912,16 +6343,18 @@ mod tests {
             .await
             .unwrap();
 
+        // Positions are relative to the first retained token, so dropping the
+        // leading "hello" stop word does not shift the remaining tokens.
         assert_eq!(
             tokens,
             vec![
                 FtsToken {
                     text: "こんにちは".to_string(),
-                    position: 1,
+                    position: 0,
                 },
                 FtsToken {
                     text: "世界".to_string(),
-                    position: 2,
+                    position: 1,
                 },
             ]
         );
@@ -6361,7 +6794,9 @@ mod tests {
     #[tokio::test]
     async fn test_add_columns(#[case] old_server: bool) {
         let table = Table::new_with_handler("my_table", move |request| {
-            if request.url().path() == "/v1/table/my_table/add_columns/" {
+            if request.url().path() == "/v1/table/my_table/describe/" {
+                simple_describe_response()
+            } else if request.url().path() == "/v1/table/my_table/add_columns/" {
                 assert_eq!(request.method(), "POST");
                 assert_eq!(
                     request.headers().get("Content-Type").unwrap(),
@@ -6385,11 +6820,14 @@ mod tests {
                 assert_eq!(expression, "cast(NULL as int32)");
 
                 if old_server {
-                    http::Response::builder().status(200).body("{}").unwrap()
+                    http::Response::builder()
+                        .status(200)
+                        .body("{}".to_string())
+                        .unwrap()
                 } else {
                     http::Response::builder()
                         .status(200)
-                        .body(r#"{"version": 43}"#)
+                        .body(r#"{"version": 43}"#.to_string())
                         .unwrap()
                 }
             } else {
@@ -6410,6 +6848,557 @@ mod tests {
         assert_eq!(result.version, if old_server { 0 } else { 43 });
     }
 
+    /// A declaration is sent as `{name, computed}` entries for the server to
+    /// plan; the client never types the expression itself.
+    #[tokio::test]
+    async fn test_add_computed_columns_sends_the_expression() {
+        let table = Table::new_with_handler("my_table", |request| match request.url().path() {
+            "/v1/table/my_table/describe/" => simple_describe_response(),
+            "/v1/table/my_table/add_columns/" => {
+                assert_eq!(request.method(), "POST");
+                let body = request.body().unwrap().as_bytes().unwrap();
+                let value: serde_json::Value = serde_json::from_slice(body).unwrap();
+                assert_eq!(
+                    value["new_columns"],
+                    serde_json::json!([{"name": "doubled", "computed": "x * 2"}])
+                );
+                http::Response::builder()
+                    .status(200)
+                    .body(r#"{"version": 7}"#.to_string())
+                    .unwrap()
+            }
+            path => panic!("Unexpected path: {path}"),
+        });
+
+        let result = table
+            .add_columns()
+            .computed("doubled", "x * 2")
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(result.version, 7);
+    }
+
+    #[tokio::test]
+    async fn test_add_scalar_function_column_sends_atomic_null_declaration() {
+        let table = Table::new_with_handler("my_table", |request| {
+            match request.url().path() {
+            "/v1/table/my_table/describe/" => http::Response::builder()
+                .status(200)
+                .body(
+                    r#"{"version":1,"schema":{"fields":[{"name":"description","nullable":true,"type":{"type":"string"}}]}}"#,
+                )
+                .unwrap(),
+            "/v1/table/my_table/add_columns/" => {
+                let actual: serde_json::Value = serde_json::from_slice(
+                    request.body().unwrap().as_bytes().unwrap(),
+                )
+                .unwrap();
+                let expected: serde_json::Value = serde_json::from_str(include_str!(
+                    "../../tests/fixtures/first_class_functions/v1/remote_scalar_declaration_request.json"
+                ))
+                .unwrap();
+                assert_eq!(actual, expected);
+                http::Response::builder()
+                    .status(200)
+                    .body(r#"{"version":8}"#)
+                    .unwrap()
+            }
+            path => panic!("Unexpected path: {path}"),
+        }
+        });
+        let application = crate::function::FunctionApplication::from_json(
+            r#"{
+                "function":{"name":"embed","version":"fv_01K3EXACT"},
+                "inputs":[{"parameter":"text","kind":"column","value":{"path":"description"}}],
+                "output":{"kind":"scalar","arrow_type":"list<float32>","nullable":false}
+            }"#,
+        )
+        .unwrap();
+
+        let result = table
+            .add_columns()
+            .function_as("embedding", application)
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(result.version, 8);
+    }
+
+    #[tokio::test]
+    async fn test_add_fixed_size_list_function_column_declares_the_vector_type() {
+        let table = Table::new_with_handler("my_table", |request| {
+            match request.url().path() {
+            "/v1/table/my_table/describe/" => http::Response::builder()
+                .status(200)
+                .body(
+                    r#"{"version":1,"schema":{"fields":[{"name":"description","nullable":true,"type":{"type":"string"}}]}}"#,
+                )
+                .unwrap(),
+            "/v1/table/my_table/add_columns/" => {
+                let actual: serde_json::Value = serde_json::from_slice(
+                    request.body().unwrap().as_bytes().unwrap(),
+                )
+                .unwrap();
+                let expected: serde_json::Value = serde_json::from_str(include_str!(
+                    "../../tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json"
+                ))
+                .unwrap();
+                assert_eq!(actual, expected);
+                http::Response::builder()
+                    .status(200)
+                    .body(r#"{"version":8}"#)
+                    .unwrap()
+            }
+            path => panic!("Unexpected path: {path}"),
+        }
+        });
+        let application = crate::function::FunctionApplication::from_json(
+            r#"{
+                "function":{"name":"embed","version":"fv_01K3EXACT"},
+                "inputs":[{"parameter":"text","kind":"column","value":{"path":"description"}}],
+                "output":{"kind":"scalar","arrow_type":"fixed_size_list<float32, 3>","nullable":false}
+            }"#,
+        )
+        .unwrap();
+
+        let result = table
+            .add_columns()
+            .function_as("embedding", application)
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(result.version, 8);
+    }
+
+    #[tokio::test]
+    async fn test_add_named_struct_function_expands_one_atomic_binding() {
+        let table = Table::new_with_handler("my_table", |request| match request.url().path() {
+            "/v1/table/my_table/describe/" => http::Response::builder()
+                .status(200)
+                .body(
+                    r#"{"version":1,"schema":{"fields":[
+                        {"name":"title","nullable":true,"type":{"type":"string"}},
+                        {"name":"body","nullable":true,"type":{"type":"string"}}
+                    ]}}"#,
+                )
+                .unwrap(),
+            "/v1/table/my_table/add_columns/" => {
+                let actual: serde_json::Value =
+                    serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
+                let expected: serde_json::Value = serde_json::from_str(include_str!(
+                    "../../tests/fixtures/first_class_functions/v1/remote_multi_output_declaration_request.json"
+                ))
+                .unwrap();
+                assert_eq!(actual, expected);
+                http::Response::builder()
+                    .status(200)
+                    .body(r#"{"version":9}"#)
+                    .unwrap()
+            }
+            path => panic!("Unexpected path: {path}"),
+        });
+        let application = crate::function::FunctionApplication::from_json(
+            r#"{
+                "function":{"name":"text_features","version":"fv_01K3TEXT"},
+                "inputs":[
+                    {"parameter":"title","kind":"column","value":{"path":"title"}},
+                    {"parameter":"body","kind":"column","value":{"path":"body"}}
+                ],
+                "output":{"kind":"named_struct","fields":[
+                    {"name":"normalized_text","arrow_type":"utf8","nullable":false},
+                    {"name":"token_count","arrow_type":"int64","nullable":false}
+                ]},
+                "columns":{"normalized_text":"search_text"}
+            }"#,
+        )
+        .unwrap();
+
+        let result = table
+            .add_columns()
+            .function(application)
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(result.version, 9);
+    }
+
+    #[tokio::test]
+    async fn test_add_columns_fails_closed_on_newer_function_binding_metadata() {
+        let table = Table::new_with_handler("my_table", |request| {
+            match request.url().path() {
+            "/v1/table/my_table/describe/" => http::Response::builder()
+                .status(200)
+                .body(
+                    r#"{"version":1,"schema":{"fields":[{"name":"x","nullable":true,"type":{"type":"int32"}}],"metadata":{"lancedb::function_bindings":"{\"version\":2,\"bindings\":[]}"}}}"#,
+                )
+                .unwrap(),
+            path => panic!("mutation request must not be sent: {path}"),
+        }
+        });
+
+        let err = table
+            .add_columns()
+            .computed("doubled", "x * 2")
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::NotSupported { .. }));
+    }
+
+    /// A remote refresh is a server job: the async form returns its handle,
+    /// and the blocking form refuses rather than invent a fill count.
+    #[tokio::test]
+    async fn test_refresh_column_async_submits_a_backfill_job() {
+        let table = Table::new_with_handler("my_table", |request| {
+            assert_eq!(request.method(), "POST");
+            assert_eq!(request.url().path(), "/v1/table/my_table/backfill_column");
+            let body = request.body().unwrap().as_bytes().unwrap();
+            let value: serde_json::Value = serde_json::from_slice(body).unwrap();
+            assert_eq!(value["column"], "doubled");
+            http::Response::builder()
+                .status(202)
+                .body(r#"{"job_id": "j-42"}"#)
+                .unwrap()
+        });
+
+        let job = table.refresh_column_async("doubled").await.unwrap();
+        assert_eq!(job.id(), Some("j-42"));
+
+        let err = table.refresh_column("doubled").await.unwrap_err();
+        assert!(
+            matches!(&err, Error::NotSupported { message }
+                if message.contains("refresh_column_async")),
+            "{err:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_refresh_submission_uses_add_columns_version_fence() {
+        let table = Table::new_with_handler("my_table", |request| match request.url().path() {
+            "/v1/table/my_table/describe/" => simple_describe_response(),
+            "/v1/table/my_table/add_columns/" => http::Response::builder()
+                .status(200)
+                .body(r#"{"version": 7}"#.to_string())
+                .unwrap(),
+            "/v1/table/my_table/backfill_column" => {
+                let min_version = request
+                    .headers()
+                    .get("x-lancedb-min-version")
+                    .and_then(|value| value.to_str().ok());
+                if min_version != Some("7") {
+                    return http::Response::builder()
+                        .status(400)
+                        .body(r#"{"error":"Column not found: doubled"}"#.to_string())
+                        .unwrap();
+                }
+                http::Response::builder()
+                    .status(202)
+                    .body(r#"{"job_id": "j-43"}"#.to_string())
+                    .unwrap()
+            }
+            path => panic!("unexpected request: {path}"),
+        });
+
+        let result = table
+            .add_columns()
+            .computed("doubled", "a * 2")
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(result.version, 7);
+
+        let job = table.refresh_column_async("doubled").await.unwrap();
+        assert_eq!(job.id(), Some("j-43"));
+    }
+
+    /// The gate's reproducer: after a successful wait, a same-handle read
+    /// must carry a freshness baseline so a stale server cache cannot serve
+    /// the pre-backfill snapshot.
+    #[tokio::test]
+    async fn test_backfill_wait_establishes_read_freshness() {
+        let saw_min_timestamp = Arc::new(std::sync::atomic::AtomicBool::new(false));
+        let saw = saw_min_timestamp.clone();
+        let table =
+            Table::new_with_handler("my_table", move |request| match request.url().path() {
+                "/v1/table/my_table/backfill_column" => http::Response::builder()
+                    .status(202)
+                    .body(r#"{"job_id": "j-7"}"#.to_string())
+                    .unwrap(),
+                "/v1/jobs/describe" => http::Response::builder()
+                    .status(200)
+                    .body(refresh_done("j-7"))
+                    .unwrap(),
+                "/v1/table/my_table/count_rows/" => {
+                    saw.store(
+                        request.headers().contains_key("x-lancedb-min-timestamp"),
+                        std::sync::atomic::Ordering::SeqCst,
+                    );
+                    http::Response::builder()
+                        .status(200)
+                        .body("1".to_string())
+                        .unwrap()
+                }
+                path => panic!("unexpected request: {path}"),
+            });
+
+        let job = table.refresh_column_async("doubled").await.unwrap();
+        let result = job.wait().await.unwrap();
+        assert_eq!(result.rows_assigned, 12);
+        assert_eq!(result.published_version, Some(8));
+        table.count_rows(None).await.unwrap();
+        assert!(
+            saw_min_timestamp.load(std::sync::atomic::Ordering::SeqCst),
+            "read after wait carried no freshness baseline"
+        );
+    }
+
+    /// A checkout after submission wins over the completion fence: the
+    /// pinned view must not regain a timestamp floor from the job.
+    #[tokio::test]
+    async fn test_checkout_after_submit_beats_the_completion_fence() {
+        let saw_min_timestamp = Arc::new(std::sync::atomic::AtomicBool::new(false));
+        let saw = saw_min_timestamp.clone();
+        let table =
+            Table::new_with_handler("my_table", move |request| match request.url().path() {
+                "/v1/table/my_table/backfill_column" => http::Response::builder()
+                    .status(202)
+                    .body(r#"{"job_id": "j-8"}"#.to_string())
+                    .unwrap(),
+                "/v1/jobs/describe" => http::Response::builder()
+                    .status(200)
+                    .body(refresh_done("j-8"))
+                    .unwrap(),
+                "/v1/table/my_table/describe/" => {
+                    let schema = Schema::new(vec![Field::new("x", DataType::Int32, true)]);
+                    http::Response::builder()
+                        .status(200)
+                        .body(describe_response(&schema))
+                        .unwrap()
+                }
+                "/v1/table/my_table/count_rows/" => {
+                    saw.store(
+                        request.headers().contains_key("x-lancedb-min-timestamp"),
+                        std::sync::atomic::Ordering::SeqCst,
+                    );
+                    http::Response::builder()
+                        .status(200)
+                        .body("1".to_string())
+                        .unwrap()
+                }
+                path => panic!("unexpected request: {path}"),
+            });
+
+        let job = table.refresh_column_async("doubled").await.unwrap();
+        table.checkout(3).await.unwrap();
+        job.wait().await.unwrap();
+        table.count_rows(None).await.unwrap();
+        assert!(
+            !saw_min_timestamp.load(std::sync::atomic::Ordering::SeqCst),
+            "completion fence overrode an explicit checkout"
+        );
+    }
+
+    /// Tag checkout resets freshness state wholesale; the fence must not
+    /// survive it.
+    #[tokio::test]
+    async fn test_tag_checkout_after_submit_beats_the_completion_fence() {
+        let saw_min_timestamp = Arc::new(std::sync::atomic::AtomicBool::new(false));
+        let saw = saw_min_timestamp.clone();
+        let table =
+            Table::new_with_handler("my_table", move |request| match request.url().path() {
+                "/v1/table/my_table/backfill_column" => http::Response::builder()
+                    .status(202)
+                    .body(r#"{"job_id": "j-9"}"#.to_string())
+                    .unwrap(),
+                "/v1/jobs/describe" => http::Response::builder()
+                    .status(200)
+                    .body(refresh_done("j-9"))
+                    .unwrap(),
+                "/v1/table/my_table/tags/version/" => http::Response::builder()
+                    .status(200)
+                    .body(r#"{"version": 5}"#.to_string())
+                    .unwrap(),
+                "/v1/table/my_table/describe/" => {
+                    let schema = Schema::new(vec![Field::new("x", DataType::Int32, true)]);
+                    http::Response::builder()
+                        .status(200)
+                        .body(describe_response(&schema))
+                        .unwrap()
+                }
+                "/v1/table/my_table/count_rows/" => {
+                    saw.store(
+                        request.headers().contains_key("x-lancedb-min-timestamp"),
+                        std::sync::atomic::Ordering::SeqCst,
+                    );
+                    http::Response::builder()
+                        .status(200)
+                        .body("1".to_string())
+                        .unwrap()
+                }
+                path => panic!("unexpected request: {path}"),
+            });
+
+        let job = table.refresh_column_async("doubled").await.unwrap();
+        table.checkout_tag("v1").await.unwrap();
+        job.wait().await.unwrap();
+        table.count_rows(None).await.unwrap();
+        assert!(
+            !saw_min_timestamp.load(std::sync::atomic::Ordering::SeqCst),
+            "completion fence overrode a tag checkout"
+        );
+    }
+
+    /// A checkout landing while the submission request is in flight advances
+    /// the epoch past the token captured at submit.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_checkout_during_submission_beats_the_completion_fence() {
+        let saw_min_timestamp = Arc::new(std::sync::atomic::AtomicBool::new(false));
+        let saw = saw_min_timestamp.clone();
+        let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
+        let release_rx = Arc::new(std::sync::Mutex::new(release_rx));
+        let (arrived_tx, arrived_rx) = std::sync::mpsc::channel::<()>();
+        let arrived_tx = Arc::new(std::sync::Mutex::new(arrived_tx));
+        let table = Table::new_with_handler("my_table", move |request| {
+            match request.url().path() {
+                "/v1/table/my_table/backfill_column" => {
+                    // Signal arrival, then hold the response until the
+                    // test's checkout completes.
+                    arrived_tx.lock().unwrap().send(()).unwrap();
+                    release_rx
+                        .lock()
+                        .unwrap()
+                        .recv_timeout(std::time::Duration::from_secs(10))
+                        .unwrap();
+                    http::Response::builder()
+                        .status(202)
+                        .body(r#"{"job_id": "j-10"}"#.to_string())
+                        .unwrap()
+                }
+                "/v1/jobs/describe" => http::Response::builder()
+                    .status(200)
+                    .body(refresh_done("j-10"))
+                    .unwrap(),
+                "/v1/table/my_table/describe/" => {
+                    let schema = Schema::new(vec![Field::new("x", DataType::Int32, true)]);
+                    http::Response::builder()
+                        .status(200)
+                        .body(describe_response(&schema))
+                        .unwrap()
+                }
+                "/v1/table/my_table/count_rows/" => {
+                    saw.store(
+                        request.headers().contains_key("x-lancedb-min-timestamp"),
+                        std::sync::atomic::Ordering::SeqCst,
+                    );
+                    http::Response::builder()
+                        .status(200)
+                        .body("1".to_string())
+                        .unwrap()
+                }
+                path => panic!("unexpected request: {path}"),
+            }
+        });
+
+        let submit = tokio::spawn({
+            let table = table.clone();
+            async move { table.refresh_column_async("doubled").await }
+        });
+        tokio::task::spawn_blocking(move || {
+            arrived_rx
+                .recv_timeout(std::time::Duration::from_secs(10))
+                .unwrap()
+        })
+        .await
+        .unwrap();
+        table.checkout(7).await.unwrap();
+        release_tx.send(()).unwrap();
+
+        let job = submit.await.unwrap().unwrap();
+        job.wait().await.unwrap();
+        table.count_rows(None).await.unwrap();
+        assert!(
+            !saw_min_timestamp.load(std::sync::atomic::Ordering::SeqCst),
+            "completion fence overrode a checkout that landed mid-submission"
+        );
+    }
+
+    /// checkout_latest keeps the handle on latest, so a completed backfill
+    /// must still establish its post-fill baseline -- strictly later than the
+    /// checkout's own, or a pre-fill cache could still serve.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_checkout_latest_during_submission_keeps_the_fence() {
+        let seen_min_timestamp = Arc::new(std::sync::Mutex::new(None::<String>));
+        let saw = seen_min_timestamp.clone();
+        let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
+        let release_rx = Arc::new(std::sync::Mutex::new(release_rx));
+        let (arrived_tx, arrived_rx) = std::sync::mpsc::channel::<()>();
+        let arrived_tx = Arc::new(std::sync::Mutex::new(arrived_tx));
+        let table =
+            Table::new_with_handler("my_table", move |request| match request.url().path() {
+                "/v1/table/my_table/backfill_column" => {
+                    arrived_tx.lock().unwrap().send(()).unwrap();
+                    release_rx
+                        .lock()
+                        .unwrap()
+                        .recv_timeout(std::time::Duration::from_secs(10))
+                        .unwrap();
+                    http::Response::builder()
+                        .status(202)
+                        .body(r#"{"job_id": "j-11"}"#.to_string())
+                        .unwrap()
+                }
+                "/v1/jobs/describe" => http::Response::builder()
+                    .status(200)
+                    .body(refresh_done("j-11"))
+                    .unwrap(),
+                "/v1/table/my_table/count_rows/" => {
+                    *saw.lock().unwrap() = request
+                        .headers()
+                        .get("x-lancedb-min-timestamp")
+                        .map(|v| v.to_str().unwrap().to_string());
+                    http::Response::builder()
+                        .status(200)
+                        .body("1".to_string())
+                        .unwrap()
+                }
+                path => panic!("unexpected request: {path}"),
+            });
+
+        let submit = tokio::spawn({
+            let table = table.clone();
+            async move { table.refresh_column_async("doubled").await }
+        });
+        tokio::task::spawn_blocking(move || {
+            arrived_rx
+                .recv_timeout(std::time::Duration::from_secs(10))
+                .unwrap()
+        })
+        .await
+        .unwrap();
+        table.checkout_latest().await.unwrap();
+        let after_checkout = SystemTime::now();
+        // Real separation between the checkout baseline and completion.
+        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
+        release_tx.send(()).unwrap();
+
+        let job = submit.await.unwrap().unwrap();
+        job.wait().await.unwrap();
+        table.count_rows(None).await.unwrap();
+        let header = seen_min_timestamp
+            .lock()
+            .unwrap()
+            .clone()
+            .expect("no baseline");
+        let sent: SystemTime = chrono::DateTime::parse_from_rfc3339(&header)
+            .unwrap()
+            .into();
+        assert!(
+            sent > after_checkout,
+            "baseline {header} did not advance past the checkout"
+        );
+    }
+
     #[tokio::test]
     async fn test_prewarm_index() {
         let table = Table::new_with_handler("my_table", |request| {
@@ -6497,6 +7486,41 @@ mod tests {
         assert!(matches!(e, Error::IndexNotFound { .. }));
     }
 
+    /// Index names are unvalidated, so reserved characters must be
+    /// percent-encoded or they restructure the request path.
+    #[tokio::test]
+    async fn test_per_index_paths_encode_reserved_characters() {
+        const NAME: &str = "my/index?a#b c";
+        const PREFIX: &str = "/v1/table/my_table/index/my%2Findex%3Fa%23b%20c";
+
+        let table = Table::new_with_handler("my_table", |request| {
+            assert_eq!(request.url().path(), format!("{PREFIX}/stats/"));
+            let body = serde_json::json!({
+              "num_indexed_rows": 1,
+              "num_unindexed_rows": 0,
+              "index_type": "IVF_PQ",
+              "distance_type": "l2"
+            });
+            http::Response::builder()
+                .status(200)
+                .body(serde_json::to_string(&body).unwrap())
+                .unwrap()
+        });
+        assert!(table.index_stats(NAME).await.unwrap().is_some());
+
+        let table = Table::new_with_handler("my_table", |request| {
+            assert_eq!(request.url().path(), format!("{PREFIX}/drop/"));
+            http::Response::builder().status(200).body("{}").unwrap()
+        });
+        table.drop_index(NAME).await.unwrap();
+
+        let table = Table::new_with_handler("my_table", |request| {
+            assert_eq!(request.url().path(), format!("{PREFIX}/prewarm/"));
+            http::Response::builder().status(200).body("{}").unwrap()
+        });
+        table.prewarm_index(NAME).await.unwrap();
+    }
+
     #[tokio::test]
     async fn test_set_lsm_write_spec_unsharded() {
         let table = Table::new_with_handler("my_table", |request| {
@@ -6519,7 +7543,7 @@ mod tests {
                 .unwrap()
         });
         let spec = crate::table::LsmWriteSpec::unsharded()
-            .with_maintained_indexes(["id_idx"])
+            .with_maintained_indexes(vec!["id_idx".to_string()])
             .with_writer_config_defaults([("max_memtable_rows", "1000")]);
         table.set_lsm_write_spec(spec).await.unwrap();
     }
@@ -6538,7 +7562,8 @@ mod tests {
                 body["sharding"],
                 serde_json::json!({ "mode": "bucket", "column": "id", "num_buckets": 16 })
             );
-            assert_eq!(body["maintained_indexes"], serde_json::json!([]));
+            // An unpinned maintained set sends null: resolve server-side.
+            assert_eq!(body["maintained_indexes"], serde_json::Value::Null);
             http::Response::builder().status(200).body("{}").unwrap()
         });
         table
@@ -6547,6 +7572,23 @@ mod tests {
             .unwrap();
     }
 
+    /// `[]` (none) must stay distinguishable on the wire from null (all).
+    #[tokio::test]
+    async fn test_set_lsm_write_spec_no_maintained_indexes() {
+        let table = Table::new_with_handler("my_table", |request| {
+            let body = request.body().unwrap().as_bytes().unwrap();
+            let body: serde_json::Value = serde_json::from_slice(body).unwrap();
+            assert_eq!(body["maintained_indexes"], serde_json::json!([]));
+            http::Response::builder().status(200).body("{}").unwrap()
+        });
+        table
+            .set_lsm_write_spec(
+                crate::table::LsmWriteSpec::bucket("id", 16).with_maintained_indexes(Vec::new()),
+            )
+            .await
+            .unwrap();
+    }
+
     #[tokio::test]
     async fn test_set_lsm_write_spec_identity() {
         let table = Table::new_with_handler("my_table", |request| {
@@ -6621,7 +7663,7 @@ mod tests {
             } => {
                 assert_eq!(column, "id");
                 assert_eq!(num_buckets, 4);
-                assert_eq!(maintained_indexes, vec!["id_idx".to_string()]);
+                assert_eq!(maintained_indexes, Some(vec!["id_idx".to_string()]));
                 assert_eq!(
                     writer_config_defaults
                         .get("durable_write")
@@ -6650,6 +7692,499 @@ mod tests {
         assert!(table.get_lsm_write_spec().await.unwrap().is_none());
     }
 
+    /// Build a `get_lsm_stats` body for one bucket holding `generations`.
+    fn stats_body(generations: &[u64], compacting: bool) -> String {
+        serde_json::json!({
+            "lsm_stats": {
+                "buckets": [{
+                    "shard_id": "b0",
+                    "status": "Active",
+                    "writer_epoch": 1,
+                    "manifest_version": 1,
+                    "current_generation": generations.iter().max().copied().unwrap_or(0) + 1,
+                    "replay_after_wal_entry_position": 0,
+                    "wal_entry_position_last_seen": 0,
+                    "generations": generations.iter()
+                        .map(|g| serde_json::json!({ "generation": g, "bytes": 1 }))
+                        .collect::<Vec<_>>(),
+                    "compacting": compacting,
+                    "memtables": [],
+                }],
+            }
+        })
+        .to_string()
+    }
+
+    /// `flush_lsm` / `compact_lsm` answer 202 with no body at all.
+    fn accepted() -> http::Response<String> {
+        http::Response::builder()
+            .status(202)
+            .body(String::new())
+            .unwrap()
+    }
+
+    fn ok_json(body: String) -> http::Response<String> {
+        http::Response::builder().status(200).body(body).unwrap()
+    }
+
+    /// A flush landing in an empty L0 finishes on the opening stats read
+    /// alone. Asserting zero compacts is the point: "it returned Ok" is also
+    /// true of a loop that ran a pointless pass.
+    #[tokio::test(start_paused = true)]
+    async fn test_checkpoint_short_circuits_on_empty_l0() {
+        let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let seen = compacts.clone();
+        let table = Table::new_with_handler("my_table", move |request| {
+            let path = request.url().path().to_string();
+            if path.contains("compact_lsm") {
+                seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                panic!("an already-converged table must issue no compact calls");
+            }
+            if path.contains("flush_lsm") {
+                return accepted();
+            }
+            assert_eq!(path, "/v1/table/my_table/get_lsm_stats/");
+            ok_json(stats_body(&[], false))
+        });
+
+        table.checkpoint_lsm().await.unwrap();
+        assert_eq!(compacts.load(std::sync::atomic::Ordering::SeqCst), 0);
+    }
+
+    /// The loop triggers compaction until every generation that existed at
+    /// the start is gone, one bounded prefix per pass.
+    #[tokio::test(start_paused = true)]
+    async fn test_checkpoint_triggers_until_targets_are_drained() {
+        let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let seen = compacts.clone();
+        let table = Table::new_with_handler("my_table", move |request| {
+            let path = request.url().path().to_string();
+            if path.contains("flush_lsm") {
+                return accepted();
+            }
+            if path.contains("compact_lsm") {
+                seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                return accepted();
+            }
+            // Each pass drains the oldest generation.
+            let drained = seen.load(std::sync::atomic::Ordering::SeqCst);
+            let left: Vec<u64> = [1u64, 2, 3].into_iter().skip(drained).collect();
+            ok_json(stats_body(&left, false))
+        });
+
+        table.checkpoint_lsm().await.unwrap();
+        assert_eq!(
+            compacts.load(std::sync::atomic::Ordering::SeqCst),
+            3,
+            "one trigger per generation prefix, then stop"
+        );
+    }
+
+    /// Generations created *during* the checkpoint are not waited on, which
+    /// is what lets the loop terminate on a table taking writes where "L0 is
+    /// empty" never becomes true.
+    #[tokio::test(start_paused = true)]
+    async fn test_checkpoint_ignores_generations_created_while_it_runs() {
+        let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let seen = compacts.clone();
+        let table = Table::new_with_handler("my_table", move |request| {
+            let path = request.url().path().to_string();
+            if path.contains("flush_lsm") {
+                return accepted();
+            }
+            if path.contains("compact_lsm") {
+                seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                return accepted();
+            }
+            // Target is 5. One pass drains it; a writer keeps adding above.
+            let n = seen.load(std::sync::atomic::Ordering::SeqCst);
+            let body = if n == 0 {
+                stats_body(&[5], false)
+            } else {
+                stats_body(&[6, 7], false)
+            };
+            ok_json(body)
+        });
+
+        table.checkpoint_lsm().await.unwrap();
+        assert_eq!(
+            compacts.load(std::sync::atomic::Ordering::SeqCst),
+            1,
+            "the loop must not chase generations written after it started"
+        );
+    }
+
+    /// Contention is a 429 and must be retried. The server keeps it off 503
+    /// precisely so the client can act on the status alone — reading it as
+    /// terminal stops the checkpoint early on a healthy node.
+    #[tokio::test(start_paused = true)]
+    async fn test_checkpoint_retries_contention() {
+        let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let seen = compacts.clone();
+        let table = Table::new_with_handler("my_table", move |request| {
+            let path = request.url().path().to_string();
+            if path.contains("flush_lsm") {
+                return accepted();
+            }
+            if path.contains("compact_lsm") {
+                // First two triggers: every bucket already latched.
+                if seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2 {
+                    return http::Response::builder()
+                        .status(429)
+                        .body(r#"{"code":21,"error":"Too many concurrent writes"}"#.to_string())
+                        .unwrap();
+                }
+                return accepted();
+            }
+            let accepted_triggers = seen
+                .load(std::sync::atomic::Ordering::SeqCst)
+                .saturating_sub(2);
+            let left: Vec<u64> = if accepted_triggers == 0 {
+                vec![1]
+            } else {
+                vec![]
+            };
+            ok_json(stats_body(&left, false))
+        });
+
+        table
+            .checkpoint_lsm()
+            .await
+            .expect("contention must not abort the checkpoint");
+        assert_eq!(
+            compacts.load(std::sync::atomic::Ordering::SeqCst),
+            3,
+            "assert the retry count, not just the outcome"
+        );
+    }
+
+    /// A transient fault on the poll must not abort the checkpoint. This route
+    /// meets the most contention — it runs every `POLL_INTERVAL` for the
+    /// checkpoint's whole life, with the transport retry layer disabled — yet
+    /// was the one call reached with a bare `?`.
+    #[tokio::test(start_paused = true)]
+    async fn test_checkpoint_retries_a_contended_stats_poll() {
+        let polls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let seen = polls.clone();
+        let table = Table::new_with_handler("my_table", move |request| {
+            let path = request.url().path().to_string();
+            if path.contains("flush_lsm") || path.contains("compact_lsm") {
+                return accepted();
+            }
+            // The opening read lands; the next two polls are latched out.
+            let n = seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+            if (1..3).contains(&n) {
+                return http::Response::builder()
+                    .status(429)
+                    .body(r#"{"code":21,"error":"Too many concurrent writes"}"#.to_string())
+                    .unwrap();
+            }
+            ok_json(stats_body(if n < 4 { &[1] } else { &[] }, false))
+        });
+
+        table
+            .checkpoint_lsm()
+            .await
+            .expect("a contended poll must be retried, not surfaced");
+        assert_eq!(
+            polls.load(std::sync::atomic::Ordering::SeqCst),
+            5,
+            "the two rejected polls must be re-issued, not skipped"
+        );
+    }
+
+    /// Contention and a lost claim draw on separate budgets: five straight
+    /// 429s on `flush`, more than `MAX_REISSUES`, must still converge. On one
+    /// shared counter this spent the re-issue cap and then reported a lost
+    /// claim nothing had ever reported.
+    #[tokio::test(start_paused = true)]
+    async fn test_contention_does_not_exhaust_the_reissue_budget() {
+        let flushes = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let seen = flushes.clone();
+        let table = Table::new_with_handler("my_table", move |request| {
+            let path = request.url().path().to_string();
+            if path.contains("flush_lsm") {
+                if seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 5 {
+                    return http::Response::builder()
+                        .status(429)
+                        .body(r#"{"code":21,"error":"Too many concurrent writes"}"#.to_string())
+                        .unwrap();
+                }
+                return accepted();
+            }
+            if path.contains("compact_lsm") {
+                return accepted();
+            }
+            ok_json(stats_body(&[], false))
+        });
+
+        table
+            .checkpoint_lsm()
+            .await
+            .expect("contention must not be reported as a lost claim");
+        assert_eq!(
+            flushes.load(std::sync::atomic::Ordering::SeqCst),
+            6,
+            "five retries against one seal, then it lands"
+        );
+    }
+
+    /// An exhausted retry budget surfaces the fault that consumed it, not a
+    /// message the loop invented: "429, nine times" points an operator at a
+    /// saturated pool, a generic runtime error points them nowhere.
+    #[tokio::test(start_paused = true)]
+    async fn test_exhausted_retries_surface_the_underlying_fault() {
+        let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let seen = calls.clone();
+        let table = Table::new_with_handler("my_table", move |_request| {
+            seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+            http::Response::builder()
+                .status(429)
+                .body(r#"{"code":21,"error":"Too many concurrent writes"}"#.to_string())
+                .unwrap()
+        });
+
+        let err = table.checkpoint_lsm().await.unwrap_err();
+        assert!(
+            matches!(&err, Error::Http { status_code: Some(s), .. } if s.as_u16() == 429),
+            "the fault that spent the budget must be the one reported: {err:?}"
+        );
+        assert_eq!(
+            calls.load(std::sync::atomic::Ordering::SeqCst),
+            9,
+            "one call plus MAX_RETRIES — the re-issue budget is not spent on top"
+        );
+    }
+
+    /// A draining node is terminal, but the client does not know that from the
+    /// status: draining and a proxy blip are both 503, and telling them apart
+    /// takes parsing the body for a namespace code. So it spends the retry
+    /// budget and then reports what the server said — the drain gate never
+    /// releases, so the answer does not change, and the operator still reads
+    /// "WAL node draining" in the error.
+    #[tokio::test(start_paused = true)]
+    async fn test_draining_surfaces_after_the_retry_budget() {
+        let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let seen = calls.clone();
+        let table = Table::new_with_handler("my_table", move |_request| {
+            seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+            http::Response::builder()
+                .status(503)
+                .body(r#"{"code":19,"error":"WAL node draining"}"#.to_string())
+                .unwrap()
+        });
+
+        let err = table.checkpoint_lsm().await.unwrap_err();
+        let message = err.to_string();
+        assert!(
+            matches!(&err, Error::Http { status_code: Some(s), .. } if s.as_u16() == 503),
+            "the 503 must surface as itself: {err:?}"
+        );
+        assert!(
+            message.contains("WAL node draining"),
+            "the server's own diagnosis must survive to the caller: {message}"
+        );
+        assert_eq!(
+            calls.load(std::sync::atomic::Ordering::SeqCst),
+            9,
+            "one call plus MAX_RETRIES, then it reports rather than spinning"
+        );
+    }
+
+    /// A long stall with nothing compacting must keep waiting, not fail. The
+    /// client cannot judge this: a checkpoint queued behind unrelated tables
+    /// on the pod-wide compactor pool reports exactly these numbers — flat
+    /// generations, an idle latch — as one whose merges are failing. The
+    /// deadline is the caller's.
+    #[tokio::test(start_paused = true)]
+    async fn test_checkpoint_waits_out_a_long_stall_rather_than_failing() {
+        let polls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let seen = polls.clone();
+        let table = Table::new_with_handler("my_table", move |request| {
+            let path = request.url().path().to_string();
+            if path.contains("flush_lsm") || path.contains("compact_lsm") {
+                return accepted();
+            }
+            // Flat for far longer than any bound this loop ever had, with
+            // `compacting: false` throughout — then it drains.
+            let n = seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+            ok_json(stats_body(if n < 40 { &[1, 2] } else { &[] }, false))
+        });
+
+        table
+            .checkpoint_lsm()
+            .await
+            .expect("a stall is the server being slow, not the client's call to make");
+        assert!(
+            polls.load(std::sync::atomic::Ordering::SeqCst) > 40,
+            "the loop must have kept polling well past the old ten-poll bound"
+        );
+    }
+
+    /// A pass already owns the latch on every outstanding bucket, so the loop
+    /// waits rather than piling on triggers it would only refuse. This is the
+    /// sole thing `compacting` is read for.
+    #[tokio::test(start_paused = true)]
+    async fn test_checkpoint_waits_while_a_pass_is_running() {
+        let polls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let seen_polls = polls.clone();
+        let seen_compacts = compacts.clone();
+        let table = Table::new_with_handler("my_table", move |request| {
+            let path = request.url().path().to_string();
+            if path.contains("flush_lsm") {
+                return accepted();
+            }
+            if path.contains("compact_lsm") {
+                seen_compacts.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                return accepted();
+            }
+            // Latched for many polls, then done.
+            let n = seen_polls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+            ok_json(if n > 15 {
+                stats_body(&[], false)
+            } else {
+                stats_body(&[1], true)
+            })
+        });
+
+        table
+            .checkpoint_lsm()
+            .await
+            .expect("a running pass is progress, not a stall");
+        assert_eq!(
+            compacts.load(std::sync::atomic::Ordering::SeqCst),
+            0,
+            "never trigger against a bucket already compacting"
+        );
+    }
+
+    /// WAL off ⇒ `None`; WAL on ⇒ a fully populated `Some` with no field
+    /// defaulting to a zero it did not measure. `include_generation_rows`
+    /// rides in the body and is off unless asked for.
+    #[tokio::test]
+    async fn test_get_lsm_stats_round_trip() {
+        let table = Table::new_with_handler("my_table", |request| {
+            assert_eq!(request.url().path(), "/v1/table/my_table/get_lsm_stats/");
+            let body = request.body().unwrap().as_bytes().unwrap();
+            let body: serde_json::Value = serde_json::from_slice(body).unwrap();
+            assert_eq!(
+                body["include_generation_rows"], true,
+                "the flag must reach the server, not be silently dropped"
+            );
+            let response = serde_json::json!({
+                "lsm_stats": {
+                    "buckets": [{
+                        "shard_id": "b0",
+                        "status": "Active",
+                        "writer_epoch": 3,
+                        "manifest_version": 11,
+                        "current_generation": 9,
+                        "replay_after_wal_entry_position": 100,
+                        "wal_entry_position_last_seen": 140,
+                        "generations": [{ "generation": 8, "bytes": 4096, "rows": 30 }],
+                        "compacting": false,
+                        "memtables": [
+                            { "generation": 9, "rows": 12, "bytes": 900, "batches": 2,
+                              "indexes": ["vec_idx"] }
+                        ],
+                    }],
+                }
+            });
+            http::Response::builder()
+                .status(200)
+                .body(response.to_string())
+                .unwrap()
+        });
+
+        let stats = table
+            .get_lsm_stats(true)
+            .await
+            .unwrap()
+            .expect("a WAL-backed table reports Some");
+        let bucket = &stats.buckets[0];
+        assert_eq!(bucket.replay_after_wal_entry_position, 100);
+        assert_eq!(bucket.wal_entry_position_last_seen, 140);
+        assert!(!bucket.compacting);
+        assert_eq!(bucket.generations[0].generation, 8);
+        assert_eq!(bucket.generations[0].rows, Some(30));
+        // The line that answers "why is my fresh-tier vector search
+        // brute-force" — an absent index name is the whole explanation.
+        let memtables = bucket.memtables.as_ref().unwrap();
+        assert_eq!(memtables[0].indexes, vec!["vec_idx".to_string()]);
+    }
+
+    /// A 404 arrives as `TableNotFound`, not as a lost claim the loop
+    /// re-issues from flush until its cap. The two are distinguished by
+    /// status: 404 is "no such table", 421 is "this node holds no claim".
+    /// They shared 404 once, and the loop chased a name that never existed.
+    #[tokio::test(start_paused = true)]
+    async fn test_missing_table_is_not_read_as_a_lost_claim() {
+        let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let seen = calls.clone();
+        let table = Table::new_with_handler("my_table", move |_request| {
+            seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+            http::Response::builder()
+                .status(404)
+                .body(r#"{"code":4,"error":"Not found: Table not found: my_table"}"#.to_string())
+                .unwrap()
+        });
+
+        let err = table.checkpoint_lsm().await.unwrap_err();
+        assert!(
+            matches!(err, Error::TableNotFound { .. }),
+            "a missing table must say so: {err:?}"
+        );
+        assert_eq!(
+            calls.load(std::sync::atomic::Ordering::SeqCst),
+            1,
+            "no point re-claiming a table that does not exist"
+        );
+    }
+
+    /// A lost claim — 421, not 404 — does re-issue from flush, the call that
+    /// re-claims and replays.
+    #[tokio::test(start_paused = true)]
+    async fn test_registry_miss_reissues_from_flush() {
+        let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let seen = calls.clone();
+        let table = Table::new_with_handler("my_table", move |request| {
+            let path = request.url().path().to_string();
+            let n = seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+            if path.contains("flush_lsm") {
+                // First flush lands; the claim is then lost, and the
+                // re-issued flush succeeds.
+                return accepted();
+            }
+            if path.contains("compact_lsm") {
+                if n < 4 {
+                    return http::Response::builder()
+                        .status(421)
+                        .body(r#"{"code":19,"error":"table not claimed"}"#.to_string())
+                        .unwrap();
+                }
+                return accepted();
+            }
+            ok_json(stats_body(if n < 6 { &[1] } else { &[] }, false))
+        });
+
+        table
+            .checkpoint_lsm()
+            .await
+            .expect("a lost claim must be recovered by re-flushing, not surfaced");
+    }
+
+    #[tokio::test]
+    async fn test_get_lsm_stats_absent_when_wal_off() {
+        let table = Table::new_with_handler("my_table", |_request| {
+            http::Response::builder()
+                .status(200)
+                .body(serde_json::json!({ "lsm_stats": null }).to_string())
+                .unwrap()
+        });
+        assert!(table.get_lsm_stats(false).await.unwrap().is_none());
+    }
+
     #[tokio::test]
     async fn test_wait_for_index() {
         let table = _make_table_with_indices(0);
@@ -7234,6 +8769,28 @@ mod tests {
         }
     }
 
+    /// A pinned snapshot should reuse the version and schema returned by its
+    /// initial describe instead of issuing two more describe requests.
+    #[tokio::test]
+    async fn test_checkout_current_seeds_schema_from_single_describe() {
+        let describe_calls = Arc::new(AtomicUsize::new(0));
+        let calls = describe_calls.clone();
+        let table = Table::new_with_handler("my_table", move |request| {
+            assert_eq!(request.url().path(), "/v1/table/my_table/describe/");
+            calls.fetch_add(1, Ordering::SeqCst);
+            http::Response::builder()
+                .status(200)
+                .body(
+                    r#"{"version":42,"schema":{"fields":[{"name":"a","type":{"type":"int32"},"nullable":false}]}}"#,
+                )
+                .unwrap()
+        });
+
+        let snapshot = table.checkout_current().await.unwrap();
+        assert_eq!(snapshot.schema().await.unwrap().fields().len(), 1);
+        assert_eq!(describe_calls.load(Ordering::SeqCst), 1);
+    }
+
     /// Test that schema cache is invalidated after checkout
     #[tokio::test]
     async fn test_schema_cache_invalidation_on_checkout() {
@@ -9115,6 +10672,20 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_materialized_view_refused_without_a_request() {
+        // Materialized views are local-only. The table-level entry the
+        // bindings use must refuse a remote table before reading its schema,
+        // so the panicking handler is the assertion.
+        let table = Table::new_with_handler("my_table", |request| -> http::Response<String> {
+            panic!("unexpected request: {}", request.url().path())
+        });
+        let err = crate::MaterializedView::from_table(table)
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::NotSupported { .. }), "got {err:?}");
+    }
+
     #[tokio::test]
     async fn test_create_branch_empty_name_rejected_client_side() {
         use lance::dataset::refs::Ref;
@@ -9204,8 +10775,7 @@ mod tests {
             "changedColumns":[],
             "addedIndexes":[],
             "removedIndexes":[],
-            "mergeable":true,
-            "mergeBlockers":[]
+            "errors":[]
         }"#
     }
 
@@ -9223,15 +10793,18 @@ mod tests {
         });
         let diff = table.diff_branch("exp").await.unwrap();
         assert_eq!(diff.from_branch, "exp");
-        assert!(diff.mergeable);
+        assert!(diff.errors.is_empty());
         assert_eq!(diff.added_columns.len(), 1);
         assert_eq!(diff.added_columns[0].name, "tag");
     }
 
     #[tokio::test]
-    async fn test_merge_branch_dry_run() {
+    async fn test_cherry_pick_dry_run() {
         let table = Table::new_with_handler("my_table", |request| {
-            assert_eq!(request.url().path(), "/v1/table/my_table/branches/merge/");
+            assert_eq!(
+                request.url().path(),
+                "/v1/table/my_table/branches/cherry_pick/"
+            );
             let body = request_body_json(&request);
             assert_eq!(body["from_branch"], "exp");
             assert_eq!(body["dry_run"], true);
@@ -9241,27 +10814,29 @@ mod tests {
             );
             http::Response::builder().status(200).body(resp).unwrap()
         });
-        let result = table.merge_branch("exp", true).await.unwrap();
-        assert_eq!(result.status, crate::table::MergeBranchStatus::Ready);
+        let result = table.cherry_pick("exp", true).await.unwrap();
+        assert_eq!(result.status, crate::table::CherryPickStatus::Ready);
         assert_eq!(result.preview.promoted_columns, vec!["tag".to_string()]);
         assert!(result.main_version_after.is_none());
     }
 
     #[tokio::test]
-    async fn test_merge_branch_rejected_returns_ok_with_body() {
+    async fn test_cherry_pick_failed_returns_ok_with_body() {
         let table = Table::new_with_handler("my_table", |request| {
-            assert_eq!(request.url().path(), "/v1/table/my_table/branches/merge/");
+            assert_eq!(
+                request.url().path(),
+                "/v1/table/my_table/branches/cherry_pick/"
+            );
             let body = request_body_json(&request);
             assert_eq!(body["dry_run"], false);
             let mut diff: serde_json::Value =
                 serde_json::from_str(sample_branch_diff_json()).unwrap();
-            diff["mergeable"] = serde_json::json!(false);
-            diff["mergeBlockers"] = serde_json::json!([{
+            diff["errors"] = serde_json::json!([{
                 "code": "baseMoved",
                 "message": "main has advanced"
             }]);
             let resp = serde_json::json!({
-                "status": "rejected",
+                "status": "failed",
                 "diff": diff,
                 "preview": { "promotedColumns": [] }
             });
@@ -9270,24 +10845,23 @@ mod tests {
                 .body(resp.to_string())
                 .unwrap()
         });
-        let result = table.merge_branch("exp", false).await.unwrap();
-        assert_eq!(result.status, crate::table::MergeBranchStatus::Rejected);
-        assert!(!result.diff.mergeable);
-        assert_eq!(result.diff.merge_blockers.len(), 1);
+        let result = table.cherry_pick("exp", false).await.unwrap();
+        assert_eq!(result.status, crate::table::CherryPickStatus::Failed);
+        assert!(!result.diff.errors.is_empty());
+        assert_eq!(result.diff.errors.len(), 1);
     }
 
     #[tokio::test]
-    async fn test_merge_branch_unknown_blocker_code_parses() {
+    async fn test_cherry_pick_unknown_error_code_parses() {
         let table = Table::new_with_handler("my_table", |_| {
             let mut diff: serde_json::Value =
                 serde_json::from_str(sample_branch_diff_json()).unwrap();
-            diff["mergeable"] = serde_json::json!(false);
-            diff["mergeBlockers"] = serde_json::json!([{
+            diff["errors"] = serde_json::json!([{
                 "code": "multipleCommits",
                 "message": "branch has more than one data commit"
             }]);
             let resp = serde_json::json!({
-                "status": "rejected",
+                "status": "failed",
                 "diff": diff,
                 "preview": { "operation": "append", "rowsAdded": 2 }
             });
@@ -9296,24 +10870,24 @@ mod tests {
                 .body(resp.to_string())
                 .unwrap()
         });
-        let result = table.merge_branch("exp", false).await.unwrap();
-        assert_eq!(result.status, crate::table::MergeBranchStatus::Rejected);
+        let result = table.cherry_pick("exp", false).await.unwrap();
+        assert_eq!(result.status, crate::table::CherryPickStatus::Failed);
         assert_eq!(
-            result.diff.merge_blockers[0].code,
-            crate::table::MergeBlockerCode::Unknown
+            result.diff.errors[0].code,
+            crate::table::CherryPickErrorCode::Unknown
         );
         assert!(result.preview.promoted_columns.is_empty());
     }
 
     #[tokio::test]
-    async fn test_merge_branch_unexpected_2xx_is_error() {
+    async fn test_cherry_pick_unexpected_2xx_is_error() {
         let table = Table::new_with_handler("my_table", |_| {
             http::Response::builder()
                 .status(204)
                 .body(String::new())
                 .unwrap()
         });
-        let err = table.merge_branch("exp", false).await.unwrap_err();
+        let err = table.cherry_pick("exp", false).await.unwrap_err();
         match err {
             Error::Http {
                 status_code: Some(code),
@@ -9870,6 +11444,7 @@ mod tests {
                     .status(200)
                     .body("{}".to_string())
                     .unwrap(),
+                "/v1/table/my_table/describe/" => simple_describe_response(),
                 "/v1/table/my_table/add_columns/"
                 | "/v1/table/my_table/alter_columns/"
                 | "/v1/table/my_table/drop_columns/" => {
diff --git a/rust/lancedb/src/remote/table/blobs.rs b/rust/lancedb/src/remote/table/blobs.rs
index bb8a5d103..387d3c6dc 100644
--- a/rust/lancedb/src/remote/table/blobs.rs
+++ b/rust/lancedb/src/remote/table/blobs.rs
@@ -90,7 +90,7 @@ struct RemoteBlobState {
 
 /// Seekable Cloud blob handle over HTTP Range.
 #[derive(Debug)]
-pub(crate) struct RemoteBlobFile {
+pub struct RemoteBlobFile {
     requester: Arc<dyn BlobRangeRequester>,
     state: Mutex<RemoteBlobState>,
     closed: AtomicBool,
diff --git a/rust/lancedb/src/remote/table/insert.rs b/rust/lancedb/src/remote/table/insert.rs
index 67ea7765d..a4a28a9c6 100644
--- a/rust/lancedb/src/remote/table/insert.rs
+++ b/rust/lancedb/src/remote/table/insert.rs
@@ -33,7 +33,7 @@ use crate::table::{AddResult, MergeResult};
 /// same Arrow-IPC streaming body and error side-channel; only the target
 /// endpoint, query parameters, and parsed result type differ.
 #[derive(Debug, Clone)]
-pub(crate) enum WriteOp {
+pub enum WriteOp {
     /// `add`: stream to `/v1/table/{id}/insert/`, optionally overwriting.
     Insert { overwrite: bool },
     /// `merge_insert`: stream to `/v1/table/{id}/merge_insert/` with the merge
@@ -49,7 +49,7 @@ pub(crate) enum WriteOp {
 /// The parsed server response for a completed write, discriminated by the
 /// operation that produced it.
 #[derive(Debug, Clone)]
-pub(crate) enum WriteResult {
+pub enum WriteResult {
     Add(AddResult),
     Merge(MergeResult),
 }
diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs
index 51bc6486b..5d794319b 100644
--- a/rust/lancedb/src/table.rs
+++ b/rust/lancedb/src/table.rs
@@ -50,7 +50,6 @@ use crate::DistanceType;
 use crate::blob::BlobRangeRequest;
 use crate::data::scannable::{PeekedScannable, Scannable, estimate_write_partitions};
 use crate::database::Database;
-use crate::database::listing::LANCE_FILE_EXTENSION;
 use crate::database::read_freshness::TableFreshness;
 use crate::embeddings::{EmbeddingDefinition, EmbeddingRegistry, MemoryRegistry};
 use crate::error::{Error, Result};
@@ -67,15 +66,19 @@ use self::merge::MergeInsertBuilder;
 
 pub mod add_columns;
 mod add_data;
-pub mod branch_merge;
+pub mod checkpoint;
+pub mod cherry_pick;
+pub mod computed_columns;
 mod create_index;
 pub mod datafusion;
 pub(crate) mod dataset;
 pub mod delete;
+pub mod lsm_stats;
 pub mod merge;
 pub mod optimize;
 mod primary_key;
 pub mod query;
+pub mod refresh;
 pub mod schema_evolution;
 pub mod update;
 pub mod write_progress;
@@ -84,18 +87,22 @@ pub use add_columns::AddColumnsBuilder;
 #[cfg(feature = "remote")]
 pub(crate) use add_data::PreprocessingOutput;
 pub use add_data::{AddDataBuilder, AddDataMode, AddResult, NaNVectorBehavior};
-pub use branch_merge::{
-    BranchDiff, ColumnChange, ColumnSummary, IndexSummary, MergeBlocker, MergeBlockerCode,
-    MergeBranchResult, MergeBranchStatus, MergePreview, RowCountSummary,
+pub use cherry_pick::{
+    BranchDiff, CherryPickError, CherryPickErrorCode, CherryPickPreview, CherryPickResult,
+    CherryPickStatus, ColumnChange, ColumnSummary, IndexSummary, RowCountSummary,
 };
 pub use chrono::Duration;
+pub use computed_columns::{
+    ComputedColumn, ComputedColumnKind, computed_column_from_field, computed_columns,
+};
 pub use delete::DeleteResult;
 use futures::future::join_all;
 pub use lance::dataset::refs::{BranchContents, Ref, TagContents, Tags as LanceTags};
 pub use lance::dataset::scanner::DatasetRecordBatchStream;
-use lance::dataset::statistics::DatasetStatisticsExt;
 pub use lance_index::optimize::OptimizeOptions;
+pub use lsm_stats::{BucketStats, GenerationStats, LsmStats, MemtableStats};
 pub use optimize::{CompactionOptions, OptimizeAction, OptimizeStats};
+pub use refresh::RefreshColumnResult;
 pub use schema_evolution::{
     AddColumnsResult, AlterColumnsResult, DropColumnsResult, FieldMetadataUpdate,
     UpdateFieldMetadataResult,
@@ -150,55 +157,6 @@ pub(crate) fn map_namespace_lance_error(err: lance::Error, table_name: &str) ->
     }
 }
 
-/// Map a `lance::Error::DatasetNotFound` for the table at `uri` into a `lancedb::Error`.
-///
-/// Lance reports "there is nothing at this location" and "there is a table directory
-/// here but nothing loadable inside it" with the same error. Only the first is a
-/// `TableNotFound`: a `<name>.lance` directory left behind by an interrupted drop and
-/// re-create is still reported by `Connection::table_names`, so callers need to be able
-/// to tell "never existed" from "exists but is broken".
-///
-/// See <https://github.com/lancedb/lancedb/issues/3127>.
-async fn map_dataset_not_found(
-    uri: &str,
-    name: &str,
-    params: ReadParams,
-    err: lance::Error,
-) -> Error {
-    let name = name.to_string();
-    let source = Box::new(err);
-    if table_dir_exists(uri, params).await.unwrap_or(false) {
-        Error::TableCorrupted { name, source }
-    } else {
-        Error::TableNotFound { name, source }
-    }
-}
-
-/// Whether a table directory is present at `uri`, even though no dataset could be
-/// loaded from it.
-///
-/// This looks for a `<name>.lance` entry in the parent directory, which is exactly what
-/// `ListingDatabase::table_names` lists, so the two APIs agree on whether a table is
-/// present. Probing `uri` itself would not work: object stores have no empty
-/// directories to probe, and on a local filesystem the interesting case is precisely an
-/// empty directory.
-async fn table_dir_exists(uri: &str, params: ReadParams) -> Result<bool> {
-    let (object_store, path, _) = DatasetBuilder::from_uri(uri)
-        .with_read_params(params)
-        .build_object_store()
-        .await?;
-    // Only `*.lance` entries are ever reported as tables, so nothing else can produce
-    // the list-then-open mismatch this guards against.
-    if path.extension() != Some(LANCE_FILE_EXTENSION) {
-        return Ok(false);
-    }
-    let (Some(parent), Some(dir_name)) = (path.parent(), path.filename()) else {
-        return Ok(false);
-    };
-    let entries = object_store.read_dir(parent).await?;
-    Ok(entries.iter().any(|entry| entry.as_str() == dir_name))
-}
-
 /// Defines the type of column
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum ColumnKind {
@@ -368,6 +326,8 @@ pub use self::merge::MergeResult;
 /// date) and [`LsmWriteSpec::with_writer_config_defaults`] (default
 /// `ShardWriter` configuration recorded in the MemWAL index).
 ///
+/// A fresh spec maintains every index on the table, resolved on install.
+///
 /// Install a spec with [`Table::set_lsm_write_spec`] and remove it with
 /// [`Table::unset_lsm_write_spec`]. The actual `merge_insert` dispatch
 /// onto the MemWAL writer is a follow-up.
@@ -382,9 +342,12 @@ pub enum LsmWriteSpec {
     Bucket {
         column: String,
         num_buckets: u32,
-        /// Names of indexes (already created on the table) that the
-        /// MemWAL should maintain in-memory as rows are appended.
-        maintained_indexes: Vec<String>,
+        /// Indexes the MemWAL maintains in-memory as rows are appended.
+        ///
+        /// `None` means every index it can maintain, resolved on install — a
+        /// snapshot, so indexes created later need the spec unset and re-set.
+        /// `Some([])` maintains nothing.
+        maintained_indexes: Option<Vec<String>>,
         /// Default `ShardWriter` configuration recorded in the MemWAL index.
         writer_config_defaults: HashMap<String, String>,
     },
@@ -394,35 +357,41 @@ pub enum LsmWriteSpec {
     /// distinct value of `column` becomes its own shard.
     Identity {
         column: String,
-        /// Names of indexes (already created on the table) that the
-        /// MemWAL should maintain in-memory as rows are appended.
-        maintained_indexes: Vec<String>,
+        /// Indexes the MemWAL maintains in-memory as rows are appended.
+        ///
+        /// `None` means every index it can maintain, resolved on install — a
+        /// snapshot, so indexes created later need the spec unset and re-set.
+        /// `Some([])` maintains nothing.
+        maintained_indexes: Option<Vec<String>>,
         /// Default `ShardWriter` configuration recorded in the MemWAL index.
         writer_config_defaults: HashMap<String, String>,
     },
     /// No sharding — every `merge_insert` call writes to a single MemWAL shard.
     Unsharded {
-        /// Names of indexes (already created on the table) that the
-        /// MemWAL should maintain in-memory as rows are appended.
-        maintained_indexes: Vec<String>,
+        /// Indexes the MemWAL maintains in-memory as rows are appended.
+        ///
+        /// `None` means every index it can maintain, resolved on install — a
+        /// snapshot, so indexes created later need the spec unset and re-set.
+        /// `Some([])` maintains nothing.
+        maintained_indexes: Option<Vec<String>>,
         /// Default `ShardWriter` configuration recorded in the MemWAL index.
         writer_config_defaults: HashMap<String, String>,
     },
 }
 
 impl LsmWriteSpec {
-    /// Construct a hash-bucket sharding spec with no maintained indexes.
+    /// Construct a hash-bucket sharding spec maintaining every index on the table.
     pub fn bucket(column: impl Into<String>, num_buckets: u32) -> Self {
         Self::Bucket {
             column: column.into(),
             num_buckets,
-            maintained_indexes: Vec::new(),
+            maintained_indexes: None,
             writer_config_defaults: HashMap::new(),
         }
     }
 
     /// Construct an identity-sharding spec (shard by the raw value of
-    /// `column`) with no maintained indexes.
+    /// `column`) maintaining every index on the table.
     ///
     /// `column` must be a deterministic function of the unenforced primary
     /// key: every row with a given primary key must always produce the same
@@ -434,28 +403,37 @@ impl LsmWriteSpec {
     pub fn identity(column: impl Into<String>) -> Self {
         Self::Identity {
             column: column.into(),
-            maintained_indexes: Vec::new(),
+            maintained_indexes: None,
             writer_config_defaults: HashMap::new(),
         }
     }
 
-    /// Construct an unsharded spec with no maintained indexes.
+    /// Construct an unsharded spec maintaining every index on the table.
     pub fn unsharded() -> Self {
         Self::Unsharded {
-            maintained_indexes: Vec::new(),
+            maintained_indexes: None,
             writer_config_defaults: HashMap::new(),
         }
     }
 
-    /// Replace the list of indexes the MemWAL should keep up to date as
-    /// rows are appended. Each name must reference an index that already
-    /// exists on the table at the time `set_lsm_write_spec` is called.
-    pub fn with_maintained_indexes<I, S>(mut self, indexes: I) -> Self
-    where
-        I: IntoIterator<Item = S>,
-        S: Into<String>,
-    {
-        let v: Vec<String> = indexes.into_iter().map(Into::into).collect();
+    /// Set which indexes the MemWAL maintains.
+    ///
+    /// `None` (the default) resolves to every index on the table at install,
+    /// failing if one cannot be maintained — name the set to install anyway. A
+    /// list is verbatim: each name must already exist and be maintainable, and
+    /// an empty list maintains nothing.
+    ///
+    /// ```
+    /// # use lancedb::table::LsmWriteSpec;
+    /// // Every index the table has when the spec is installed:
+    /// LsmWriteSpec::unsharded().with_maintained_indexes(None);
+    /// // Exactly these:
+    /// LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_idx".to_string()]);
+    /// // None at all:
+    /// LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new());
+    /// ```
+    pub fn with_maintained_indexes(mut self, indexes: impl Into<Option<Vec<String>>>) -> Self {
+        let indexes = indexes.into();
         match &mut self {
             Self::Bucket {
                 maintained_indexes, ..
@@ -465,7 +443,7 @@ impl LsmWriteSpec {
             }
             | Self::Unsharded {
                 maintained_indexes, ..
-            } => *maintained_indexes = v,
+            } => *maintained_indexes = indexes,
         }
         self
     }
@@ -501,8 +479,9 @@ impl LsmWriteSpec {
         self
     }
 
-    /// Borrow the list of index names this spec asks MemWAL to maintain.
-    pub fn maintained_indexes(&self) -> &[String] {
+    /// Borrow the list of index names this spec asks MemWAL to maintain, or
+    /// `None` when it asks for every index on the table.
+    pub fn maintained_indexes(&self) -> Option<&[String]> {
         match self {
             Self::Bucket {
                 maintained_indexes, ..
@@ -512,7 +491,7 @@ impl LsmWriteSpec {
             }
             | Self::Unsharded {
                 maintained_indexes, ..
-            } => maintained_indexes,
+            } => maintained_indexes.as_deref(),
         }
     }
 
@@ -692,6 +671,31 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
             message: "get_lsm_write_spec is not supported on this table type".into(),
         })
     }
+    /// Seal every bucket's active memtable into L0.
+    ///
+    /// The default implementation returns `NotSupported`.
+    async fn flush_lsm(&self) -> Result<()> {
+        Err(Error::NotSupported {
+            message: "flush_lsm is not supported on this table type".into(),
+        })
+    }
+    /// Trigger a background L0 → base compaction pass per bucket.
+    ///
+    /// The default implementation returns `NotSupported`.
+    async fn compact_lsm(&self) -> Result<()> {
+        Err(Error::NotSupported {
+            message: "compact_lsm is not supported on this table type".into(),
+        })
+    }
+    /// Read live LSM state, or `None` when the LSM write path is not
+    /// enabled for this table.
+    ///
+    /// The default implementation returns `NotSupported`.
+    async fn get_lsm_stats(&self, _include_generation_rows: bool) -> Result<Option<LsmStats>> {
+        Err(Error::NotSupported {
+            message: "get_lsm_stats is not supported on this table type".into(),
+        })
+    }
     /// Drain and close any cached MemWAL shard writers for this table.
     ///
     /// The default implementation is a no-op; table types that maintain
@@ -741,12 +745,59 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
         transforms: NewColumnTransform,
         read_columns: Option<Vec<String>>,
     ) -> Result<AddColumnsResult>;
+    /// Declare computed columns, each defined by a SQL expression.
+    ///
+    /// Where the declaration is planned depends on the backend: a local table
+    /// validates and types the expression itself, a remote one sends the text
+    /// for the server to plan.
+    async fn add_computed_columns(
+        &self,
+        _columns: &[(String, String)],
+    ) -> Result<AddColumnsResult> {
+        Err(Error::NotSupported {
+            message: "computed columns are not supported on this table type".into(),
+        })
+    }
+    /// Declare one immutable registered-Function binding.
+    async fn add_function_columns(
+        &self,
+        _application: &crate::function::FunctionApplication,
+        _output_name: Option<&str>,
+    ) -> Result<AddColumnsResult> {
+        Err(Error::NotSupported {
+            message: "Function columns are supported only on LanceDB Cloud and Enterprise".into(),
+        })
+    }
+    /// Fill a computed column's unfilled rows.
+    ///
+    /// The default returns `NotSupported`; Lance-backed tables override it.
+    async fn refresh_column(&self, _column: &str) -> Result<RefreshColumnResult> {
+        Err(Error::NotSupported {
+            message: "computed columns are supported only on local tables".into(),
+        })
+    }
+    /// Fill a computed column's unfilled rows, returning a [`Job`] tracking
+    /// the operation.
+    async fn refresh_column_async(
+        &self,
+        _column: &str,
+    ) -> Result<Job<crate::function::RefreshColumnResult>> {
+        Err(Error::NotSupported {
+            message: "computed columns are supported only on local tables".into(),
+        })
+    }
     /// Alter columns in the table.
     async fn alter_columns(&self, alterations: &[ColumnAlteration]) -> Result<AlterColumnsResult>;
     /// Drop columns from the table.
     async fn drop_columns(&self, columns: &[&str]) -> Result<DropColumnsResult>;
     /// Get the version of the table.
     async fn version(&self) -> Result<u64>;
+    /// Return a new table handle pinned to the exact revision currently visible.
+    async fn checkout_current(&self) -> Result<Arc<dyn BaseTable>> {
+        Err(Error::NotSupported {
+            message: "checkout_current is not supported on this table type".into(),
+        })
+    }
     /// Checkout a specific version of the table.
     async fn checkout(&self, version: u64) -> Result<()>;
     /// Checkout a table version referenced by a tag.
@@ -754,6 +805,21 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
     async fn checkout_tag(&self, tag: &str) -> Result<()>;
     /// Checkout the latest version of the table.
     async fn checkout_latest(&self) -> Result<()>;
+    /// Return an independent handle pinned to the version currently selected.
+    ///
+    /// Backends that can advance between requests should override this for
+    /// multi-request operations that need snapshot consistency. Backends whose
+    /// existing handles already provide the desired behavior return `None`.
+    async fn snapshot_at_current_version(&self) -> Result<Option<Arc<dyn BaseTable>>> {
+        Ok(None)
+    }
+    /// Whether repeated identical scans return rows in the same order.
+    ///
+    /// Callers that assign meaning to a row's position must order the results
+    /// themselves when this is false.  Defaults to false so a table type opts in.
+    fn scan_order_is_deterministic(&self) -> bool {
+        false
+    }
     /// Restore the table to the currently checked out version.
     async fn restore(&self) -> Result<()>;
     /// List the versions of the table.
@@ -790,14 +856,14 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
     /// Diff a branch against main. Remote only.
     async fn diff_branch(&self, _from_branch: &str) -> Result<BranchDiff> {
         Err(Error::NotSupported {
-            message: "diff_branch is only supported on remote tables".into(),
+            message: "Branch diffs are only supported on Enterprise tables.".into(),
         })
     }
-    /// Merge a branch into main, or dry-run. Remote only.
-    /// HTTP 409 still returns [`Ok`] with [`MergeBranchStatus::Rejected`].
-    async fn merge_branch(&self, _from_branch: &str, _dry_run: bool) -> Result<MergeBranchResult> {
+    /// Cherry-pick a branch onto main, or dry-run. Remote only.
+    /// HTTP 409 still returns [`Ok`] with [`CherryPickStatus::Failed`].
+    async fn cherry_pick(&self, _from_branch: &str, _dry_run: bool) -> Result<CherryPickResult> {
         Err(Error::NotSupported {
-            message: "merge_branch is only supported on remote tables".into(),
+            message: "Cherry-picking branches is only supported on Enterprise tables.".into(),
         })
     }
     /// The branch this handle is scoped to, or `None` for `main`.
@@ -1024,6 +1090,11 @@ impl Table {
         self.database.as_ref().unwrap()
     }
 
+    /// The database this handle was opened through, when it was.
+    pub fn database_opt(&self) -> Option<&Arc<dyn Database>> {
+        self.database.as_ref()
+    }
+
     pub fn embedding_registry(&self) -> &Arc<dyn EmbeddingRegistry> {
         &self.embedding_registry
     }
@@ -1643,6 +1714,59 @@ impl Table {
         AddColumnsBuilder::new(self.inner.clone())
     }
 
+    /// Fill the fragments of a computed column that hold no values yet.
+    ///
+    /// Declared with
+    /// [`AddColumnsBuilder::computed`](add_columns::AddColumnsBuilder::computed),
+    /// a column starts empty and gets its values here. Fragments appended
+    /// since the last refresh are filled by the next one; fragments already
+    /// filled are left as they are, so the call is idempotent and does not
+    /// observe a mutated input.
+    ///
+    /// Local tables only: a remote refresh runs as a server job, through
+    /// [`Table::refresh_column_async`].
+    ///
+    /// ```
+    /// # use lancedb::Table;
+    /// # async fn refresh(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
+    /// let result = table.refresh_column("doubled").await?;
+    /// println!("filled {} rows at version {}", result.rows_filled, result.version);
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub async fn refresh_column(&self, column: impl AsRef<str>) -> Result<RefreshColumnResult> {
+        self.inner.refresh_column(column.as_ref()).await
+    }
+
+    /// Like [`Table::refresh_column`], but returns a [`Job`] tracking the
+    /// operation instead of blocking until it completes.
+    ///
+    /// The job may already be complete when returned, and callers must not
+    /// assume the column is filled until [`Job::wait`] returns. A successful
+    /// wait returns the durable [`crate::function::RefreshColumnResult`] for
+    /// both expression-backed and Function-backed columns. Invalid input
+    /// -- an unknown column, or one that is not computed -- is reported by
+    /// this call rather than by the job. On local tables the job runs as an
+    /// in-process task; on LanceDB Cloud and Enterprise it is the server's
+    /// backfill job.
+    ///
+    /// ```
+    /// # use lancedb::Table;
+    /// # async fn refresh_in_background(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
+    /// let job = table.refresh_column_async("doubled").await?;
+    /// println!("refresh running: {:?}", job.status().await?);
+    /// let result = job.wait().await?;
+    /// println!("assigned {} rows", result.rows_assigned);
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub async fn refresh_column_async(
+        &self,
+        column: impl AsRef<str>,
+    ) -> Result<Job<crate::function::RefreshColumnResult>> {
+        self.inner.refresh_column_async(column.as_ref()).await
+    }
+
     /// Change a column's name or nullability.
     pub async fn alter_columns(
         &self,
@@ -1702,7 +1826,7 @@ impl Table {
     /// # async fn example(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
     /// table
     ///     .set_lsm_write_spec(
-    ///         LsmWriteSpec::bucket("id", 16).with_maintained_indexes(["id_idx"]),
+    ///         LsmWriteSpec::bucket("id", 16).with_maintained_indexes(vec!["id_idx".to_string()]),
     ///     )
     ///     .await?;
     /// # Ok(())
@@ -1724,9 +1848,10 @@ impl Table {
     ///
     /// Returns `Ok(None)` when the MemWAL LSM write path is not enabled (no
     /// spec has been set, or it was removed with [`Table::unset_lsm_write_spec`]).
-    /// The returned spec — including its [`LsmWriteSpec::maintained_indexes`] and
-    /// [`LsmWriteSpec::writer_config_defaults`] — mirrors what was passed to
-    /// [`Table::set_lsm_write_spec`].
+    /// The returned spec mirrors what was passed to
+    /// [`Table::set_lsm_write_spec`], except that
+    /// [`LsmWriteSpec::maintained_indexes`] always reports the concrete list
+    /// resolved when the spec was set — `None` never round-trips.
     ///
     /// # Example
     ///
@@ -1743,6 +1868,85 @@ impl Table {
         self.inner.get_lsm_write_spec().await
     }
 
+    /// Converge this table's LSM write path into its base table.
+    ///
+    /// One `flush` to seal every memtable into L0, then compaction triggers
+    /// until every generation that existed at that moment has reached base.
+    /// The loop runs client-side, reading progress from `get_lsm_stats`, so
+    /// there is no held socket and nothing to reconcile if you drop this
+    /// future partway through.
+    ///
+    /// **Best-effort.** Generations created *after* the opening flush are
+    /// deliberately not waited on — that is what lets this terminate on a
+    /// table taking writes. Idempotent and safe on a cadence: an
+    /// already-converged table costs two round trips and triggers nothing.
+    ///
+    /// **No deadline, and the caller owns that.** It returns when the target
+    /// generations are gone, propagates a terminal server fault, and
+    /// otherwise waits however long the server takes. A slow table and a
+    /// stuck one are the same picture from here: the compactor pool is shared
+    /// across every table on the node, so a checkpoint queued behind
+    /// unrelated work is indistinguishable from one that is merging. Wrap
+    /// this in `tokio::time::timeout` for a wall-clock bound; abandoning it
+    /// partway costs nothing.
+    ///
+    /// # Example
+    ///
+    /// ```no_run
+    /// # use lancedb::Table;
+    /// # async fn example(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
+    /// let before = table.get_lsm_stats(false).await?;
+    /// table.checkpoint_lsm().await?;
+    /// let after = table.get_lsm_stats(false).await?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub async fn checkpoint_lsm(&self) -> Result<()> {
+        checkpoint::checkpoint_lsm(self).await
+    }
+
+    /// Seal every bucket's active memtable into L0 without touching the
+    /// base table.
+    ///
+    /// Independently useful: flushing makes memtable rows readable from L0 at
+    /// a lower per-query cost. On a node that has not claimed this table it
+    /// claims it and replays the WAL log first — reporting "nothing to flush"
+    /// without replaying would lie about durable data.
+    pub async fn flush_lsm(&self) -> Result<()> {
+        self.inner.flush_lsm().await
+    }
+
+    /// Run one bounded L0 → base compaction pass per bucket, reporting what
+    /// it merged and what is left.
+    ///
+    /// One pass, not convergence: that bounds each request's cost and gives a
+    /// caller driving its own cadence a progress signal per round trip.
+    pub async fn compact_lsm(&self) -> Result<()> {
+        self.inner.compact_lsm().await
+    }
+
+    /// Read live per-bucket LSM state.
+    ///
+    /// Answers "how far behind is my fresh tier", "which bucket is hot", and
+    /// "why is my fresh-tier vector search brute-force". Mutates no table
+    /// state, though on a node that has not claimed this table it claims it,
+    /// exactly as a read would.
+    ///
+    /// `include_generation_rows` reports a row count per L0 generation. Off by
+    /// default: each count opens an uncached Lance dataset, and
+    /// `checkpoint_lsm` polls this needing only generation numbers.
+    ///
+    /// `Ok(None)` only when the LSM write path is not enabled, matching
+    /// [`Table::get_lsm_write_spec`]. Stats is fresh-tier only, so with the
+    /// WAL off there is no manifest to report and a struct of zeros would
+    /// read as measurements.
+    ///
+    /// Do not build a checkpoint's termination on this: the completion
+    /// predicate lives in the `flush` and `compact` responses.
+    pub async fn get_lsm_stats(&self, include_generation_rows: bool) -> Result<Option<LsmStats>> {
+        self.inner.get_lsm_stats(include_generation_rows).await
+    }
+
     /// Drain and close any cached MemWAL shard writers held for this table.
     ///
     /// When an [`LsmWriteSpec`] is installed, `merge_insert` opens MemWAL shard
@@ -1763,6 +1967,20 @@ impl Table {
         self.inner.version().await
     }
 
+    /// Return a new table handle pinned to the exact revision currently visible.
+    ///
+    /// This is used when asynchronous preparation must remain consistent with
+    /// the revision used for a later read.
+    #[doc(hidden)]
+    pub async fn checkout_current(&self) -> Result<Self> {
+        let inner = self.inner.checkout_current().await?;
+        Ok(Self {
+            inner,
+            database: self.database.clone(),
+            embedding_registry: self.embedding_registry.clone(),
+        })
+    }
+
     /// Checks out a specific version of the Table
     ///
     /// Any read operation on the table will now access the data at the checked out version.
@@ -2099,14 +2317,10 @@ impl Table {
         self.inner.diff_branch(from_branch).await
     }
 
-    /// Merge a branch into main, or dry-run. Remote only.
-    /// HTTP 409 still returns [`Ok`] with [`MergeBranchStatus::Rejected`].
-    pub async fn merge_branch(
-        &self,
-        from_branch: &str,
-        dry_run: bool,
-    ) -> Result<MergeBranchResult> {
-        self.inner.merge_branch(from_branch, dry_run).await
+    /// Cherry-pick a branch onto main, or dry-run. Remote only.
+    /// HTTP 409 still returns [`Ok`] with [`CherryPickStatus::Failed`].
+    pub async fn cherry_pick(&self, from_branch: &str, dry_run: bool) -> Result<CherryPickResult> {
+        self.inner.cherry_pick(from_branch, dry_run).await
     }
 
     /// The branch this handle is scoped to, or `None` for `main`.
@@ -2309,8 +2523,6 @@ impl NativeTable {
             None => false,
         };
 
-        // Kept so that a `DatasetNotFound` can be re-checked against storage below.
-        let recovery_params = params.clone();
         let mut builder = DatasetBuilder::from_uri(uri).with_read_params(params);
 
         // Set up commit handler when managed_versioning is enabled
@@ -2329,7 +2541,12 @@ impl NativeTable {
         let dataset = match builder.load().await {
             Ok(dataset) => dataset,
             Err(e @ lance::Error::DatasetNotFound { .. }) => {
-                return Err(map_dataset_not_found(uri, name, recovery_params, e).await);
+                // The manifest load is the existence check. A physical prefix may be
+                // from a concurrent or abandoned create, so it cannot refine this error.
+                return Err(Error::TableNotFound {
+                    name: name.to_string(),
+                    source: Box::new(e),
+                });
             }
             Err(e) => return Err(e.into()),
         };
@@ -2541,6 +2758,7 @@ impl NativeTable {
         namespace_client: Option<Arc<dyn LanceNamespace>>,
         pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
     ) -> Result<Self> {
+        computed_columns::ensure_no_foreign_declarations(batches.arrow_schema().fields())?;
         // Default params uses format v1.
         let params = params.unwrap_or(WriteParams {
             ..Default::default()
@@ -2840,6 +3058,12 @@ impl BaseTable for NativeTable {
         self
     }
 
+    /// Lance scans fragments in order (`Scanner::ordered` defaults to true, and we
+    /// never clear it), so repeated identical scans agree.
+    fn scan_order_is_deterministic(&self) -> bool {
+        true
+    }
+
     fn name(&self) -> &str {
         self.name.as_str()
     }
@@ -2868,6 +3092,18 @@ impl BaseTable for NativeTable {
         Ok(self.dataset.get().await?.version().version)
     }
 
+    async fn checkout_current(&self) -> Result<Arc<dyn BaseTable>> {
+        let current = self.dataset.get().await?;
+        let dataset = dataset::DatasetConsistencyWrapper::new_time_travel(
+            current.as_ref().clone(),
+            self.read_consistency_interval,
+        );
+        Ok(Arc::new(Self {
+            dataset,
+            ..self.clone()
+        }))
+    }
+
     async fn checkout(&self, version: u64) -> Result<()> {
         self.dataset.as_time_travel(version).await
     }
@@ -3001,6 +3237,14 @@ impl BaseTable for NativeTable {
         let ds = self.dataset.get().await?;
 
         let table_schema = Schema::from(&ds.schema().clone());
+        computed_columns::ensure_supported_function_metadata(&table_schema)?;
+        computed_columns::ensure_not_written(
+            &table_schema,
+            add.data.schema().fields().iter().map(|f| f.name().as_str()),
+        )?;
+        if matches!(add.mode, AddDataMode::Overwrite) {
+            computed_columns::ensure_no_foreign_declarations(add.data.schema().fields())?;
+        }
 
         let num_partitions = if let Some(parallelism) = add.write_parallelism {
             parallelism
@@ -3025,7 +3269,7 @@ impl BaseTable for NativeTable {
 
         let output = add.into_plan(&table_schema, &table_def)?;
 
-        let lance_params = output
+        let mut lance_params = output
             .write_options
             .lance_write_params
             .unwrap_or(WriteParams {
@@ -3035,6 +3279,9 @@ impl BaseTable for NativeTable {
                 },
                 ..Default::default()
             });
+        if output.allow_external_blob_outside_bases {
+            lance_params.allow_external_blob_outside_bases = true;
+        }
 
         // Repartition for write parallelism if beneficial.
         let plan = if num_partitions > 1 {
@@ -3161,6 +3408,11 @@ impl BaseTable for NativeTable {
         params: MergeInsertBuilder,
         new_data: Box<dyn RecordBatchReader + Send>,
     ) -> Result<MergeResult> {
+        let source_schema = arrow_array::RecordBatchReader::schema(&new_data);
+        computed_columns::ensure_not_written(
+            &Schema::from(self.dataset.get().await?.schema()),
+            source_schema.fields().iter().map(|f| f.name().as_str()),
+        )?;
         let result = merge::execute_merge_insert(self, params, new_data).await?;
         self.bump_freshness();
         Ok(result)
@@ -3242,6 +3494,25 @@ impl BaseTable for NativeTable {
         Ok(result)
     }
 
+    async fn add_computed_columns(&self, columns: &[(String, String)]) -> Result<AddColumnsResult> {
+        let result = schema_evolution::execute_declare(self, columns).await?;
+        self.bump_freshness();
+        Ok(result)
+    }
+
+    async fn refresh_column(&self, column: &str) -> Result<RefreshColumnResult> {
+        let result = refresh::execute_refresh_column(self, column).await?;
+        self.bump_freshness();
+        Ok(result)
+    }
+
+    async fn refresh_column_async(
+        &self,
+        column: &str,
+    ) -> Result<Job<crate::function::RefreshColumnResult>> {
+        refresh::execute_refresh_column_async(self, column).await
+    }
+
     async fn alter_columns(&self, alterations: &[ColumnAlteration]) -> Result<AlterColumnsResult> {
         let result = schema_evolution::execute_alter_columns(self, alterations).await?;
         self.bump_freshness();
@@ -3470,9 +3741,24 @@ impl BaseTable for NativeTable {
         let num_rows = self.count_rows(None).await?;
         let num_indices = self.list_indices().await?.len();
         let ds = self.dataset.get().await?;
-        let ds_clone = (*ds).clone();
-        let ds_stats = Arc::new(ds_clone).calculate_data_stats().await?;
-        let total_bytes = ds_stats.fields.iter().map(|f| f.bytes_on_disk).sum::<u64>() as usize;
+        // Sizes come from the manifest. Summing per-field `bytes_on_disk` instead
+        // would open every data file to read its column metadata, which costs one
+        // IO per fragment and reports 0 for legacy v1 storage.
+        //
+        // The manifest summary covers only the fragments' base data files, so
+        // overlay files (recorded on each fragment) and index files (recorded in
+        // the manifest's index section) are added separately.
+        let mut total_bytes = ds.manifest().summary().total_files_size as usize;
+        for frag in ds.manifest().fragments.iter() {
+            for overlay in &frag.overlays {
+                if let Some(size) = overlay.data_file.file_size_bytes.get() {
+                    total_bytes += size.get() as usize;
+                }
+            }
+        }
+        for index in ds.load_indices().await?.iter() {
+            total_bytes += index.total_size_bytes().unwrap_or(0) as usize;
+        }
 
         let frags = ds.get_fragments();
         let mut sorted_sizes = join_all(
@@ -3544,7 +3830,12 @@ impl BaseTable for NativeTable {
 #[skip_serializing_none]
 #[derive(Debug, Deserialize, PartialEq)]
 pub struct TableStatistics {
-    /// The total number of bytes in the table
+    /// The total size, in bytes, of the table's data files, index files, and
+    /// overlay files
+    ///
+    /// Read from the manifest, so this excludes deletion files and manifests,
+    /// and it excludes any file whose size the manifest does not record
+    /// (tables and indices written before writers persisted file sizes).
     pub total_bytes: usize,
 
     /// The number of rows in the table
@@ -3589,7 +3880,7 @@ pub struct FragmentSummaryStats {
 #[allow(deprecated)]
 mod tests {
     use std::sync::Arc;
-    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
     use std::time::Duration;
 
     use arrow_array::{
@@ -3605,6 +3896,7 @@ mod tests {
     use super::*;
     use crate::connect;
     use crate::connection::ConnectBuilder;
+    use crate::io::object_store::io_tracking::IoTrackingStore;
     use crate::query::Select;
     use crate::query::{ExecutableQuery, QueryBase};
     use crate::test_utils::connection::new_test_connection;
@@ -3670,73 +3962,50 @@ mod tests {
         );
     }
 
-    /// Write a table and then break it, leaving the `<name>.lance` directory in place.
-    ///
-    /// `remove_all` reproduces an interrupted drop + re-create (the directory is left
-    /// empty); otherwise only the manifests are removed, leaving the data files behind.
-    async fn write_then_corrupt_table(dir: &std::path::Path, remove_all: bool) -> String {
-        let dataset_path = dir.join("test.lance");
-        let uri = dataset_path.to_str().unwrap().to_string();
-
-        let batch = make_test_batches();
-        let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema());
-        Dataset::write(reader, &uri, None).await.unwrap();
-
-        if remove_all {
-            for entry in std::fs::read_dir(&dataset_path).unwrap() {
-                let entry = entry.unwrap();
-                if entry.file_type().unwrap().is_dir() {
-                    std::fs::remove_dir_all(entry.path()).unwrap();
-                } else {
-                    std::fs::remove_file(entry.path()).unwrap();
-                }
-            }
-            assert_eq!(std::fs::read_dir(&dataset_path).unwrap().count(), 0);
-        } else {
-            let versions = dataset_path.join("_versions");
-            assert!(versions.is_dir(), "expected manifests under {versions:?}");
-            std::fs::remove_dir_all(&versions).unwrap();
-            assert!(std::fs::read_dir(&dataset_path).unwrap().count() > 0);
-        }
-
-        uri
-    }
-
     #[tokio::test]
-    async fn test_open_corrupt_empty_dir() {
+    async fn test_open_not_found_when_empty_directory_exists() {
         let tmp_dir = tempdir().unwrap();
-        let uri = write_then_corrupt_table(tmp_dir.path(), true).await;
+        let dataset_path = tmp_dir.path().join("test.lance");
+        std::fs::create_dir(&dataset_path).unwrap();
 
-        let err = NativeTable::open(&uri).await.unwrap_err();
+        let err = NativeTable::open(dataset_path.to_str().unwrap())
+            .await
+            .unwrap_err();
         assert!(
-            matches!(&err, Error::TableCorrupted { name, .. } if name == "test"),
+            matches!(&err, Error::TableNotFound { name, .. } if name == "test"),
             "got {err:?}"
         );
     }
 
     #[tokio::test]
-    async fn test_open_corrupt_missing_manifest() {
+    async fn test_open_not_found_when_only_uncommitted_storage_exists() {
         let tmp_dir = tempdir().unwrap();
-        let uri = write_then_corrupt_table(tmp_dir.path(), false).await;
+        let dataset_path = tmp_dir.path().join("test.lance");
+        let data_dir = dataset_path.join("data");
+        std::fs::create_dir_all(&data_dir).unwrap();
+        std::fs::write(data_dir.join("orphan.lance"), b"uncommitted").unwrap();
 
-        let err = NativeTable::open(&uri).await.unwrap_err();
+        let err = NativeTable::open(dataset_path.to_str().unwrap())
+            .await
+            .unwrap_err();
         assert!(
-            matches!(&err, Error::TableCorrupted { name, .. } if name == "test"),
+            matches!(&err, Error::TableNotFound { name, .. } if name == "test"),
             "got {err:?}"
         );
     }
 
-    /// A table listed by `table_names()` must not be reported as missing by
-    /// `open_table()`. See <https://github.com/lancedb/lancedb/issues/3127>.
+    /// Listing databases discover physical `*.lance` entries. That snapshot is not an
+    /// authoritative table-existence check: only a committed manifest makes a table
+    /// openable, and the entry could also be concurrently created or dropped.
     #[tokio::test]
-    async fn test_open_table_corrupt_is_still_listed() {
+    async fn test_table_names_may_include_uncommitted_storage() {
         let tmp_dir = tempdir().unwrap();
         let db = connect(tmp_dir.path().to_str().unwrap())
             .execute()
             .await
             .unwrap();
 
-        write_then_corrupt_table(tmp_dir.path(), true).await;
+        std::fs::create_dir(tmp_dir.path().join("test.lance")).unwrap();
 
         assert_eq!(
             db.table_names().execute().await.unwrap(),
@@ -3744,12 +4013,177 @@ mod tests {
         );
         let err = db.open_table("test").execute().await.unwrap_err();
         assert!(
-            matches!(&err, Error::TableCorrupted { name, .. } if name == "test"),
+            matches!(&err, Error::TableNotFound { name, .. } if name == "test"),
+            "physical storage without a committed manifest is not a table: {err:?}"
+        );
+    }
+
+    #[derive(Debug)]
+    struct ParentListGuardStore {
+        inner: Arc<dyn object_store::ObjectStore>,
+        parent: object_store::path::Path,
+        parent_list_calls: Arc<AtomicUsize>,
+    }
+
+    impl std::fmt::Display for ParentListGuardStore {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            f.write_str("ParentListGuardStore")
+        }
+    }
+
+    #[async_trait::async_trait]
+    #[deny(clippy::missing_trait_methods)]
+    impl object_store::ObjectStore for ParentListGuardStore {
+        async fn put_opts(
+            &self,
+            location: &object_store::path::Path,
+            payload: object_store::PutPayload,
+            opts: object_store::PutOptions,
+        ) -> object_store::Result<object_store::PutResult> {
+            self.inner.put_opts(location, payload, opts).await
+        }
+
+        async fn put_multipart_opts(
+            &self,
+            location: &object_store::path::Path,
+            opts: object_store::PutMultipartOptions,
+        ) -> object_store::Result<Box<dyn object_store::MultipartUpload>> {
+            self.inner.put_multipart_opts(location, opts).await
+        }
+
+        async fn get_opts(
+            &self,
+            location: &object_store::path::Path,
+            options: object_store::GetOptions,
+        ) -> object_store::Result<object_store::GetResult> {
+            self.inner.get_opts(location, options).await
+        }
+
+        async fn get_ranges(
+            &self,
+            location: &object_store::path::Path,
+            ranges: &[std::ops::Range<u64>],
+        ) -> object_store::Result<Vec<bytes::Bytes>> {
+            self.inner.get_ranges(location, ranges).await
+        }
+
+        fn delete_stream(
+            &self,
+            locations: futures::stream::BoxStream<
+                'static,
+                object_store::Result<object_store::path::Path>,
+            >,
+        ) -> futures::stream::BoxStream<'static, object_store::Result<object_store::path::Path>>
+        {
+            self.inner.delete_stream(locations)
+        }
+
+        fn list(
+            &self,
+            prefix: Option<&object_store::path::Path>,
+        ) -> futures::stream::BoxStream<'static, object_store::Result<object_store::ObjectMeta>>
+        {
+            if prefix == Some(&self.parent) {
+                self.parent_list_calls.fetch_add(1, Ordering::Relaxed);
+            }
+            self.inner.list(prefix)
+        }
+
+        fn list_with_offset(
+            &self,
+            prefix: Option<&object_store::path::Path>,
+            offset: &object_store::path::Path,
+        ) -> futures::stream::BoxStream<'static, object_store::Result<object_store::ObjectMeta>>
+        {
+            if prefix == Some(&self.parent) {
+                self.parent_list_calls.fetch_add(1, Ordering::Relaxed);
+            }
+            self.inner.list_with_offset(prefix, offset)
+        }
+
+        async fn list_with_delimiter(
+            &self,
+            prefix: Option<&object_store::path::Path>,
+        ) -> object_store::Result<object_store::ListResult> {
+            if prefix == Some(&self.parent) {
+                self.parent_list_calls.fetch_add(1, Ordering::Relaxed);
+            }
+            self.inner.list_with_delimiter(prefix).await
+        }
+
+        async fn copy_opts(
+            &self,
+            from: &object_store::path::Path,
+            to: &object_store::path::Path,
+            options: object_store::CopyOptions,
+        ) -> object_store::Result<()> {
+            self.inner.copy_opts(from, to, options).await
+        }
+
+        async fn rename_opts(
+            &self,
+            from: &object_store::path::Path,
+            to: &object_store::path::Path,
+            options: object_store::RenameOptions,
+        ) -> object_store::Result<()> {
+            self.inner.rename_opts(from, to, options).await
+        }
+    }
+
+    #[derive(Debug)]
+    struct ParentListGuardWrapper {
+        parent_list_calls: Arc<AtomicUsize>,
+    }
+
+    impl WrappingObjectStore for ParentListGuardWrapper {
+        fn wrap(
+            &self,
+            _store_prefix: &str,
+            inner: Arc<dyn object_store::ObjectStore>,
+        ) -> Arc<dyn object_store::ObjectStore> {
+            Arc::new(ParentListGuardStore {
+                inner,
+                parent: object_store::path::Path::from("database"),
+                parent_list_calls: self.parent_list_calls.clone(),
+            })
+        }
+    }
+
+    #[tokio::test]
+    async fn test_open_missing_never_lists_database_parent() {
+        let parent_list_calls = Arc::new(AtomicUsize::new(0));
+        let params = ReadParams {
+            store_options: Some(ObjectStoreParams {
+                object_store_wrapper: Some(Arc::new(ParentListGuardWrapper {
+                    parent_list_calls: parent_list_calls.clone(),
+                })),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let err = NativeTable::open_with_params(
+            "memory:///database/missing.lance",
+            "missing",
+            Vec::new(),
+            None,
+            Some(params),
+            None,
+            None,
+            HashSet::new(),
+            None,
+        )
+        .await
+        .unwrap_err();
+
+        assert!(
+            matches!(&err, Error::TableNotFound { name, .. } if name == "missing"),
             "got {err:?}"
         );
-        assert!(
-            err.to_string().contains("exists but could not be loaded"),
-            "got {err}"
+        assert_eq!(
+            parent_list_calls.load(Ordering::Relaxed),
+            0,
+            "opening one missing table must not enumerate sibling tables"
         );
     }
 
@@ -4987,7 +5421,7 @@ mod tests {
         // Bucket spec round-trips exactly, including the routing column (recovered
         // from its field id), maintained indexes, and writer config defaults.
         let spec = LsmWriteSpec::bucket("id", 4)
-            .with_maintained_indexes([idx_name])
+            .with_maintained_indexes(vec![idx_name.clone()])
             .with_writer_config_defaults([("durable_write", "false")]);
         table.set_lsm_write_spec(spec.clone()).await.unwrap();
         assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
@@ -4997,15 +5431,125 @@ mod tests {
         assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
 
         // Identity sharding round-trips (column recovered from the schema).
+        // A spec left at its default maintains every index on the table, so it
+        // reads back naming the one on the table rather than as "infer".
         let spec = LsmWriteSpec::identity("region");
         table.set_lsm_write_spec(spec.clone()).await.unwrap();
-        assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
+        assert_eq!(
+            table.get_lsm_write_spec().await.unwrap(),
+            Some(spec.with_maintained_indexes(vec![idx_name.clone()]))
+        );
         table.unset_lsm_write_spec().await.unwrap();
 
         // Unsharded round-trips (no routing column).
         let spec = LsmWriteSpec::unsharded();
         table.set_lsm_write_spec(spec.clone()).await.unwrap();
-        assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
+        assert_eq!(
+            table.get_lsm_write_spec().await.unwrap(),
+            Some(spec.with_maintained_indexes(vec![idx_name]))
+        );
+    }
+
+    /// The maintained set defaults to every index on the table, resolved at
+    /// install. An index the memtable cannot build fails the install rather
+    /// than being dropped: maintaining it would take the table offline for
+    /// writes, dropping it would hide that from the caller.
+    #[tokio::test]
+    async fn test_set_lsm_write_spec_infers_maintained_indexes() {
+        let tmp_dir = tempdir().unwrap();
+        let uri = tmp_dir.path().to_str().unwrap();
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int64, false),
+            Field::new("tag", DataType::Utf8, true),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(arrow_array::Int64Array::from(vec![1, 2, 3])),
+                Arc::new(StringArray::from(vec!["a", "b", "c"])),
+            ],
+        )
+        .unwrap();
+        let reader: Box<dyn arrow_array::RecordBatchReader + Send> =
+            Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema.clone()));
+        let conn = ConnectBuilder::new(uri)
+            .read_consistency_interval(Duration::from_secs(0))
+            .execute()
+            .await
+            .unwrap();
+        let table = conn.create_table("t", reader).execute().await.unwrap();
+
+        table
+            .create_index(&["id"], Index::BTree(Default::default()))
+            .name("id_btree".to_string())
+            .execute()
+            .await
+            .unwrap();
+        table
+            .create_index(&["tag"], Index::Bitmap(Default::default()))
+            .name("tag_bitmap".to_string())
+            .execute()
+            .await
+            .unwrap();
+
+        // Explicitly naming the bitmap index fails before anything commits.
+        let err = table
+            .set_lsm_write_spec(
+                LsmWriteSpec::unsharded().with_maintained_indexes(vec!["tag_bitmap".to_string()]),
+            )
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(err, Error::InvalidInput { ref message } if message.contains("tag_bitmap")),
+            "expected the bitmap index to be rejected, got {err:?}"
+        );
+        assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
+
+        // The default covers every index, so the bitmap fails it too.
+        let err = table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded())
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(err, Error::InvalidInput { ref message }
+                if message.contains("tag_bitmap") && message.contains("maintained_indexes")),
+            "expected the inferred set to be rejected, got {err:?}"
+        );
+        assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
+
+        // Naming the maintainable subset installs.
+        table
+            .set_lsm_write_spec(
+                LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_btree".to_string()]),
+            )
+            .await
+            .unwrap();
+        assert_eq!(
+            table
+                .get_lsm_write_spec()
+                .await
+                .unwrap()
+                .unwrap()
+                .maintained_indexes(),
+            Some(["id_btree".to_string()].as_slice())
+        );
+
+        // Opting out entirely is distinct from the default.
+        table.unset_lsm_write_spec().await.unwrap();
+        table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new()))
+            .await
+            .unwrap();
+        assert_eq!(
+            table
+                .get_lsm_write_spec()
+                .await
+                .unwrap()
+                .unwrap()
+                .maintained_indexes(),
+            Some([].as_slice())
+        );
     }
 
     #[tokio::test]
@@ -5053,12 +5597,16 @@ mod tests {
 
         let res = table.stats().await.unwrap();
         println!("{:#?}", res);
+        // `total_bytes` is the full on-disk size of the 11 data files (this table
+        // has no index or overlay files), so it is well above the 2000 bytes of
+        // column data these 250 int32 pairs hold: each file carries its own footer
+        // and metadata.
         assert_eq!(
             res,
             TableStatistics {
                 num_rows: 250,
                 num_indices: 0,
-                total_bytes: 2300,
+                total_bytes: 8925,
                 fragment_stats: FragmentStatistics {
                     num_fragments: 11,
                     num_small_fragments: 11,
@@ -5098,4 +5646,196 @@ mod tests {
             }
         )
     }
+
+    /// `total_bytes` counts more than the base data files: index files and
+    /// overlay files recorded in the manifest are included too.
+    #[tokio::test]
+    pub async fn test_stats_includes_index_and_overlay_files() {
+        use lance::dataset::WriteDestination;
+        use lance::dataset::transaction::{DataOverlayGroup, Operation};
+        use lance_file::version::stable_file_version;
+        use lance_file::writer::FileWriterOptions;
+        use lance_io::utils::CachedFileSize;
+        use lance_table::format::DataFile;
+        use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
+        use roaring::RoaringBitmap;
+
+        let tmp_dir = tempdir().unwrap();
+        let uri = tmp_dir.path().to_str().unwrap();
+        let conn = ConnectBuilder::new(uri)
+            .read_consistency_interval(Duration::from_secs(0))
+            .execute()
+            .await
+            .unwrap();
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("foo", DataType::Int32, true),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..100)),
+                Arc::new(Int32Array::from_iter_values(0..100)),
+            ],
+        )
+        .unwrap();
+        let table = conn
+            .create_table("test_stats_extra_files", batch)
+            .execute()
+            .await
+            .unwrap();
+
+        let data_only = table.stats().await.unwrap().total_bytes;
+        assert!(data_only > 0);
+
+        // A scalar index adds index files whose sizes are recorded in the
+        // manifest's index section.
+        table
+            .create_index(&["id"], Index::Auto)
+            .execute()
+            .await
+            .unwrap();
+        let with_index = table.stats().await.unwrap().total_bytes;
+        let dataset = {
+            let native = table.as_native().unwrap();
+            (*native.dataset.get().await.unwrap()).clone()
+        };
+        let index_bytes: usize = dataset
+            .load_indices()
+            .await
+            .unwrap()
+            .iter()
+            .map(|idx| idx.total_size_bytes().unwrap_or(0) as usize)
+            .sum();
+        assert!(index_bytes > 0);
+        assert_eq!(with_index, data_only + index_bytes);
+
+        // Commit an overlay file supplying new `foo` values for the first three
+        // rows of fragment 0. There is no high-level API that writes overlays
+        // yet, so write the overlay's data file and commit the `DataOverlay`
+        // operation by hand.
+        let read_version = dataset.version().version;
+        let fragment_id = dataset.get_fragments()[0].id() as u64;
+        let foo_field_id = dataset.schema().field("foo").unwrap().id;
+        let overlay_schema = dataset.schema().project_by_ids(&[foo_field_id], true);
+        let file_version = stable_file_version();
+
+        let filename = "overlay.lance".to_string();
+        let store = dataset.object_store(None).await.unwrap();
+        let path = dataset.data_dir().child(filename.clone());
+        let obj_writer = store.create(&path).await.unwrap();
+        let mut writer = lance_file::versions::create_writer(
+            file_version,
+            obj_writer,
+            overlay_schema,
+            FileWriterOptions::default(),
+        )
+        .unwrap();
+        writer
+            .write_column(0, Arc::new(Int32Array::from(vec![1000, 1001, 1002])) as _)
+            .await
+            .unwrap();
+        let summary = writer.finish().await.unwrap();
+        let overlay_bytes = summary.size_bytes as usize;
+        assert!(overlay_bytes > 0);
+
+        let mut data_file = DataFile::new_unstarted(filename, file_version);
+        data_file.fields = writer
+            .field_id_to_column_indices()
+            .iter()
+            .map(|(field_id, _)| *field_id as i32)
+            .collect::<Vec<_>>()
+            .into();
+        data_file.column_indices = writer
+            .field_id_to_column_indices()
+            .iter()
+            .map(|(_, column_index)| *column_index as i32)
+            .collect::<Vec<_>>()
+            .into();
+        data_file.file_size_bytes = CachedFileSize::new(summary.size_bytes);
+
+        let overlay = DataOverlayFile {
+            data_file,
+            coverage: OverlayCoverage::dense(RoaringBitmap::from_iter(0..3)),
+            committed_version: 0,
+        };
+        Dataset::commit(
+            WriteDestination::Dataset(Arc::new(dataset)),
+            Operation::DataOverlay {
+                groups: vec![DataOverlayGroup {
+                    fragment_id,
+                    overlays: vec![overlay],
+                }],
+            },
+            Some(read_version),
+            None,
+            None,
+            Arc::new(Default::default()),
+            false,
+        )
+        .await
+        .unwrap();
+
+        table.checkout_latest().await.unwrap();
+        let with_overlay = table.stats().await.unwrap().total_bytes;
+        assert_eq!(with_overlay, with_index + overlay_bytes);
+    }
+
+    /// `stats()` must stay manifest-only. Summing per-field `bytes_on_disk`
+    /// instead opens every data file, so cost would grow with fragment count.
+    #[tokio::test]
+    pub async fn test_stats_does_not_read_data_files() {
+        let tmp_dir = tempdir().unwrap();
+        let uri = tmp_dir.path().to_str().unwrap();
+
+        let conn = ConnectBuilder::new(uri).execute().await.unwrap();
+
+        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int32Array::from_iter_values(0..10))],
+        )
+        .unwrap();
+
+        conn.create_table("test_stats_io", batch.clone())
+            .execute()
+            .await
+            .unwrap();
+        let table = conn.open_table("test_stats_io").execute().await.unwrap();
+        const NUM_APPENDS: usize = 20;
+        for _ in 0..NUM_APPENDS {
+            table.add(batch.clone()).execute().await.unwrap();
+        }
+
+        // Reopen through a tracking store so the counters cover `stats()` alone and
+        // not the writes above.
+        let (wrapper, io_stats) = IoTrackingStore::new_wrapper();
+        let table = conn
+            .open_table("test_stats_io")
+            .lance_read_params(ReadParams {
+                store_options: Some(ObjectStoreParams {
+                    object_store_wrapper: Some(wrapper),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            })
+            .execute()
+            .await
+            .unwrap();
+        io_stats.lock().unwrap().read_iops = 0;
+
+        let stats = table.stats().await.unwrap();
+        let read_iops = io_stats.lock().unwrap().read_iops;
+
+        assert_eq!(stats.fragment_stats.num_fragments, NUM_APPENDS + 1);
+        assert!(stats.total_bytes > 0);
+        // Reading the fragments' data files would take at least one IOP each.
+        assert!(
+            read_iops < stats.fragment_stats.num_fragments as u64,
+            "stats() issued {} read IOPs across {} fragments",
+            read_iops,
+            stats.fragment_stats.num_fragments
+        );
+    }
 }
diff --git a/rust/lancedb/src/table/add_columns.rs b/rust/lancedb/src/table/add_columns.rs
index 0d410cd04..1ac0c6b4f 100644
--- a/rust/lancedb/src/table/add_columns.rs
+++ b/rust/lancedb/src/table/add_columns.rs
@@ -9,12 +9,15 @@ use lance::dataset::NewColumnTransform;
 
 use super::BaseTable;
 use super::schema_evolution::AddColumnsResult;
+use crate::function::FunctionApplication;
 use crate::{Error, Result};
 
 /// Adds columns to a table. See [`Table::add_columns`](super::Table::add_columns).
 pub struct AddColumnsBuilder {
     parent: Arc<dyn BaseTable>,
     transform: Option<NewColumnTransform>,
+    computed: Vec<(String, String)>,
+    function: Option<(FunctionApplication, Option<String>)>,
     read_columns: Option<Vec<String>>,
 }
 
@@ -23,6 +26,8 @@ impl std::fmt::Debug for AddColumnsBuilder {
         f.debug_struct("AddColumnsBuilder")
             .field("parent", &self.parent)
             .field("has_transform", &self.transform.is_some())
+            .field("computed", &self.computed)
+            .field("has_function", &self.function.is_some())
             .field("read_columns", &self.read_columns)
             .finish()
     }
@@ -33,19 +38,101 @@ impl AddColumnsBuilder {
         Self {
             parent,
             transform: None,
+            computed: Vec::new(),
+            function: None,
             read_columns: None,
         }
     }
 
-    /// Set how the new columns' values are produced. Required.
+    /// Set how the new columns' values are produced.
     pub fn transform(mut self, transform: NewColumnTransform) -> Self {
         self.transform = Some(transform);
         self
     }
 
+    /// Add a column defined by `expression`, evaluated by a later refresh
+    /// rather than by this commit. Its type and inputs are derived from the
+    /// expression.
+    ///
+    /// The column is committed with no values, so declaring one costs the same
+    /// on an empty table as on a large one. Rows get values from
+    /// [`Table::refresh_column`](super::Table::refresh_column), which fills
+    /// every fragment that has none -- including fragments appended since the
+    /// last refresh.
+    ///
+    /// Refresh does not revisit a fragment it has filled, so mutating an input
+    /// leaves the value computed at fill time; recomputing means dropping the
+    /// column and declaring it again. An input cannot be renamed, retyped or
+    /// dropped while a declaration reads it, since the expression names it.
+    ///
+    /// On LanceDB Cloud and Enterprise the expression is planned by the
+    /// server, and the refresh runs as a server job -- see
+    /// [`Table::refresh_column_async`](super::Table::refresh_column_async).
+    ///
+    /// ```
+    /// # use lancedb::Table;
+    /// # async fn declare(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
+    /// table
+    ///     .add_columns()
+    ///     .computed("doubled", "x * 2")
+    ///     .execute()
+    ///     .await?;
+    /// let filled = table.refresh_column("doubled").await?;
+    /// println!("filled {} rows", filled.rows_filled);
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn computed(mut self, name: impl Into<String>, expression: impl Into<String>) -> Self {
+        self.computed.push((name.into(), expression.into()));
+        self
+    }
+
+    /// Declare every field of a named-struct Function result as one atomic
+    /// binding. Result-field aliases come from
+    /// [`FunctionApplication::columns`](crate::function::FunctionApplication::columns).
+    ///
+    /// ```
+    /// # use lancedb::Table;
+    /// # use lancedb::function::FunctionApplication;
+    /// # async fn declare(table: &Table, application: FunctionApplication) -> lancedb::Result<()> {
+    /// table.add_columns().function(application).execute().await?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn function(mut self, application: FunctionApplication) -> Self {
+        self.function = Some((application, None));
+        self
+    }
+
+    /// Declare a scalar or entire named-struct Function result as one table
+    /// column. The physical column starts all-null and is materialized by the
+    /// remote Function refresh path.
+    ///
+    /// ```
+    /// # use lancedb::Table;
+    /// # use lancedb::function::FunctionApplication;
+    /// # async fn declare(table: &Table, application: FunctionApplication) -> lancedb::Result<()> {
+    /// table
+    ///     .add_columns()
+    ///     .function_as("embedding", application)
+    ///     .execute()
+    ///     .await?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn function_as(
+        mut self,
+        name: impl Into<String>,
+        application: FunctionApplication,
+    ) -> Self {
+        self.function = Some((application, Some(name.into())));
+        self
+    }
+
     /// Limit which existing columns a [`NewColumnTransform::BatchUDF`] mapper
-    /// receives. Every other transform determines what it reads, so setting
-    /// this alongside one is an error rather than a silent no-op.
+    /// receives. Every other transform, and a computed column, determines what
+    /// it reads, so setting this alongside one is an error rather than a silent
+    /// no-op.
     pub fn read_columns(mut self, columns: impl IntoIterator<Item = impl Into<String>>) -> Self {
         self.read_columns = Some(columns.into_iter().map(Into::into).collect());
         self
@@ -56,24 +143,56 @@ impl AddColumnsBuilder {
         let Self {
             parent,
             transform,
+            computed,
+            function,
             read_columns,
         } = self;
 
-        let Some(transform) = transform else {
+        let declaration_count = usize::from(!computed.is_empty()) + usize::from(function.is_some());
+        if transform.is_some() && declaration_count != 0 || declaration_count > 1 {
             return Err(Error::InvalidInput {
-                message: "add_columns requires a transform".into(),
-            });
-        };
-
-        if read_columns.is_some() && !matches!(transform, NewColumnTransform::BatchUDF(_)) {
-            return Err(Error::InvalidInput {
-                message: "read_columns applies only to a BatchUDF transform; \
-                          every other transform determines what it reads"
+                message: "add_columns cannot mix transforms, SQL computed columns, and a Function application; they cannot be added atomically in one call"
                     .into(),
             });
         }
 
-        parent.add_columns(transform, read_columns).await
+        match (transform, computed.is_empty(), function) {
+            (None, true, None) => Err(Error::InvalidInput {
+                message: "add_columns requires a transform or a computed column".into(),
+            }),
+            (Some(transform), true, None) => {
+                if read_columns.is_some() && !matches!(transform, NewColumnTransform::BatchUDF(_)) {
+                    return Err(Error::InvalidInput {
+                        message: "read_columns applies only to a BatchUDF transform; \
+                                  every other transform determines what it reads"
+                            .into(),
+                    });
+                }
+                parent.add_columns(transform, read_columns).await
+            }
+            (None, false, None) => {
+                if read_columns.is_some() {
+                    return Err(Error::InvalidInput {
+                        message: "read_columns applies only to a BatchUDF transform; \
+                                  a computed column's inputs come from its expression"
+                            .into(),
+                    });
+                }
+                parent.add_computed_columns(&computed).await
+            }
+            (None, true, Some((application, output_name))) => {
+                if read_columns.is_some() {
+                    return Err(Error::InvalidInput {
+                        message: "read_columns does not apply to a Function application; its inputs are already bound"
+                            .into(),
+                    });
+                }
+                parent
+                    .add_function_columns(&application, output_name.as_deref())
+                    .await
+            }
+            _ => unreachable!("mixed add_columns modes were rejected above"),
+        }
     }
 }
 
@@ -85,8 +204,8 @@ mod tests {
     use arrow_schema::{DataType, Field, Schema};
     use lance::dataset::{BatchUDF, NewColumnTransform};
 
-    use crate::Table;
     use crate::connect;
+    use crate::{Error, Table};
 
     async fn table_with_two_columns(name: &str) -> Table {
         let conn = connect("memory://").execute().await.unwrap();
@@ -98,10 +217,7 @@ mod tests {
     async fn test_requires_a_transform() {
         let table = table_with_two_columns("no_transform").await;
         let err = table.add_columns().execute().await.unwrap_err();
-        assert!(
-            err.to_string().contains("requires a transform"),
-            "got: {err}"
-        );
+        assert!(matches!(err, Error::InvalidInput { .. }));
     }
 
     #[tokio::test]
@@ -117,7 +233,7 @@ mod tests {
             .execute()
             .await
             .unwrap_err();
-        assert!(err.to_string().contains("BatchUDF"), "got: {err}");
+        assert!(matches!(err, Error::InvalidInput { .. }));
 
         let schema = table.schema().await.unwrap();
         assert!(
@@ -126,6 +242,47 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_mixing_transform_and_computed_is_rejected() {
+        let table = table_with_two_columns("mixed_add").await;
+        let err = table
+            .add_columns()
+            .transform(NewColumnTransform::SqlExpressions(vec![(
+                "eager".into(),
+                "x * 2".into(),
+            )]))
+            .computed("lazy", "x * 3")
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::InvalidInput { .. }));
+
+        let schema = table.schema().await.unwrap();
+        assert!(schema.field_with_name("eager").is_err());
+        assert!(schema.field_with_name("lazy").is_err());
+    }
+
+    #[tokio::test]
+    async fn test_read_columns_with_computed_is_rejected() {
+        let table = table_with_two_columns("read_cols_computed").await;
+        let err = table
+            .add_columns()
+            .computed("doubled", "x * 2")
+            .read_columns(["x"])
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::InvalidInput { .. }));
+        assert!(
+            table
+                .schema()
+                .await
+                .unwrap()
+                .field_with_name("doubled")
+                .is_err()
+        );
+    }
+
     #[tokio::test]
     async fn test_read_columns_limits_what_a_batch_udf_sees() {
         let table = table_with_two_columns("read_cols_udf").await;
diff --git a/rust/lancedb/src/table/add_data.rs b/rust/lancedb/src/table/add_data.rs
index 11ba43dd6..15ccf9f66 100644
--- a/rust/lancedb/src/table/add_data.rs
+++ b/rust/lancedb/src/table/add_data.rs
@@ -60,6 +60,7 @@ pub struct AddDataBuilder {
     pub(crate) embedding_registry: Option<Arc<dyn EmbeddingRegistry>>,
     pub(crate) progress_callback: Option<ProgressCallback>,
     pub(crate) write_parallelism: Option<usize>,
+    pub(crate) allow_external_blob_outside_bases: bool,
 }
 
 impl std::fmt::Debug for AddDataBuilder {
@@ -87,6 +88,7 @@ impl AddDataBuilder {
             embedding_registry,
             progress_callback: None,
             write_parallelism: None,
+            allow_external_blob_outside_bases: false,
         }
     }
 
@@ -141,6 +143,16 @@ impl AddDataBuilder {
         self
     }
 
+    /// Store blob URIs that sit outside registered blob bases.
+    ///
+    /// The row keeps a reference, so the object has to stay readable.
+    /// [`crate::table::Table::fetch_blobs`] reads from that location.
+    /// Defaults to `false`. Local tables only.
+    pub fn allow_external_blob_outside_bases(mut self, allow: bool) -> Self {
+        self.allow_external_blob_outside_bases = allow;
+        self
+    }
+
     pub async fn execute(self) -> Result<AddResult> {
         if self.write_parallelism.map(|p| p == 0).unwrap_or(false) {
             return Err(Error::InvalidInput {
@@ -199,6 +211,7 @@ impl AddDataBuilder {
             write_options: self.write_options,
             mode: self.mode,
             tracker,
+            allow_external_blob_outside_bases: self.allow_external_blob_outside_bases,
         })
     }
 }
@@ -212,6 +225,7 @@ pub struct PreprocessingOutput {
     pub write_options: WriteOptions,
     pub mode: AddDataMode,
     pub tracker: Option<Arc<WriteProgressTracker>>,
+    pub allow_external_blob_outside_bases: bool,
 }
 
 /// Check that the input schema is valid for insert.
diff --git a/rust/lancedb/src/table/checkpoint.rs b/rust/lancedb/src/table/checkpoint.rs
new file mode 100644
index 000000000..bb76604ed
--- /dev/null
+++ b/rust/lancedb/src/table/checkpoint.rs
@@ -0,0 +1,315 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+//! Converging a table's LSM write path into its base table.
+//!
+//! `checkpoint_lsm` seals once, then triggers compaction and watches
+//! generation numbers until the L0 that existed at the start is gone.
+//!
+//! The loop runs in the client, not the server: `compact_lsm` dispatches a
+//! pass and returns, so nothing holds a socket and a client can vanish
+//! mid-operation with nothing to reconcile. Completion is read from
+//! generation numbers in the shard manifest — durable state, unlike a count
+//! in a compact response, which a concurrent write invalidates.
+//!
+//! The target set is fixed at the start, so generations created *during* the
+//! checkpoint are ignored. That is what lets it terminate under write load,
+//! and what makes it best-effort: it converges the fresh tier as of some
+//! instant. Idempotent, abandonable at any point, safe on a cadence.
+//!
+//! No liveness bound — the caller owns the deadline. The compactor pool is
+//! shared pod-wide, so a checkpoint queued behind unrelated tables looks
+//! exactly like one that is merging.
+
+use std::collections::HashMap;
+use std::future::Future;
+use std::time::Duration;
+
+use crate::{Error, Result, Table};
+
+/// The HTTP status a failed request carried, if it carried one.
+///
+/// `None` for anything with no retry story: a `TableNotFound` that
+/// `check_table_response` already translated, or a connection failure that
+/// never reached the server. Both are terminal.
+fn status_of(e: &Error) -> Option<u16> {
+    #[cfg(feature = "remote")]
+    {
+        match e {
+            Error::Http {
+                status_code: Some(status),
+                ..
+            } => Some(status.as_u16()),
+            _ => None,
+        }
+    }
+    #[cfg(not(feature = "remote"))]
+    {
+        let _ = e;
+        None
+    }
+}
+
+/// 429 (latch held, pool saturated, or the pod replaying its WAL) and 503 (a
+/// draining node, or a proxy between here and it).
+///
+/// The status is the whole signal: the server deliberately keeps contention
+/// off 503, so a latch collision is a 429. A draining node *is* terminal, but
+/// it is also a 503 that stays a 503, so retrying spends one budget and then
+/// reports the server's own message — cheaper than parsing the body for the
+/// namespace code it would take to tell the two apart.
+fn is_retryable(e: &Error) -> bool {
+    matches!(status_of(e), Some(429 | 503))
+}
+
+/// 421: the owning node holds no claim. Only `flush` re-claims and replays,
+/// so this cannot be retried in place — the caller has to start over.
+fn is_lost_claim(e: &Error) -> bool {
+    status_of(e) == Some(421)
+}
+
+/// Interval between `get_lsm_stats` polls. One interval is roughly one
+/// compaction pass, the granularity at which the answer can change.
+///
+/// Fixed rather than configurable, matching `wait_for_index`. It costs
+/// nothing on an already-converged table and at most one interval of tail
+/// latency after the final pass lands.
+const POLL_INTERVAL: Duration = Duration::from_secs(5);
+
+/// Cap on re-issues from `flush` after a 421, so a crash-looping node cannot
+/// turn flush → compact → 421 → flush into a spin.
+///
+/// Deliberately not shared with [`MAX_RETRIES`]: a claim that keeps
+/// evaporating is a broken node, while contention is routine and wants a real
+/// budget. One shared counter let a merely contended table exhaust this cap
+/// and then blame a claim it never lost.
+const MAX_REISSUES: usize = 3;
+
+/// Retryable faults tolerated on a *single* request, reset on every success —
+/// scattered contention across a long checkpoint must not accumulate toward a
+/// cap. Roughly 16s of retrying against the backoff below.
+const MAX_RETRIES: usize = 8;
+
+/// Backoff between retries, doubling up to [`RETRY_BACKOFF_MAX`]. Latch
+/// contention clears in about the time one pass takes, so start small; a
+/// saturated pool wants the ceiling.
+const RETRY_BACKOFF_BASE: Duration = Duration::from_millis(100);
+const RETRY_BACKOFF_MAX: Duration = Duration::from_secs(5);
+
+/// Sleep before re-issuing a retryable request.
+async fn backoff(attempt: usize) {
+    let delay = RETRY_BACKOFF_BASE
+        .saturating_mul(1u32 << attempt.min(8) as u32)
+        .min(RETRY_BACKOFF_MAX);
+    tokio::time::sleep(delay).await;
+}
+
+/// Whether the drain loop finished or needs the table re-claimed first.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum CheckpointOutcome {
+    Done,
+    ReissueFromFlush,
+}
+
+/// What one LSM request produced: its value, or word that the owning node
+/// holds no claim and only `flush` can get it back.
+enum Attempt<T> {
+    Ok(T),
+    ReissueFromFlush,
+}
+
+/// Issue one LSM request, retrying in place while the fault is retryable.
+///
+/// The two recoverable faults have separate budgets: contention clears on its
+/// own and retries here against [`MAX_RETRIES`], while a 421 needs `flush` to
+/// re-claim, which only the caller can drive.
+///
+/// An exhausted budget propagates the last error *as itself* rather than a
+/// synthesized one — "429 after nine tries" beats "checkpoint failed", and a
+/// draining node arrives carrying the server's own message.
+async fn issue<T, F, Fut>(mut call: F) -> Result<Attempt<T>>
+where
+    F: FnMut() -> Fut,
+    Fut: Future<Output = Result<T>>,
+{
+    let mut retries = 0;
+    loop {
+        let e = match call().await {
+            Ok(value) => return Ok(Attempt::Ok(value)),
+            Err(e) => e,
+        };
+        if is_lost_claim(&e) {
+            return Ok(Attempt::ReissueFromFlush);
+        }
+        if !is_retryable(&e) || retries >= MAX_RETRIES {
+            return Err(e);
+        }
+        backoff(retries).await;
+        retries += 1;
+    }
+}
+
+/// Drive [`Table::checkpoint_lsm`]: seal once, fix the target watermark
+/// from the resulting L0, then trigger and poll until it drains.
+pub(crate) async fn checkpoint_lsm(table: &Table) -> Result<()> {
+    for reissue in 0..=MAX_REISSUES {
+        // The seal turns everything written before this call into a
+        // generation, so the watermark has to be read after it. Idempotent:
+        // sealing an empty memtable is a no-op, so a re-issue does not churn
+        // empty generations.
+        match issue(|| table.flush_lsm()).await? {
+            Attempt::Ok(()) => {}
+            Attempt::ReissueFromFlush => {
+                backoff(reissue).await;
+                continue;
+            }
+        }
+
+        let stats = match issue(|| table.get_lsm_stats(false)).await? {
+            Attempt::Ok(stats) => stats,
+            Attempt::ReissueFromFlush => {
+                backoff(reissue).await;
+                continue;
+            }
+        };
+        let Some(stats) = stats else {
+            // Not WAL-backed; `flush_lsm` would have errored first but for a race.
+            return Ok(());
+        };
+        let targets: HashMap<String, u64> = stats
+            .buckets
+            .iter()
+            .filter_map(|b| Some((b.shard_id.clone(), b.newest_generation()?)))
+            .collect();
+        if targets.is_empty() {
+            return Ok(());
+        }
+
+        match drain_to_targets(table, &targets).await? {
+            CheckpointOutcome::Done => return Ok(()),
+            CheckpointOutcome::ReissueFromFlush => {
+                backoff(reissue).await;
+                continue;
+            }
+        }
+    }
+    Err(Error::Runtime {
+        message: "checkpoint_lsm: the owning node kept losing its claim; \
+                  re-issued from flush the maximum number of times"
+            .into(),
+    })
+}
+
+/// Trigger and poll until no bucket holds a generation at or below its
+/// target.
+///
+/// No liveness bound, deliberately. The pod-wide compactor pool (a semaphore
+/// of 2 by default, shared across every table on the node) is taken *inside*
+/// the pass, after the bucket latch, so a checkpoint queued behind unrelated
+/// tables is indistinguishable from one that is merging. An idle-poll counter
+/// here could only ever have fired on a table that would have finished.
+async fn drain_to_targets(
+    table: &Table,
+    targets: &HashMap<String, u64>,
+) -> Result<CheckpointOutcome> {
+    loop {
+        let stats = match issue(|| table.get_lsm_stats(false)).await? {
+            Attempt::Ok(stats) => stats,
+            Attempt::ReissueFromFlush => return Ok(CheckpointOutcome::ReissueFromFlush),
+        };
+        let Some(stats) = stats else {
+            return Ok(CheckpointOutcome::Done);
+        };
+        // `compacting` is the bucket's compaction latch, held from dispatch
+        // until the pass ends — including while it waits on the pod-wide
+        // permit. So it answers one question only: do not pile on. Buckets
+        // with nothing outstanding are skipped, not counted as idle.
+        let mut outstanding = 0;
+        let mut all_compacting = true;
+        for b in &stats.buckets {
+            let Some(target) = targets.get(&b.shard_id) else {
+                continue;
+            };
+            let n = b.outstanding_generations(*target);
+            if n > 0 {
+                outstanding += n;
+                all_compacting &= b.compacting;
+            }
+        }
+        if outstanding == 0 {
+            return Ok(CheckpointOutcome::Done);
+        }
+
+        if !all_compacting {
+            match table.compact_lsm().await {
+                Ok(()) => {}
+                Err(e) if is_lost_claim(&e) => return Ok(CheckpointOutcome::ReissueFromFlush),
+                Err(e) if !is_retryable(&e) => return Err(e),
+                // A 429 here means the server could latch no bucket at all,
+                // which the poll above already handles. Not retried in place:
+                // the latch it would contend for is the one doing the work, so
+                // fall through and re-read — `POLL_INTERVAL` is the backoff.
+                Err(_) => {}
+            }
+        }
+        tokio::time::sleep(POLL_INTERVAL).await;
+    }
+}
+
+#[cfg(all(test, feature = "remote"))]
+mod tests {
+    use super::*;
+
+    fn http(status: u16) -> Error {
+        Error::Http {
+            source: "server said no".into(),
+            request_id: "rid".into(),
+            status_code: reqwest::StatusCode::from_u16(status).ok(),
+        }
+    }
+
+    /// Every status the loop acts on. The two predicates are checked together
+    /// because their overlap is what would be wrong: a status must never be
+    /// both, and 421 in particular must not read as retryable — retrying it in
+    /// place re-issues the call that just said the node holds no claim.
+    #[test]
+    fn taxonomy_round_trips() {
+        for status in [429, 503] {
+            assert!(is_retryable(&http(status)), "{status} must retry");
+            assert!(
+                !is_lost_claim(&http(status)),
+                "{status} is not a lost claim"
+            );
+        }
+        assert!(is_lost_claim(&http(421)), "a lost claim must re-claim");
+        assert!(
+            !is_retryable(&http(421)),
+            "retrying a lost claim in place only asks the same node again"
+        );
+        for status in [400, 404, 409, 500] {
+            assert!(!is_retryable(&http(status)), "{status} is terminal");
+            assert!(!is_lost_claim(&http(status)), "{status} is terminal");
+        }
+    }
+
+    /// An error carrying no status has no retry story and must be terminal —
+    /// a connection that never reached the server, or a `TableNotFound` that
+    /// `check_table_response` translated before the loop saw it.
+    #[test]
+    fn errors_without_a_status_are_terminal() {
+        let no_status = Error::Http {
+            source: "connection reset".into(),
+            request_id: "rid".into(),
+            status_code: None,
+        };
+        assert!(!is_retryable(&no_status));
+        assert!(!is_lost_claim(&no_status));
+
+        let translated = Error::TableNotFound {
+            name: "t".into(),
+            source: "gone".into(),
+        };
+        assert!(!is_retryable(&translated));
+        assert!(!is_lost_claim(&translated));
+    }
+}
diff --git a/rust/lancedb/src/table/branch_merge.rs b/rust/lancedb/src/table/cherry_pick.rs
similarity index 86%
rename from rust/lancedb/src/table/branch_merge.rs
rename to rust/lancedb/src/table/cherry_pick.rs
index ad81ab64c..93bc76637 100644
--- a/rust/lancedb/src/table/branch_merge.rs
+++ b/rust/lancedb/src/table/cherry_pick.rs
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-//! Types for remote branch diff / merge against main.
+//! Types for remote branch diff / cherry-pick onto main.
 
 use serde::{Deserialize, Serialize};
 
@@ -44,13 +44,13 @@ pub struct RowCountSummary {
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
 #[serde(rename_all = "camelCase")]
-pub enum MergeBlockerCode {
+pub enum CherryPickErrorCode {
     BaseMoved,
     RowCountMismatch,
     RowsChanged,
     ColumnRemoved,
     ColumnChanged,
-    NoMergeableChanges,
+    NothingToApply,
     NoColumnChanges,
     InputColumnDependency,
     ParentNotMain,
@@ -60,8 +60,8 @@ pub enum MergeBlockerCode {
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
 #[serde(rename_all = "camelCase")]
-pub struct MergeBlocker {
-    pub code: MergeBlockerCode,
+pub struct CherryPickError {
+    pub code: CherryPickErrorCode,
     pub message: String,
 }
 
@@ -81,34 +81,33 @@ pub struct BranchDiff {
     pub changed_columns: Vec<ColumnChange>,
     pub added_indexes: Vec<IndexSummary>,
     pub removed_indexes: Vec<IndexSummary>,
-    pub mergeable: bool,
-    pub merge_blockers: Vec<MergeBlocker>,
+    pub errors: Vec<CherryPickError>,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
 #[serde(rename_all = "camelCase")]
-pub struct MergePreview {
+pub struct CherryPickPreview {
     #[serde(default)]
     pub promoted_columns: Vec<String>,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
 #[serde(rename_all = "camelCase")]
-pub enum MergeBranchStatus {
+pub enum CherryPickStatus {
     Ready,
-    Rejected,
+    Failed,
     NotImplemented,
-    Merged,
+    CherryPicked,
     #[serde(other)]
     Unknown,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
 #[serde(rename_all = "camelCase")]
-pub struct MergeBranchResult {
-    pub status: MergeBranchStatus,
+pub struct CherryPickResult {
+    pub status: CherryPickStatus,
     pub diff: BranchDiff,
-    pub preview: MergePreview,
+    pub preview: CherryPickPreview,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub main_version_after: Option<u64>,
 }
diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs
new file mode 100644
index 000000000..2b95cb34c
--- /dev/null
+++ b/rust/lancedb/src/table/computed_columns.rs
@@ -0,0 +1,2507 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+//! Computed columns.
+//!
+//! A computed column is defined by a rule rather than by values supplied at
+//! write time. Declaring one commits the column carrying that rule in field
+//! metadata but no data, so the cost does not scale with the table; a later
+//! refresh fills the rows.
+//!
+//! The rule is tagged by kind ([`ComputedColumnKind`]) because kinds differ in
+//! where the column's type and inputs come from. A SQL expression is
+//! self-describing -- both are derived from the expression, so a caller writes
+//! neither -- while a kind resolved through a registry cannot be typed without
+//! consulting it. Registered Functions use an exact remote version plus a
+//! schema-level Function binding; unknown newer kinds remain readable and fail
+//! closed before mutation.
+//!
+//! [`computed_columns`] and [`computed_column_from_field`] read declarations
+//! back off a schema.
+
+use std::collections::{BTreeSet, HashMap};
+use std::sync::Arc;
+
+use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema, SchemaRef};
+use datafusion_common::tree_node::TreeNode;
+use datafusion_physical_plan::PhysicalExpr;
+use lance::dataset::NewColumnTransform;
+use lance_datafusion::planner::Planner;
+use lance_namespace::models::{JsonArrowDataType, JsonArrowField, JsonArrowSchema};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
+use crate::function::{FunctionApplication, FunctionBinding};
+use crate::{Error, Result};
+
+/// Field metadata key marking a column as computed. The value is `"true"`.
+pub const COMPUTED_COLUMN_META_KEY: &str = "computed_column";
+
+/// Field metadata key naming the kind of rule that defines the column.
+pub const KIND_META_KEY: &str = "computed_column.kind";
+
+/// Field metadata key holding the SQL expression that defines the column.
+pub const EXPRESSION_META_KEY: &str = "computed_column.expression";
+
+/// Field metadata key holding the column's inputs, as a JSON array of names.
+pub const INPUTS_META_KEY: &str = "computed_column.inputs";
+
+/// Field metadata key holding the Function binding identity.
+pub const FUNCTION_BINDING_ID_META_KEY: &str = "computed_column.function.binding_id";
+
+/// Field metadata key holding this sibling's ordered Function output ordinal.
+pub const FUNCTION_OUTPUT_ORDINAL_META_KEY: &str = "computed_column.function.output_ordinal";
+
+/// Schema metadata key holding all immutable Function bindings.
+pub const FUNCTION_BINDINGS_META_KEY: &str = "lancedb::function_bindings";
+
+/// Version of the schema-level Function binding envelope.
+pub const FUNCTION_BINDINGS_VERSION: u32 = 1;
+
+/// Value of [`KIND_META_KEY`] for a column defined by a SQL expression.
+pub const SQL_KIND: &str = "sql";
+
+/// Value of [`KIND_META_KEY`] for a registered Function binding.
+pub const FUNCTION_KIND: &str = "function";
+
+/// Synthetic result identity used when the entire Function result maps to one
+/// table column (scalar or struct-as-one-column).
+pub const WHOLE_RESULT_FIELD: &str = "$value";
+
+/// The rule that defines a computed column's values.
+///
+/// Non-exhaustive: a kind added later is an additive change, and a caller that
+/// only handles the kinds it knows keeps compiling.
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum ComputedColumnKind {
+    /// A SQL expression evaluated by DataFusion. It is the whole definition:
+    /// the column's type and its inputs are both derived from it.
+    Sql {
+        /// The expression.
+        expression: String,
+    },
+    /// One physical output in an immutable registered-Function
+    /// binding. The full binding lives in schema metadata.
+    Function {
+        /// Shared immutable binding identity.
+        binding_id: String,
+        /// Position of this field in the binding's ordered sibling outputs.
+        output_ordinal: u32,
+    },
+    /// A kind this version does not understand, written by a newer one.
+    ///
+    /// Reported rather than hidden so a caller can tell a column it cannot
+    /// refresh apart from one that was never computed. Nothing produces this.
+    Unrecognized {
+        /// The kind as it was found in the metadata.
+        kind: String,
+    },
+}
+
+/// A computed column's declaration, as read back from field metadata.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ComputedColumn {
+    /// Name of the computed column.
+    pub name: String,
+    /// The rule that defines it.
+    pub kind: ComputedColumnKind,
+    /// Columns the rule reads, recorded at declaration time.
+    ///
+    /// Outside the kind because every kind has inputs and the consumers that
+    /// use them -- refresh planning, dependency ordering -- do not care which
+    /// kind produced them. Where they come from does differ, and that is
+    /// settled at declaration: derived from a SQL expression, supplied by the
+    /// caller for a kind that cannot be parsed.
+    pub inputs: Vec<String>,
+}
+
+/// Build the field metadata recording a SQL binding.
+fn computed_column_metadata(expression: &str, inputs: &[String]) -> HashMap<String, String> {
+    HashMap::from([
+        (COMPUTED_COLUMN_META_KEY.to_string(), "true".to_string()),
+        (KIND_META_KEY.to_string(), SQL_KIND.to_string()),
+        (EXPRESSION_META_KEY.to_string(), expression.to_string()),
+        (
+            INPUTS_META_KEY.to_string(),
+            serde_json::to_string(inputs).unwrap_or_else(|_| "[]".to_string()),
+        ),
+    ])
+}
+
+/// Build field metadata for one physical sibling of a Function binding.
+pub fn function_computed_column_metadata(
+    binding_id: &str,
+    output_ordinal: u32,
+    inputs: &[String],
+) -> HashMap<String, String> {
+    HashMap::from([
+        (COMPUTED_COLUMN_META_KEY.to_string(), "true".to_string()),
+        (KIND_META_KEY.to_string(), FUNCTION_KIND.to_string()),
+        (
+            FUNCTION_BINDING_ID_META_KEY.to_string(),
+            binding_id.to_string(),
+        ),
+        (
+            FUNCTION_OUTPUT_ORDINAL_META_KEY.to_string(),
+            output_ordinal.to_string(),
+        ),
+        (
+            INPUTS_META_KEY.to_string(),
+            serde_json::to_string(inputs).unwrap_or_else(|_| "[]".to_string()),
+        ),
+    ])
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct FunctionBindingEnvelope {
+    version: u32,
+    bindings: Vec<Value>,
+}
+
+/// Encode immutable Function bindings for schema-level persistence.
+pub fn function_bindings_metadata(bindings: &[FunctionBinding]) -> Result<String> {
+    let bindings = bindings
+        .iter()
+        .map(serde_json::to_value)
+        .collect::<std::result::Result<Vec<_>, _>>()
+        .map_err(|e| Error::InvalidInput {
+            message: format!("invalid Function binding metadata: {e}"),
+        })?;
+    serde_json::to_string(&FunctionBindingEnvelope {
+        version: FUNCTION_BINDINGS_VERSION,
+        bindings,
+    })
+    .map_err(|e| Error::InvalidInput {
+        message: format!("invalid Function binding metadata: {e}"),
+    })
+}
+
+/// Decode known Function bindings without rewriting their raw schema
+/// metadata. Unknown envelope versions fail closed.
+pub fn function_bindings(schema: &ArrowSchema) -> Result<Vec<FunctionBinding>> {
+    let Some(envelope) = function_binding_envelope(schema)? else {
+        return Ok(Vec::new());
+    };
+    envelope
+        .bindings
+        .into_iter()
+        .map(|binding| {
+            serde_json::from_value(binding).map_err(|e| Error::InvalidInput {
+                message: format!("invalid Function binding metadata: {e}"),
+            })
+        })
+        .collect()
+}
+
+fn function_binding_envelope(schema: &ArrowSchema) -> Result<Option<FunctionBindingEnvelope>> {
+    let Some(raw) = schema.metadata().get(FUNCTION_BINDINGS_META_KEY) else {
+        return Ok(None);
+    };
+    let envelope: FunctionBindingEnvelope =
+        serde_json::from_str(raw).map_err(|e| Error::InvalidInput {
+            message: format!("invalid Function binding metadata: {e}"),
+        })?;
+    if envelope.version != FUNCTION_BINDINGS_VERSION {
+        return Err(Error::NotSupported {
+            message: format!(
+                "Function binding metadata version {} is not supported by this client",
+                envelope.version
+            ),
+        });
+    }
+    Ok(Some(envelope))
+}
+
+/// Validate metadata before a schema mutation. Read-only access remains
+/// possible for older datasets, while incomplete or newer contracts cannot be
+/// silently rewritten by this client.
+pub(crate) fn ensure_supported_function_metadata(schema: &ArrowSchema) -> Result<()> {
+    let raw_bindings = function_binding_envelope(schema)?
+        .map(|envelope| envelope.bindings)
+        .unwrap_or_default();
+    for value in &raw_bindings {
+        ensure_known_binding_shape(value)?;
+    }
+    let bindings = raw_bindings
+        .into_iter()
+        .map(|binding| {
+            serde_json::from_value(binding).map_err(|e| Error::InvalidInput {
+                message: format!("invalid Function binding metadata: {e}"),
+            })
+        })
+        .collect::<Result<Vec<FunctionBinding>>>()?;
+    let mut binding_ids = BTreeSet::new();
+    for binding in &bindings {
+        if !binding_ids.insert(binding.binding_id().to_string()) {
+            return Err(Error::InvalidInput {
+                message: format!("duplicate Function binding '{}'", binding.binding_id()),
+            });
+        }
+        if binding.outputs().is_empty() {
+            return Err(Error::InvalidInput {
+                message: format!("Function binding '{}' has no outputs", binding.binding_id()),
+            });
+        }
+        if binding.function().name.is_empty() || binding.function().version.is_empty() {
+            return Err(Error::InvalidInput {
+                message: format!(
+                    "Function binding '{}' has no exact version",
+                    binding.binding_id()
+                ),
+            });
+        }
+        if binding.input_schema().is_none() || binding.output_schema().is_none() {
+            return Err(Error::NotSupported {
+                message: format!(
+                    "Function binding '{}' does not contain exact Arrow schemas",
+                    binding.binding_id()
+                ),
+            });
+        }
+        for (ordinal, output) in binding.outputs().iter().enumerate() {
+            if output.output_ordinal != ordinal as u32 {
+                return Err(Error::InvalidInput {
+                    message: format!(
+                        "Function binding '{}' has non-canonical output ordinals",
+                        binding.binding_id()
+                    ),
+                });
+            }
+        }
+        ensure_binding_matches_schema(schema, binding)?;
+    }
+
+    let bindings_by_id = bindings
+        .iter()
+        .map(|binding| (binding.binding_id(), binding))
+        .collect::<HashMap<_, _>>();
+    for field in schema.fields() {
+        if field
+            .metadata()
+            .get(COMPUTED_COLUMN_META_KEY)
+            .map(String::as_str)
+            != Some("true")
+        {
+            continue;
+        }
+        match computed_column_from_field(field) {
+            Some(ComputedColumn {
+                kind:
+                    ComputedColumnKind::Function {
+                        binding_id,
+                        output_ordinal,
+                    },
+                ..
+            }) => {
+                let binding =
+                    bindings_by_id
+                        .get(binding_id.as_str())
+                        .ok_or_else(|| Error::InvalidInput {
+                            message: format!(
+                                "Function output '{}' references missing binding '{}'",
+                                field.name(),
+                                binding_id
+                            ),
+                        })?;
+                let output = binding
+                    .outputs()
+                    .get(output_ordinal as usize)
+                    .ok_or_else(|| Error::InvalidInput {
+                        message: format!(
+                            "Function output '{}' has invalid ordinal {}",
+                            field.name(),
+                            output_ordinal
+                        ),
+                    })?;
+                if output.output_name != field.name().as_str() {
+                    return Err(Error::InvalidInput {
+                        message: format!(
+                            "Function output '{}' does not match binding destination '{}'",
+                            field.name(),
+                            output.output_name
+                        ),
+                    });
+                }
+            }
+            Some(ComputedColumn {
+                kind: ComputedColumnKind::Sql { .. },
+                ..
+            }) => {}
+            Some(ComputedColumn {
+                kind: ComputedColumnKind::Unrecognized { kind },
+                ..
+            }) => {
+                return Err(Error::NotSupported {
+                    message: format!(
+                        "computed column '{}' uses unsupported kind '{}'",
+                        field.name(),
+                        kind
+                    ),
+                });
+            }
+            None => {
+                return Err(Error::InvalidInput {
+                    message: format!(
+                        "computed column '{}' has incomplete declaration metadata",
+                        field.name()
+                    ),
+                });
+            }
+        }
+    }
+    Ok(())
+}
+
+pub(crate) fn ensure_no_function_bindings_for_mutation(
+    schema: &ArrowSchema,
+    operation: &str,
+) -> Result<()> {
+    ensure_supported_function_metadata(schema)?;
+    if !function_bindings(schema)?.is_empty() {
+        return Err(Error::NotSupported {
+            message: format!(
+                "{operation} is not supported on a table with registered Function bindings"
+            ),
+        });
+    }
+    Ok(())
+}
+
+/// Read a field's computed-column declaration, if it carries one.
+///
+/// A field flagged computed but carrying no kind, or a SQL one missing its
+/// expression, is not a computed column here: without the rule there is
+/// nothing to refresh from, so it is reported as absent rather than as a
+/// half-formed declaration. An unrecognized kind is different -- the rule is
+/// there and intact, this version just cannot act on it -- and comes back as
+/// [`ComputedColumnKind::Unrecognized`].
+pub fn computed_column_from_field(field: &ArrowField) -> Option<ComputedColumn> {
+    let metadata = field.metadata();
+    if metadata.get(COMPUTED_COLUMN_META_KEY).map(String::as_str) != Some("true") {
+        return None;
+    }
+    let kind = match metadata.get(KIND_META_KEY)?.as_str() {
+        SQL_KIND => ComputedColumnKind::Sql {
+            expression: metadata.get(EXPRESSION_META_KEY)?.clone(),
+        },
+        FUNCTION_KIND => match (
+            metadata.get(FUNCTION_BINDING_ID_META_KEY),
+            metadata
+                .get(FUNCTION_OUTPUT_ORDINAL_META_KEY)
+                .and_then(|value| value.parse::<u32>().ok()),
+        ) {
+            (Some(binding_id), Some(output_ordinal)) if !binding_id.is_empty() => {
+                ComputedColumnKind::Function {
+                    binding_id: binding_id.clone(),
+                    output_ordinal,
+                }
+            }
+            _ => ComputedColumnKind::Unrecognized {
+                kind: FUNCTION_KIND.to_string(),
+            },
+        },
+        other => ComputedColumnKind::Unrecognized {
+            kind: other.to_string(),
+        },
+    };
+    let inputs = metadata
+        .get(INPUTS_META_KEY)
+        .and_then(|raw| serde_json::from_str::<Vec<String>>(raw).ok())
+        .unwrap_or_default();
+    Some(ComputedColumn {
+        name: field.name().clone(),
+        kind,
+        inputs,
+    })
+}
+
+/// Read every computed-column declaration carried by `schema`, in field order.
+///
+/// Introspection is a pure read of the schema the caller already holds, the
+/// way a SQL catalog reports a generation expression as another column of
+/// `information_schema.columns`.
+pub fn computed_columns(schema: &ArrowSchema) -> Vec<ComputedColumn> {
+    schema
+        .fields()
+        .iter()
+        .filter_map(|field| computed_column_from_field(field))
+        .collect()
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub(crate) struct FunctionOutputTarget {
+    pub result_field: String,
+    pub output_name: String,
+    pub output_ordinal: u32,
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub(crate) struct FunctionInputTarget {
+    pub parameter: String,
+    pub field_path: String,
+    pub arrow_type: String,
+    pub nullable: bool,
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub(crate) struct FunctionDeclarationPlan {
+    pub application: FunctionApplication,
+    pub binding_metadata_version: u32,
+    pub input_bindings: Vec<FunctionInputTarget>,
+    pub input_schema: JsonArrowSchema,
+    pub output_schema: JsonArrowSchema,
+    pub outputs: Vec<FunctionOutputTarget>,
+}
+
+fn invalid_function(message: impl Into<String>) -> Error {
+    Error::InvalidInput {
+        message: message.into(),
+    }
+}
+
+fn reject_unknown_object_fields(value: &Value, allowed: &[&str], context: &str) -> Result<()> {
+    let object = value.as_object().ok_or_else(|| {
+        invalid_function(format!(
+            "invalid Function binding metadata: {context} must be an object"
+        ))
+    })?;
+    let unknown = object
+        .keys()
+        .filter(|key| !allowed.contains(&key.as_str()))
+        .cloned()
+        .collect::<Vec<_>>();
+    if unknown.is_empty() {
+        Ok(())
+    } else {
+        Err(Error::NotSupported {
+            message: format!(
+                "Function binding metadata contains newer {context} fields: {unknown:?}"
+            ),
+        })
+    }
+}
+
+fn ensure_known_binding_shape(value: &Value) -> Result<()> {
+    reject_unknown_object_fields(
+        value,
+        &[
+            "binding_id",
+            "function",
+            "inputs",
+            "outputs",
+            "input_schema",
+            "output_schema",
+        ],
+        "binding",
+    )?;
+    let object = value.as_object().unwrap();
+    reject_unknown_object_fields(
+        object
+            .get("function")
+            .ok_or_else(|| invalid_function("Function binding is missing its exact version"))?,
+        &["name", "version"],
+        "version reference",
+    )?;
+    for input in object
+        .get("inputs")
+        .and_then(Value::as_array)
+        .ok_or_else(|| invalid_function("Function binding inputs must be an array"))?
+    {
+        reject_unknown_object_fields(
+            input,
+            &[
+                "parameter",
+                "field_id",
+                "field_path",
+                "arrow_type",
+                "nullable",
+            ],
+            "input binding",
+        )?;
+    }
+    for output in object
+        .get("outputs")
+        .and_then(Value::as_array)
+        .ok_or_else(|| invalid_function("Function binding outputs must be an array"))?
+    {
+        reject_unknown_object_fields(
+            output,
+            &[
+                "result_field",
+                "output_name",
+                "output_field_id",
+                "output_ordinal",
+                "arrow_type",
+                "nullable",
+            ],
+            "output mapping",
+        )?;
+    }
+    Ok(())
+}
+
+fn resolve_field_path<'a>(schema: &'a ArrowSchema, path: &str) -> Result<&'a ArrowField> {
+    let parts = lance_core::datatypes::parse_field_path(path).map_err(|e| {
+        invalid_function(format!("invalid Function input field path '{path}': {e}"))
+    })?;
+    let Some((root, children)) = parts.split_first() else {
+        return Err(invalid_function(
+            "Function input field path cannot be empty",
+        ));
+    };
+    let mut field = schema
+        .field_with_name(root)
+        .map_err(|_| invalid_function(format!("unknown Function input column '{path}'")))?;
+    for child in children {
+        let DataType::Struct(fields) = field.data_type() else {
+            return Err(invalid_function(format!(
+                "Function input field path '{path}' traverses a non-struct field"
+            )));
+        };
+        field = fields
+            .iter()
+            .find(|field| field.name() == child)
+            .map(AsRef::as_ref)
+            .ok_or_else(|| invalid_function(format!("unknown Function input column '{path}'")))?;
+    }
+    Ok(field)
+}
+
+fn canonical_input_arrow_type(field: &JsonArrowField) -> Result<String> {
+    if field.r#type.fields.is_none() && field.r#type.length.is_none() {
+        Ok(field.r#type.r#type.clone())
+    } else {
+        serde_json::to_string(field.r#type.as_ref()).map_err(|e| {
+            invalid_function(format!("could not encode exact Function input type: {e}"))
+        })
+    }
+}
+
+/// `fixed_size_list<item, size>` -> (`item`, `size`); the comma must sit outside
+/// any nested `<...>`.
+fn split_fixed_size_list(raw: &str) -> Option<(&str, i32)> {
+    let inner = raw.strip_prefix("fixed_size_list<")?.strip_suffix('>')?;
+    let mut depth = 0_u32;
+    let mut separator = None;
+    for (index, byte) in inner.bytes().enumerate() {
+        match byte {
+            b'<' => depth += 1,
+            b'>' => depth = depth.checked_sub(1)?,
+            b',' if depth == 0 => separator = Some(index),
+            _ => {}
+        }
+    }
+    let (item, size) = inner.split_at(separator?);
+    let size: i32 = size[1..].trim().parse().ok()?;
+    (size > 0).then_some((item.trim(), size))
+}
+
+fn parse_output_arrow_type(raw: &str) -> Result<JsonArrowDataType> {
+    fn parse(raw: &str) -> Result<JsonArrowDataType> {
+        let raw = raw.trim();
+        if raw.starts_with('{') {
+            return serde_json::from_str(raw).map_err(|e| {
+                invalid_function(format!("invalid Function Arrow type '{raw}': {e}"))
+            });
+        }
+        if let Some(inner) = raw
+            .strip_prefix("list<")
+            .and_then(|value| value.strip_suffix('>'))
+        {
+            let mut data_type = JsonArrowDataType::new("list".to_string());
+            data_type.fields = Some(vec![JsonArrowField::new(
+                "item".to_string(),
+                false,
+                parse(inner)?,
+            )]);
+            return Ok(data_type);
+        }
+        if let Some(inner) = raw
+            .strip_prefix("large_list<")
+            .and_then(|value| value.strip_suffix('>'))
+        {
+            let mut data_type = JsonArrowDataType::new("large_list".to_string());
+            data_type.fields = Some(vec![JsonArrowField::new(
+                "item".to_string(),
+                false,
+                parse(inner)?,
+            )]);
+            return Ok(data_type);
+        }
+        if let Some((inner, size)) = split_fixed_size_list(raw) {
+            let mut data_type = JsonArrowDataType::new("fixed_size_list".to_string());
+            data_type.fields = Some(vec![JsonArrowField::new(
+                "item".to_string(),
+                false,
+                parse(inner)?,
+            )]);
+            data_type.length = Some(i64::from(size));
+            return Ok(data_type);
+        }
+        let normalized = match raw {
+            "boolean" => "bool",
+            "string" => "utf8",
+            "large_string" => "large_utf8",
+            "halffloat" => "float16",
+            "float" => "float32",
+            "double" => "float64",
+            other => other,
+        };
+        Ok(JsonArrowDataType::new(normalized.to_string()))
+    }
+
+    let data_type = parse(raw)?;
+    lance_namespace::schema::convert_json_arrow_type(&data_type)
+        .map_err(|e| invalid_function(format!("unsupported Function Arrow type '{raw}': {e}")))?;
+    Ok(data_type)
+}
+
+fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding) -> Result<()> {
+    let mut input_fields = Vec::with_capacity(binding.inputs().len());
+    for input in binding.inputs() {
+        let field = resolve_field_path(schema, &input.field_path)?;
+        if field
+            .metadata()
+            .get(COMPUTED_COLUMN_META_KEY)
+            .map(String::as_str)
+            == Some("true")
+        {
+            return Err(invalid_function(format!(
+                "Function input '{}' is computed",
+                input.field_path
+            )));
+        }
+        // A non-null source is within a nullable parameter's domain. The
+        // reverse can pass nulls to a Function that does not accept them.
+        if field.is_nullable() && !input.nullable {
+            return Err(invalid_function(format!(
+                "Function input column '{}' is nullable, but parameter '{}' in binding '{}' is non-nullable",
+                input.field_path,
+                input.parameter,
+                binding.binding_id()
+            )));
+        }
+        let parameter_field = ArrowField::new(
+            input.parameter.clone(),
+            field.data_type().clone(),
+            input.nullable,
+        )
+        .with_metadata(field.metadata().clone());
+        let json = lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![
+            parameter_field.clone(),
+        ]))
+        .map_err(|e| invalid_function(format!("invalid Function input schema: {e}")))?;
+        let json_field = json.fields.into_iter().next().unwrap();
+        if canonical_input_arrow_type(&json_field)? != input.arrow_type {
+            return Err(invalid_function(format!(
+                "Function input '{}' type no longer matches binding '{}'",
+                input.field_path,
+                binding.binding_id()
+            )));
+        }
+        input_fields.push(parameter_field);
+    }
+    let input_schema =
+        lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(input_fields))
+            .map_err(|e| invalid_function(format!("invalid Function input schema: {e}")))?;
+    let input_schema = serde_json::to_value(input_schema).map_err(|e| {
+        invalid_function(format!("could not encode exact Function input schema: {e}"))
+    })?;
+    if binding.input_schema() != Some(&input_schema) {
+        return Err(invalid_function(format!(
+            "Function binding '{}' input schema does not match its inputs",
+            binding.binding_id()
+        )));
+    }
+
+    let mut output_fields = Vec::with_capacity(binding.outputs().len());
+    for output in binding.outputs() {
+        let field = schema.field_with_name(&output.output_name).map_err(|_| {
+            invalid_function(format!(
+                "Function binding '{}' output '{}' is missing",
+                binding.binding_id(),
+                output.output_name
+            ))
+        })?;
+        if field.name() != &output.output_name || !field.is_nullable() || output.nullable {
+            return Err(invalid_function(format!(
+                "Function output '{}' no longer matches binding '{}'",
+                output.output_name,
+                binding.binding_id()
+            )));
+        }
+        let expected_type = parse_output_arrow_type(&output.arrow_type)?;
+        let expected_type = lance_namespace::schema::convert_json_arrow_type(&expected_type)
+            .map_err(|e| invalid_function(format!("invalid Function output type: {e}")))?;
+        if field.data_type() != &expected_type {
+            return Err(invalid_function(format!(
+                "Function output '{}' type no longer matches binding '{}'",
+                output.output_name,
+                binding.binding_id()
+            )));
+        }
+        output_fields.push(ArrowField::new(
+            field.name().clone(),
+            field.data_type().clone(),
+            true,
+        ));
+    }
+    let output_schema =
+        lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(output_fields))
+            .map_err(|e| invalid_function(format!("invalid Function output schema: {e}")))?;
+    let output_schema = serde_json::to_value(output_schema).map_err(|e| {
+        invalid_function(format!(
+            "could not encode exact Function output schema: {e}"
+        ))
+    })?;
+    if binding.output_schema() != Some(&output_schema) {
+        return Err(invalid_function(format!(
+            "Function binding '{}' output schema does not match physical siblings",
+            binding.binding_id()
+        )));
+    }
+    Ok(())
+}
+
+/// Resolve a Function application against a table schema before any request is
+/// serialized. Input paths and the complete sibling output schema are fixed in
+/// one plan.
+pub(crate) fn plan_function_application(
+    schema: &ArrowSchema,
+    application: &FunctionApplication,
+    output_name: Option<&str>,
+) -> Result<FunctionDeclarationPlan> {
+    ensure_no_function_bindings_for_mutation(schema, "Function binding declaration")?;
+    if application.has_unknown_fields() {
+        return Err(Error::NotSupported {
+            message: "Function application contains fields from a newer contract".into(),
+        });
+    }
+    if application.function().name.is_empty() || application.function().version.is_empty() {
+        return Err(invalid_function(
+            "Function application requires an exact version",
+        ));
+    }
+
+    let mut parameters = BTreeSet::new();
+    let mut input_bindings = Vec::with_capacity(application.inputs().len());
+    let mut input_fields = Vec::with_capacity(application.inputs().len());
+    for input in application.inputs() {
+        if !parameters.insert(input.parameter.as_str()) {
+            return Err(invalid_function(format!(
+                "duplicate Function parameter '{}'",
+                input.parameter
+            )));
+        }
+        if input.kind != "column" {
+            return Err(Error::NotSupported {
+                message: format!(
+                    "Function input kind '{}' is not supported for column declaration",
+                    input.kind
+                ),
+            });
+        }
+        let source = input.value.as_object().ok_or_else(|| {
+            invalid_function(format!(
+                "Function parameter '{}' has an invalid column source",
+                input.parameter
+            ))
+        })?;
+        if source.len() != 1 {
+            return Err(Error::NotSupported {
+                message: format!(
+                    "Function parameter '{}' uses a newer column source contract",
+                    input.parameter
+                ),
+            });
+        }
+        let path = source.get("path").and_then(Value::as_str).ok_or_else(|| {
+            invalid_function(format!(
+                "Function parameter '{}' requires a column path",
+                input.parameter
+            ))
+        })?;
+        let field = resolve_field_path(schema, path)?;
+        if field
+            .metadata()
+            .get(COMPUTED_COLUMN_META_KEY)
+            .map(String::as_str)
+            == Some("true")
+        {
+            return Err(invalid_function(format!(
+                "Function input '{path}' is computed; computed-on-computed bindings are not supported"
+            )));
+        }
+        let parameter_field = ArrowField::new(
+            input.parameter.clone(),
+            field.data_type().clone(),
+            field.is_nullable(),
+        )
+        .with_metadata(field.metadata().clone());
+        let input_schema = lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![
+            parameter_field.clone(),
+        ]))
+        .map_err(|e| invalid_function(format!("invalid Function input schema: {e}")))?;
+        let json_field = input_schema.fields.into_iter().next().unwrap();
+        input_bindings.push(FunctionInputTarget {
+            parameter: input.parameter.clone(),
+            field_path: path.to_string(),
+            arrow_type: canonical_input_arrow_type(&json_field)?,
+            nullable: field.is_nullable(),
+        });
+        input_fields.push(parameter_field);
+    }
+    let input_schema =
+        lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(input_fields))
+            .map_err(|e| invalid_function(format!("invalid Function input schema: {e}")))?;
+
+    let output = application.output();
+    let mut outputs = Vec::new();
+    let mut output_fields = Vec::new();
+    match output.kind.as_str() {
+        "scalar" => {
+            if !application.columns().is_empty() {
+                return Err(invalid_function(
+                    "scalar Function applications cannot rename result fields",
+                ));
+            }
+            let name = output_name.ok_or_else(|| {
+                invalid_function(
+                    "a scalar Function application must be mapped to one output column",
+                )
+            })?;
+            if output.nullable != Some(false) {
+                return Err(invalid_function(
+                    "Function logical outputs must be non-nullable during NULL assignment",
+                ));
+            }
+            let data_type =
+                parse_output_arrow_type(output.arrow_type.as_deref().ok_or_else(|| {
+                    invalid_function("scalar Function output is missing its Arrow type")
+                })?)?;
+            outputs.push(FunctionOutputTarget {
+                result_field: WHOLE_RESULT_FIELD.to_string(),
+                output_name: name.to_string(),
+                output_ordinal: 0,
+            });
+            output_fields.push(JsonArrowField::new(name.to_string(), true, data_type));
+        }
+        "named_struct" => {
+            if output.fields.is_empty() {
+                return Err(invalid_function(
+                    "named-struct Function output requires at least one field",
+                ));
+            }
+            let result_names = output
+                .fields
+                .iter()
+                .map(|field| field.name.as_str())
+                .collect::<BTreeSet<_>>();
+            if result_names.len() != output.fields.len() {
+                return Err(invalid_function(
+                    "named-struct Function result field names must be unique",
+                ));
+            }
+            if output.fields.iter().any(|field| field.nullable) {
+                return Err(invalid_function(
+                    "Function logical outputs must be non-nullable during NULL assignment",
+                ));
+            }
+            let unknown = application
+                .columns()
+                .keys()
+                .filter(|name| !result_names.contains(name.as_str()))
+                .cloned()
+                .collect::<Vec<_>>();
+            if !unknown.is_empty() {
+                return Err(invalid_function(format!(
+                    "unknown Function result fields: {unknown:?}"
+                )));
+            }
+
+            if let Some(name) = output_name {
+                if !application.columns().is_empty() {
+                    return Err(invalid_function(
+                        "a named-struct mapped to one column cannot also rename expanded fields",
+                    ));
+                }
+                let fields = output
+                    .fields
+                    .iter()
+                    .map(|field| {
+                        Ok(JsonArrowField::new(
+                            field.name.clone(),
+                            false,
+                            parse_output_arrow_type(&field.arrow_type)?,
+                        ))
+                    })
+                    .collect::<Result<Vec<_>>>()?;
+                let mut data_type = JsonArrowDataType::new("struct".to_string());
+                data_type.fields = Some(fields);
+                outputs.push(FunctionOutputTarget {
+                    result_field: WHOLE_RESULT_FIELD.to_string(),
+                    output_name: name.to_string(),
+                    output_ordinal: 0,
+                });
+                output_fields.push(JsonArrowField::new(name.to_string(), true, data_type));
+            } else {
+                let mut destinations = BTreeSet::new();
+                for (ordinal, field) in output.fields.iter().enumerate() {
+                    let name = application
+                        .columns()
+                        .get(&field.name)
+                        .unwrap_or(&field.name);
+                    if !destinations.insert(name.as_str()) {
+                        return Err(invalid_function(
+                            "Function output destinations must be unique",
+                        ));
+                    }
+                    outputs.push(FunctionOutputTarget {
+                        result_field: field.name.clone(),
+                        output_name: name.clone(),
+                        output_ordinal: ordinal as u32,
+                    });
+                    output_fields.push(JsonArrowField::new(
+                        name.clone(),
+                        true,
+                        parse_output_arrow_type(&field.arrow_type)?,
+                    ));
+                }
+            }
+        }
+        kind => {
+            return Err(Error::NotSupported {
+                message: format!(
+                    "Function output kind '{kind}' is not supported for column declaration"
+                ),
+            });
+        }
+    }
+
+    for output in &outputs {
+        if output.output_name.is_empty() {
+            return Err(invalid_function(
+                "Function output column name cannot be empty",
+            ));
+        }
+        if schema.field_with_name(&output.output_name).is_ok() {
+            return Err(Error::ColumnAlreadyExists {
+                name: output.output_name.clone(),
+            });
+        }
+    }
+
+    Ok(FunctionDeclarationPlan {
+        application: application.clone(),
+        binding_metadata_version: FUNCTION_BINDINGS_VERSION,
+        input_bindings,
+        input_schema,
+        output_schema: JsonArrowSchema::new(output_fields),
+        outputs,
+    })
+}
+
+/// Reject a schema change to a column some declaration reads.
+///
+/// A binding is SQL text naming its inputs, so renaming, retyping or dropping
+/// one leaves an expression that no longer resolves. Refusing the change keeps
+/// a declaration that survived [`plan`] evaluable for as long as it exists.
+///
+/// Paths are compared at their root: a declaration reading `metadata` is
+/// invalidated by a change to `metadata.age` just as surely.
+pub(crate) fn ensure_not_an_input(schema: &SchemaRef, paths: &[&str]) -> Result<()> {
+    for declaration in computed_columns(schema) {
+        // The expression, not stored inputs, is the source of truth; an
+        // expression that no longer parses proves nothing, so refuse.
+        let inputs = match &declaration.kind {
+            ComputedColumnKind::Sql { expression } => Planner::new(schema.clone())
+                .parse_expr(expression)
+                .map(|parsed| Planner::column_names_in_expr(&parsed))
+                .map_err(|e| Error::InvalidInput {
+                    message: format!(
+                        "computed column '{}' has an unevaluable expression ({e}); drop it \
+                         before changing the schema",
+                        declaration.name
+                    ),
+                })?,
+            _ => declaration.inputs.clone(),
+        };
+        for path in paths {
+            // Exact target only: the binding travels with the whole column,
+            // not with a nested field the expression still shapes.
+            if declaration.name == *path {
+                continue;
+            }
+            if declaration.name == root(path) {
+                return Err(Error::InvalidInput {
+                    message: format!(
+                        "'{}' is part of computed column '{}'; drop the column and declare \
+                         it again",
+                        path, declaration.name
+                    ),
+                });
+            }
+            if inputs.iter().any(|input| root(input) == root(path)) {
+                return Err(Error::InvalidInput {
+                    message: format!(
+                        "column '{}' is read by computed column '{}'; drop that column first",
+                        path, declaration.name
+                    ),
+                });
+            }
+        }
+    }
+    Ok(())
+}
+
+/// Reject a write that supplies values for a computed column directly:
+/// only refresh materializes one, and refresh never revisits a filled row.
+pub(crate) fn ensure_not_written<'a>(
+    schema: &ArrowSchema,
+    written: impl IntoIterator<Item = &'a str>,
+) -> Result<()> {
+    let declared: Vec<String> = computed_columns(schema)
+        .into_iter()
+        .map(|declaration| declaration.name)
+        .collect();
+    for name in written {
+        if declared.iter().any(|declared| declared == root(name)) {
+            return Err(Error::InvalidInput {
+                message: format!(
+                    "column '{}' is computed; its values come from refresh and cannot be \
+                     written directly",
+                    root(name)
+                ),
+            });
+        }
+    }
+    Ok(())
+}
+
+/// Reject a batch holding values for a computed column. Null slots are the
+/// declared state, so planner-padded placeholders pass.
+pub(crate) fn ensure_batch_writes_no_computed_values(
+    declared: &[String],
+    batch: &arrow_array::RecordBatch,
+) -> Result<()> {
+    for name in declared {
+        if let Some(column) = batch.column_by_name(name)
+            && column.null_count() != column.len()
+        {
+            return Err(Error::InvalidInput {
+                message: format!(
+                    "column '{name}' is computed; its values come from refresh and cannot \
+                     be written directly"
+                ),
+            });
+        }
+    }
+    Ok(())
+}
+
+/// Reject fields carrying declaration metadata that did not come through
+/// [`plan`]. One authority for creation, overwrite and raw transforms.
+pub(crate) fn ensure_no_foreign_declarations<'a>(
+    fields: impl IntoIterator<Item = &'a Arc<ArrowField>>,
+) -> Result<()> {
+    for field in fields {
+        if field.metadata().keys().any(|k| is_declaration_key(k)) {
+            return Err(Error::InvalidInput {
+                message: format!(
+                    "field '{}' carries computed-column metadata; declare computed columns \
+                     with add_columns().computed()",
+                    field.name()
+                ),
+            });
+        }
+    }
+    Ok(())
+}
+
+/// True for field-metadata keys that belong to a computed-column declaration.
+///
+/// A declaration is immutable through metadata edits: it is validated as a
+/// whole at declare time, and rewriting any piece of it -- the flag, the
+/// kind, the expression, the inputs -- would bypass that validation or move
+/// a binding out from under a refresh. Drop the column and declare it again.
+pub(crate) fn is_declaration_key(key: &str) -> bool {
+    key == COMPUTED_COLUMN_META_KEY || key.starts_with("computed_column.")
+}
+
+/// Reject retyping a computed column itself.
+///
+/// A cast keeps the stored expression while changing the type it must yield
+/// -- and lance's cast rewrites the field without its metadata, so the
+/// declaration silently stops being one. Dropping and redeclaring is the
+/// coherent way to change a computed column's type.
+pub(crate) fn ensure_not_retyped(schema: &ArrowSchema, paths: &[&str]) -> Result<()> {
+    for declaration in computed_columns(schema) {
+        for path in paths {
+            if declaration.name == root(path) {
+                return Err(Error::InvalidInput {
+                    message: format!(
+                        "column '{}' is computed; drop it and declare it again to change \
+                         its type",
+                        declaration.name
+                    ),
+                });
+            }
+        }
+    }
+    Ok(())
+}
+
+/// The top-level column a possibly nested input path reads.
+pub(crate) fn root(path: &str) -> &str {
+    path.split('.').next().unwrap_or(path)
+}
+
+/// A declaration's expression bound to a schema, ready to evaluate.
+pub(crate) struct BoundExpression {
+    /// The columns the expression names, as written; nested inputs keep
+    /// their dotted path.
+    pub inputs: Vec<String>,
+    /// The top-level columns evaluation reads, in [`Self::read_schema`]
+    /// order. A nested input appears through its root.
+    pub roots: Vec<String>,
+    /// The projected schema evaluation runs against.
+    pub read_schema: SchemaRef,
+    /// The compiled expression.
+    pub physical: Arc<dyn PhysicalExpr>,
+    /// The type the expression yields.
+    pub data_type: DataType,
+}
+
+/// Parse, resolve and compile `expression` against `schema`.
+///
+/// Inputs come from the expression as written, before optimization: the
+/// simplifier can fold a referenced column out entirely (`true OR x > 0`),
+/// and the guard protecting the stored SQL has to see every column the text
+/// names, not just the ones the simplified form still reads.
+pub(crate) fn bind(schema: SchemaRef, column: &str, expression: &str) -> Result<BoundExpression> {
+    let invalid = |message: String| Error::InvalidExpression {
+        column: column.to_string(),
+        message,
+    };
+
+    let planner = Planner::new(schema.clone());
+    let parsed = planner
+        .parse_expr(expression)
+        .map_err(|e| invalid(e.to_string()))?;
+
+    // A declaration is evaluated more than once -- staging and writing are
+    // separate passes, and a refresh years later replays the same text -- so
+    // a function that can answer differently each time has no coherent value
+    // to declare.
+    let mut volatile = None;
+    parsed
+        .apply(|expr| {
+            use datafusion_common::tree_node::TreeNodeRecursion;
+            if let datafusion_expr::Expr::ScalarFunction(function) = expr
+                && function.func.signature().volatility != datafusion_expr::Volatility::Immutable
+            {
+                volatile = Some(function.func.name().to_string());
+                return Ok(TreeNodeRecursion::Stop);
+            }
+            Ok(TreeNodeRecursion::Continue)
+        })
+        .map_err(|e| invalid(e.to_string()))?;
+    if let Some(function) = volatile {
+        return Err(invalid(format!(
+            "'{function}' is not deterministic; a computed column's expression must \
+             yield the same value every time it is evaluated"
+        )));
+    }
+
+    let mut inputs = Planner::column_names_in_expr(&parsed);
+    inputs.sort();
+    inputs.dedup();
+
+    // A nested input is recorded by its path but read through its root
+    // column; Schema::index_of resolves top-level names only. Resolved here
+    // rather than left to the planner so an unknown column names itself in
+    // the error instead of surfacing as a plan failure.
+    let mut indices = Vec::with_capacity(inputs.len());
+    for input in &inputs {
+        let index = schema
+            .index_of(root(input))
+            .map_err(|_| invalid(format!("unknown column '{input}'")))?;
+        if !indices.contains(&index) {
+            indices.push(index);
+        }
+    }
+    indices.sort_unstable();
+
+    // Physical expressions address columns by position, so the planner that
+    // compiles the expression has to be built on the projected schema
+    // evaluation will actually read.
+    let read_schema = Arc::new(
+        schema
+            .project(&indices)
+            .map_err(|e| invalid(e.to_string()))?,
+    );
+    let roots = read_schema
+        .fields()
+        .iter()
+        .map(|field| field.name().clone())
+        .collect();
+
+    let optimized = planner
+        .optimize_expr(parsed)
+        .map_err(|e| invalid(e.to_string()))?;
+    let physical = Planner::new(read_schema.clone())
+        .create_physical_expr(&optimized)
+        .map_err(|e| invalid(e.to_string()))?;
+    let data_type = physical
+        .data_type(read_schema.as_ref())
+        .map_err(|e| invalid(e.to_string()))?;
+
+    Ok(BoundExpression {
+        inputs,
+        roots,
+        read_schema,
+        physical,
+        data_type,
+    })
+}
+
+/// Resolve `(name, expression)` pairs against `schema` into fields carrying
+/// their bindings.
+///
+/// Everything that can be known statically is checked here rather than at
+/// refresh time: that the expression parses, that every column it reads
+/// exists, and that the target name is free. A declaration that survives this
+/// is one a refresh can always act on.
+pub(crate) fn plan(schema: SchemaRef, columns: &[(String, String)]) -> Result<Vec<ArrowField>> {
+    if columns.is_empty() {
+        return Err(Error::InvalidInput {
+            message: "at least one computed column is required".into(),
+        });
+    }
+
+    let mut fields = Vec::with_capacity(columns.len());
+    let mut declared: Vec<&str> = Vec::with_capacity(columns.len());
+
+    for (name, expression) in columns {
+        if schema.field_with_name(name).is_ok() || declared.contains(&name.as_str()) {
+            return Err(Error::ColumnAlreadyExists { name: name.clone() });
+        }
+
+        let bound = bind(schema.clone(), name, expression)?;
+
+        // Declared columns start entirely null, so nullability is a property
+        // of the declaration rather than of what the expression yields.
+        fields.push(
+            ArrowField::new(name, bound.data_type, true)
+                .with_metadata(computed_column_metadata(expression, &bound.inputs)),
+        );
+        declared.push(name);
+    }
+
+    Ok(fields)
+}
+
+/// Build the transform that declares `columns` against `schema`.
+///
+/// An all-null column is how a binding with no values yet is carried into a
+/// commit; that it is spelled `AllNulls` is a detail of the commit, not of the
+/// column, which is why this is internal and
+/// [`AddColumnsBuilder::computed`](super::AddColumnsBuilder::computed) is the
+/// public way in.
+pub(crate) fn declare(
+    schema: SchemaRef,
+    columns: &[(String, String)],
+) -> Result<NewColumnTransform> {
+    let fields = plan(schema, columns)?;
+    Ok(NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new(
+        fields,
+    ))))
+}
+
+/// Commit a declaration of a kind this version does not produce, the way a
+/// newer lancedb would leave one behind. Bypasses admission, which exists to
+/// stop exactly this through the public API.
+#[cfg(test)]
+pub(super) async fn add_foreign_kind(table: &crate::Table, name: &str, kind: &str) {
+    let field = ArrowField::new(name, DataType::Int32, true).with_metadata(HashMap::from([
+        (COMPUTED_COLUMN_META_KEY.to_string(), "true".to_string()),
+        (KIND_META_KEY.to_string(), kind.to_string()),
+        (INPUTS_META_KEY.to_string(), r#"["x"]"#.to_string()),
+    ]));
+    super::schema_evolution::commit_add_columns(
+        table.as_native().unwrap(),
+        NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new(vec![field]))),
+        None,
+    )
+    .await
+    .unwrap();
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn output_arrow_type_grammar_matches_the_shared_golden() {
+        let golden: serde_json::Value = serde_json::from_str(include_str!(
+            "../../tests/fixtures/first_class_functions/v1/arrow_types.json"
+        ))
+        .unwrap();
+        let valid = golden["valid"].as_array().unwrap().iter();
+        for case in valid.chain(golden["server_only"].as_array().unwrap()) {
+            let raw = case["arrow_type"].as_str().unwrap();
+            let parsed = super::parse_output_arrow_type(raw)
+                .unwrap_or_else(|error| panic!("{raw}: {error}"));
+            assert_eq!(
+                serde_json::to_value(&parsed).unwrap(),
+                case["json"],
+                "{raw}"
+            );
+        }
+        for raw in golden["invalid"].as_array().unwrap() {
+            let raw = raw.as_str().unwrap();
+            assert!(
+                super::parse_output_arrow_type(raw).is_err(),
+                "{raw:?} should be rejected"
+            );
+        }
+    }
+
+    use arrow_array::record_batch;
+    use arrow_schema::DataType;
+    use futures::TryStreamExt;
+    use lance::dataset::ColumnAlteration;
+
+    use super::*;
+    use crate::connect;
+    use crate::query::{ExecutableQuery, QueryBase, Select};
+    use crate::{Error, Table};
+
+    async fn table_with_ints(name: &str) -> Table {
+        let conn = connect("memory://").execute().await.unwrap();
+        let batch = record_batch!(("x", Int32, [1, 2, 3])).unwrap();
+        conn.create_table(name, batch).execute().await.unwrap()
+    }
+
+    /// Declare `columns` the way a caller would: plan the expressions, then
+    /// add them through the ordinary column API.
+    async fn add_computed(table: &Table, columns: &[(String, String)]) -> Result<u64> {
+        let mut builder = table.add_columns();
+        for (name, expression) in columns {
+            builder = builder.computed(name, expression);
+        }
+        Ok(builder.execute().await?.version)
+    }
+
+    async fn declared(table: &Table) -> Vec<ComputedColumn> {
+        computed_columns(table.schema().await.unwrap().as_ref())
+    }
+
+    #[tokio::test]
+    async fn test_declare_infers_type_and_inputs() {
+        let table = table_with_ints("declare_infers").await;
+        let initial = table.version().await.unwrap();
+
+        let version = add_computed(&table, &[("doubled".into(), "x * 2".into())])
+            .await
+            .unwrap();
+        assert!(version > initial);
+
+        let schema = table.schema().await.unwrap();
+        let field = schema.field_with_name("doubled").unwrap();
+        assert_eq!(field.data_type(), &DataType::Int32);
+        assert!(field.is_nullable());
+
+        assert_eq!(
+            declared(&table).await,
+            vec![ComputedColumn {
+                name: "doubled".into(),
+                kind: ComputedColumnKind::Sql {
+                    expression: "x * 2".into()
+                },
+                inputs: vec!["x".into()],
+            }]
+        );
+    }
+
+    /// The binding reaches the schema only if `AllNulls` carries per-field
+    /// metadata through the commit. The whole representation rests on it.
+    #[tokio::test]
+    async fn test_all_nulls_preserves_field_metadata() {
+        let table = table_with_ints("metadata_survives").await;
+        add_computed(&table, &[("doubled".into(), "x * 2".into())])
+            .await
+            .unwrap();
+
+        let schema = table.schema().await.unwrap();
+        let metadata = schema.field_with_name("doubled").unwrap().metadata();
+        assert_eq!(
+            metadata.get(COMPUTED_COLUMN_META_KEY).map(String::as_str),
+            Some("true")
+        );
+        assert_eq!(metadata.get(KIND_META_KEY).map(String::as_str), Some("sql"));
+        assert_eq!(
+            metadata.get(EXPRESSION_META_KEY).map(String::as_str),
+            Some("x * 2")
+        );
+        assert_eq!(
+            metadata.get(INPUTS_META_KEY).map(String::as_str),
+            Some(r#"["x"]"#)
+        );
+    }
+
+    #[tokio::test]
+    async fn test_declared_column_is_all_null() {
+        let table = table_with_ints("declare_is_null").await;
+        add_computed(&table, &[("doubled".into(), "x * 2".into())])
+            .await
+            .unwrap();
+
+        let batches = table
+            .query()
+            .select(Select::columns(&["doubled"]))
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+
+        let total: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert_eq!(total, 3);
+        for batch in &batches {
+            assert_eq!(batch["doubled"].null_count(), batch.num_rows());
+        }
+    }
+
+    #[tokio::test]
+    async fn test_unknown_column_fails_at_declare_time() {
+        let table = table_with_ints("unknown_input").await;
+        let err = add_computed(&table, &[("bad".into(), "missing + 1".into())])
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::InvalidExpression { column, .. } if column == "bad"));
+
+        let schema = table.schema().await.unwrap();
+        assert!(schema.field_with_name("bad").is_err());
+    }
+
+    #[tokio::test]
+    async fn test_unparsable_expression_fails_at_declare_time() {
+        let table = table_with_ints("bad_syntax").await;
+        let err = add_computed(&table, &[("bad".into(), "x *".into())])
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::InvalidExpression { column, .. } if column == "bad"));
+        assert!(
+            table
+                .schema()
+                .await
+                .unwrap()
+                .field_with_name("bad")
+                .is_err()
+        );
+    }
+
+    /// A user-defined function is an expression like any other; only its
+    /// resolution is missing. When a registry-aware planner exists this
+    /// becomes a supported declaration rather than a new API.
+    #[tokio::test]
+    async fn test_unregistered_function_is_rejected_for_now() {
+        let table = table_with_ints("udf_not_yet").await;
+        let err = add_computed(&table, &[("vec".into(), "embed(x)".into())])
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::InvalidExpression { column, .. } if column == "vec"));
+        assert!(
+            table
+                .schema()
+                .await
+                .unwrap()
+                .field_with_name("vec")
+                .is_err()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_existing_column_name_is_rejected() {
+        let table = table_with_ints("name_taken").await;
+        let err = add_computed(&table, &[("x".into(), "x * 2".into())])
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::ColumnAlreadyExists { name } if name == "x"));
+        assert!(declared(&table).await.is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_constant_expression_needs_no_inputs() {
+        let table = table_with_ints("constant").await;
+        add_computed(&table, &[("answer".into(), "42".into())])
+            .await
+            .unwrap();
+
+        let declared = declared(&table).await;
+        assert_eq!(declared.len(), 1);
+        assert!(declared[0].inputs.is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_multiple_columns_in_one_commit() {
+        let table = table_with_ints("multi").await;
+        let initial = table.version().await.unwrap();
+
+        add_computed(
+            &table,
+            &[
+                ("plus".into(), "x + 1".into()),
+                ("squared".into(), "x * x".into()),
+            ],
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(table.version().await.unwrap(), initial + 1);
+        let declared = declared(&table).await;
+        assert_eq!(declared.len(), 2);
+        assert_eq!(declared[0].name, "plus");
+        assert_eq!(declared[1].name, "squared");
+    }
+
+    #[tokio::test]
+    async fn test_duplicate_declaration_in_one_call_is_rejected() {
+        let table = table_with_ints("dupe").await;
+        let err = add_computed(
+            &table,
+            &[
+                ("dup".into(), "x + 1".into()),
+                ("dup".into(), "x + 2".into()),
+            ],
+        )
+        .await
+        .unwrap_err();
+        assert!(matches!(err, Error::ColumnAlreadyExists { name } if name == "dup"));
+        assert!(declared(&table).await.is_empty());
+    }
+
+    /// A column added by an ordinary transform is materialized, not bound, so
+    /// it carries no declaration to report.
+    #[tokio::test]
+    async fn test_ordinary_columns_are_not_reported_as_computed() {
+        let table = table_with_ints("plain").await;
+        assert!(declared(&table).await.is_empty());
+
+        table
+            .add_columns()
+            .transform(NewColumnTransform::SqlExpressions(vec![(
+                "eager".into(),
+                "x * 2".into(),
+            )]))
+            .execute()
+            .await
+            .unwrap();
+        assert!(declared(&table).await.is_empty());
+    }
+
+    /// Built-in functions type the column the same way an operator does.
+    #[tokio::test]
+    async fn test_builtin_function_inference() {
+        let conn = connect("memory://").execute().await.unwrap();
+        let batch = record_batch!(("name", Utf8, ["ada", "grace"]), ("n", Int32, [-1, 2])).unwrap();
+        let table = conn
+            .create_table("builtins", batch)
+            .execute()
+            .await
+            .unwrap();
+
+        add_computed(
+            &table,
+            &[
+                ("shout".into(), "upper(name)".into()),
+                ("width".into(), "length(name)".into()),
+                ("magnitude".into(), "abs(n)".into()),
+            ],
+        )
+        .await
+        .unwrap();
+
+        let schema = table.schema().await.unwrap();
+        assert_eq!(
+            schema.field_with_name("shout").unwrap().data_type(),
+            &DataType::Utf8
+        );
+        assert_eq!(
+            schema.field_with_name("magnitude").unwrap().data_type(),
+            &DataType::Int32
+        );
+        // length() returns a width-dependent integer type; assert it is one
+        // rather than pinning which.
+        assert!(
+            schema
+                .field_with_name("width")
+                .unwrap()
+                .data_type()
+                .is_integer()
+        );
+
+        let declared = declared(&table).await;
+        assert_eq!(declared.len(), 3);
+        assert_eq!(declared[0].inputs, vec!["name".to_string()]);
+        assert_eq!(declared[2].inputs, vec!["n".to_string()]);
+    }
+
+    /// The reason the kind is tagged: a declaration written by a newer version
+    /// has to read back as a computed column this one cannot evaluate, not as
+    /// an ordinary column. Reported as absent it would be refreshable by
+    /// nothing and redeclarable over, silently.
+    #[tokio::test]
+    async fn test_unrecognized_kind_is_reported_rather_than_hidden() {
+        let table = table_with_ints("foreign_kind").await;
+        super::add_foreign_kind(&table, "embedding", "udf").await;
+
+        assert_eq!(
+            declared(&table).await,
+            vec![ComputedColumn {
+                name: "embedding".into(),
+                kind: ComputedColumnKind::Unrecognized { kind: "udf".into() },
+                inputs: vec!["x".into()],
+            }]
+        );
+
+        let err = add_computed(&table, &[("embedding".into(), "x * 2".into())])
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::NotSupported { .. }));
+    }
+
+    /// A kind is what makes a declaration readable at all, so the flag alone
+    /// is half-formed in the same way a missing expression is.
+    #[test]
+    fn test_flag_without_a_kind_is_not_a_declaration() {
+        let field =
+            ArrowField::new("half", DataType::Int32, true).with_metadata(HashMap::from([(
+                COMPUTED_COLUMN_META_KEY.to_string(),
+                "true".to_string(),
+            )]));
+        assert_eq!(computed_column_from_field(&field), None);
+    }
+
+    /// A SQL declaration is its expression; without one there is nothing to
+    /// refresh from.
+    #[test]
+    fn test_sql_kind_without_an_expression_is_not_a_declaration() {
+        let field = ArrowField::new("half", DataType::Int32, true).with_metadata(HashMap::from([
+            (COMPUTED_COLUMN_META_KEY.to_string(), "true".to_string()),
+            (KIND_META_KEY.to_string(), SQL_KIND.to_string()),
+        ]));
+        assert_eq!(computed_column_from_field(&field), None);
+    }
+
+    #[tokio::test]
+    async fn test_inputs_are_deduplicated_and_sorted() {
+        let conn = connect("memory://").execute().await.unwrap();
+        let batch = record_batch!(("b", Int32, [1, 2]), ("a", Int32, [3, 4])).unwrap();
+        let table = conn.create_table("dedupe", batch).execute().await.unwrap();
+
+        add_computed(&table, &[("total".into(), "b + a + b".into())])
+            .await
+            .unwrap();
+
+        assert_eq!(
+            declared(&table).await[0].inputs,
+            vec!["a".to_string(), "b".to_string()]
+        );
+    }
+
+    #[tokio::test]
+    async fn test_dropping_an_input_is_refused() {
+        let table = table_with_ints("drop_input").await;
+        add_computed(&table, &[("doubled".into(), "x * 2".into())])
+            .await
+            .unwrap();
+
+        let err = table.drop_columns(&["x"]).await.unwrap_err();
+        assert!(
+            matches!(&err, Error::InvalidInput { message } if message.contains("doubled")),
+            "{err:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_renaming_an_input_is_refused() {
+        let table = table_with_ints("rename_input").await;
+        add_computed(&table, &[("doubled".into(), "x * 2".into())])
+            .await
+            .unwrap();
+
+        let err = table
+            .alter_columns(&[ColumnAlteration::new("x".into()).rename("y".into())])
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(&err, Error::InvalidInput { message } if message.contains("doubled")),
+            "{err:?}"
+        );
+    }
+
+    /// Nothing resolves against nullability, so it is not a rebinding.
+    #[tokio::test]
+    async fn test_altering_an_input_nullability_is_allowed() {
+        let table = table_with_ints("nullable_input").await;
+        add_computed(&table, &[("doubled".into(), "x * 2".into())])
+            .await
+            .unwrap();
+
+        table
+            .alter_columns(&[ColumnAlteration::new("x".into()).set_nullable(true)])
+            .await
+            .unwrap();
+    }
+
+    /// The gate's reproducer: a volatile function evaluates differently in
+    /// the counting and writing passes, so the declared value is incoherent.
+    /// Refused at declare time.
+    #[tokio::test]
+    async fn test_a_volatile_expression_is_refused() {
+        let table = table_with_ints("volatile_expr").await;
+        let err = add_computed(&table, &[("maybe".into(), "random() < 0.5".into())])
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(&err, Error::InvalidExpression { message, .. }
+                if message.contains("random") && message.contains("deterministic")),
+            "{err:?}"
+        );
+    }
+
+    /// The gate's reproducer: the simplifier folds `true OR x > 0` to a
+    /// constant, but the stored SQL still names `x`, so the recorded inputs
+    /// must too -- otherwise dropping `x` is allowed and refresh breaks.
+    #[tokio::test]
+    async fn test_inputs_survive_expression_optimization() {
+        let table = table_with_ints("optimized_inputs").await;
+        add_computed(&table, &[("flag".into(), "true OR x > 0".into())])
+            .await
+            .unwrap();
+
+        assert_eq!(declared(&table).await[0].inputs, vec!["x".to_string()]);
+        let err = table.drop_columns(&["x"]).await.unwrap_err();
+        assert!(
+            matches!(&err, Error::InvalidInput { message } if message.contains("flag")),
+            "{err:?}"
+        );
+    }
+
+    /// The gate's reproducer: casting a computed column rewrites the field
+    /// without its metadata, silently destroying the declaration.
+    #[tokio::test]
+    async fn test_retyping_the_computed_column_is_refused() {
+        use arrow_schema::DataType as ArrowDataType;
+
+        let table = table_with_ints("retype_computed").await;
+        add_computed(&table, &[("doubled".into(), "x * 2".into())])
+            .await
+            .unwrap();
+
+        let err = table
+            .alter_columns(&[ColumnAlteration::new("doubled".into()).cast_to(ArrowDataType::Int64)])
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(&err, Error::InvalidInput { message } if message.contains("computed")),
+            "{err:?}"
+        );
+
+        // The declaration survives the refused change.
+        table.refresh_column("doubled").await.unwrap();
+    }
+
+    /// A declaration cannot be edited, fabricated or erased through field
+    /// metadata: it is validated as a whole at declare time.
+    #[tokio::test]
+    async fn test_declaration_metadata_is_immutable() {
+        use crate::table::FieldMetadataUpdate;
+
+        let table = table_with_ints("metadata_tamper").await;
+        add_computed(&table, &[("doubled".into(), "x * 2".into())])
+            .await
+            .unwrap();
+
+        // Moving the binding.
+        let err = table
+            .update_field_metadata(&[
+                FieldMetadataUpdate::new("doubled").set(EXPRESSION_META_KEY, "x * 3")
+            ])
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::InvalidInput { .. }), "{err:?}");
+
+        // Fabricating a declaration on a plain column.
+        let err = table
+            .update_field_metadata(&[FieldMetadataUpdate::new("x")
+                .set(COMPUTED_COLUMN_META_KEY, "true")
+                .set(KIND_META_KEY, SQL_KIND)
+                .set(EXPRESSION_META_KEY, "x")])
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::InvalidInput { .. }), "{err:?}");
+
+        // Erasing the declaration wholesale.
+        let err = table
+            .update_field_metadata(&[FieldMetadataUpdate::new("doubled")
+                .set("note", "hi")
+                .replace()])
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::InvalidInput { .. }), "{err:?}");
+
+        // Ordinary metadata on a computed column still merges.
+        table
+            .update_field_metadata(&[FieldMetadataUpdate::new("doubled").set("note", "hi")])
+            .await
+            .unwrap();
+        table.refresh_column("doubled").await.unwrap();
+    }
+
+    /// The gate's reproducer: only refresh materializes a declared column;
+    /// a direct write would store an arbitrary durable value.
+    #[tokio::test]
+    async fn test_a_computed_column_cannot_be_written_directly() {
+        let table = table_with_ints("direct_write").await;
+        add_computed(&table, &[("doubled".into(), "x * 2".into())])
+            .await
+            .unwrap();
+
+        let batch = record_batch!(("x", Int32, [4]), ("doubled", Int32, [999])).unwrap();
+        let err = table.add(batch.clone()).execute().await.unwrap_err();
+        assert!(
+            matches!(&err, Error::InvalidInput { message } if message.contains("refresh")),
+            "{err:?}"
+        );
+
+        let err = table
+            .update()
+            .column("doubled", "999")
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::InvalidInput { .. }));
+
+        let mut merge = table.merge_insert(&["x"]);
+        merge
+            .when_matched_update_all(None)
+            .when_not_matched_insert_all();
+        let err = merge
+            .execute(Box::new(arrow_array::RecordBatchIterator::new(
+                vec![Ok(batch.clone())],
+                batch.schema(),
+            )))
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::InvalidInput { .. }));
+
+        // The append that omits the column still works.
+        let plain = record_batch!(("x", Int32, [4])).unwrap();
+        table.add(plain).execute().await.unwrap();
+    }
+
+    /// The gate's reproducer: the reciprocal of the declare-under-spec check.
+    #[tokio::test]
+    async fn test_installing_an_lsm_spec_over_computed_columns_is_refused() {
+        use crate::table::LsmWriteSpec;
+
+        let tmp_dir = tempfile::tempdir().unwrap();
+        let conn = connect(tmp_dir.path().to_str().unwrap())
+            .execute()
+            .await
+            .unwrap();
+        let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
+            "x",
+            DataType::Int32,
+            false,
+        )]));
+        let batch = arrow_array::RecordBatch::try_new(
+            schema,
+            vec![Arc::new(arrow_array::Int32Array::from(vec![1, 2])) as _],
+        )
+        .unwrap();
+        let table = conn
+            .create_table("lsm_after", batch)
+            .execute()
+            .await
+            .unwrap();
+        table.set_unenforced_primary_key(["x"]).await.unwrap();
+        add_computed(&table, &[("doubled".into(), "x * 2".into())])
+            .await
+            .unwrap();
+
+        let err = table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded())
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(&err, Error::NotSupported { message } if message.contains("computed")),
+            "{err:?}"
+        );
+        assert!(table.get_lsm_write_spec().await.unwrap().is_none());
+    }
+
+    /// The gate's reproducer: declaration metadata is admitted only through
+    /// the validated declare path, never smuggled through a raw transform.
+    #[tokio::test]
+    async fn test_forged_declaration_metadata_is_rejected() {
+        let table = table_with_ints("forged_metadata").await;
+        let field =
+            ArrowField::new("doubled", DataType::Int32, true).with_metadata(HashMap::from([
+                (COMPUTED_COLUMN_META_KEY.to_string(), "true".to_string()),
+                (KIND_META_KEY.to_string(), SQL_KIND.to_string()),
+                (EXPRESSION_META_KEY.to_string(), "x * 2".to_string()),
+                (INPUTS_META_KEY.to_string(), "[]".to_string()),
+            ]));
+        let err = table
+            .add_columns()
+            .transform(NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new(
+                vec![field],
+            ))))
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(&err, Error::InvalidInput { message } if message.contains("computed()")),
+            "{err:?}"
+        );
+        assert!(declared(&table).await.is_empty());
+    }
+
+    /// The gate's reproducer: SQL INSERT is a write path too.
+    #[tokio::test]
+    async fn test_sql_insert_cannot_write_a_computed_column() {
+        use datafusion::prelude::SessionContext;
+
+        let table = table_with_ints("sql_insert").await;
+        add_computed(&table, &[("doubled".into(), "x * 2".into())])
+            .await
+            .unwrap();
+
+        let ctx = SessionContext::new();
+        let provider =
+            crate::table::datafusion::BaseTableAdapter::try_new(table.base_table().clone())
+                .await
+                .unwrap();
+        ctx.register_table("t", Arc::new(provider)).unwrap();
+
+        let result = async {
+            ctx.sql("INSERT INTO t (x, doubled) VALUES (4, 999)")
+                .await?
+                .collect()
+                .await
+        }
+        .await;
+        let err = result.unwrap_err().to_string();
+        assert!(err.contains("refresh"), "{err}");
+    }
+
+    /// The gate's reproducer: an overwrite must not smuggle in a filled
+    /// declaration.
+    #[tokio::test]
+    async fn test_overwrite_cannot_inject_a_declaration() {
+        use crate::table::AddDataMode;
+
+        let table = table_with_ints("overwrite_inject").await;
+        let field =
+            ArrowField::new("doubled", DataType::Int32, true).with_metadata(HashMap::from([
+                (COMPUTED_COLUMN_META_KEY.to_string(), "true".to_string()),
+                (KIND_META_KEY.to_string(), SQL_KIND.to_string()),
+                (EXPRESSION_META_KEY.to_string(), "x * 2".to_string()),
+            ]));
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("x", DataType::Int32, true),
+            field,
+        ]));
+        let batch = arrow_array::RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(arrow_array::Int32Array::from(vec![1])) as _,
+                Arc::new(arrow_array::Int32Array::from(vec![999])) as _,
+            ],
+        )
+        .unwrap();
+
+        let err = table
+            .add(batch)
+            .mode(AddDataMode::Overwrite)
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(&err, Error::InvalidInput { message } if message.contains("declare")),
+            "{err:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_create_table_cannot_inject_a_declaration() {
+        let conn = connect("memory://").execute().await.unwrap();
+        let field =
+            ArrowField::new("doubled", DataType::Int32, true).with_metadata(HashMap::from([
+                (COMPUTED_COLUMN_META_KEY.to_string(), "true".to_string()),
+                (KIND_META_KEY.to_string(), SQL_KIND.to_string()),
+                (EXPRESSION_META_KEY.to_string(), "x * 2".to_string()),
+            ]));
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("x", DataType::Int32, true),
+            field,
+        ]));
+        let batch = arrow_array::RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(arrow_array::Int32Array::from(vec![1])) as _,
+                Arc::new(arrow_array::Int32Array::from(vec![999])) as _,
+            ],
+        )
+        .unwrap();
+        let err = conn
+            .create_table("forged_create", batch)
+            .execute()
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(&err, Error::InvalidInput { message } if message.contains("computed()")),
+            "{err:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_sql_insert_omitting_computed_is_allowed() {
+        use datafusion::prelude::SessionContext;
+
+        let table = table_with_ints("sql_insert_omitted").await;
+        add_computed(&table, &[("doubled".into(), "x * 2".into())])
+            .await
+            .unwrap();
+
+        let ctx = SessionContext::new();
+        let provider =
+            crate::table::datafusion::BaseTableAdapter::try_new(table.base_table().clone())
+                .await
+                .unwrap();
+        ctx.register_table("t", Arc::new(provider)).unwrap();
+        ctx.sql("INSERT INTO t (x) VALUES (4)")
+            .await
+            .unwrap()
+            .collect()
+            .await
+            .unwrap();
+
+        table.checkout_latest().await.unwrap();
+        assert_eq!(table.count_rows(None).await.unwrap(), 4);
+    }
+
+    #[tokio::test]
+    async fn test_a_nested_computed_field_cannot_be_renamed() {
+        let table = table_with_ints("computed_struct_rename").await;
+        add_computed(&table, &[("payload".into(), "named_struct('a', x)".into())])
+            .await
+            .unwrap();
+
+        let err = table
+            .alter_columns(&[ColumnAlteration::new("payload.a".into()).rename("b".into())])
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(&err, Error::InvalidInput { message } if message.contains("payload")),
+            "{err:?}"
+        );
+    }
+
+    /// Stale handles must not commit the computed/LSM state in either order.
+    #[tokio::test]
+    async fn test_stale_handles_cannot_mix_computed_and_lsm() {
+        use crate::table::LsmWriteSpec;
+
+        let tmp_dir = tempfile::tempdir().unwrap();
+        let uri = tmp_dir.path().to_str().unwrap();
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "x",
+            DataType::Int32,
+            false,
+        )]));
+        let batch = arrow_array::RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(arrow_array::Int32Array::from(vec![1])) as _],
+        )
+        .unwrap();
+        let conn = connect(uri).execute().await.unwrap();
+        let table = conn.create_table("mix", batch).execute().await.unwrap();
+        table.set_unenforced_primary_key(["x"]).await.unwrap();
+        let stale = conn.open_table("mix").execute().await.unwrap();
+
+        // Declare on one handle; the stale handle must not install a spec.
+        add_computed(&table, &[("doubled".into(), "x * 2".into())])
+            .await
+            .unwrap();
+        let err = stale
+            .set_lsm_write_spec(LsmWriteSpec::unsharded())
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::NotSupported { .. }), "install won");
+
+        // Reverse order on fresh tables.
+        let batch = arrow_array::RecordBatch::try_new(
+            schema,
+            vec![Arc::new(arrow_array::Int32Array::from(vec![1])) as _],
+        )
+        .unwrap();
+        let table = conn.create_table("mix2", batch).execute().await.unwrap();
+        table.set_unenforced_primary_key(["x"]).await.unwrap();
+        let stale = conn.open_table("mix2").execute().await.unwrap();
+        table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded())
+            .await
+            .unwrap();
+        let err = add_computed(&stale, &[("doubled".into(), "x * 2".into())])
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::NotSupported { .. }), "declare won");
+    }
+
+    /// The gate's reproducer: after catch-up activation, an LSM write, and
+    /// unset, retained SSTable rows survive without a live spec. The catch-up
+    /// flag is the durable marker; declaration refuses on it.
+    #[tokio::test]
+    async fn test_unset_with_retained_lsm_rows_cannot_admit_a_declaration() {
+        use crate::table::LsmWriteSpec;
+        use arrow_array::{Int64Array, RecordBatchIterator};
+
+        let tmp_dir = tempfile::tempdir().unwrap();
+        let conn = connect(tmp_dir.path().to_str().unwrap())
+            .execute()
+            .await
+            .unwrap();
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("id", DataType::Int64, false),
+            ArrowField::new("value", DataType::Int64, false),
+        ]));
+        let batch = arrow_array::RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int64Array::from(vec![1, 2])) as _,
+                Arc::new(Int64Array::from(vec![10, 20])) as _,
+            ],
+        )
+        .unwrap();
+        let table = conn
+            .create_table("t", batch.clone())
+            .execute()
+            .await
+            .unwrap();
+        table.set_unenforced_primary_key(["id"]).await.unwrap();
+        table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded())
+            .await
+            .unwrap();
+
+        let mut merge = table.merge_insert(&["id"]);
+        merge
+            .when_matched_update_all(None)
+            .when_not_matched_insert_all()
+            .use_lsm(true);
+        merge
+            .execute(Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema)))
+            .await
+            .unwrap();
+        table.unset_lsm_write_spec().await.unwrap();
+
+        let err = add_computed(&table, &[("doubled".into(), "value * 2".into())])
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(&err, Error::NotSupported { message } if message.contains("LSM")),
+            "{err:?}"
+        );
+    }
+
+    /// A declaration does not read itself, so it travels with its binding.
+    #[tokio::test]
+    async fn test_dropping_the_computed_column_is_allowed() {
+        let table = table_with_ints("drop_computed").await;
+        add_computed(&table, &[("doubled".into(), "x * 2".into())])
+            .await
+            .unwrap();
+
+        table.drop_columns(&["doubled"]).await.unwrap();
+        assert!(declared(&table).await.is_empty());
+    }
+
+    fn function_input_schema() -> ArrowSchema {
+        ArrowSchema::new(vec![
+            ArrowField::new("title", DataType::Utf8, true),
+            ArrowField::new("body", DataType::Utf8, true),
+        ])
+    }
+
+    fn named_struct_application(columns: &str) -> FunctionApplication {
+        FunctionApplication::from_json(&format!(
+            r#"{{
+                "function":{{"name":"text_features","version":"fv_exact"}},
+                "inputs":[
+                    {{"parameter":"title","kind":"column","value":{{"path":"title"}}}},
+                    {{"parameter":"body","kind":"column","value":{{"path":"body"}}}}
+                ],
+                "output":{{"kind":"named_struct","fields":[
+                    {{"name":"normalized_text","arrow_type":"utf8","nullable":false}},
+                    {{"name":"token_count","arrow_type":"int64","nullable":false}}
+                ]}},
+                "columns":{columns}
+            }}"#
+        ))
+        .unwrap()
+    }
+
+    fn function_binding_schema(title_nullable: bool, body_nullable: bool) -> ArrowSchema {
+        ArrowSchema::new(vec![
+            ArrowField::new("title", DataType::Utf8, title_nullable),
+            ArrowField::new("body", DataType::Utf8, body_nullable),
+            ArrowField::new("search_text", DataType::Utf8, true),
+            ArrowField::new("search_token_count", DataType::Int64, true),
+        ])
+    }
+
+    #[test]
+    fn test_non_nullable_function_inputs_can_bind_to_nullable_parameters() {
+        let binding = FunctionBinding::from_json(include_str!(
+            "../../tests/fixtures/first_class_functions/v1/remote_function_binding.json"
+        ))
+        .unwrap();
+
+        ensure_binding_matches_schema(&function_binding_schema(false, false), &binding).unwrap();
+    }
+
+    #[test]
+    fn test_nullable_function_input_cannot_bind_to_non_nullable_parameter() {
+        let mut raw_binding: Value = serde_json::from_str(include_str!(
+            "../../tests/fixtures/first_class_functions/v1/remote_function_binding.json"
+        ))
+        .unwrap();
+        raw_binding["inputs"][0]["nullable"] = Value::Bool(false);
+        raw_binding["input_schema"]["fields"][0]["nullable"] = Value::Bool(false);
+        let binding: FunctionBinding = serde_json::from_value(raw_binding).unwrap();
+
+        let err = ensure_binding_matches_schema(&function_binding_schema(true, false), &binding)
+            .unwrap_err();
+        assert!(
+            matches!(&err, Error::InvalidInput { message }
+                if message.contains("input column 'title' is nullable")
+                    && message.contains("parameter 'title'")
+                    && message.contains("binding 'fb_01K3TEXT'")
+                    && message.contains("non-nullable")),
+            "{err:?}"
+        );
+    }
+
+    #[test]
+    fn test_function_binding_metadata_survives_schema_round_trip() {
+        let binding = FunctionBinding::from_json(include_str!(
+            "../../tests/fixtures/first_class_functions/v1/remote_function_binding.json"
+        ))
+        .unwrap();
+        let raw = function_bindings_metadata(std::slice::from_ref(&binding)).unwrap();
+        let mut fields = vec![
+            ArrowField::new("title", DataType::Utf8, true),
+            ArrowField::new("body", DataType::Utf8, true),
+        ];
+        fields.extend(
+            binding
+                .outputs()
+                .iter()
+                .map(|output| {
+                    let data_type = match output.arrow_type.as_str() {
+                        "utf8" => DataType::Utf8,
+                        "int64" => DataType::Int64,
+                        other => panic!("unexpected fixture output type {other}"),
+                    };
+                    let metadata = function_computed_column_metadata(
+                        binding.binding_id(),
+                        output.output_ordinal,
+                        &["title".into(), "body".into()],
+                    );
+                    ArrowField::new(&output.output_name, data_type, true).with_metadata(metadata)
+                })
+                .collect::<Vec<_>>(),
+        );
+        let schema = ArrowSchema::new_with_metadata(
+            fields,
+            HashMap::from([(FUNCTION_BINDINGS_META_KEY.to_string(), raw)]),
+        );
+
+        let reopened =
+            ArrowSchema::new_with_metadata(schema.fields().to_vec(), schema.metadata().clone());
+        let bindings = function_bindings(&reopened).unwrap();
+        assert_eq!(bindings, vec![binding.clone()]);
+        assert!(bindings[0].input_schema().is_some());
+        assert!(bindings[0].output_schema().is_some());
+        assert!(matches!(
+            computed_column_from_field(reopened.field(3)).unwrap().kind,
+            ComputedColumnKind::Function {
+                ref binding_id,
+                output_ordinal: 1,
+            } if binding_id == "fb_01K3TEXT"
+        ));
+        let err = plan_function_application(&reopened, &named_struct_application("{}"), None)
+            .unwrap_err();
+        assert!(matches!(err, Error::NotSupported { .. }));
+    }
+
+    #[test]
+    fn test_newer_binding_fields_remain_readable_but_fail_closed_on_mutation() {
+        let raw_binding: Value = serde_json::from_str(include_str!(
+            "../../tests/fixtures/first_class_functions/v1/remote_function_binding.json"
+        ))
+        .unwrap();
+        let binding: FunctionBinding = serde_json::from_value(raw_binding.clone()).unwrap();
+        assert_eq!(binding.binding_id(), "fb_01K3TEXT");
+
+        let schema = ArrowSchema::new_with_metadata(
+            Vec::<ArrowField>::new(),
+            HashMap::from([(
+                FUNCTION_BINDINGS_META_KEY.to_string(),
+                serde_json::json!({
+                    "version": FUNCTION_BINDINGS_VERSION,
+                    "bindings": [raw_binding],
+                })
+                .to_string(),
+            )]),
+        );
+        let err = ensure_supported_function_metadata(&schema).unwrap_err();
+        assert!(matches!(err, Error::NotSupported { .. }));
+    }
+
+    #[test]
+    fn test_named_struct_can_be_kept_as_one_nullable_physical_column() {
+        let application = named_struct_application("{}");
+        let plan =
+            plan_function_application(&function_input_schema(), &application, Some("features"))
+                .unwrap();
+
+        assert_eq!(plan.outputs.len(), 1);
+        assert_eq!(plan.outputs[0].result_field, WHOLE_RESULT_FIELD);
+        assert_eq!(plan.output_schema.fields.len(), 1);
+        assert!(plan.output_schema.fields[0].nullable);
+        assert_eq!(plan.output_schema.fields[0].r#type.r#type, "struct");
+        assert_eq!(
+            plan.output_schema.fields[0]
+                .r#type
+                .fields
+                .as_ref()
+                .unwrap()
+                .len(),
+            2
+        );
+    }
+
+    #[test]
+    fn test_function_mapping_and_sibling_collisions_fail_before_request() {
+        let unknown = named_struct_application(r#"{"missing":"renamed"}"#);
+        let err = plan_function_application(&function_input_schema(), &unknown, None).unwrap_err();
+        assert!(matches!(&err, Error::InvalidInput { message } if message.contains("unknown")));
+
+        let duplicate =
+            named_struct_application(r#"{"normalized_text":"same","token_count":"same"}"#);
+        let err =
+            plan_function_application(&function_input_schema(), &duplicate, None).unwrap_err();
+        assert!(
+            matches!(&err, Error::InvalidInput { message } if message.contains("destinations"))
+        );
+
+        let mut fields = function_input_schema().fields().to_vec();
+        fields.push(Arc::new(ArrowField::new(
+            "token_count",
+            DataType::Int64,
+            true,
+        )));
+        let collision_schema = ArrowSchema::new(fields);
+        let err =
+            plan_function_application(&collision_schema, &named_struct_application("{}"), None)
+                .unwrap_err();
+        assert!(matches!(err, Error::ColumnAlreadyExists { name } if name == "token_count"));
+    }
+
+    #[test]
+    fn test_unknown_and_mixed_version_function_contracts_fail_closed() {
+        let application = FunctionApplication::from_json(
+            r#"{
+                "function":{"name":"f","version":"fv"},
+                "inputs":[{"parameter":"title","kind":"future_source","value":{"path":"title"}}],
+                "output":{"kind":"scalar","arrow_type":"int64","nullable":false}
+            }"#,
+        )
+        .unwrap();
+        let err = plan_function_application(&function_input_schema(), &application, Some("out"))
+            .unwrap_err();
+        assert!(matches!(err, Error::NotSupported { .. }));
+
+        let future_application = FunctionApplication::from_json(
+            r#"{
+                "function":{"name":"f","version":"fv"},
+                "inputs":[],
+                "output":{"kind":"scalar","arrow_type":"int64","nullable":false},
+                "future_declaration":{"mode":"managed"}
+            }"#,
+        )
+        .unwrap();
+        let err =
+            plan_function_application(&function_input_schema(), &future_application, Some("out"))
+                .unwrap_err();
+        assert!(matches!(err, Error::NotSupported { .. }));
+
+        let nested_future_application = FunctionApplication::from_json(
+            r#"{
+                "function":{"name":"f","version":"fv"},
+                "inputs":[],
+                "output":{"kind":"scalar","arrow_type":"int64","nullable":false,"assignment":"cell_flag"}
+            }"#,
+        )
+        .unwrap();
+        let err = plan_function_application(
+            &function_input_schema(),
+            &nested_future_application,
+            Some("out"),
+        )
+        .unwrap_err();
+        assert!(matches!(err, Error::NotSupported { .. }));
+
+        let mixed_schema = ArrowSchema::new_with_metadata(
+            function_input_schema().fields().to_vec(),
+            HashMap::from([(
+                FUNCTION_BINDINGS_META_KEY.to_string(),
+                r#"{"version":2,"bindings":[]}"#.to_string(),
+            )]),
+        );
+        let err = plan_function_application(&mixed_schema, &named_struct_application("{}"), None)
+            .unwrap_err();
+        assert!(matches!(err, Error::NotSupported { .. }));
+    }
+
+    #[test]
+    fn test_function_inputs_use_paths_and_cannot_be_computed() {
+        let mut schema = function_input_schema();
+        let plan =
+            plan_function_application(&schema, &named_struct_application("{}"), None).unwrap();
+        assert_eq!(plan.input_bindings[0].field_path, "title");
+        assert_eq!(plan.input_bindings[1].field_path, "body");
+
+        let title = schema
+            .field(0)
+            .as_ref()
+            .clone()
+            .with_metadata(HashMap::from([
+                (COMPUTED_COLUMN_META_KEY.to_string(), "true".to_string()),
+                (KIND_META_KEY.to_string(), SQL_KIND.to_string()),
+                (EXPRESSION_META_KEY.to_string(), "title".to_string()),
+            ]));
+        schema = ArrowSchema::new(vec![title, schema.field(1).as_ref().clone()]);
+        let err =
+            plan_function_application(&schema, &named_struct_application("{}"), None).unwrap_err();
+        assert!(
+            matches!(&err, Error::InvalidInput { message } if message.contains("computed-on-computed"))
+        );
+    }
+}
diff --git a/rust/lancedb/src/table/create_index.rs b/rust/lancedb/src/table/create_index.rs
index 144c6dbfb..e373522bc 100644
--- a/rust/lancedb/src/table/create_index.rs
+++ b/rust/lancedb/src/table/create_index.rs
@@ -12,6 +12,7 @@ use arrow_schema::{DataType, Field};
 use lance::index::DatasetIndexExt;
 use lance::index::vector::VectorIndexParams;
 use lance::index::vector::utils::infer_vector_dim;
+use lance_arrow::json::is_json_field;
 use lance_index::IndexType;
 use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
 use lance_index::vector::bq::RQBuildParams;
@@ -219,6 +220,14 @@ impl NativeTable {
                 )))
             }
             Index::Bitmap(_) => {
+                if is_json_field(field) {
+                    return Err(Error::Schema {
+                        message: format!(
+                            "A BITMAP index cannot be created on the whole-document lance.json field `{}`. Create a JSON-path scalar index for structured equality or range predicates, or use FTS for document search",
+                            field.name()
+                        ),
+                    });
+                }
                 Self::validate_index_type(field, "Bitmap", supported_bitmap_data_type)?;
                 Ok(Box::new(ScalarIndexParams::for_builtin(
                     BuiltinIndexType::Bitmap,
@@ -1465,6 +1474,35 @@ mod tests {
         assert_eq!(stats.distance_type, None);
     }
 
+    #[tokio::test]
+    async fn test_create_bitmap_index_rejects_lance_json() {
+        let conn = connect("memory://").execute().await.unwrap();
+        let schema = Arc::new(Schema::new(vec![lance_arrow::json::json_field(
+            "metadata", true,
+        )]));
+        let table = conn
+            .create_empty_table("json_bitmap", schema)
+            .execute()
+            .await
+            .unwrap();
+
+        let err = table
+            .create_index(&["metadata"], Index::Bitmap(Default::default()))
+            .execute()
+            .await
+            .expect_err("a whole-document lance.json field must not support a bitmap index");
+        let message = err.to_string();
+        assert!(
+            message.contains("lance.json"),
+            "unexpected error: {message}"
+        );
+        assert!(
+            message.contains("JSON-path scalar index"),
+            "unexpected error: {message}"
+        );
+        assert!(message.contains("FTS"), "unexpected error: {message}");
+    }
+
     #[tokio::test]
     async fn test_create_label_list_index() {
         let conn = connect("memory://").execute().await.unwrap();
diff --git a/rust/lancedb/src/table/datafusion/blob_coerce.rs b/rust/lancedb/src/table/datafusion/blob_coerce.rs
index b29b2423b..cb984f7f4 100644
--- a/rust/lancedb/src/table/datafusion/blob_coerce.rs
+++ b/rust/lancedb/src/table/datafusion/blob_coerce.rs
@@ -7,7 +7,7 @@
 
 use std::sync::Arc;
 
-use arrow_schema::{DataType, Field, FieldRef};
+use arrow_schema::{DataType, Field, FieldRef, Fields};
 use datafusion::functions::core::{get_field, named_struct};
 use datafusion_common::ScalarValue;
 use datafusion_common::config::ConfigOptions;
@@ -35,8 +35,9 @@ pub(super) fn coerce_blob_expr(
         });
     };
 
-    let input_struct_children = match input_field.data_type() {
-        DataType::Binary | DataType::LargeBinary | DataType::BinaryView => None,
+    let input_shape = match input_field.data_type() {
+        DataType::Binary | DataType::LargeBinary | DataType::BinaryView => BlobInputShape::Bytes,
+        DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => BlobInputShape::String,
         DataType::Struct(children) => {
             if !children
                 .iter()
@@ -49,13 +50,15 @@ pub(super) fn coerce_blob_expr(
                     ),
                 });
             }
-            Some(children)
+            BlobInputShape::Struct(children)
         }
         other => {
             return Err(Error::InvalidInput {
                 message: format!(
                     "cannot coerce column '{}' with type {} into a blob v2 struct. \
-                     expected Binary, LargeBinary, BinaryView, or a Struct with a 'data' or 'uri' child",
+                     expected binary bytes (Binary, LargeBinary, BinaryView), \
+                     strings (Utf8, LargeUtf8, Utf8View), \
+                     or a Struct with a 'data' or 'uri' child",
                     table_field.name(),
                     other,
                 ),
@@ -69,9 +72,8 @@ pub(super) fn coerce_blob_expr(
             declared.name().as_str(),
         ))));
 
-        let value: Arc<dyn PhysicalExpr> = match input_struct_children {
-            // Raw binary lands in `data` and everything else is a typed null.
-            None => {
+        let value: Arc<dyn PhysicalExpr> = match &input_shape {
+            BlobInputShape::Bytes => {
                 if declared.name() == "data" {
                     Arc::new(CastExpr::new(
                         input_expr.clone(),
@@ -82,30 +84,43 @@ pub(super) fn coerce_blob_expr(
                     typed_null(declared.data_type())?
                 }
             }
-            Some(children) => match children.iter().find(|c| c.name() == declared.name()) {
-                Some(child) => {
-                    let field_expr: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::new(
-                        &format!("get_field({})", declared.name()),
-                        get_field(),
-                        vec![
-                            input_expr.clone(),
-                            Arc::new(Literal::new(ScalarValue::from(declared.name().as_str()))),
-                        ],
-                        Arc::new(child.as_ref().clone()),
-                        config.clone(),
-                    ));
-                    if child.data_type() == declared.data_type() {
-                        field_expr
-                    } else {
-                        Arc::new(CastExpr::new(
-                            field_expr,
-                            declared.data_type().clone(),
-                            None,
-                        ))
-                    }
+            BlobInputShape::String => {
+                if declared.name() == "uri" {
+                    Arc::new(CastExpr::new(
+                        input_expr.clone(),
+                        declared.data_type().clone(),
+                        None,
+                    ))
+                } else {
+                    typed_null(declared.data_type())?
                 }
-                None => typed_null(declared.data_type())?,
-            },
+            }
+            BlobInputShape::Struct(children) => {
+                match children.iter().find(|c| c.name() == declared.name()) {
+                    Some(child) => {
+                        let field_expr: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::new(
+                            &format!("get_field({})", declared.name()),
+                            get_field(),
+                            vec![
+                                input_expr.clone(),
+                                Arc::new(Literal::new(ScalarValue::from(declared.name().as_str()))),
+                            ],
+                            Arc::new(child.as_ref().clone()),
+                            config.clone(),
+                        ));
+                        if child.data_type() == declared.data_type() {
+                            field_expr
+                        } else {
+                            Arc::new(CastExpr::new(
+                                field_expr,
+                                declared.data_type().clone(),
+                                None,
+                            ))
+                        }
+                    }
+                    None => typed_null(declared.data_type())?,
+                }
+            }
         };
         ns_args.push(value);
     }
@@ -120,6 +135,12 @@ pub(super) fn coerce_blob_expr(
     Ok((expr, table_field.clone()))
 }
 
+enum BlobInputShape<'a> {
+    Bytes,
+    String,
+    Struct(&'a Fields),
+}
+
 fn typed_null(data_type: &DataType) -> Result<Arc<dyn PhysicalExpr>> {
     let scalar = ScalarValue::try_from(data_type).map_err(|e| Error::InvalidInput {
         message: format!("cannot build null literal for blob child type {data_type}: {e}"),
@@ -134,7 +155,7 @@ mod tests {
     use crate::blob::blob;
     use arrow_array::{
         Array, ArrayRef, BinaryArray, BinaryViewArray, Int32Array, Int64Array, LargeBinaryArray,
-        RecordBatch, StringArray, StructArray, UInt8Array, UInt64Array,
+        RecordBatch, StringArray, StringViewArray, StructArray, UInt8Array, UInt64Array,
     };
     use arrow_schema::Schema;
     use datafusion::prelude::SessionContext;
@@ -436,14 +457,78 @@ mod tests {
     #[tokio::test]
     async fn unsupported_input_type_is_rejected_with_column_name() {
         let batch = batch_with_image(
-            Field::new("image", DataType::Utf8, true),
-            Arc::new(StringArray::from(vec!["not bytes"])),
+            Field::new("image", DataType::Int64, true),
+            Arc::new(Int64Array::from(vec![42])),
         );
         let err = coerce_err(batch, &blob_table_schema()).await;
         assert!(matches!(err, Error::InvalidInput { .. }), "got {err:?}");
         assert!(err.to_string().contains("image"));
     }
 
+    #[tokio::test]
+    async fn utf8_string_coerces_to_uri_child() {
+        let batch = batch_with_image(
+            Field::new("image", DataType::Utf8, true),
+            Arc::new(StringArray::from(vec![Some("s3://bucket/key"), None])),
+        );
+        let coerced = coerce(batch, &blob_table_schema()).await;
+        let image = image_struct(&coerced);
+        let uri: &StringArray = image
+            .column_by_name("uri")
+            .unwrap()
+            .as_any()
+            .downcast_ref()
+            .unwrap();
+        assert_eq!(uri.value(0), "s3://bucket/key");
+        assert!(image.column_by_name("data").unwrap().is_null(0));
+        assert!(uri.is_null(1));
+    }
+
+    #[tokio::test]
+    async fn large_utf8_string_coerces_into_four_child_blob_layout() {
+        use arrow_array::LargeStringArray;
+
+        let table_schema = Schema::new(vec![
+            Field::new("id", DataType::Int64, false),
+            wide_blob_field("image"),
+        ]);
+        let batch = batch_with_image(
+            Field::new("image", DataType::LargeUtf8, true),
+            Arc::new(LargeStringArray::from(vec!["file:///tmp/blob.bin"])),
+        );
+        let coerced = coerce(batch, &table_schema).await;
+        let image = image_struct(&coerced);
+        assert_eq!(image.num_columns(), 4);
+        let uri: &StringArray = image
+            .column_by_name("uri")
+            .unwrap()
+            .as_any()
+            .downcast_ref()
+            .unwrap();
+        assert_eq!(uri.value(0), "file:///tmp/blob.bin");
+        assert!(image.column_by_name("data").unwrap().is_null(0));
+        assert!(image.column_by_name("position").unwrap().is_null(0));
+        assert!(image.column_by_name("size").unwrap().is_null(0));
+    }
+
+    #[tokio::test]
+    async fn utf8_view_string_coerces_to_uri_child() {
+        let batch = batch_with_image(
+            Field::new("image", DataType::Utf8View, true),
+            Arc::new(StringViewArray::from(vec![Some("s3://bucket/view-key")])),
+        );
+        let coerced = coerce(batch, &blob_table_schema()).await;
+        let image = image_struct(&coerced);
+        let uri: &StringArray = image
+            .column_by_name("uri")
+            .unwrap()
+            .as_any()
+            .downcast_ref()
+            .unwrap();
+        assert_eq!(uri.value(0), "s3://bucket/view-key");
+        assert!(image.column_by_name("data").unwrap().is_null(0));
+    }
+
     #[tokio::test]
     async fn blob_metadata_survives_cast_of_sibling_column() {
         let batch = RecordBatch::try_new(
diff --git a/rust/lancedb/src/table/datafusion/insert.rs b/rust/lancedb/src/table/datafusion/insert.rs
index e176c228b..b9bd2396e 100644
--- a/rust/lancedb/src/table/datafusion/insert.rs
+++ b/rust/lancedb/src/table/datafusion/insert.rs
@@ -17,7 +17,7 @@ use datafusion_physical_plan::stream::RecordBatchStreamAdapter;
 use datafusion_physical_plan::{
     DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
 };
-use futures::TryStreamExt;
+use futures::StreamExt;
 use lance::Dataset;
 use lance::dataset::transaction::{Operation, Transaction};
 use lance::dataset::{CommitBuilder, InsertBuilder, WriteParams, WriteProgressFn};
@@ -194,12 +194,23 @@ impl ExecutionPlan for InsertExec {
 
         let output_bytes = MetricBuilder::new(&self.metrics).output_bytes(partition);
         let input_schema = input_stream.schema();
+        let declared: Vec<String> = crate::table::computed_columns::computed_columns(
+            &arrow_schema::Schema::from(self.dataset.schema()),
+        )
+        .into_iter()
+        .map(|declaration| declaration.name)
+        .collect();
         let input_stream: SendableRecordBatchStream =
             Box::pin(InstrumentedRecordBatchStreamAdapter::new(
                 input_schema,
-                input_stream.map_ok(move |batch| {
+                input_stream.map(move |batch| {
+                    let batch = batch?;
+                    crate::table::computed_columns::ensure_batch_writes_no_computed_values(
+                        &declared, &batch,
+                    )
+                    .map_err(|e| datafusion::error::DataFusionError::External(Box::new(e)))?;
                     output_bytes.add(batch.get_array_memory_size());
-                    batch
+                    Ok(batch)
                 }),
                 partition,
                 &self.metrics,
diff --git a/rust/lancedb/src/table/lsm_stats.rs b/rust/lancedb/src/table/lsm_stats.rs
new file mode 100644
index 000000000..953aea90f
--- /dev/null
+++ b/rust/lancedb/src/table/lsm_stats.rs
@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+//! Live per-bucket LSM state — the shape [`crate::Table::get_lsm_stats`]
+//! returns and [`super::checkpoint`] polls.
+//!
+//! Nothing here is derived: sums and differences (total L0 bytes, WAL lag)
+//! are the caller's to compute. There is no "WAL is off" shape — that case is
+//! `None`, because a struct of zeros would read as measurements.
+
+use serde::Deserialize;
+
+/// One flushed L0 generation.
+#[derive(Debug, Clone, Deserialize)]
+pub struct GenerationStats {
+    pub generation: u64,
+    pub bytes: u64,
+    /// Present only when `include_generation_rows` was requested. Off by
+    /// default because each count opens an uncached Lance dataset, and the
+    /// checkpoint loop polls this route needing only generation numbers.
+    #[serde(default)]
+    pub rows: Option<u64>,
+}
+
+/// One in-memory memtable.
+#[derive(Debug, Clone, Deserialize)]
+pub struct MemtableStats {
+    pub generation: u64,
+    pub rows: u64,
+    pub bytes: u64,
+    pub batches: u64,
+    /// Names of the indexes this memtable carries. An absent name is the whole
+    /// answer to "why is my fresh-tier search on that column brute-force".
+    pub indexes: Vec<String>,
+}
+
+/// Live state of one bucket. A table is N buckets on one node; flattening to
+/// a single number hides the one hot bucket that is usually why someone
+/// opened this endpoint.
+#[derive(Debug, Clone, Deserialize)]
+pub struct BucketStats {
+    pub shard_id: String,
+    /// `Active` | `Sealed` (drop-table 2PC in flight).
+    pub status: String,
+    pub writer_epoch: u64,
+    pub manifest_version: u64,
+    pub current_generation: u64,
+    pub replay_after_wal_entry_position: u64,
+    pub wal_entry_position_last_seen: u64,
+    pub generations: Vec<GenerationStats>,
+    /// Whether a pass owns this bucket's compaction latch right now. Says *a*
+    /// driver is running, not *whose*, and the latch is held from dispatch —
+    /// including while the pass queues for a pod-wide compactor permit. Read
+    /// it as "do not pile on", never as "mine is progressing".
+    pub compacting: bool,
+    /// Oldest first, active last. Absent for a `Sealed` bucket, whose
+    /// in-memory state is torn down.
+    #[serde(default)]
+    pub memtables: Option<Vec<MemtableStats>>,
+}
+
+impl BucketStats {
+    /// The newest flushed generation, or `None` when L0 is empty.
+    pub(crate) fn newest_generation(&self) -> Option<u64> {
+        self.generations.iter().map(|g| g.generation).max()
+    }
+
+    /// How many generations at or below `target` are still in L0.
+    ///
+    /// A count, not a boolean: one pass drains a bounded prefix rather than
+    /// the whole target set, so a boolean would read as "no progress" for
+    /// every pass but the last. Compaction drains oldest-first, so this
+    /// decreases monotonically.
+    pub(crate) fn outstanding_generations(&self, target: u64) -> usize {
+        self.generations
+            .iter()
+            .filter(|g| g.generation <= target)
+            .count()
+    }
+}
+
+/// Live LSM state, one entry per bucket.
+#[derive(Debug, Clone, Deserialize)]
+pub struct LsmStats {
+    pub buckets: Vec<BucketStats>,
+}
+
+/// Server-side JSON envelope for `get_lsm_stats`. `lsm_stats` is null when
+/// the table has no LSM write path.
+#[derive(Debug, Deserialize)]
+pub(crate) struct GetLsmStatsResponse {
+    #[serde(default)]
+    pub lsm_stats: Option<LsmStats>,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn bucket(shard: &str, generations: &[u64], compacting: bool) -> BucketStats {
+        BucketStats {
+            shard_id: shard.into(),
+            status: "Active".into(),
+            writer_epoch: 1,
+            manifest_version: 1,
+            current_generation: generations.iter().max().copied().unwrap_or(0) + 1,
+            replay_after_wal_entry_position: 0,
+            wal_entry_position_last_seen: 0,
+            generations: generations
+                .iter()
+                .map(|g| GenerationStats {
+                    generation: *g,
+                    bytes: 1,
+                    rows: None,
+                })
+                .collect(),
+            compacting,
+            memtables: None,
+        }
+    }
+
+    /// The target watermark is the newest generation at the start, and a
+    /// generation created after it must not hold the loop open — that is why
+    /// the predicate terminates under write load.
+    #[test]
+    fn newer_generations_do_not_extend_the_target() {
+        let start = bucket("b0", &[7, 8], false);
+        let target = start.newest_generation().expect("L0 is non-empty");
+        assert_eq!(target, 8);
+
+        // Compaction drained 7 and 8; 9 and 10 arrived while it ran.
+        let later = bucket("b0", &[9, 10], false);
+        assert_eq!(
+            later.outstanding_generations(target),
+            0,
+            "generations above the target are somebody else's problem"
+        );
+
+        // Still holding 8 means still outstanding.
+        assert_eq!(
+            bucket("b0", &[8, 9], false).outstanding_generations(target),
+            1
+        );
+    }
+
+    /// The metric counts generations, not buckets: a pass drains a bounded
+    /// prefix, so one bucket going 3 → 2 → 1 → 0 is three steps.
+    #[test]
+    fn progress_is_measured_in_generations() {
+        let target = 3;
+        let counts: Vec<usize> = [&[1u64, 2, 3][..], &[2, 3][..], &[3][..], &[][..]]
+            .iter()
+            .map(|gens| bucket("b0", gens, false).outstanding_generations(target))
+            .collect();
+        assert_eq!(counts, vec![3, 2, 1, 0]);
+    }
+
+    #[test]
+    fn empty_l0_has_no_target() {
+        assert!(bucket("b0", &[], false).newest_generation().is_none());
+    }
+}
diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs
index 480527bc0..3227e3edf 100644
--- a/rust/lancedb/src/table/merge.rs
+++ b/rust/lancedb/src/table/merge.rs
@@ -233,6 +233,10 @@ pub(crate) async fn execute_merge_insert(
     params: MergeInsertBuilder,
     new_data: Box<dyn RecordBatchReader + Send>,
 ) -> Result<MergeResult> {
+    super::computed_columns::ensure_no_function_bindings_for_mutation(
+        table.schema().await?.as_ref(),
+        "merge_insert",
+    )?;
     match lsm::lsm_dispatch_decision(table, &params).await? {
         lsm::LsmDispatch::Lsm(plan) => {
             let future =
@@ -315,7 +319,11 @@ pub(crate) async fn execute_merge_insert(
 
 #[cfg(test)]
 mod tests {
-    use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader};
+    use arrow_array::builder::FixedSizeBinaryBuilder;
+    use arrow_array::{
+        FixedSizeListArray, Int32Array, NullArray, RecordBatch, RecordBatchIterator,
+        RecordBatchReader, StringArray, UInt32Array, UInt64Array,
+    };
     use arrow_schema::{DataType, Field, Schema};
     use std::sync::Arc;
 
@@ -337,6 +345,42 @@ mod tests {
         Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema))
     }
 
+    fn fixed_size_binary_merge_batch(
+        id_range: std::ops::Range<u64>,
+        price: u64,
+    ) -> Box<dyn RecordBatchReader + Send> {
+        let ids = id_range.collect::<Vec<_>>();
+        let mut id_builder = FixedSizeBinaryBuilder::new(16);
+        for id in &ids {
+            let mut bytes = [0; 16];
+            bytes[..8].copy_from_slice(&id.to_le_bytes());
+            id_builder.append_value(bytes).unwrap();
+        }
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::FixedSizeBinary(16), false),
+            Field::new("id_as_int", DataType::UInt64, false),
+            Field::new("name", DataType::Utf8, false),
+            Field::new("market", DataType::Utf8, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(id_builder.finish()),
+                Arc::new(UInt64Array::from_iter_values(ids.iter().copied())),
+                Arc::new(StringArray::from_iter_values(
+                    ids.iter().map(|id| format!("name{id}")),
+                )),
+                Arc::new(StringArray::from_iter_values(std::iter::repeat_n(
+                    format!("market_{price}"),
+                    ids.len(),
+                ))),
+            ],
+        )
+        .unwrap();
+        Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema))
+    }
+
     #[tokio::test]
     async fn test_merge_insert() {
         let conn = connect("memory://").execute().await.unwrap();
@@ -388,6 +432,36 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_merge_insert_fixed_size_binary_non_nullable() {
+        // Regression test for #2869: an unrelated FixedSizeBinary column used to corrupt the
+        // outer join that implements when_not_matched_by_source_delete.
+        let conn = connect("memory://").execute().await.unwrap();
+        let table = conn
+            .create_table(
+                "fixed_size_binary_merge",
+                fixed_size_binary_merge_batch(0..256, 100),
+            )
+            .execute()
+            .await
+            .unwrap();
+
+        let mut merge_insert = table.merge_insert(&["id_as_int"]);
+        merge_insert
+            .when_matched_update_all(None)
+            .when_not_matched_insert_all()
+            .when_not_matched_by_source_delete(None);
+        let result = merge_insert
+            .execute(fixed_size_binary_merge_batch(100..356, 200))
+            .await
+            .unwrap();
+
+        assert_eq!(result.num_updated_rows, 156);
+        assert_eq!(result.num_inserted_rows, 100);
+        assert_eq!(result.num_deleted_rows, 100);
+        assert_eq!(table.count_rows(None).await.unwrap(), 256);
+    }
+
     #[tokio::test]
     async fn test_merge_insert_use_index() {
         let conn = connect("memory://").execute().await.unwrap();
@@ -456,6 +530,74 @@ mod tests {
         assert_eq!(result.num_deleted_rows, 5);
         assert_eq!(table.count_rows(None).await.unwrap(), 5);
     }
+
+    #[tokio::test]
+    async fn test_merge_insert_fixed_size_list_above_u32_child_count() {
+        // Arrow's FixedSizeList take kernel uses u32 child indices. Previously,
+        // delete-by-source materialized the target payload in a full outer join,
+        // causing the final list below to overflow those indices and panic.
+        // A Null child keeps this boundary test small in memory.
+        const LIST_SIZE: i32 = 65_536;
+        const ROW_COUNT: usize = (u32::MAX as usize / LIST_SIZE as usize) + 1;
+        const BATCH_SIZE: usize = 8_192;
+
+        let item = Arc::new(Field::new("item", DataType::Null, true));
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::UInt32, false),
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(item.clone(), LIST_SIZE),
+                false,
+            ),
+        ]));
+        let batch = |start: usize, len: usize| {
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(UInt32Array::from_iter_values(
+                        start as u32..(start + len) as u32,
+                    )),
+                    Arc::new(FixedSizeListArray::new(
+                        item.clone(),
+                        LIST_SIZE,
+                        Arc::new(NullArray::new(len * LIST_SIZE as usize)),
+                        None,
+                    )),
+                ],
+            )
+            .unwrap()
+        };
+
+        let target_batches = (0..ROW_COUNT)
+            .step_by(BATCH_SIZE)
+            .map(|start| {
+                let len = (ROW_COUNT - start).min(BATCH_SIZE);
+                Ok(batch(start, len))
+            })
+            .collect::<Vec<_>>();
+        let target_data: Box<dyn RecordBatchReader + Send> =
+            Box::new(RecordBatchIterator::new(target_batches, schema.clone()));
+        let conn = connect("memory://").execute().await.unwrap();
+        let table = conn
+            .create_table("fixed_size_list_overflow", target_data)
+            .execute()
+            .await
+            .unwrap();
+
+        let source = batch(ROW_COUNT - 1, 1);
+        let mut merge = table.merge_insert(&["id"]);
+        merge
+            .when_matched_update_all(None)
+            .when_not_matched_by_source_delete(None);
+        let result = merge
+            .execute(Box::new(RecordBatchIterator::new([Ok(source)], schema)))
+            .await
+            .unwrap();
+
+        assert_eq!(result.num_updated_rows, 1);
+        assert_eq!(result.num_deleted_rows, (ROW_COUNT - 1) as u64);
+        assert_eq!(table.count_rows(None).await.unwrap(), 1);
+    }
 }
 
 #[cfg(test)]
@@ -1130,7 +1272,7 @@ mod lsm_tests {
             .unwrap();
         let fts_index = table.list_indices().await.unwrap()[0].name.clone();
         table
-            .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes([fts_index]))
+            .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![fts_index]))
             .await
             .unwrap();
 
@@ -1223,7 +1365,7 @@ mod lsm_tests {
             .unwrap();
         let vec_index = table.list_indices().await.unwrap()[0].name.clone();
         table
-            .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes([vec_index]))
+            .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![vec_index]))
             .await
             .unwrap();
 
diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs
index 0eb7c0231..a06507ba0 100644
--- a/rust/lancedb/src/table/merge/lsm.rs
+++ b/rust/lancedb/src/table/merge/lsm.rs
@@ -29,6 +29,7 @@ use arrow_schema::{DataType, Schema as ArrowSchema, SchemaRef};
 use lance::Dataset;
 use lance::dataset::mem_wal::{
     DatasetMemWalExt, ShardWriter, ShardWriterConfig, evaluate_sharding_spec,
+    validate_maintained_indexes,
 };
 use lance::index::DatasetIndexExt;
 use lance_core::datatypes::Schema as LanceSchema;
@@ -37,8 +38,9 @@ use tokio::sync::RwLock;
 use uuid::Uuid;
 
 use crate::error::{Error, Result};
+use crate::index::IndexConfig;
 use crate::table::merge::{MergeInsertBuilder, MergeResult};
-use crate::table::{LsmWriteSpec, NativeTable};
+use crate::table::{BaseTable, LsmWriteSpec, NativeTable};
 
 /// Spec id of the sole sharding spec installed by [`set_lsm_write_spec`].
 /// Must match Lance's `InitializeMemWalBuilder` (`SHARDING_SPEC_ID`).
@@ -80,32 +82,60 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
         }
     }
 
+    // Before the builder borrows the dataset clone. `list_indices` merges an
+    // index's segments into one entry, so the result needs no dedup.
+    let maintained_indexes = {
+        let dataset = table.dataset.get().await?;
+        resolve_maintained_indexes(
+            &dataset,
+            &table.list_indices().await?,
+            spec.maintained_indexes(),
+        )
+        .await?
+    };
+
+    table.checkout_latest().await?;
     let mut dataset = (*table.dataset.get().await?).clone();
+    let schema = arrow_schema::Schema::from(dataset.schema());
+    if !crate::table::computed_columns::computed_columns(&schema).is_empty() {
+        return Err(Error::NotSupported {
+            message: "an LSM write spec cannot be installed on a table with computed \
+                      columns: rows in un-compacted tiers are invisible to refresh"
+                .into(),
+        });
+    }
+    if crate::materialized_view::materialized_view_kind(&dataset.schema().metadata)?.is_some() {
+        return Err(Error::NotSupported {
+            message: "an LSM write spec cannot be installed on a materialized view: \
+                      rows in un-compacted tiers are invisible to refresh"
+                .into(),
+        });
+    }
     let mut builder = dataset.initialize_mem_wal();
-    let (maintained_indexes, writer_config_defaults) = match spec {
+    let writer_config_defaults = match spec {
         LsmWriteSpec::Bucket {
             column,
             num_buckets,
-            maintained_indexes,
             writer_config_defaults,
+            ..
         } => {
             builder = builder.bucket_sharding(column, num_buckets);
-            (maintained_indexes, writer_config_defaults)
+            writer_config_defaults
         }
         LsmWriteSpec::Identity {
             column,
-            maintained_indexes,
             writer_config_defaults,
+            ..
         } => {
             builder = builder.identity_sharding(column);
-            (maintained_indexes, writer_config_defaults)
+            writer_config_defaults
         }
         LsmWriteSpec::Unsharded {
-            maintained_indexes,
             writer_config_defaults,
+            ..
         } => {
             builder = builder.unsharded();
-            (maintained_indexes, writer_config_defaults)
+            writer_config_defaults
         }
     };
     builder = builder.maintained_indexes(maintained_indexes);
@@ -117,6 +147,58 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
     Ok(())
 }
 
+/// Resolve a spec's maintained-index selection against `indices`, as reported
+/// by [`Table::list_indices`](crate::Table::list_indices).
+///
+/// `None` means every index on the table, snapshotted now. Lance validates
+/// either selection against its shard-writer rules, so a spec that installs is
+/// one the MemWAL can open.
+///
+/// An unmaintainable index fails an inferred set rather than being dropped from
+/// it — dropping would leave the caller believing it is maintained.
+async fn resolve_maintained_indexes(
+    dataset: &Dataset,
+    indices: &[IndexConfig],
+    requested: Option<&[String]>,
+) -> Result<Vec<String>> {
+    let Some(requested) = requested else {
+        let all: Vec<String> = indices.iter().map(|index| index.name.clone()).collect();
+        validate_maintained_indexes(dataset, &all)
+            .await
+            .map_err(|source| Error::InvalidInput {
+                message: format!(
+                    "cannot maintain every index on this table: {source}. Set \
+                     maintained_indexes explicitly to choose from {}",
+                    index_name_list(indices),
+                ),
+            })?;
+        return Ok(all);
+    };
+    for name in requested {
+        if !indices.iter().any(|index| &index.name == name) {
+            return Err(Error::InvalidInput {
+                message: format!(
+                    "maintained index '{}' does not exist on this table; it has {}",
+                    name,
+                    index_name_list(indices),
+                ),
+            });
+        }
+    }
+    validate_maintained_indexes(dataset, requested).await?;
+    Ok(requested.to_vec())
+}
+
+/// Index names for an error message.
+fn index_name_list(indices: &[IndexConfig]) -> String {
+    if indices.is_empty() {
+        return "no indexes".to_string();
+    }
+    let mut names: Vec<&str> = indices.iter().map(|index| index.name.as_str()).collect();
+    names.sort_unstable();
+    format!("[{}]", names.join(", "))
+}
+
 // =============================================================================
 // unset_lsm_write_spec
 // =============================================================================
diff --git a/rust/lancedb/src/table/optimize.rs b/rust/lancedb/src/table/optimize.rs
index e29445b2d..4ad58cffb 100644
--- a/rust/lancedb/src/table/optimize.rs
+++ b/rust/lancedb/src/table/optimize.rs
@@ -214,12 +214,17 @@ pub(crate) async fn execute_optimize(
 
 #[cfg(test)]
 mod tests {
-    use arrow_array::{Int32Array, RecordBatch, StringArray};
+    use arrow_array::{
+        Array, FixedSizeListArray, Float32Array, Int32Array, RecordBatch, StringArray,
+    };
     use arrow_schema::{DataType, Field, Schema};
+    use lance_arrow::FixedSizeListArrayExt;
     use rstest::rstest;
     use std::sync::Arc;
 
     use crate::connect;
+    use crate::database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS;
+    use crate::index::vector::IvfRqIndexBuilder;
     use crate::index::{Index, scalar::BTreeIndexBuilder};
     use crate::query::ExecutableQuery;
     use crate::table::{CompactionOptions, OptimizeAction, OptimizeStats};
@@ -304,6 +309,96 @@ mod tests {
         assert_eq!(all_values, expected);
     }
 
+    #[tokio::test]
+    async fn test_compact_with_concurrent_add() {
+        const NUM_FRAGMENTS: usize = 5;
+        const ROWS_PER_FRAGMENT: i32 = 300;
+
+        let tmpdir = tempfile::tempdir().unwrap();
+        let conn = connect(tmpdir.path().to_str().unwrap())
+            .execute()
+            .await
+            .unwrap();
+        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![Arc::new(Int32Array::from_iter_values(0..ROWS_PER_FRAGMENT))],
+        )
+        .unwrap();
+
+        let table = conn
+            .create_table("test_concurrent_compact", batch.clone())
+            .execute()
+            .await
+            .unwrap();
+        table
+            .create_index(&["id"], Index::BTree(BTreeIndexBuilder::default()))
+            .execute()
+            .await
+            .unwrap();
+        for _ in 0..NUM_FRAGMENTS {
+            table.add(batch.clone()).execute().await.unwrap();
+        }
+
+        // Use separate handles so the two writes actually overlap, as they can
+        // when different Node connections operate on the same S3 table.
+        let compact_table = conn
+            .open_table("test_concurrent_compact")
+            .execute()
+            .await
+            .unwrap();
+        let append_table = conn
+            .open_table("test_concurrent_compact")
+            .execute()
+            .await
+            .unwrap();
+        let compact_task = tokio::spawn(async move {
+            compact_table
+                .optimize(OptimizeAction::Compact {
+                    options: CompactionOptions {
+                        target_rows_per_fragment: 1_000,
+                        ..Default::default()
+                    },
+                    remap_options: None,
+                })
+                .await
+        });
+        tokio::task::yield_now().await;
+        for _ in 0..NUM_FRAGMENTS {
+            append_table.add(batch.clone()).execute().await.unwrap();
+        }
+        compact_task.await.unwrap().unwrap();
+
+        let table = conn
+            .open_table("test_concurrent_compact")
+            .execute()
+            .await
+            .unwrap();
+        let dataset = table.dataset().unwrap().get().await.unwrap();
+        let fragment_ids = dataset
+            .get_fragments()
+            .iter()
+            .map(|fragment| fragment.id())
+            .collect::<Vec<_>>();
+        assert!(fragment_ids.windows(2).all(|ids| ids[0] < ids[1]));
+
+        // A second compaction exposed the original out-of-order row-id bug.
+        table
+            .optimize(OptimizeAction::Compact {
+                options: CompactionOptions {
+                    target_rows_per_fragment: 1_000,
+                    ..Default::default()
+                },
+                remap_options: None,
+            })
+            .await
+            .unwrap();
+        assert_eq!(
+            table.count_rows(None).await.unwrap(),
+            ROWS_PER_FRAGMENT as usize * (NUM_FRAGMENTS * 2 + 1)
+        );
+    }
+
     #[tokio::test]
     async fn test_optimize_prune_versions() {
         let conn = connect("memory://").execute().await.unwrap();
@@ -442,6 +537,58 @@ mod tests {
         assert_eq!(final_row_count, 200);
     }
 
+    #[tokio::test]
+    async fn test_optimize_vector_index_after_delete_with_stable_row_ids() {
+        const NUM_ROWS: i32 = 400;
+        const DIMENSION: i32 = 32;
+
+        let conn = connect("memory://").execute().await.unwrap();
+        let vectors = FixedSizeListArray::try_new_from_values(
+            Float32Array::from_iter_values((0..NUM_ROWS).flat_map(|id| {
+                (0..DIMENSION).map(move |offset| ((id as f32 * 0.1) + (offset as f32 * 0.3)).sin())
+            })),
+            DIMENSION,
+        )
+        .unwrap();
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("vector", vectors.data_type().clone(), false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..NUM_ROWS)),
+                Arc::new(vectors),
+            ],
+        )
+        .unwrap();
+        let table = conn
+            .create_table("test_vector_index_optimize_after_delete", batch)
+            .storage_option(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, "true")
+            .execute()
+            .await
+            .unwrap();
+
+        table
+            .create_index(
+                &["vector"],
+                Index::IvfRq(IvfRqIndexBuilder::default().num_partitions(4)),
+            )
+            .execute()
+            .await
+            .unwrap();
+        table.delete("id % 3 = 0").await.unwrap();
+
+        // Regression test for #3330: deleted stable row IDs used to become
+        // misaligned with row addresses while joining small IVF partitions.
+        table
+            .optimize(OptimizeAction::Index(Default::default()))
+            .await
+            .unwrap();
+
+        assert_eq!(table.count_rows(None).await.unwrap(), 266);
+    }
+
     #[tokio::test]
     async fn test_optimize_all() {
         let conn = connect("memory://").execute().await.unwrap();
diff --git a/rust/lancedb/src/table/query.rs b/rust/lancedb/src/table/query.rs
index 5708a0fbb..629cb4e6f 100644
--- a/rust/lancedb/src/table/query.rs
+++ b/rust/lancedb/src/table/query.rs
@@ -70,6 +70,9 @@ async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> R
         .contains(&NamespaceClientPushdownOperation::QueryTable)
         && table.namespace_client.is_some()
         && table.dataset.current_branch().is_none()
+        // NsQueryTableRequest has no version field, so a pushed-down query would
+        // read latest and ignore the pin.
+        && table.dataset.time_travel_version().is_none()
         && !requires_local_namespace_execution(query))
     {
         return Ok(false);
@@ -694,6 +697,7 @@ mod tests {
 
     use super::*;
     use crate::query::{QueryExecutionOptions, QueryRequest};
+    use crate::table::BaseTable;
 
     fn fixed_size_list_array(values: Vec<f32>, dimension: i32) -> FixedSizeListArray {
         FixedSizeListArray::try_new_from_values(Float32Array::from(values), dimension).unwrap()
@@ -886,10 +890,56 @@ mod tests {
 
         async fn query_table(&self, _request: NsQueryTableRequest) -> lance::Result<bytes::Bytes> {
             self.query_table_calls.fetch_add(1, Ordering::SeqCst);
-            panic!("approx_mode queries must not be pushed down to namespace query_table");
+            panic!("query must not be pushed down to namespace query_table");
         }
     }
 
+    #[tokio::test]
+    async fn test_execute_query_pinned_snapshot_with_namespace_pushdown_runs_locally() {
+        use crate::connect;
+        use arrow_array::{Int32Array, RecordBatch};
+        use arrow_schema::{DataType, Field, Schema};
+
+        let conn = connect("memory://").execute().await.unwrap();
+        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))],
+        )
+        .unwrap();
+        let table = conn
+            .create_table("test_pinned_namespace_fallback", vec![batch])
+            .execute()
+            .await
+            .unwrap();
+
+        let namespace_client = Arc::new(CountingNamespaceClient::default());
+        let mut native_table = table.as_native().unwrap().clone();
+        native_table.namespace_client = Some(namespace_client.clone());
+        native_table
+            .pushdown_operations
+            .insert(NamespaceClientPushdownOperation::QueryTable);
+
+        let snapshot = native_table.checkout_current().await.unwrap();
+        let snapshot = snapshot.as_any().downcast_ref::<NativeTable>().unwrap();
+        assert!(snapshot.dataset.time_travel_version().is_some());
+
+        let query = AnyQuery::Query(QueryRequest {
+            filter: Some(QueryFilter::Sql("id > 3".to_string())),
+            ..Default::default()
+        });
+        let stream = execute_query(snapshot, &query, QueryExecutionOptions::default())
+            .await
+            .unwrap();
+        let batches = stream.try_collect::<Vec<_>>().await.unwrap();
+
+        assert_eq!(
+            batches.iter().map(|batch| batch.num_rows()).sum::<usize>(),
+            2
+        );
+        assert_eq!(namespace_client.query_table_calls.load(Ordering::SeqCst), 0);
+    }
+
     #[tokio::test]
     async fn test_execute_query_approx_mode_with_namespace_pushdown_runs_locally() {
         use crate::connect;
diff --git a/rust/lancedb/src/table/query/lsm.rs b/rust/lancedb/src/table/query/lsm.rs
index 074d13476..255d649b1 100644
--- a/rust/lancedb/src/table/query/lsm.rs
+++ b/rust/lancedb/src/table/query/lsm.rs
@@ -84,9 +84,8 @@ pub(super) async fn create_lsm_plan(
     let pk_columns = pk_columns(&ds_ref)?;
     // The base index an indexed arm relies on may lag compaction; resolve it so the
     // snapshot retains SSTables the index has not yet caught up to.
-    let arm_index = arm_maintained_index_name(&ds_ref, &query, &details).await?;
-    let (snapshots, in_memory) =
-        build_read_context(table, &ds_ref, &details, arm_index.as_deref()).await?;
+    let arm_indexes = arm_maintained_index_names(&ds_ref, &query, &details).await?;
+    let (snapshots, in_memory) = build_read_context(table, &ds_ref, &details, &arm_indexes).await?;
 
     let limit = query.base.limit;
     let offset = query.base.offset;
@@ -232,28 +231,40 @@ fn pk_columns(dataset: &Dataset) -> Result<Vec<String>> {
     Ok(pk)
 }
 
-/// Per-shard SSTable exclusion watermark: the generation at or below which SSTables
-/// are safe to drop for this arm. A generation is droppable only once it is
-/// compacted into the base table AND covered by `index_name`'s catch-up (for an
-/// indexed arm); a plain scan (`index_name == None`) uses the compaction watermark
-/// alone. Capping at the index catch-up keeps rows the base index has not yet
-/// indexed visible through their SSTable. First occurrence per shard mirrors Lance's
-/// `compacted_generation_for_shard`.
+/// Per-shard SSTable exclusion watermark: the generation at or below which
+/// SSTables are safe to drop for this query.
+///
+/// A generation is droppable only once it is compacted into the base table AND
+/// covered by the catch-up of every index the query relies on, so the watermark
+/// is the minimum across `index_names`. Gating on fewer than all of them would
+/// drop SSTables holding rows an uncounted index has not yet indexed, and that
+/// arm would silently return fewer rows.
+///
+/// See [`arm_maintained_index_names`] for which indexes are collected today: a
+/// vector search with a scalar prefilter is not yet among them.
+///
+/// An empty `index_names` (a plain scan) uses the compaction watermark alone.
+/// First occurrence per shard mirrors Lance's `compacted_generation_for_shard`.
 fn exclusion_watermarks(
     details: &MemWalIndexDetails,
-    index_name: Option<&str>,
+    index_names: &[String],
 ) -> HashMap<Uuid, u64> {
     let mut exclude: HashMap<Uuid, u64> = HashMap::new();
     for entry in &details.compacted_sstables {
         let mut watermark = entry.generation;
-        if let Some(name) = index_name
-            && let Some(caught_up) = details
+        for name in index_names {
+            match details
                 .index_catchup
                 .iter()
-                .find(|icp| icp.index_name == name)
+                .find(|icp| icp.index_name == *name)
                 .and_then(|icp| icp.caught_up_generation_for_shard(&entry.shard_id))
-        {
-            watermark = watermark.min(caught_up);
+            {
+                Some(caught_up) => watermark = watermark.min(caught_up),
+                // No entry means the index is *not* known to hold these rows,
+                // and the base arm is index-only -- so every generation stays
+                // readable from its SSTable.
+                None => watermark = 0,
+            }
         }
         exclude.entry(entry.shard_id).or_insert(watermark);
     }
@@ -271,9 +282,9 @@ async fn build_read_context(
     table: &NativeTable,
     dataset: &Dataset,
     details: &MemWalIndexDetails,
-    index_name: Option<&str>,
+    index_names: &[String],
 ) -> Result<(Vec<ShardSnapshot>, HashMap<Uuid, InMemoryMemTables>)> {
-    let exclude = exclusion_watermarks(details, index_name);
+    let exclude = exclusion_watermarks(details, index_names);
 
     let shard_ids = dataset.list_mem_wal_latest_shard_ids().await?;
     // Use the dataset's own object store (not `ObjectStore::from_uri`, which
@@ -487,19 +498,33 @@ async fn index_maintained(
     }))
 }
 
-/// The maintained base index the query's arm relies on (vector index for ANN, FTS
-/// index for full-text), used to gate SSTable compaction exclusion by index catch-up.
-/// `None` for a plain scan or when no maintained index covers the searched column.
-async fn arm_maintained_index_name(
+/// Every maintained base index this query relies on, used to gate SSTable
+/// exclusion by index catch-up.
+///
+/// Returns a list because the watermark must be the lowest across every index a
+/// query relies on. Today it never holds more than one: `reject_unsupported`
+/// refuses hybrid search, so the vector and full-text arms are mutually
+/// exclusive.
+///
+/// The case that is genuinely multi-index -- a vector search with a scalar or
+/// bitmap prefilter -- is **not collected yet**. Identifying those needs the
+/// planner's chosen indexes, not the columns the filter names, and no Lance API
+/// exposes them. Until it does, such a query is gated on its vector index alone.
+///
+/// Empty for a plain scan, or when no maintained index covers the searched
+/// column.
+async fn arm_maintained_index_names(
     dataset: &Dataset,
     query: &VectorQueryRequest,
     details: &MemWalIndexDetails,
-) -> Result<Option<String>> {
+) -> Result<Vec<String>> {
     use lance::index::DatasetIndexExt;
-    // Resolve the arm's searched column, the index-detail type it relies on, and a
+
+    // Each arm's searched column, the index-detail type it relies on, and a
     // label for diagnostics — catch-up is taken from the vector/FTS index
     // specifically, not a BTree on the same column.
-    let (column, type_url_suffix, arm) = if !query.query_vector.is_empty() {
+    let mut arms: Vec<(String, &str, &str)> = Vec::new();
+    if !query.query_vector.is_empty() {
         let arrow_schema = ArrowSchema::from(dataset.schema());
         let column = match &query.column {
             Some(column) => column.clone(),
@@ -508,31 +533,43 @@ async fn arm_maintained_index_name(
                 default_vector_column(&arrow_schema, dim)?
             }
         };
-        (column, "VectorIndexDetails", "vector")
-    } else if let Some(fts) = &query.base.full_text_search {
-        match fts.columns().into_iter().next() {
-            Some(column) => (column, "InvertedIndexDetails", "full-text"),
-            None => return Ok(None),
-        }
-    } else {
-        return Ok(None);
-    };
-    let Some(field) = dataset.schema().field(&column) else {
-        return Ok(None);
-    };
+        arms.push((column, "VectorIndexDetails", "vector"));
+    }
+    if let Some(fts) = &query.base.full_text_search
+        && let Some(column) = fts.columns().into_iter().next()
+    {
+        arms.push((column, "InvertedIndexDetails", "full-text"));
+    }
+    if arms.is_empty() {
+        return Ok(Vec::new());
+    }
+
     let indices = dataset.load_indices().await?;
-    let segment_names: Vec<String> = indices
-        .iter()
-        .filter(|idx| {
-            idx.fields.contains(&field.id)
-                && idx
-                    .index_details
-                    .as_ref()
-                    .is_some_and(|d| d.type_url.ends_with(type_url_suffix))
-        })
-        .map(|idx| idx.name.clone())
-        .collect();
-    resolve_single_index(segment_names, &details.maintained_indexes, arm, &column)
+    let mut names = Vec::with_capacity(arms.len());
+    for (column, type_url_suffix, arm) in arms {
+        let Some(field) = dataset.schema().field(&column) else {
+            continue;
+        };
+        let segment_names: Vec<String> = indices
+            .iter()
+            .filter(|idx| {
+                idx.fields.contains(&field.id)
+                    && idx
+                        .index_details
+                        .as_ref()
+                        .is_some_and(|d| d.type_url.ends_with(type_url_suffix))
+            })
+            .map(|idx| idx.name.clone())
+            .collect();
+        if let Some(name) =
+            resolve_single_index(segment_names, &details.maintained_indexes, arm, &column)?
+        {
+            names.push(name);
+        }
+    }
+    names.sort();
+    names.dedup();
+    Ok(names)
 }
 
 /// Resolve the single logical index from the names of its matching physical
@@ -734,21 +771,96 @@ mod tests {
         };
 
         // Plain scan: drop every compacted generation (through 5).
-        assert_eq!(exclusion_watermarks(&details, None).get(&shard), Some(&5));
+        assert_eq!(exclusion_watermarks(&details, &[]).get(&shard), Some(&5));
 
         // FTS arm with a lagging index: exclusion is capped at the index catch-up
         // (2), so SSTable generations 3..=5 are retained until the index covers
         // them — otherwise those documents would silently vanish from FTS results.
         assert_eq!(
-            exclusion_watermarks(&details, Some("fts_idx")).get(&shard),
+            exclusion_watermarks(&details, &["fts_idx".to_string()]).get(&shard),
             Some(&2)
         );
 
-        // A caught-up index — or one untracked in index_catchup — falls back to the
-        // compaction watermark.
+        // An index absent from index_catchup is *not* known to hold these rows,
+        // so nothing may be excluded and every generation stays readable from
+        // its SSTable. An indexed query against an index that has not caught up
+        // must not silently lose rows.
         assert_eq!(
-            exclusion_watermarks(&details, Some("caught_up_idx")).get(&shard),
-            Some(&5)
+            exclusion_watermarks(&details, &["untracked_idx".to_string()]).get(&shard),
+            Some(&0)
+        );
+
+        // One missing entry is enough to hold everything back, even alongside an
+        // index that has caught up.
+        let mixed = vec!["fts_idx".to_string(), "untracked_idx".to_string()];
+        assert_eq!(exclusion_watermarks(&details, &mixed).get(&shard), Some(&0));
+    }
+
+    /// A hybrid search reads a vector and a full-text index, and either may lag.
+    /// Retaining to the lower of the two is what keeps both arms complete;
+    /// gating on one alone would drop SSTables the other has not indexed.
+    #[test]
+    fn exclusion_watermark_takes_the_minimum_across_every_index_used() {
+        let shard = Uuid::from_u128(1);
+        let details = MemWalIndexDetails {
+            compacted_sstables: vec![CompactedSsTable::new(shard, 9)],
+            index_catchup: vec![
+                IndexCatchupProgress::new(
+                    "vec_idx".to_string(),
+                    vec![CompactedSsTable::new(shard, 7)],
+                ),
+                IndexCatchupProgress::new(
+                    "fts_idx".to_string(),
+                    vec![CompactedSsTable::new(shard, 4)],
+                ),
+            ],
+            maintained_indexes: vec!["vec_idx".to_string(), "fts_idx".to_string()],
+            ..Default::default()
+        };
+
+        // Each index alone stops at its own catch-up.
+        assert_eq!(
+            exclusion_watermarks(&details, &["vec_idx".to_string()]).get(&shard),
+            Some(&7)
+        );
+        assert_eq!(
+            exclusion_watermarks(&details, &["fts_idx".to_string()]).get(&shard),
+            Some(&4)
+        );
+
+        // Used together, the lower one governs regardless of order.
+        let both = ["vec_idx".to_string(), "fts_idx".to_string()];
+        assert_eq!(exclusion_watermarks(&details, &both).get(&shard), Some(&4));
+        let reversed = ["fts_idx".to_string(), "vec_idx".to_string()];
+        assert_eq!(
+            exclusion_watermarks(&details, &reversed).get(&shard),
+            Some(&4)
+        );
+    }
+
+    /// An index with no catch-up entry is not known to hold these rows, so it
+    /// retains everything -- it is never widened by a tracked sibling that has
+    /// caught up further.
+    #[test]
+    fn an_untracked_index_retains_everything() {
+        let shard = Uuid::from_u128(1);
+        let details = MemWalIndexDetails {
+            compacted_sstables: vec![CompactedSsTable::new(shard, 9)],
+            index_catchup: vec![IndexCatchupProgress::new(
+                "fts_idx".to_string(),
+                vec![CompactedSsTable::new(shard, 4)],
+            )],
+            maintained_indexes: vec!["fts_idx".to_string(), "untracked_idx".to_string()],
+            ..Default::default()
+        };
+
+        let both = ["fts_idx".to_string(), "untracked_idx".to_string()];
+        assert_eq!(exclusion_watermarks(&details, &both).get(&shard), Some(&0));
+
+        // The tracked sibling still caps on its own.
+        assert_eq!(
+            exclusion_watermarks(&details, &["fts_idx".to_string()]).get(&shard),
+            Some(&4)
         );
     }
 
diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs
new file mode 100644
index 000000000..35f883411
--- /dev/null
+++ b/rust/lancedb/src/table/refresh.rs
@@ -0,0 +1,1011 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+//! Filling computed columns.
+//!
+//! A row without a value gets one; a row that has one keeps it. Refresh is
+//! therefore idempotent and does not observe input mutation -- once a row is
+//! filled, changing what the expression reads leaves the stored result alone.
+//!
+//! Two passes per fragment. The first scans only the unfilled live rows and
+//! evaluates the expression over them, which yields the exact fill count and
+//! decides whether the fragment is staged at all -- a fragment where nothing
+//! would change stages nothing, which is what lets an expression yielding
+//! null settle instead of restaging forever. The second streams the
+//! fragment's physical rows into `write_columns` a batch at a time, so peak
+//! memory is bounded by a scan batch. The expression is evaluated by this
+//! module, never through a projection alias, and only over rows being
+//! filled: every other row -- deleted, or already holding a value -- has its
+//! inputs masked to null first, so a poison value in a row nobody is filling
+//! cannot fail the refresh.
+
+use std::sync::Arc;
+
+use arrow_array::{ArrayRef, BooleanArray, RecordBatch, RecordBatchOptions};
+use arrow_schema::Schema as ArrowSchema;
+use datafusion_expr::ColumnarValue;
+use futures::{Stream, StreamExt, TryStreamExt};
+use lance::Dataset;
+use lance::dataset::WriteDestination;
+use lance::dataset::fragment::FileFragment;
+use lance::dataset::transaction::Operation;
+use lance_core::ROW_ID;
+use lance_core::datatypes::Schema as LanceSchema;
+use serde::{Deserialize, Serialize};
+
+use super::computed_columns::{BoundExpression, ComputedColumnKind, computed_column_from_field};
+use super::{BaseTable, NativeTable};
+use crate::job::Job;
+use crate::{Error, Result};
+
+/// The result of refreshing a computed column.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
+pub struct RefreshColumnResult {
+    /// Rows that had a value computed.
+    #[serde(default)]
+    pub rows_filled: u64,
+    /// The commit version associated with the operation.
+    #[serde(default)]
+    pub version: u64,
+}
+
+struct RefreshExecution {
+    result: RefreshColumnResult,
+    source_version: u64,
+}
+
+/// Internal implementation of the refresh logic.
+pub(crate) async fn execute_refresh_column(
+    table: &NativeTable,
+    column: &str,
+) -> Result<RefreshColumnResult> {
+    Ok(execute_refresh_column_with_source(table, column)
+        .await?
+        .result)
+}
+
+async fn execute_refresh_column_with_source(
+    table: &NativeTable,
+    column: &str,
+) -> Result<RefreshExecution> {
+    table.dataset.ensure_mutable()?;
+    ensure_no_lsm_write_spec(table).await?;
+    let dataset = table.dataset.get().await?;
+
+    let expression = declared_expression(&dataset, column)?;
+    let schema = Arc::new(ArrowSchema::from(dataset.schema()));
+    let bound = Arc::new(super::computed_columns::bind(schema, column, &expression)?);
+    let field = dataset
+        .schema()
+        .field(column)
+        .ok_or_else(|| Error::ColumnNotFound {
+            name: column.to_string(),
+        })?;
+    // The dataset's own field, so the identity write_columns checks against the
+    // manifest holds by construction.
+    let column_schema = LanceSchema {
+        fields: vec![field.clone()],
+        metadata: Default::default(),
+    };
+
+    let mut rows_filled = 0u64;
+    let mut replacements = Vec::new();
+    for fragment in dataset.get_fragments() {
+        let gained = count_fragment_gains(&dataset, &fragment, &bound, column).await?;
+        if gained == 0 {
+            continue;
+        }
+        rows_filled += gained;
+        let values = fill_stream(&dataset, &fragment, bound.clone(), column).await?;
+        replacements.push(fragment.write_columns(values, &column_schema).await?);
+    }
+
+    if replacements.is_empty() {
+        let source_version = dataset.version().version;
+        return Ok(RefreshExecution {
+            result: RefreshColumnResult {
+                rows_filled: 0,
+                version: source_version,
+            },
+            source_version,
+        });
+    }
+
+    let read_version = dataset.version().version;
+    // The dataset's own session, so registrations and caches survive the
+    // commit being installed on the handle.
+    let session = dataset.session();
+    let new_dataset = Dataset::commit(
+        WriteDestination::Dataset(dataset.clone()),
+        Operation::DataReplacement { replacements },
+        Some(read_version),
+        None,
+        None,
+        session,
+        false,
+    )
+    .await?;
+
+    let version = new_dataset.version().version;
+    table.dataset.update(new_dataset);
+    Ok(RefreshExecution {
+        result: RefreshColumnResult {
+            rows_filled,
+            version,
+        },
+        source_version: read_version,
+    })
+}
+
+/// Run the refresh as a [`Job`] in this process.
+pub(crate) async fn execute_refresh_column_async(
+    table: &NativeTable,
+    column: &str,
+) -> Result<Job<crate::function::RefreshColumnResult>> {
+    // Validate before spawning so bad input is reported by this call rather
+    // than only by the job.
+    table.dataset.ensure_mutable()?;
+    ensure_no_lsm_write_spec(table).await?;
+    let dataset = table.dataset.get().await?;
+    declared_expression(&dataset, column)?;
+    drop(dataset);
+
+    let table = table.clone();
+    let column = column.to_string();
+    Ok(Job::spawned(tokio::spawn(async move {
+        let execution = execute_refresh_column_with_source(&table, &column).await?;
+        table.bump_freshness();
+        Ok(crate::function::RefreshColumnResult {
+            rows_assigned: execution.result.rows_filled,
+            rows_failed: 0,
+            rows_remaining: 0,
+            source_version: execution.source_version,
+            published_version: (execution.result.rows_filled > 0)
+                .then_some(execution.result.version),
+        })
+    })))
+}
+
+/// Refuse to refresh under an LSM write spec.
+///
+/// Refresh enumerates base fragments, and a write spec keeps visible rows in
+/// un-compacted MemWAL tiers it cannot reach -- success would silently omit
+/// readable rows.
+async fn ensure_no_lsm_write_spec(table: &NativeTable) -> Result<()> {
+    use lance::dataset::mem_wal::DatasetMemWalExt;
+
+    // Unset drops the MemWAL index, so the spec alone stops describing a table
+    // whose SSTables still hold rows. The shard directories outlive it and are
+    // the durable evidence.
+    let retained_sstables = !table
+        .dataset
+        .get()
+        .await?
+        .list_mem_wal_latest_shard_ids()
+        .await?
+        .is_empty();
+    if retained_sstables || table.get_lsm_write_spec().await?.is_some() {
+        return Err(Error::NotSupported {
+            message: "refresh_column is not supported on a table with an LSM write \
+                      spec: rows in un-compacted tiers are invisible to refresh"
+                .into(),
+        });
+    }
+    Ok(())
+}
+
+/// The SQL expression `column` is declared with.
+fn declared_expression(dataset: &Dataset, column: &str) -> Result<String> {
+    let schema = ArrowSchema::from(dataset.schema());
+    let field = schema
+        .field_with_name(column)
+        .map_err(|_| Error::ColumnNotFound {
+            name: column.to_string(),
+        })?;
+    let declaration =
+        computed_column_from_field(field).ok_or_else(|| Error::NotAComputedColumn {
+            name: column.to_string(),
+        })?;
+    match declaration.kind {
+        ComputedColumnKind::Sql { expression } => Ok(expression),
+        ComputedColumnKind::Function { .. } => Err(Error::NotSupported {
+            message: "registered Function columns are refreshed only by a remote server Job".into(),
+        }),
+        ComputedColumnKind::Unrecognized { kind } => Err(Error::NotSupported {
+            message: format!(
+                "computed column '{column}' is defined by '{kind}', which this version of \
+                 lancedb cannot evaluate"
+            ),
+        }),
+    }
+}
+
+/// Quote `name` as a lance SQL identifier.
+///
+/// Lance's dialect delimits with backticks, so a double-quoted name would
+/// parse as a string literal rather than a column.
+pub(crate) fn quote_identifier(name: &str) -> String {
+    format!("`{}`", name.replace('`', "``"))
+}
+
+/// Assemble the batch evaluation runs against: the bound roots, in read-schema
+/// order. Built by name so scan-side column order never matters.
+fn evaluation_batch(
+    batch: &RecordBatch,
+    bound: &BoundExpression,
+    mask_out: Option<&BooleanArray>,
+) -> lance_core::Result<RecordBatch> {
+    let mut columns = Vec::with_capacity(bound.roots.len());
+    for name in &bound.roots {
+        let column = batch.column_by_name(name).ok_or_else(|| {
+            lance_core::Error::invalid_input(format!(
+                "refreshing a computed column read no {name} column"
+            ))
+        })?;
+        // Rows outside the mask must not reach the expression: a value in a
+        // deleted or already-filled row can be one it would choke on.
+        columns.push(match mask_out {
+            Some(mask) => arrow::compute::nullif(column, mask)?,
+            None => column.clone(),
+        });
+    }
+    Ok(RecordBatch::try_new_with_options(
+        bound.read_schema.clone(),
+        columns,
+        &RecordBatchOptions::new().with_row_count(Some(batch.num_rows())),
+    )?)
+}
+
+/// Evaluate the expression over `batch`, materializing a constant result to
+/// the batch's length.
+fn evaluate(bound: &BoundExpression, batch: &RecordBatch) -> lance_core::Result<ArrayRef> {
+    let value = bound
+        .physical
+        .evaluate(batch)
+        .map_err(lance_core::Error::from)?;
+    match value {
+        ColumnarValue::Array(array) => Ok(array),
+        scalar => scalar
+            .into_array(batch.num_rows())
+            .map_err(lance_core::Error::from),
+    }
+}
+
+/// How many rows of one fragment would gain a value.
+///
+/// Scans only the unfilled live rows -- deleted rows never reach the
+/// expression here, the filter having already excluded them -- and counts the
+/// non-null results. Exact, so it is both the staging decision and the
+/// fragment's contribution to `rows_filled`.
+async fn count_fragment_gains(
+    dataset: &Dataset,
+    fragment: &FileFragment,
+    bound: &BoundExpression,
+    column: &str,
+) -> Result<u64> {
+    let mut scanner = dataset.scan();
+    scanner
+        .with_fragments(vec![fragment.metadata().clone()])
+        .with_row_id()
+        .filter(&format!("{} IS NULL", quote_identifier(column)))?
+        .project(&bound.roots)?;
+
+    let mut gained = 0u64;
+    let mut batches = scanner.try_into_stream().await?;
+    while let Some(batch) = batches.try_next().await? {
+        let evaluated = evaluate(bound, &evaluation_batch(&batch, bound, None)?)?;
+        gained += (batch.num_rows() - evaluated.null_count()) as u64;
+    }
+    Ok(gained)
+}
+
+/// Stream one fragment's column in physical order, filling the unfilled live
+/// rows and keeping every other value.
+///
+/// Deleted rows are carried through so the values line up positionally with
+/// the fragment's data files; they are never read back, but the column file
+/// has to cover them.
+async fn fill_stream(
+    dataset: &Dataset,
+    fragment: &FileFragment,
+    bound: Arc<BoundExpression>,
+    column: &str,
+) -> Result<impl Stream<Item = lance_core::Result<RecordBatch>> + Send + use<>> {
+    let mut projection: Vec<String> = bound.roots.clone();
+    projection.push(column.to_string());
+    let mut scanner = dataset.scan();
+    scanner
+        .with_fragments(vec![fragment.metadata().clone()])
+        .with_row_id()
+        .include_deleted_rows()
+        .project(&projection)?;
+
+    let projected = Arc::new(ArrowSchema::new(vec![
+        ArrowSchema::from(dataset.schema())
+            .field_with_name(column)
+            .map_err(|_| Error::ColumnNotFound {
+                name: column.to_string(),
+            })?
+            .clone(),
+    ]));
+
+    let column = column.to_string();
+    let batches = scanner.try_into_stream().await?;
+    Ok(batches.map(move |batch| {
+        let batch = batch?;
+        let missing = |name: &str| {
+            lance_core::Error::invalid_input(format!(
+                "refreshing a computed column read no {name} column"
+            ))
+        };
+        let existing = batch
+            .column_by_name(&column)
+            .ok_or_else(|| missing(&column))?;
+        let row_ids = batch
+            .column_by_name(ROW_ID)
+            .ok_or_else(|| missing(ROW_ID))?;
+
+        // Only an unfilled live row gains a value; a deleted row has a null
+        // row id and keeps its (null) slot.
+        let unfilled = arrow::compute::is_null(existing.as_ref())?;
+        let live = arrow::compute::is_not_null(row_ids.as_ref())?;
+        let fill = arrow::compute::and(&unfilled, &live)?;
+        let keep = arrow::compute::not(&fill)?;
+
+        let computed = evaluate(&bound, &evaluation_batch(&batch, &bound, Some(&keep))?)?;
+        let merged = arrow_select::zip::zip(&fill, &computed, existing)?;
+        Ok(RecordBatch::try_new(projected.clone(), vec![merged])?)
+    }))
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow_array::{Int32Array, record_batch};
+    use futures::TryStreamExt;
+
+    use crate::connect;
+    use crate::query::{ExecutableQuery, QueryBase, Select};
+    use crate::{Error, Result, Table};
+
+    async fn table_with(name: &str, values: Vec<i32>) -> Table {
+        let conn = connect("memory://").execute().await.unwrap();
+        let batch = record_batch!(("x", Int32, values)).unwrap();
+        conn.create_table(name, batch).execute().await.unwrap()
+    }
+
+    async fn declare_doubled(table: &Table) -> Result<u64> {
+        Ok(table
+            .add_columns()
+            .computed("doubled", "x * 2")
+            .execute()
+            .await?
+            .version)
+    }
+
+    async fn read(table: &Table, column: &str) -> Vec<Option<i32>> {
+        let batches = table
+            .query()
+            .select(Select::columns(&[column]))
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        let mut values: Vec<Option<i32>> = batches
+            .iter()
+            .flat_map(|batch| {
+                batch[column]
+                    .as_any()
+                    .downcast_ref::<Int32Array>()
+                    .unwrap()
+                    .iter()
+                    .collect::<Vec<_>>()
+            })
+            .collect();
+        values.sort();
+        values
+    }
+
+    async fn append(table: &Table, values: Vec<i32>) {
+        let batch = record_batch!(("x", Int32, values)).unwrap();
+        table.add(batch).execute().await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_refresh_fills_a_declared_column() {
+        let table = table_with("refresh_fills", vec![1, 2, 3]).await;
+        let declared = declare_doubled(&table).await.unwrap();
+        assert_eq!(read(&table, "doubled").await, vec![None, None, None]);
+
+        let result = table.refresh_column("doubled").await.unwrap();
+        assert!(result.version > declared);
+        assert_eq!(result.rows_filled, 3);
+        assert_eq!(
+            read(&table, "doubled").await,
+            vec![Some(2), Some(4), Some(6)]
+        );
+
+        let no_op = table
+            .refresh_column_async("doubled")
+            .await
+            .unwrap()
+            .wait()
+            .await
+            .unwrap();
+        assert_eq!(no_op.rows_assigned, 0);
+        assert_eq!(no_op.source_version, 3);
+        assert_eq!(no_op.published_version, None);
+    }
+
+    /// Values written after the last refresh must be reachable by another one.
+    #[tokio::test]
+    async fn test_refresh_fills_rows_appended_since_the_last_refresh() {
+        let table = table_with("refresh_appended", vec![1, 2]).await;
+        declare_doubled(&table).await.unwrap();
+        table.refresh_column("doubled").await.unwrap();
+
+        append(&table, vec![5, 6]).await;
+        assert_eq!(
+            read(&table, "doubled").await,
+            vec![None, None, Some(2), Some(4)]
+        );
+
+        let result = table.refresh_column("doubled").await.unwrap();
+        assert_eq!(result.rows_filled, 2);
+        assert_eq!(
+            read(&table, "doubled").await,
+            vec![Some(2), Some(4), Some(10), Some(12)]
+        );
+    }
+
+    #[tokio::test]
+    async fn test_refresh_with_nothing_to_fill() {
+        let table = table_with("refresh_noop", vec![1, 2, 3]).await;
+        declare_doubled(&table).await.unwrap();
+        table.refresh_column("doubled").await.unwrap();
+
+        let again = table.refresh_column("doubled").await.unwrap();
+        assert_eq!(again.rows_filled, 0);
+        assert_eq!(
+            read(&table, "doubled").await,
+            vec![Some(2), Some(4), Some(6)]
+        );
+    }
+
+    /// A row is filled only by gaining a value, so an expression yielding null
+    /// settles at once instead of re-selecting the same rows forever. Nothing
+    /// is staged, so the version does not move either.
+    #[tokio::test]
+    async fn test_refresh_converges_on_a_null_result() {
+        let table = table_with("refresh_null_result", vec![1, 2, 3]).await;
+        let declared = table
+            .add_columns()
+            .computed("maybe", "nullif(x, x)")
+            .execute()
+            .await
+            .unwrap()
+            .version;
+
+        let first = table.refresh_column("maybe").await.unwrap();
+        assert_eq!(first.rows_filled, 0);
+        assert_eq!(first.version, declared);
+        assert_eq!(read(&table, "maybe").await, vec![None, None, None]);
+
+        let again = table.refresh_column("maybe").await.unwrap();
+        assert_eq!(again.rows_filled, 0);
+        assert_eq!(again.version, declared);
+    }
+
+    /// The contract's boundary: a filled fragment is not revisited, so
+    /// mutating an input leaves the value computed at fill time.
+    #[tokio::test]
+    async fn test_refresh_does_not_observe_input_mutation() {
+        let table = table_with("refresh_mutation", vec![1]).await;
+        declare_doubled(&table).await.unwrap();
+        table.refresh_column("doubled").await.unwrap();
+        assert_eq!(read(&table, "doubled").await, vec![Some(2)]);
+
+        table.update().column("x", "3").execute().await.unwrap();
+
+        let again = table.refresh_column("doubled").await.unwrap();
+        assert_eq!(again.rows_filled, 0);
+        assert_eq!(read(&table, "doubled").await, vec![Some(2)]);
+    }
+
+    /// A row rewrite before the first refresh materializes the declared
+    /// column as null behind a covering data file. Those rows are still
+    /// unfilled and a later refresh has to reach them.
+    #[tokio::test]
+    async fn test_update_before_the_first_refresh() {
+        let table = table_with("refresh_update_first", vec![1]).await;
+        declare_doubled(&table).await.unwrap();
+
+        table.update().column("x", "3").execute().await.unwrap();
+
+        let result = table.refresh_column("doubled").await.unwrap();
+        assert_eq!(result.rows_filled, 1);
+        assert_eq!(read(&table, "doubled").await, vec![Some(6)]);
+    }
+
+    /// The contract holds row by row, not fragment by fragment: revisiting a
+    /// fragment to fill one row must not recompute a filled row sitting beside
+    /// it, even where the input behind it has since changed.
+    #[tokio::test]
+    async fn test_refresh_does_not_recompute_a_filled_row_beside_an_unfilled_one() {
+        let table = table_with("refresh_mixed", vec![1, 2]).await;
+        declare_doubled(&table).await.unwrap();
+        table.refresh_column("doubled").await.unwrap();
+
+        append(&table, vec![5]).await;
+        table
+            .update()
+            .column("x", "100")
+            .only_if("x = 1")
+            .execute()
+            .await
+            .unwrap();
+        table
+            .optimize(crate::table::OptimizeAction::Compact {
+                options: crate::table::CompactionOptions::default(),
+                remap_options: None,
+            })
+            .await
+            .unwrap();
+
+        let result = table.refresh_column("doubled").await.unwrap();
+        assert_eq!(result.rows_filled, 1);
+        // 2 is the mutated row keeping the value it was filled with, not 200.
+        assert_eq!(
+            read(&table, "doubled").await,
+            vec![Some(2), Some(4), Some(10)]
+        );
+    }
+
+    /// Filling a fragment must not disturb the values it already holds, which
+    /// is what makes a compaction-mixed fragment safe to revisit.
+    #[tokio::test]
+    async fn test_refresh_preserves_already_filled_rows() {
+        let table = table_with("refresh_preserves", vec![1, 2]).await;
+        declare_doubled(&table).await.unwrap();
+        table.refresh_column("doubled").await.unwrap();
+
+        append(&table, vec![5]).await;
+        table
+            .optimize(crate::table::OptimizeAction::Compact {
+                options: crate::table::CompactionOptions::default(),
+                remap_options: None,
+            })
+            .await
+            .unwrap();
+
+        let result = table.refresh_column("doubled").await.unwrap();
+        assert_eq!(result.rows_filled, 1);
+        assert_eq!(
+            read(&table, "doubled").await,
+            vec![Some(2), Some(4), Some(10)]
+        );
+    }
+
+    #[tokio::test]
+    async fn test_refresh_leaves_deleted_rows_alone() {
+        let table = table_with("refresh_deleted", vec![1, 2, 3, 4]).await;
+        declare_doubled(&table).await.unwrap();
+        table.delete("x = 2").await.unwrap();
+
+        let result = table.refresh_column("doubled").await.unwrap();
+        assert_eq!(result.rows_filled, 3);
+        assert_eq!(
+            read(&table, "doubled").await,
+            vec![Some(2), Some(6), Some(8)]
+        );
+    }
+
+    #[tokio::test]
+    async fn test_refresh_a_constant_expression() {
+        let table = table_with("refresh_constant", vec![1, 2, 3]).await;
+        table
+            .add_columns()
+            .computed("answer", "42")
+            .execute()
+            .await
+            .unwrap();
+
+        let result = table.refresh_column("answer").await.unwrap();
+        assert_eq!(result.rows_filled, 3);
+    }
+
+    /// A name needing quotes reaches the evaluator intact: it is carried as a
+    /// projection alias, never spliced into SQL text.
+    #[tokio::test]
+    async fn test_refresh_a_column_whose_name_needs_quoting() {
+        let table = table_with("refresh_quoted", vec![1, 2, 3]).await;
+        table
+            .add_columns()
+            .computed("double value", "x * 2")
+            .execute()
+            .await
+            .unwrap();
+
+        let result = table.refresh_column("double value").await.unwrap();
+        assert_eq!(result.rows_filled, 3);
+        assert_eq!(
+            read(&table, "double value").await,
+            vec![Some(2), Some(4), Some(6)]
+        );
+    }
+
+    /// A fragment spanning several scan batches exercises the streamed fill:
+    /// the probe buffers only until the first gained value and the rest flows
+    /// through write_columns a batch at a time.
+    #[tokio::test]
+    async fn test_refresh_streams_a_multi_batch_fragment() {
+        let values: Vec<i32> = (0..20_000).collect();
+        let table = table_with("refresh_multi_batch", values.clone()).await;
+        declare_doubled(&table).await.unwrap();
+
+        let result = table.refresh_column("doubled").await.unwrap();
+        assert_eq!(result.rows_filled, 20_000);
+
+        let read_back = read(&table, "doubled").await;
+        assert_eq!(read_back.len(), 20_000);
+        let mut expected: Vec<Option<i32>> = values.iter().map(|v| Some(v * 2)).collect();
+        expected.sort();
+        assert_eq!(read_back, expected);
+    }
+
+    /// The gate's reproducer: the commit must reuse the configured session,
+    /// or registrations and caches vanish from the handle after a refresh.
+    #[tokio::test]
+    async fn test_refresh_preserves_the_configured_session() {
+        let session = Arc::new(lance::session::Session::default());
+        let conn = crate::connect("memory://")
+            .session(session.clone())
+            .execute()
+            .await
+            .unwrap();
+        let batch = record_batch!(("x", Int32, [1, 2])).unwrap();
+        let table = conn
+            .create_table("session_kept", batch)
+            .execute()
+            .await
+            .unwrap();
+        declare_doubled(&table).await.unwrap();
+        table.refresh_column("doubled").await.unwrap();
+
+        let dataset = table.as_native().unwrap().dataset.get().await.unwrap();
+        assert!(Arc::ptr_eq(&dataset.session(), &session));
+    }
+
+    /// The async form's job settles with the fill visible, like
+    /// create_index's execute_async.
+    #[tokio::test]
+    async fn test_refresh_async_job_waits_for_the_fill() {
+        let table = table_with("refresh_async", vec![1, 2, 3]).await;
+        declare_doubled(&table).await.unwrap();
+
+        let job = table.refresh_column_async("doubled").await.unwrap();
+        assert!(job.id().is_none(), "in-process jobs have no server id");
+        let result = job.wait().await.unwrap();
+        assert_eq!(result.rows_assigned, 3);
+        assert_eq!(result.rows_failed, 0);
+        assert_eq!(result.rows_remaining, 0);
+        assert_eq!(result.source_version, 2);
+        assert_eq!(result.published_version, Some(3));
+        assert_eq!(job.status().await.unwrap(), "finished");
+        assert_eq!(
+            read(&table, "doubled").await,
+            vec![Some(2), Some(4), Some(6)]
+        );
+    }
+
+    /// Bad input is reported by the call, not by the job.
+    #[tokio::test]
+    async fn test_refresh_async_rejects_bad_input_before_spawning() {
+        let table = table_with("refresh_async_bad", vec![1, 2, 3]).await;
+
+        let err = table.refresh_column_async("x").await.unwrap_err();
+        assert!(matches!(err, Error::NotAComputedColumn { name } if name == "x"));
+
+        let err = table.refresh_column_async("nope").await.unwrap_err();
+        assert!(matches!(err, Error::ColumnNotFound { name } if name == "nope"));
+    }
+
+    #[tokio::test]
+    async fn test_refresh_async_job_reports_success_to_every_waiter() {
+        let table = table_with("refresh_async_waiters", vec![1, 2]).await;
+        declare_doubled(&table).await.unwrap();
+
+        let job = table.refresh_column_async("doubled").await.unwrap();
+        let first = job.wait().await.unwrap();
+        // A second wait after completion observes the same outcome.
+        assert_eq!(job.wait().await.unwrap(), first);
+        assert_eq!(job.status().await.unwrap(), "finished");
+    }
+
+    #[tokio::test]
+    async fn test_refresh_rejects_a_plain_column() {
+        let table = table_with("refresh_plain", vec![1, 2, 3]).await;
+        let err = table.refresh_column("x").await.unwrap_err();
+        assert!(matches!(err, Error::NotAComputedColumn { name } if name == "x"));
+    }
+
+    #[tokio::test]
+    async fn test_refresh_rejects_an_unknown_column() {
+        let table = table_with("refresh_missing", vec![1, 2, 3]).await;
+        let err = table.refresh_column("nope").await.unwrap_err();
+        assert!(matches!(err, Error::ColumnNotFound { name } if name == "nope"));
+    }
+
+    /// The gate's reproducer: a poison value in a deleted row must not
+    /// abort filling the live rows, since nobody can read it.
+    #[tokio::test]
+    async fn test_a_deleted_rows_value_is_never_evaluated() {
+        let table = table_with("refresh_deleted_poison", vec![1, 0]).await;
+        table
+            .add_columns()
+            .computed("quotient", "10 / x")
+            .execute()
+            .await
+            .unwrap();
+        table.delete("x = 0").await.unwrap();
+
+        let result = table.refresh_column("quotient").await.unwrap();
+        assert_eq!(result.rows_filled, 1);
+        assert_eq!(read(&table, "quotient").await, vec![Some(10)]);
+    }
+
+    /// The gate's reproducer: an already-filled row's value must not be
+    /// re-evaluated either -- its input may have mutated into one the
+    /// expression chokes on.
+    #[tokio::test]
+    async fn test_a_filled_rows_value_is_never_evaluated() {
+        let table = table_with("refresh_filled_poison", vec![1, 2]).await;
+        table
+            .add_columns()
+            .computed("quotient", "10 / x")
+            .execute()
+            .await
+            .unwrap();
+        table.refresh_column("quotient").await.unwrap();
+
+        table
+            .update()
+            .column("x", "0")
+            .only_if("x = 1")
+            .execute()
+            .await
+            .unwrap();
+        append(&table, vec![5]).await;
+
+        let result = table.refresh_column("quotient").await.unwrap();
+        assert_eq!(result.rows_filled, 1);
+        assert_eq!(
+            read(&table, "quotient").await,
+            vec![Some(2), Some(5), Some(10)]
+        );
+    }
+
+    /// The gate's reproducer: the old internal projection alias is an
+    /// ordinary column name; a computed column may use it.
+    #[tokio::test]
+    async fn test_refresh_a_column_named_like_the_old_alias() {
+        let table = table_with("refresh_alias_name", vec![1, 2]).await;
+        table
+            .add_columns()
+            .computed("__lancedb_computed", "x * 2")
+            .execute()
+            .await
+            .unwrap();
+
+        let result = table.refresh_column("__lancedb_computed").await.unwrap();
+        assert_eq!(result.rows_filled, 2);
+        assert_eq!(
+            read(&table, "__lancedb_computed").await,
+            vec![Some(2), Some(4)]
+        );
+    }
+
+    /// The gate's reproducer: a late-gain fragment (filled, then one null row
+    /// compacted onto the end) fills without the old probe's buffering, which
+    /// this pins behaviorally; the memory bound is structural -- the fill
+    /// stream retains no batches at all.
+    #[tokio::test]
+    async fn test_refresh_fills_a_late_gain_fragment() {
+        let values: Vec<i32> = (0..20_000).collect();
+        let table = table_with("refresh_late_gain", values).await;
+        declare_doubled(&table).await.unwrap();
+        table.refresh_column("doubled").await.unwrap();
+
+        append(&table, vec![2_000_000]).await;
+        table
+            .optimize(crate::table::OptimizeAction::Compact {
+                options: crate::table::CompactionOptions::default(),
+                remap_options: None,
+            })
+            .await
+            .unwrap();
+
+        let result = table.refresh_column("doubled").await.unwrap();
+        assert_eq!(result.rows_filled, 1);
+        let read_back = read(&table, "doubled").await;
+        assert_eq!(read_back.len(), 20_001);
+        assert_eq!(read_back.last().unwrap(), &Some(4_000_000));
+    }
+
+    /// The gate's reproducer: a nested input declares, refreshes, and guards
+    /// its root against invalidating schema changes.
+    #[tokio::test]
+    async fn test_a_nested_input_declares_and_refreshes() {
+        use arrow_array::{Int32Array, StructArray};
+        use arrow_schema::{DataType, Field, Fields};
+
+        let conn = connect("memory://").execute().await.unwrap();
+        let age = Arc::new(Int32Array::from(vec![30, 40]));
+        let fields = Fields::from(vec![Field::new("age", DataType::Int32, true)]);
+        let metadata = StructArray::new(fields.clone(), vec![age as _], None);
+        let schema = Arc::new(arrow_schema::Schema::new(vec![Field::new(
+            "metadata",
+            DataType::Struct(fields),
+            true,
+        )]));
+        let batch =
+            arrow_array::RecordBatch::try_new(schema, vec![Arc::new(metadata) as _]).unwrap();
+        let table = conn
+            .create_table("refresh_nested", batch)
+            .execute()
+            .await
+            .unwrap();
+
+        table
+            .add_columns()
+            .computed("next_age", "metadata.age + 1")
+            .execute()
+            .await
+            .unwrap();
+        let declaration =
+            &crate::table::computed_columns(table.schema().await.unwrap().as_ref())[0];
+        assert_eq!(declaration.inputs, vec!["metadata.age".to_string()]);
+
+        let result = table.refresh_column("next_age").await.unwrap();
+        assert_eq!(result.rows_filled, 2);
+        assert_eq!(read(&table, "next_age").await, vec![Some(31), Some(41)]);
+
+        // The dotted input guards its root.
+        let err = table.drop_columns(&["metadata"]).await.unwrap_err();
+        assert!(
+            matches!(&err, Error::InvalidInput { message } if message.contains("next_age")),
+            "{err:?}"
+        );
+
+        // Masking a struct input for a deleted row goes through the same
+        // nullif path as a primitive; a nested input plus deletions must not
+        // be the combination that breaks it.
+        table.delete("next_age = 31").await.unwrap();
+        append_struct_row(&table, 50).await;
+        let result = table.refresh_column("next_age").await.unwrap();
+        assert_eq!(result.rows_filled, 1);
+        assert_eq!(read(&table, "next_age").await, vec![Some(41), Some(51)]);
+    }
+
+    /// Append one `metadata: {age}` row to the nested-input table.
+    async fn append_struct_row(table: &Table, age: i32) {
+        use arrow_array::{Int32Array, StructArray};
+        use arrow_schema::{DataType, Field, Fields};
+
+        let ages = Arc::new(Int32Array::from(vec![age]));
+        let fields = Fields::from(vec![Field::new("age", DataType::Int32, true)]);
+        let metadata = StructArray::new(fields.clone(), vec![ages as _], None);
+        let schema = Arc::new(arrow_schema::Schema::new(vec![Field::new(
+            "metadata",
+            DataType::Struct(fields),
+            true,
+        )]));
+        let batch =
+            arrow_array::RecordBatch::try_new(schema, vec![Arc::new(metadata) as _]).unwrap();
+        table.add(batch).execute().await.unwrap();
+    }
+
+    /// Both orders of declare+spec are refused at the source (see the
+    /// schema_evolution tests); refresh's own check covers a dataset another
+    /// writer left in that state.
+    #[tokio::test]
+    async fn test_refresh_refuses_a_foreign_lsm_state() {
+        use crate::table::LsmWriteSpec;
+
+        let tmp_dir = tempfile::tempdir().unwrap();
+        let conn = connect(tmp_dir.path().to_str().unwrap())
+            .execute()
+            .await
+            .unwrap();
+        let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
+            "x",
+            arrow_schema::DataType::Int32,
+            false,
+        )]));
+        let batch =
+            arrow_array::RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1]))])
+                .unwrap();
+        let table = conn.create_table("lsm", batch).execute().await.unwrap();
+        table.set_unenforced_primary_key(["x"]).await.unwrap();
+        table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded())
+            .await
+            .unwrap();
+        super::super::computed_columns::add_foreign_kind(&table, "doubled", "sql").await;
+
+        let err = table.refresh_column("doubled").await.unwrap_err();
+        assert!(
+            matches!(&err, Error::NotSupported { message } if message.contains("LSM")),
+            "{err:?}"
+        );
+        let err = table.refresh_column_async("doubled").await.unwrap_err();
+        assert!(matches!(err, Error::NotSupported { .. }));
+    }
+
+    /// After catch-up activation and unset, no spec remains but the catch-up
+    /// flag still marks retained SSTable rows; refresh refuses on the flag.
+    #[tokio::test]
+    async fn test_refresh_refuses_retained_catchup_state() {
+        use crate::table::LsmWriteSpec;
+
+        let tmp_dir = tempfile::tempdir().unwrap();
+        let conn = connect(tmp_dir.path().to_str().unwrap())
+            .execute()
+            .await
+            .unwrap();
+        let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
+            "x",
+            arrow_schema::DataType::Int32,
+            false,
+        )]));
+        let batch = arrow_array::RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int32Array::from(vec![1]))],
+        )
+        .unwrap();
+        let table = conn
+            .create_table("catchup", batch.clone())
+            .execute()
+            .await
+            .unwrap();
+        table.set_unenforced_primary_key(["x"]).await.unwrap();
+        table
+            .set_lsm_write_spec(LsmWriteSpec::unsharded())
+            .await
+            .unwrap();
+        let mut merge = table.merge_insert(&["x"]);
+        merge
+            .when_matched_update_all(None)
+            .when_not_matched_insert_all()
+            .use_lsm(true);
+        merge
+            .execute(Box::new(arrow_array::RecordBatchIterator::new(
+                vec![Ok(batch)],
+                schema,
+            )))
+            .await
+            .unwrap();
+        table.unset_lsm_write_spec().await.unwrap();
+        super::super::computed_columns::add_foreign_kind(&table, "doubled", "sql").await;
+
+        let err = table.refresh_column("doubled").await.unwrap_err();
+        assert!(
+            matches!(&err, Error::NotSupported { message } if message.contains("LSM")),
+            "{err:?}"
+        );
+    }
+
+    /// A declaration of a kind this version cannot evaluate is refused by
+    /// name, rather than mistaken for a plain column or fed to the SQL path.
+    #[tokio::test]
+    async fn test_refresh_rejects_a_kind_it_cannot_evaluate() {
+        let table = table_with("refresh_foreign", vec![1, 2, 3]).await;
+        super::super::computed_columns::add_foreign_kind(&table, "embedding", "udf").await;
+
+        let err = table.refresh_column("embedding").await.unwrap_err();
+        assert!(matches!(err, Error::NotSupported { message } if message.contains("udf")));
+    }
+}
diff --git a/rust/lancedb/src/table/schema_evolution.rs b/rust/lancedb/src/table/schema_evolution.rs
index ce208111a..d10a45eea 100644
--- a/rust/lancedb/src/table/schema_evolution.rs
+++ b/rust/lancedb/src/table/schema_evolution.rs
@@ -8,12 +8,14 @@
 //! - [`alter_columns`](execute_alter_columns): Rename columns, change types, or modify nullability
 //! - [`drop_columns`](execute_drop_columns): Remove columns from the table
 
+use arrow_schema::Schema as ArrowSchema;
 use lance::dataset::{ColumnAlteration, NewColumnTransform};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 
-use super::NativeTable;
-use crate::Result;
+use super::computed_columns;
+use super::{BaseTable, NativeTable};
+use crate::{Error, Result};
 
 /// The result of an add columns operation.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
@@ -98,6 +100,64 @@ pub(crate) async fn execute_add_columns(
     table: &NativeTable,
     transforms: NewColumnTransform,
     read_columns: Option<Vec<String>>,
+) -> Result<AddColumnsResult> {
+    computed_columns::ensure_no_function_bindings_for_mutation(
+        table.schema().await?.as_ref(),
+        "schema evolution",
+    )?;
+    // Declarations are admitted only through [`execute_declare`].
+    match &transforms {
+        NewColumnTransform::AllNulls(schema) => {
+            computed_columns::ensure_no_foreign_declarations(schema.fields())?
+        }
+        NewColumnTransform::BatchUDF(udf) => {
+            computed_columns::ensure_no_foreign_declarations(udf.output_schema.fields())?
+        }
+        _ => {}
+    }
+    commit_add_columns(table, transforms, read_columns).await
+}
+
+/// Declare validated computed columns. The only admission path for
+/// declaration metadata.
+pub(crate) async fn execute_declare(
+    table: &NativeTable,
+    columns: &[(String, String)],
+) -> Result<AddColumnsResult> {
+    use lance::dataset::mem_wal::DatasetMemWalExt;
+
+    // An LSM write spec keeps visible rows in tiers refresh cannot reach;
+    // checked against latest committed state, not this handle's snapshot.
+    table.checkout_latest().await?;
+    computed_columns::ensure_no_function_bindings_for_mutation(
+        table.schema().await?.as_ref(),
+        "schema evolution",
+    )?;
+    // Unset drops the MemWAL index, so the spec alone stops describing a table
+    // whose SSTables still hold rows. The shard directories outlive it and are
+    // the durable evidence.
+    let retained_sstables = !table
+        .dataset
+        .get()
+        .await?
+        .list_mem_wal_latest_shard_ids()
+        .await?
+        .is_empty();
+    if retained_sstables || table.get_lsm_write_spec().await?.is_some() {
+        return Err(Error::NotSupported {
+            message: "computed columns are not supported on a table with an LSM write \
+                      spec: rows in un-compacted tiers are invisible to refresh"
+                .into(),
+        });
+    }
+    let transform = computed_columns::declare(table.schema().await?, columns)?;
+    commit_add_columns(table, transform, None).await
+}
+
+pub(crate) async fn commit_add_columns(
+    table: &NativeTable,
+    transforms: NewColumnTransform,
+    read_columns: Option<Vec<String>>,
 ) -> Result<AddColumnsResult> {
     table.dataset.ensure_mutable()?;
     let mut dataset = (*table.dataset.get().await?).clone();
@@ -116,6 +176,25 @@ pub(crate) async fn execute_alter_columns(
 ) -> Result<AlterColumnsResult> {
     table.dataset.ensure_mutable()?;
     let mut dataset = (*table.dataset.get().await?).clone();
+    // Nullability is not part of what an expression resolves against, so only
+    // a rename or a retype can invalidate a binding.
+    let schema = std::sync::Arc::new(ArrowSchema::from(dataset.schema()));
+    computed_columns::ensure_no_function_bindings_for_mutation(
+        schema.as_ref(),
+        "schema evolution",
+    )?;
+    let rebinding = alterations
+        .iter()
+        .filter(|alteration| alteration.rename.is_some() || alteration.data_type.is_some())
+        .map(|alteration| alteration.path.as_str())
+        .collect::<Vec<_>>();
+    computed_columns::ensure_not_an_input(&schema, &rebinding)?;
+    let retyped = alterations
+        .iter()
+        .filter(|alteration| alteration.data_type.is_some())
+        .map(|alteration| alteration.path.as_str())
+        .collect::<Vec<_>>();
+    computed_columns::ensure_not_retyped(schema.as_ref(), &retyped)?;
     dataset.alter_columns(alterations).await?;
     let version = dataset.version().version;
     table.dataset.update(dataset);
@@ -131,6 +210,14 @@ pub(crate) async fn execute_drop_columns(
 ) -> Result<DropColumnsResult> {
     table.dataset.ensure_mutable()?;
     let mut dataset = (*table.dataset.get().await?).clone();
+    computed_columns::ensure_no_function_bindings_for_mutation(
+        &ArrowSchema::from(dataset.schema()),
+        "schema evolution",
+    )?;
+    computed_columns::ensure_not_an_input(
+        &std::sync::Arc::new(ArrowSchema::from(dataset.schema())),
+        columns,
+    )?;
     dataset.drop_columns(columns).await?;
     let version = dataset.version().version;
     table.dataset.update(dataset);
@@ -147,6 +234,45 @@ pub(crate) async fn execute_update_field_metadata(
     table.dataset.ensure_mutable()?;
     let mut dataset = (*table.dataset.get().await?).clone();
 
+    // A declaration is validated as a whole at declare time; editing its keys
+    // here would bypass that, fabricate one on a plain column, or move a
+    // binding out from under a refresh. A replace on a declared column would
+    // silently erase it.
+    let schema = ArrowSchema::from(dataset.schema());
+    computed_columns::ensure_no_function_bindings_for_mutation(&schema, "schema evolution")?;
+    let declared: Vec<String> = computed_columns::computed_columns(&schema)
+        .into_iter()
+        .map(|declaration| declaration.name)
+        .collect();
+    for update in updates {
+        if update
+            .metadata
+            .keys()
+            .any(|key| computed_columns::is_declaration_key(key))
+        {
+            return Err(Error::InvalidInput {
+                message: format!(
+                    "metadata keys of a computed-column declaration cannot be edited \
+                     (path '{}'); drop the column and declare it again",
+                    update.path
+                ),
+            });
+        }
+        if update.replace
+            && declared
+                .iter()
+                .any(|name| name == computed_columns::root(&update.path))
+        {
+            return Err(Error::InvalidInput {
+                message: format!(
+                    "replacing all metadata of computed column '{}' would erase its \
+                     declaration; drop the column and declare it again",
+                    update.path
+                ),
+            });
+        }
+    }
+
     let mut builder = dataset.update_field_metadata();
     for update in updates {
         let entries = update.metadata.iter().map(|(k, v)| (k.clone(), v.clone()));
diff --git a/rust/lancedb/src/table/update.rs b/rust/lancedb/src/table/update.rs
index 61eb93992..98050dfe8 100644
--- a/rust/lancedb/src/table/update.rs
+++ b/rust/lancedb/src/table/update.rs
@@ -82,6 +82,14 @@ pub(crate) async fn execute_update(
 
     // 1. Snapshot the current dataset
     let dataset = table.dataset.get().await?;
+    super::computed_columns::ensure_no_function_bindings_for_mutation(
+        &arrow_schema::Schema::from(dataset.schema()),
+        "update",
+    )?;
+    super::computed_columns::ensure_not_written(
+        &arrow_schema::Schema::from(dataset.schema()),
+        update.columns.iter().map(|(name, _)| name.as_str()),
+    )?;
 
     // 2. Initialize the Lance Core builder
     let mut builder = LanceUpdateBuilder::new(dataset);
diff --git a/rust/lancedb/tests/blob_integration.rs b/rust/lancedb/tests/blob_integration.rs
index 3d327766e..7b709b645 100644
--- a/rust/lancedb/tests/blob_integration.rs
+++ b/rust/lancedb/tests/blob_integration.rs
@@ -5,12 +5,14 @@ use std::sync::Arc;
 
 use arrow_array::{
     Array, ArrayRef, BinaryArray, Int64Array, LargeBinaryArray, RecordBatch, StringArray,
-    StructArray, UInt64Array,
+    StructArray, UInt64Array, new_null_array,
 };
 use arrow_schema::{DataType, Field, Fields, Schema};
 use futures::TryStreamExt;
 use lance::Dataset;
-use lance_encoding::version::LanceFileVersion;
+use lance::dataset::WriteParams;
+use lance_file::version::{ConcreteFileVersion, LanceFileVersion};
+use lance_table::format::BasePath;
 use lancedb::{
     Connection, Error, Result, Table,
     blob::{BlobRangeRequest, blob},
@@ -19,7 +21,7 @@ use lancedb::{
         ListingDatabaseOptions, NewTableConfig, OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS,
     },
     query::{ExecutableQuery, QueryBase},
-    table::{AddDataMode, CompactionOptions, OptimizeAction, OptimizeStats},
+    table::{AddDataMode, CompactionOptions, OptimizeAction, OptimizeStats, WriteOptions},
 };
 use tempfile::tempdir;
 
@@ -61,7 +63,7 @@ async fn create_inline_blob_table(
     Ok(table)
 }
 
-async fn storage_format_version(table: &Table) -> LanceFileVersion {
+async fn storage_format_version(table: &Table) -> ConcreteFileVersion {
     table
         .as_native()
         .unwrap()
@@ -69,9 +71,14 @@ async fn storage_format_version(table: &Table) -> LanceFileVersion {
         .await
         .unwrap()
         .data_storage_format
-        .lance_file_version()
-        .unwrap()
-        .resolve()
+        .lance_file_format()
+}
+
+fn supports_blob_v2(version: ConcreteFileVersion) -> bool {
+    matches!(
+        version,
+        ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3
+    )
 }
 
 async fn uses_stable_row_ids(table: &Table) -> bool {
@@ -112,7 +119,7 @@ async fn declaring_blob_column_bumps_format_and_enables_stable_row_ids() -> Resu
         .execute()
         .await?;
 
-    assert!(storage_format_version(&table).await >= LanceFileVersion::V2_2);
+    assert!(supports_blob_v2(storage_format_version(&table).await));
     assert!(uses_stable_row_ids(&table).await);
     Ok(())
 }
@@ -127,7 +134,7 @@ async fn explicit_stable_row_id_setting_wins_over_blob_default() -> Result<()> {
         .execute()
         .await?;
 
-    assert!(storage_format_version(&table).await >= LanceFileVersion::V2_2);
+    assert!(supports_blob_v2(storage_format_version(&table).await));
     assert!(!uses_stable_row_ids(&table).await);
     Ok(())
 }
@@ -139,7 +146,7 @@ async fn non_blob_table_keeps_default_format_and_row_id_setting() -> Result<()>
     let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
     let table = db.create_empty_table("t", schema).execute().await?;
 
-    assert!(storage_format_version(&table).await < LanceFileVersion::V2_2);
+    assert!(!supports_blob_v2(storage_format_version(&table).await));
     assert!(!uses_stable_row_ids(&table).await);
     Ok(())
 }
@@ -171,7 +178,7 @@ async fn creating_with_blob_data_bumps_format() -> Result<()> {
     .unwrap();
     let table = db.create_table("t", batch).execute().await?;
 
-    assert!(storage_format_version(&table).await >= LanceFileVersion::V2_2);
+    assert!(supports_blob_v2(storage_format_version(&table).await));
     assert!(uses_stable_row_ids(&table).await);
     assert_eq!(table.count_rows(None).await?, 1);
     Ok(())
@@ -256,11 +263,11 @@ async fn add_rejects_uncoercible_blob_input() -> Result<()> {
     let batch = RecordBatch::try_new(
         Arc::new(Schema::new(vec![
             Field::new("id", DataType::Int64, false),
-            Field::new("image", DataType::Utf8, true),
+            Field::new("image", DataType::Int64, true),
         ])),
         vec![
             Arc::new(Int64Array::from(vec![1])),
-            Arc::new(StringArray::from(vec!["not bytes"])),
+            Arc::new(Int64Array::from(vec![42])),
         ],
     )
     .unwrap();
@@ -281,7 +288,7 @@ async fn connection_level_stable_row_id_setting_wins_over_blob_default() -> Resu
         .execute()
         .await?;
 
-    assert!(storage_format_version(&table).await >= LanceFileVersion::V2_2);
+    assert!(supports_blob_v2(storage_format_version(&table).await));
     assert!(!uses_stable_row_ids(&table).await);
     Ok(())
 }
@@ -297,7 +304,7 @@ async fn namespace_create_applies_blob_defaults() -> Result<()> {
         .execute()
         .await?;
 
-    assert!(storage_format_version(&table).await >= LanceFileVersion::V2_2);
+    assert!(supports_blob_v2(storage_format_version(&table).await));
     assert!(uses_stable_row_ids(&table).await);
     Ok(())
 }
@@ -474,7 +481,7 @@ async fn fetch_blobs_round_trips_nested_blob_column() -> Result<()> {
     let batch = RecordBatch::try_new(schema, vec![Arc::new(info_array) as ArrayRef]).unwrap();
     let table = db.create_table("t", batch).execute().await?;
 
-    assert!(storage_format_version(&table).await >= LanceFileVersion::V2_2);
+    assert!(supports_blob_v2(storage_format_version(&table).await));
     assert!(uses_stable_row_ids(&table).await);
 
     let ids = collect_row_ids(&table).await?;
@@ -1305,7 +1312,7 @@ async fn optimize_preserves_blob_v2_null_and_empty_distinction() -> Result<()> {
         .await?;
     table.add(null_empty_input_batch()).execute().await?;
     assert!(
-        storage_format_version(&table).await >= LanceFileVersion::V2_2,
+        supports_blob_v2(storage_format_version(&table).await),
         "blob v2 columns require storage >= 2.2"
     );
 
@@ -1327,3 +1334,223 @@ async fn optimize_preserves_blob_v2_null_and_empty_distinction() -> Result<()> {
     );
     Ok(())
 }
+
+fn uri_struct_batch(id: i64, uri: &str) -> RecordBatch {
+    let image_field = blob("image", true);
+    let DataType::Struct(child_fields) = image_field.data_type().clone() else {
+        unreachable!("blob field is a struct");
+    };
+    let children: Vec<ArrayRef> = child_fields
+        .iter()
+        .map(|field| match field.name().as_str() {
+            "uri" => Arc::new(StringArray::from(vec![Some(uri)])) as ArrayRef,
+            _ => new_null_array(field.data_type(), 1),
+        })
+        .collect();
+    let image = StructArray::new(child_fields, children, None);
+    RecordBatch::try_new(
+        Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int64, false),
+            image_field,
+        ])),
+        vec![Arc::new(Int64Array::from(vec![id])), Arc::new(image)],
+    )
+    .unwrap()
+}
+
+fn uri_string_batch(id: i64, uri: &str) -> RecordBatch {
+    RecordBatch::try_new(
+        Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int64, false),
+            Field::new("image", DataType::Utf8, true),
+        ])),
+        vec![
+            Arc::new(Int64Array::from(vec![id])),
+            Arc::new(StringArray::from(vec![Some(uri)])),
+        ],
+    )
+    .unwrap()
+}
+
+fn write_payload_file_uri(dir: &std::path::Path, name: &str, payload: &[u8]) -> String {
+    let path = dir.join(name);
+    std::fs::write(&path, payload).unwrap();
+    url::Url::from_file_path(&path).unwrap().to_string()
+}
+
+#[tokio::test]
+async fn external_uri_struct_round_trips_with_flag() -> Result<()> {
+    let tmp = tempdir().unwrap();
+    let db = connect(tmp.path().join("db").to_str().unwrap())
+        .execute()
+        .await?;
+    let payload: &[u8] = b"external-struct-payload";
+    let uri = write_payload_file_uri(tmp.path(), "payload.bin", payload);
+    let table = db
+        .create_empty_table("t", blob_table_schema())
+        .execute()
+        .await?;
+
+    table
+        .add(uri_struct_batch(1, &uri))
+        .allow_external_blob_outside_bases(true)
+        .execute()
+        .await?;
+
+    let ids = collect_row_ids(&table).await?;
+    let bytes = table.fetch_blobs("image", &ids).await?;
+    assert_eq!(bytes.value(0), payload);
+    Ok(())
+}
+
+#[tokio::test]
+async fn external_uri_add_requires_opt_in() -> Result<()> {
+    let tmp = tempdir().unwrap();
+    let db = connect(tmp.path().join("db").to_str().unwrap())
+        .execute()
+        .await?;
+    let uri = write_payload_file_uri(tmp.path(), "payload.bin", b"unreachable");
+    let table = db
+        .create_empty_table("t", blob_table_schema())
+        .execute()
+        .await?;
+
+    let err = table
+        .add(uri_struct_batch(1, &uri))
+        .execute()
+        .await
+        .unwrap_err();
+
+    assert!(
+        err.to_string()
+            .contains("allow_external_blob_outside_bases"),
+        "got: {err}"
+    );
+    assert_eq!(table.count_rows(None).await?, 0);
+    Ok(())
+}
+
+#[tokio::test]
+async fn string_uri_input_round_trips_as_external_reference() -> Result<()> {
+    let tmp = tempdir().unwrap();
+    let db = connect(tmp.path().join("db").to_str().unwrap())
+        .execute()
+        .await?;
+    let payload: &[u8] = b"external-string-payload";
+    let uri = write_payload_file_uri(tmp.path(), "payload.bin", payload);
+    let table = db
+        .create_empty_table("t", blob_table_schema())
+        .execute()
+        .await?;
+
+    table
+        .add(uri_string_batch(1, &uri))
+        .allow_external_blob_outside_bases(true)
+        .execute()
+        .await?;
+
+    let ids = collect_row_ids(&table).await?;
+    let bytes = table.fetch_blobs("image", &ids).await?;
+    assert_eq!(bytes.value(0), payload);
+
+    let files = table.fetch_blob_files("image", &ids).await?;
+    let file = files[0].as_ref().expect("missing blob file");
+    assert_eq!(file.uri(), Some(uri.as_str()));
+    Ok(())
+}
+
+#[tokio::test]
+async fn string_uri_inside_registered_base_does_not_need_the_flag() -> Result<()> {
+    let tmp = tempdir().unwrap();
+    let db_path = tmp.path().join("db");
+    let external_base = tmp.path().join("external_base");
+    let object_dir = external_base.join("objects");
+    std::fs::create_dir_all(&object_dir).unwrap();
+    let payload: &[u8] = b"mapped-in-base";
+    let object_path = object_dir.join("mapped.bin");
+    std::fs::write(&object_path, payload).unwrap();
+    let object_uri = url::Url::from_file_path(&object_path).unwrap().to_string();
+    let base_uri = url::Url::from_file_path(&external_base)
+        .unwrap()
+        .to_string();
+
+    let db = connect(db_path.to_str().unwrap()).execute().await?;
+    let table = db
+        .create_empty_table("t", blob_table_schema())
+        .write_options(WriteOptions {
+            lance_write_params: Some(WriteParams {
+                initial_bases: Some(vec![BasePath {
+                    id: 1,
+                    name: Some("external".to_string()),
+                    path: base_uri,
+                    is_dataset_root: false,
+                }]),
+                ..Default::default()
+            }),
+        })
+        .execute()
+        .await?;
+
+    table
+        .add(uri_string_batch(1, &object_uri))
+        .execute()
+        .await?;
+
+    let ids = collect_row_ids(&table).await?;
+    let bytes = table.fetch_blobs("image", &ids).await?;
+    assert_eq!(bytes.value(0), payload);
+    Ok(())
+}
+
+#[tokio::test]
+async fn external_uri_rows_mix_with_inline_rows() -> Result<()> {
+    let tmp = tempdir().unwrap();
+    let db = connect(tmp.path().join("db").to_str().unwrap())
+        .execute()
+        .await?;
+    let external_payload: &[u8] = b"external-bytes";
+    let uri = write_payload_file_uri(tmp.path(), "payload.bin", external_payload);
+    let table =
+        create_inline_blob_table(&db, "t", &[1], &[Some(b"inline-bytes".as_slice())]).await?;
+
+    table
+        .add(uri_string_batch(2, &uri))
+        .allow_external_blob_outside_bases(true)
+        .execute()
+        .await?;
+
+    let pairs = collect_id_rowid(&table).await?;
+    let row_ids: Vec<u64> = pairs.iter().map(|(_, r)| *r).collect();
+    let bytes = table.fetch_blobs("image", &row_ids).await?;
+    for (i, (id, _)) in pairs.iter().enumerate() {
+        match id {
+            1 => assert_eq!(bytes.value(i), b"inline-bytes"),
+            2 => assert_eq!(bytes.value(i), external_payload),
+            _ => unreachable!(),
+        }
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn malformed_string_uri_is_rejected_at_write() -> Result<()> {
+    let tmp = tempdir().unwrap();
+    let db = connect(tmp.path().join("db").to_str().unwrap())
+        .execute()
+        .await?;
+    let table = db
+        .create_empty_table("t", blob_table_schema())
+        .execute()
+        .await?;
+
+    let err = table
+        .add(uri_string_batch(1, "not a uri"))
+        .allow_external_blob_outside_bases(true)
+        .execute()
+        .await
+        .unwrap_err();
+
+    assert!(err.to_string().contains("not a uri"), "got: {err}");
+    assert_eq!(table.count_rows(None).await?, 0);
+    Ok(())
+}
diff --git a/rust/lancedb/tests/first_class_function_slice1.rs b/rust/lancedb/tests/first_class_function_slice1.rs
new file mode 100644
index 000000000..ce020bd53
--- /dev/null
+++ b/rust/lancedb/tests/first_class_function_slice1.rs
@@ -0,0 +1,144 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+use std::fs;
+use std::path::PathBuf;
+
+use lancedb::function::{
+    FunctionApplication, FunctionBinding, FunctionVersion, RefreshColumnResult,
+};
+use serde_json::Value;
+
+fn fixture(name: &str) -> String {
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("tests/fixtures/first_class_functions/v1")
+        .join(name);
+    fs::read_to_string(path).expect("fixture must be readable")
+}
+
+fn job_result(name: &str) -> Value {
+    serde_json::from_str::<Value>(&fixture(name)).expect("remote Job fixture")["result"].clone()
+}
+
+#[test]
+fn function_version_job_result_matches_shared_canonical_golden() {
+    let result = job_result("remote_function_job.json");
+    let version = FunctionVersion::from_json(&result.to_string()).expect("FunctionVersion result");
+
+    assert_eq!(version.name(), "embed");
+    assert_eq!(version.version(), "fv_01K3EXACT");
+    assert_eq!(version.runtime_digest(), "sha256:runtime");
+    assert_eq!(
+        version.to_canonical_json().expect("canonical JSON"),
+        fixture("remote_function_version.canonical.json").trim()
+    );
+}
+
+#[test]
+fn version_identity_is_immutable_and_exact() {
+    let original = job_result("remote_function_job.json");
+    let version =
+        FunctionVersion::from_json(&original.to_string()).expect("FunctionVersion result");
+    let reopened = version.clone();
+    assert_eq!(reopened, version);
+    assert_eq!(reopened.name(), version.name());
+    assert_eq!(reopened.version(), version.version());
+
+    let mut changed = original;
+    changed["version"] = Value::String("fv_01K3DIFFERENT".to_string());
+    let changed = FunctionVersion::from_json(&changed.to_string()).expect("changed version");
+    assert_ne!(changed, version);
+}
+
+#[test]
+fn application_and_binding_match_shared_remote_goldens() {
+    let application = FunctionApplication::from_json(&fixture("remote_function_application.json"))
+        .expect("application fixture");
+    assert_eq!(application.function().version, "fv_01K3TEXT");
+    assert_eq!(application.output().kind, "named_struct");
+    assert_eq!(application.inputs().len(), 2);
+    assert_eq!(
+        application.to_canonical_json().expect("canonical JSON"),
+        fixture("remote_function_application.canonical.json").trim()
+    );
+
+    let binding = FunctionBinding::from_json(&fixture("remote_function_binding.json"))
+        .expect("binding fixture");
+    assert_eq!(binding.function().version, "fv_01K3TEXT");
+    assert_eq!(binding.outputs()[0].output_ordinal, 0);
+    assert_eq!(binding.outputs()[1].output_ordinal, 1);
+    assert!(binding.input_schema().is_some());
+    assert!(binding.output_schema().is_some());
+    assert_eq!(
+        binding.to_canonical_json().expect("canonical JSON"),
+        fixture("remote_function_binding.canonical.json").trim()
+    );
+}
+
+#[test]
+fn refresh_job_result_matches_shared_canonical_golden() {
+    let result = job_result("remote_refresh_job.json");
+    let result = RefreshColumnResult::from_json(&result.to_string()).expect("refresh result");
+    assert_eq!(result.rows_assigned, 999_998_800);
+    assert_eq!(result.rows_filled(), result.rows_assigned);
+    assert_eq!(result.version(), result.published_version);
+    assert_eq!(
+        result.to_canonical_json().expect("canonical JSON"),
+        fixture("remote_refresh_result.canonical.json").trim()
+    );
+
+    let result = RefreshColumnResult::from_json(&fixture(
+        "remote_refresh_result_without_published_version.json",
+    ))
+    .expect("optional version");
+    assert_eq!(result.published_version, None);
+    assert_eq!(
+        result
+            .to_canonical_json()
+            .expect("canonical result without version"),
+        fixture("remote_refresh_result_without_published_version.canonical.json").trim()
+    );
+    assert_eq!(
+        RefreshColumnResult::from_json(
+            &result
+                .to_canonical_json()
+                .expect("canonical result without version")
+        )
+        .expect("round-trip result without version"),
+        result
+    );
+}
+
+#[test]
+fn unknown_fields_and_discriminators_are_forward_decodable() {
+    let mut result = job_result("remote_function_job.json");
+    result["future_version_metadata"] = serde_json::json!({"retention_class": "catalog"});
+    result["runtime"] = serde_json::json!({
+        "kind": "wasm",
+        "module_digest": "sha256:wasm"
+    });
+    result["signature"]["output"]["kind"] = Value::String("future_output_shape".to_string());
+
+    let version = FunctionVersion::from_json(&result.to_string()).expect("future remote value");
+    assert_eq!(version.runtime().kind(), "wasm");
+    assert_eq!(version.runtime().python_version(), None);
+    assert_eq!(version.signature().output.kind, "future_output_shape");
+    assert_eq!(
+        serde_json::from_str::<Value>(
+            &version.to_canonical_json().expect("canonical future value")
+        )
+        .expect("canonical JSON")["runtime"],
+        serde_json::json!({"kind": "wasm"})
+    );
+}
+
+#[test]
+fn floating_point_application_literals_are_rejected_consistently() {
+    let error = FunctionApplication::from_json(&fixture("remote_function_application_float.json"))
+        .unwrap_err();
+    assert!(
+        error
+            .to_string()
+            .contains("floating-point Function literals")
+    );
+}
diff --git a/rust/lancedb/tests/first_class_function_slice2.rs b/rust/lancedb/tests/first_class_function_slice2.rs
new file mode 100644
index 000000000..93252dde4
--- /dev/null
+++ b/rust/lancedb/tests/first_class_function_slice2.rs
@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+use std::fs;
+use std::path::PathBuf;
+
+use lancedb::Error;
+use lancedb::function::FunctionRegistrationRequest;
+
+fn fixture(name: &str) -> String {
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("tests/fixtures/first_class_functions/v1")
+        .join(name);
+    fs::read_to_string(path).expect("fixture must be readable")
+}
+
+#[test]
+fn registration_request_matches_shared_canonical_golden() {
+    let request = FunctionRegistrationRequest::from_json(&fixture(
+        "remote_function_registration_request.json",
+    ))
+    .expect("registration request");
+    assert_eq!(request.name, "normalize_score");
+    assert_eq!(request.artifact.adapter.kind, "scalar_to_arrow_batch");
+    assert_eq!(
+        request.to_canonical_json().expect("canonical request"),
+        fixture("remote_function_registration_request.canonical.json").trim()
+    );
+}
+
+#[tokio::test]
+async fn local_function_catalog_operations_return_stable_not_supported() {
+    let directory = tempfile::tempdir().unwrap();
+    let connection = lancedb::connect(directory.path().to_str().unwrap())
+        .execute()
+        .await
+        .unwrap();
+    let request = FunctionRegistrationRequest::from_json(&fixture(
+        "remote_function_registration_request.json",
+    ))
+    .unwrap();
+
+    let create_error = connection.create_function_async(request).await.unwrap_err();
+    let lookup_error = connection
+        .get_function("normalize_score", "fv_exact")
+        .await
+        .unwrap_err();
+    for error in [create_error, lookup_error] {
+        assert!(matches!(
+            error,
+            Error::NotSupported { message }
+                if message == "Function catalog operations are not supported by this database"
+        ));
+    }
+}
diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json
new file mode 100644
index 000000000..ff26e4c08
--- /dev/null
+++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json
@@ -0,0 +1,333 @@
+{
+  "valid": [
+    {
+      "arrow_type": "bool",
+      "json": {
+        "type": "bool"
+      }
+    },
+    {
+      "arrow_type": "int8",
+      "json": {
+        "type": "int8"
+      }
+    },
+    {
+      "arrow_type": "int16",
+      "json": {
+        "type": "int16"
+      }
+    },
+    {
+      "arrow_type": "int32",
+      "json": {
+        "type": "int32"
+      }
+    },
+    {
+      "arrow_type": "int64",
+      "json": {
+        "type": "int64"
+      }
+    },
+    {
+      "arrow_type": "uint8",
+      "json": {
+        "type": "uint8"
+      }
+    },
+    {
+      "arrow_type": "uint16",
+      "json": {
+        "type": "uint16"
+      }
+    },
+    {
+      "arrow_type": "uint32",
+      "json": {
+        "type": "uint32"
+      }
+    },
+    {
+      "arrow_type": "uint64",
+      "json": {
+        "type": "uint64"
+      }
+    },
+    {
+      "arrow_type": "float16",
+      "json": {
+        "type": "float16"
+      }
+    },
+    {
+      "arrow_type": "float32",
+      "json": {
+        "type": "float32"
+      }
+    },
+    {
+      "arrow_type": "float64",
+      "json": {
+        "type": "float64"
+      }
+    },
+    {
+      "arrow_type": "utf8",
+      "json": {
+        "type": "utf8"
+      }
+    },
+    {
+      "arrow_type": "binary",
+      "json": {
+        "type": "binary"
+      }
+    },
+    {
+      "arrow_type": "date32",
+      "json": {
+        "type": "date32"
+      }
+    },
+    {
+      "arrow_type": "date64",
+      "json": {
+        "type": "date64"
+      }
+    },
+    {
+      "arrow_type": "list<float32>",
+      "json": {
+        "type": "list",
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "float32"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "large_list<float32>",
+      "json": {
+        "type": "large_list",
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "float32"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "list<int64>",
+      "json": {
+        "type": "list",
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "int64"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "large_list<int64>",
+      "json": {
+        "type": "large_list",
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "int64"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "list<utf8>",
+      "json": {
+        "type": "list",
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "utf8"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "large_list<utf8>",
+      "json": {
+        "type": "large_list",
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "utf8"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "fixed_size_list<float32, 384>",
+      "json": {
+        "type": "fixed_size_list",
+        "length": 384,
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "float32"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "fixed_size_list<float16, 8>",
+      "json": {
+        "type": "fixed_size_list",
+        "length": 8,
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "float16"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "fixed_size_list<uint8, 1>",
+      "json": {
+        "type": "fixed_size_list",
+        "length": 1,
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "uint8"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "list<list<float32>>",
+      "json": {
+        "type": "list",
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "list",
+              "fields": [
+                {
+                  "name": "item",
+                  "nullable": false,
+                  "type": {
+                    "type": "float32"
+                  }
+                }
+              ]
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "fixed_size_list<list<int32>, 2>",
+      "json": {
+        "type": "fixed_size_list",
+        "length": 2,
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "list",
+              "fields": [
+                {
+                  "name": "item",
+                  "nullable": false,
+                  "type": {
+                    "type": "int32"
+                  }
+                }
+              ]
+            }
+          }
+        ]
+      }
+    },
+    {
+      "arrow_type": "list<fixed_size_list<float32, 3>>",
+      "json": {
+        "type": "list",
+        "fields": [
+          {
+            "name": "item",
+            "nullable": false,
+            "type": {
+              "type": "fixed_size_list",
+              "length": 3,
+              "fields": [
+                {
+                  "name": "item",
+                  "nullable": false,
+                  "type": {
+                    "type": "float32"
+                  }
+                }
+              ]
+            }
+          }
+        ]
+      }
+    }
+  ],
+  "server_only": [
+    {
+      "arrow_type": "null",
+      "json": {
+        "type": "null"
+      }
+    }
+  ],
+  "invalid": [
+    "",
+    "list<>",
+    "list<float32",
+    "fixed_size_list<float32>[3]",
+    "fixed_size_list<float32>",
+    "fixed_size_list<float32, 0>",
+    "fixed_size_list<float32, x>",
+    "map<utf8, int32>",
+    "decimal128(10, 2)",
+    "timestamp[us]",
+    "struct<a: int32>"
+  ]
+}
\ No newline at end of file
diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json
new file mode 100644
index 000000000..fd3944412
--- /dev/null
+++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json
@@ -0,0 +1,78 @@
+{
+  "new_columns": [
+    {
+      "name": "embedding",
+      "all_null": true
+    }
+  ],
+  "function": {
+    "application": {
+      "function": {
+        "name": "embed",
+        "version": "fv_01K3EXACT"
+      },
+      "inputs": [
+        {
+          "parameter": "text",
+          "kind": "column",
+          "value": {
+            "path": "description"
+          }
+        }
+      ],
+      "output": {
+        "kind": "scalar",
+        "arrow_type": "fixed_size_list<float32, 3>",
+        "nullable": false
+      }
+    },
+    "binding_metadata_version": 1,
+    "input_bindings": [
+      {
+        "parameter": "text",
+        "field_path": "description",
+        "arrow_type": "utf8",
+        "nullable": true
+      }
+    ],
+    "input_schema": {
+      "fields": [
+        {
+          "name": "text",
+          "nullable": true,
+          "type": {
+            "type": "utf8"
+          }
+        }
+      ]
+    },
+    "output_schema": {
+      "fields": [
+        {
+          "name": "embedding",
+          "nullable": true,
+          "type": {
+            "type": "fixed_size_list",
+            "length": 3,
+            "fields": [
+              {
+                "name": "item",
+                "nullable": false,
+                "type": {
+                  "type": "float32"
+                }
+              }
+            ]
+          }
+        }
+      ]
+    },
+    "outputs": [
+      {
+        "result_field": "$value",
+        "output_name": "embedding",
+        "output_ordinal": 0
+      }
+    ]
+  }
+}
diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.canonical.json
new file mode 100644
index 000000000..b91dd1061
--- /dev/null
+++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.canonical.json
@@ -0,0 +1 @@
+{"columns":{"normalized_text":"search_text","token_count":"search_token_count"},"function":{"name":"text_features","version":"fv_01K3TEXT"},"inputs":[{"kind":"column","parameter":"title","value":{"path":"title"}},{"kind":"column","parameter":"body","value":{"path":"body"}}],"output":{"fields":[{"arrow_type":"utf8","name":"normalized_text","nullable":false},{"arrow_type":"int64","name":"token_count","nullable":false}],"kind":"named_struct"}}
diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.json
new file mode 100644
index 000000000..177821aaa
--- /dev/null
+++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.json
@@ -0,0 +1,19 @@
+{
+  "function": {"name": "text_features", "version": "fv_01K3TEXT"},
+  "inputs": [
+    {"parameter": "title", "kind": "column", "value": {"path": "title"}},
+    {"parameter": "body", "kind": "column", "value": {"path": "body"}}
+  ],
+  "output": {
+    "kind": "named_struct",
+    "fields": [
+      {"name": "normalized_text", "arrow_type": "utf8", "nullable": false},
+      {"name": "token_count", "arrow_type": "int64", "nullable": false}
+    ]
+  },
+  "columns": {
+    "normalized_text": "search_text",
+    "token_count": "search_token_count"
+  },
+  "future_application": {"declaration_mode": "managed"}
+}
diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application_float.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application_float.json
new file mode 100644
index 000000000..c23c8f3ca
--- /dev/null
+++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application_float.json
@@ -0,0 +1,7 @@
+{
+  "function": {"name": "score", "version": "fv_01K3FLOAT"},
+  "inputs": [
+    {"parameter": "threshold", "kind": "literal", "value": 1e-7}
+  ],
+  "output": {"kind": "scalar", "arrow_type": "bool", "nullable": false}
+}
diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.canonical.json
new file mode 100644
index 000000000..143190f78
--- /dev/null
+++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.canonical.json
@@ -0,0 +1 @@
+{"binding_id":"fb_01K3TEXT","function":{"name":"text_features","version":"fv_01K3TEXT"},"input_schema":{"fields":[{"name":"title","nullable":true,"type":{"type":"utf8"}},{"name":"body","nullable":true,"type":{"type":"utf8"}}]},"inputs":[{"arrow_type":"utf8","field_id":11,"field_path":"title","nullable":true,"parameter":"title"},{"arrow_type":"utf8","field_id":12,"field_path":"body","nullable":true,"parameter":"body"}],"output_schema":{"fields":[{"name":"search_text","nullable":true,"type":{"type":"utf8"}},{"name":"search_token_count","nullable":true,"type":{"type":"int64"}}]},"outputs":[{"arrow_type":"utf8","nullable":false,"output_field_id":21,"output_name":"search_text","output_ordinal":0,"result_field":"normalized_text"},{"arrow_type":"int64","nullable":false,"output_field_id":22,"output_name":"search_token_count","output_ordinal":1,"result_field":"token_count"}]}
diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.json
new file mode 100644
index 000000000..dec3fa3f8
--- /dev/null
+++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.json
@@ -0,0 +1,25 @@
+{
+  "binding_id": "fb_01K3TEXT",
+  "function": {"name": "text_features", "version": "fv_01K3TEXT"},
+  "inputs": [
+    {"parameter": "title", "field_id": 11, "field_path": "title", "arrow_type": "utf8", "nullable": true},
+    {"parameter": "body", "field_id": 12, "field_path": "body", "arrow_type": "utf8", "nullable": true}
+  ],
+  "outputs": [
+    {"result_field": "normalized_text", "output_name": "search_text", "output_field_id": 21, "output_ordinal": 0, "arrow_type": "utf8", "nullable": false},
+    {"result_field": "token_count", "output_name": "search_token_count", "output_field_id": 22, "output_ordinal": 1, "arrow_type": "int64", "nullable": false}
+  ],
+  "input_schema": {
+    "fields": [
+      {"name": "title", "nullable": true, "type": {"type": "utf8"}},
+      {"name": "body", "nullable": true, "type": {"type": "utf8"}}
+    ]
+  },
+  "output_schema": {
+    "fields": [
+      {"name": "search_text", "nullable": true, "type": {"type": "utf8"}},
+      {"name": "search_token_count", "nullable": true, "type": {"type": "int64"}}
+    ]
+  },
+  "future_binding": {"mode": "managed"}
+}
diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json
new file mode 100644
index 000000000..39a279692
--- /dev/null
+++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json
@@ -0,0 +1,30 @@
+{
+  "job_id": "job_function_01K3",
+  "job_type": "create_function",
+  "job_state": "DONE",
+  "creation_ms": 1787270400000,
+  "spec": {"name": "embed"},
+  "result": {
+    "name": "embed",
+    "version": "fv_01K3EXACT",
+    "artifact": {
+      "kind": "python_callable",
+      "digest": "sha256:code",
+      "entrypoint": "embed"
+    },
+    "signature": {
+      "inputs": [{"name": "text", "arrow_type": "utf8", "nullable": true}],
+      "output": {"kind": "scalar", "arrow_type": "list<float32>", "nullable": false}
+    },
+    "runtime": {
+      "kind": "python",
+      "python_version": "3.12",
+      "environment": {"kind": "pip", "packages": ["sentence-transformers>=3"]},
+      "env": {"TOKENIZERS_PARALLELISM": "false"}
+    },
+    "runtime_digest": "sha256:runtime",
+    "environment_digest": "sha256:environment",
+    "created_at": "2026-08-21T00:00:00Z"
+  },
+  "future_job": {"trace_id": "trace-1"}
+}
diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_registration_request.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_registration_request.canonical.json
new file mode 100644
index 000000000..a2f2c4c21
--- /dev/null
+++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_registration_request.canonical.json
@@ -0,0 +1 @@
+{"artifact":{"adapter":{"kind":"scalar_to_arrow_batch","version":1},"content":{"data":"ZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKZGVmIG5vcm1hbGl6ZV9zY29yZSh2YWx1ZTogZmxvYXQpIC0+IGZsb2F0OgogICAgcmV0dXJuIHZhbHVlIC8gMTAwLjAK","encoding":"base64"},"digest":"sha256:760784bdcef57b802f389b97804cc0b618aae39e86044733451bcd0b13089a7f","entrypoint":"normalize_score","kind":"python_callable"},"name":"normalize_score","runtime":{"env":{"MODE":"test"},"environment":{"kind":"pip","packages":["numpy>=2"]},"kind":"python","python_version":"3.12"},"signature":{"inputs":[{"arrow_type":"float64","name":"value","nullable":false}],"output":{"arrow_type":"float64","kind":"scalar","nullable":false}}}
diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_registration_request.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_registration_request.json
new file mode 100644
index 000000000..092d76dc2
--- /dev/null
+++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_registration_request.json
@@ -0,0 +1,43 @@
+{
+  "name": "normalize_score",
+  "artifact": {
+    "kind": "python_callable",
+    "digest": "sha256:760784bdcef57b802f389b97804cc0b618aae39e86044733451bcd0b13089a7f",
+    "entrypoint": "normalize_score",
+    "content": {
+      "encoding": "base64",
+      "data": "ZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKZGVmIG5vcm1hbGl6ZV9zY29yZSh2YWx1ZTogZmxvYXQpIC0+IGZsb2F0OgogICAgcmV0dXJuIHZhbHVlIC8gMTAwLjAK"
+    },
+    "adapter": {
+      "kind": "scalar_to_arrow_batch",
+      "version": 1
+    }
+  },
+  "signature": {
+    "inputs": [
+      {
+        "name": "value",
+        "arrow_type": "float64",
+        "nullable": false
+      }
+    ],
+    "output": {
+      "kind": "scalar",
+      "arrow_type": "float64",
+      "nullable": false
+    }
+  },
+  "runtime": {
+    "kind": "python",
+    "python_version": "3.12",
+    "environment": {
+      "kind": "pip",
+      "packages": [
+        "numpy>=2"
+      ]
+    },
+    "env": {
+      "MODE": "test"
+    }
+  }
+}
diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json
new file mode 100644
index 000000000..2670ad0b2
--- /dev/null
+++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json
@@ -0,0 +1 @@
+{"artifact":{"digest":"sha256:code","entrypoint":"embed","kind":"python_callable"},"created_at":"2026-08-21T00:00:00Z","environment_digest":"sha256:environment","name":"embed","runtime":{"env":{"TOKENIZERS_PARALLELISM":"false"},"environment":{"kind":"pip","packages":["sentence-transformers>=3"]},"kind":"python","python_version":"3.12"},"runtime_digest":"sha256:runtime","signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list<float32>","kind":"scalar","nullable":false}},"version":"fv_01K3EXACT"}
diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_multi_output_declaration_request.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_multi_output_declaration_request.json
new file mode 100644
index 000000000..c4ef5009f
--- /dev/null
+++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_multi_output_declaration_request.json
@@ -0,0 +1,44 @@
+{
+  "new_columns": [
+    {"name": "search_text", "all_null": true},
+    {"name": "token_count", "all_null": true}
+  ],
+  "function": {
+    "application": {
+      "function": {"name": "text_features", "version": "fv_01K3TEXT"},
+      "inputs": [
+        {"parameter": "title", "kind": "column", "value": {"path": "title"}},
+        {"parameter": "body", "kind": "column", "value": {"path": "body"}}
+      ],
+      "output": {
+        "kind": "named_struct",
+        "fields": [
+          {"name": "normalized_text", "arrow_type": "utf8", "nullable": false},
+          {"name": "token_count", "arrow_type": "int64", "nullable": false}
+        ]
+      },
+      "columns": {"normalized_text": "search_text"}
+    },
+    "binding_metadata_version": 1,
+    "input_bindings": [
+      {"parameter": "title", "field_path": "title", "arrow_type": "utf8", "nullable": true},
+      {"parameter": "body", "field_path": "body", "arrow_type": "utf8", "nullable": true}
+    ],
+    "input_schema": {
+      "fields": [
+        {"name": "title", "nullable": true, "type": {"type": "utf8"}},
+        {"name": "body", "nullable": true, "type": {"type": "utf8"}}
+      ]
+    },
+    "output_schema": {
+      "fields": [
+        {"name": "search_text", "nullable": true, "type": {"type": "utf8"}},
+        {"name": "token_count", "nullable": true, "type": {"type": "int64"}}
+      ]
+    },
+    "outputs": [
+      {"result_field": "normalized_text", "output_name": "search_text", "output_ordinal": 0},
+      {"result_field": "token_count", "output_name": "token_count", "output_ordinal": 1}
+    ]
+  }
+}
diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_refresh_job.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_refresh_job.json
new file mode 100644
index 000000000..c2d49827d
--- /dev/null
+++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_refresh_job.json
@@ -0,0 +1,15 @@
+{
+  "job_id": "job_refresh_01K3",
+  "job_type": "refresh_function_columns",
+  "job_state": "DONE",
+  "creation_ms": 1787270400001,
+  "spec": {"table": "documents", "binding_id": "fb_01K3TEXT"},
+  "result": {
+    "rows_assigned": 999998800,
+    "rows_failed": 0,
+    "rows_remaining": 0,
+    "source_version": 812,
+    "published_version": 919,
+    "future_result": {"committed_fragment_groups": 100}
+  }
+}
diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_refresh_result.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_refresh_result.canonical.json
new file mode 100644
index 000000000..cda9287a3
--- /dev/null
+++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_refresh_result.canonical.json
@@ -0,0 +1 @@
+{"published_version":919,"rows_assigned":999998800,"rows_failed":0,"rows_remaining":0,"source_version":812}
diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_refresh_result_without_published_version.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_refresh_result_without_published_version.canonical.json
new file mode 100644
index 000000000..69c9c96c1
--- /dev/null
+++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_refresh_result_without_published_version.canonical.json
@@ -0,0 +1 @@
+{"rows_assigned":120,"rows_failed":0,"rows_remaining":0,"source_version":812}
diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_refresh_result_without_published_version.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_refresh_result_without_published_version.json
new file mode 100644
index 000000000..27231411e
--- /dev/null
+++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_refresh_result_without_published_version.json
@@ -0,0 +1,6 @@
+{
+  "rows_assigned": 120,
+  "rows_failed": 0,
+  "rows_remaining": 0,
+  "source_version": 812
+}
diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_scalar_declaration_request.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_scalar_declaration_request.json
new file mode 100644
index 000000000..357834de0
--- /dev/null
+++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_scalar_declaration_request.json
@@ -0,0 +1,40 @@
+{
+  "new_columns": [
+    {"name": "embedding", "all_null": true}
+  ],
+  "function": {
+    "application": {
+      "function": {"name": "embed", "version": "fv_01K3EXACT"},
+      "inputs": [
+        {"parameter": "text", "kind": "column", "value": {"path": "description"}}
+      ],
+      "output": {"kind": "scalar", "arrow_type": "list<float32>", "nullable": false}
+    },
+    "binding_metadata_version": 1,
+    "input_bindings": [
+      {"parameter": "text", "field_path": "description", "arrow_type": "utf8", "nullable": true}
+    ],
+    "input_schema": {
+      "fields": [
+        {"name": "text", "nullable": true, "type": {"type": "utf8"}}
+      ]
+    },
+    "output_schema": {
+      "fields": [
+        {
+          "name": "embedding",
+          "nullable": true,
+          "type": {
+            "type": "list",
+            "fields": [
+              {"name": "item", "nullable": false, "type": {"type": "float32"}}
+            ]
+          }
+        }
+      ]
+    },
+    "outputs": [
+      {"result_field": "$value", "output_name": "embedding", "output_ordinal": 0}
+    ]
+  }
+}
diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_unit_job.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_unit_job.json
new file mode 100644
index 000000000..7f2b4c6de
--- /dev/null
+++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_unit_job.json
@@ -0,0 +1,9 @@
+{
+  "job_id": "job_index_01K3",
+  "job_type": "create_index",
+  "job_state": "DONE",
+  "creation_ms": 1787270400002,
+  "spec": {"column": "vector"},
+  "result": {"future_information": "ignored by Job<()>"},
+  "future_job": {"trace_id": "trace-2"}
+}