Bump version: 0.39.0-beta.3 → 0.39.0-beta.4

chore: update lance dependency to v12.0.0-beta.14 (#4141 )
Update the Rust workspace Lance dependencies and Java lance-core from v12.0.0-beta.11 to [v12.0.0-beta.14](https://github.com/lance-format/lance/releases/tag/v12.0.0-beta.14), refreshing Cargo.lock. Resolve two Clippy diagnostics by making an internal Node.js helper private and using a byte string literal in a remote-table test fixture. Validation: `cargo clippy --quiet --workspace --tests --all-features -- -D warnings`, `cargo fmt --all --quiet`, `git diff --check`, and `pnpm build` in nodejs. --------- Co-authored-by: Jack Ye <yezhaoqin@gmail.com>
2026-09-09 14:52:25 +00:00 · 2026-09-06 20:46:25 +00:00 · 2026-09-06 13:43:35 -07:00 · 2026-09-06 09:41:24 -07:00 · 2026-09-06 08:59:56 +00:00 · 2026-09-06 01:58:40 -07:00
133 changed files with 14224 additions and 13151 deletions
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.38.0-beta.11"
+current_version = "0.39.0-beta.4"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
@@ -44,3 +44,27 @@ updates:
      python-deps:
        patterns:
          - "*"
+
+  # The npm ecosystem covers pnpm lockfiles. There are two separate installs:
+  # the bindings themselves and the examples, which have their own lockfile.
+  # As with cargo and pip above, only bump the lockfile — the version ranges
+  # in package.json are our consumers' constraints, not ours.
+  - package-ecosystem: npm
+    directory: /nodejs
+    schedule:
+      interval: weekly
+    versioning-strategy: lockfile-only
+    groups:
+      nodejs-deps:
+        patterns:
+          - "*"
+
+  - package-ecosystem: npm
+    directory: /nodejs/examples
+    schedule:
+      interval: weekly
+    versioning-strategy: lockfile-only
+    groups:
+      nodejs-examples-deps:
+        patterns:
+          - "*"
@@ -29,12 +29,14 @@ jobs:
    steps:
      - uses: actions/setup-node@v6
        with:
-          node-version: "18"
+          node-version: "24"
+      - uses: pnpm/action-setup@v6
+        with:
+          version: 11.1.1
      # These rules are disabled because Github will always ensure there
      # is a blank line between the title and the body and Github will
      # word wrap the description field to ensure a reasonable max line
      # length.
-      - run: npm install @commitlint/config-conventional
      - run: >
          echo 'module.exports = {
            "rules": {
@@ -43,7 +45,11 @@ jobs:
              "body-leading-blank": [0, "always"]
            }
          }' > .commitlintrc.js
-      - run: npx commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
+      - run: >
+          pnpm dlx
+          --package @commitlint/cli@21.2.2
+          --package @commitlint/config-conventional@21.2.2
+          commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
        env:
          COMMIT_MSG: >
            ${{ github.event.pull_request.title }}
@@ -54,7 +60,7 @@ jobs:
        with:
          script: |
            const message = `**ACTION NEEDED**
-              
+
              Lance follows the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) for release automation.

              The PR title and description are used as the merge commit message.\
@@ -56,7 +56,7 @@ jobs:
        uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0
        with:
          # Restricted to http(s) on purpose. Much of docs/src is generated
-          # API reference (the js/ tree comes from `npm run docs` in nodejs)
+          # API reference (the js/ tree comes from `pnpm run docs` in nodejs)
          # and the hand-written pages use mkdocstrings cross-references and
          # nav-relative paths that only resolve in the site mkdocs builds,
          # not in this checkout, so relative links would be reported as
@@ -55,9 +55,7 @@ jobs:
      - name: Set up node
        uses: actions/setup-node@v6
        with:
-          node-version: 20
-          cache: 'npm'
-          cache-dependency-path: docs/package-lock.json
+          node-version: 24
      - name: Install node dependencies
        working-directory: nodejs
        run: |
@@ -47,9 +47,8 @@ jobs:
        version: 11.1.1
    - uses: actions/setup-node@v6
      with:
-        # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
-        # in October. The library itself still supports Node >= 18
-        # (see test matrix below).
+        # Build on a supported LTS; the matrix job below covers every
+        # Node version the library claims to support.
        node-version: 24
        cache: 'pnpm'
        cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -84,7 +83,7 @@ jobs:
    timeout-minutes: 30
    strategy:
      matrix:
-        node-version: [ "18", "20" ]
+        node-version: [ "22", "24", "26" ]
    runs-on: "ubuntu-22.04"
    defaults:
      run:
@@ -101,9 +100,9 @@ jobs:
    - uses: actions/setup-node@v6
      name: Setup Node.js 24 for build
      with:
-        # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
-        # in October. Build/install runs on Node 24; tests run on the
-        # matrix version below using direct jest invocation.
+        # Build and install once on a fixed version so the generated docs
+        # are identical across matrix legs; the tests below then run on each
+        # supported Node version.
        node-version: 24
        cache: 'pnpm'
        cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -152,9 +151,9 @@ jobs:
        S3_TEST: "1"
        # Newer @smithy/core uses dynamic ESM imports.
        NODE_OPTIONS: "--experimental-vm-modules"
-      # Invoke jest directly because pnpm 11 itself requires Node 22+
-      # while the matrix tests on older Node versions.
-      run: npx jest --verbose
+      # Invoke the installed jest binary directly; the pnpm shim is set up
+      # against the build-phase Node, not the version selected above.
+      run: node_modules/.bin/jest --verbose
    - name: Test examples
      working-directory: ./
      env:
@@ -164,7 +163,7 @@ jobs:
      run: |
        python ci/mock_openai.py &
        cd nodejs/examples
-        npx jest --testEnvironment jest-environment-node-single-context --verbose
+        node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose
  macos:
    timeout-minutes: 30
    # macos-15 ships a newer linker; the older macos-14 linker fails to insert
@@ -185,8 +184,7 @@ jobs:
        version: 11.1.1
    - uses: actions/setup-node@v6
      with:
-        # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
-        # in October.
+        # pnpm 11 requires Node >= 22.13.
        node-version: 24
        cache: 'pnpm'
        cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -40,40 +40,31 @@ jobs:
          - target: aarch64-apple-darwin
            host: macos-latest
            features: fp16kernels
+            # Fat LTO was ~111 of this job's ~113 minutes.
+            lto: thin
+            codegen_units: 16
            pre_build: |-
              brew install protobuf
-              # Fat LTO (the workspace default in .cargo/config.toml) is
-              # single-threaded and is the peak-memory step of the build. On
-              # this runner it accounted for ~111 of the job's ~113 minutes,
-              # making it the critical path of the entire publish pipeline.
-              # ThinLTO parallelizes it across the runner's cores, for a few
-              # percent of runtime performance.
-              export CARGO_PROFILE_RELEASE_LTO=thin
-              export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
          - target: x86_64-pc-windows-msvc
            host: windows-2025
            features: ","
+            # The lower peak also keeps this on the standard 4-core runner.
+            lto: thin
+            codegen_units: 16
            pre_build: |-
              choco install --no-progress protoc ninja nasm
              tail -n 1000 /c/ProgramData/chocolatey/logs/chocolatey.log
              # There is an issue where choco doesn't add nasm to the path
              export PATH="$PATH:/c/Program Files/NASM"
              nasm -v
-              # See the ThinLTO note on aarch64-apple-darwin above. Keeping
-              # peak memory down is also what lets this run on the standard
-              # 4-core runner: the 8-core larger runner was only needed to
-              # stop fat LTO from OOMing rustc-LLVM.
-              export CARGO_PROFILE_RELEASE_LTO=thin
-              export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
          - target: aarch64-pc-windows-msvc
            host: windows-2025
            features: ","
+            lto: thin
+            codegen_units: 16
            pre_build: |-
                choco install --no-progress protoc
                rustup target add aarch64-pc-windows-msvc
-                # See the ThinLTO note on aarch64-apple-darwin above.
-                export CARGO_PROFILE_RELEASE_LTO=thin
-                export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
          - target: x86_64-unknown-linux-gnu
            host: ubuntu-latest
            features: fp16kernels
@@ -103,6 +94,14 @@ jobs:
            # https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
            features: "fp16kernels"
+            # Fat LTO OOM-killed rustc every nightly; even with lld it peaked
+            # at 31391 MiB of the runner's 32 GiB.
+            lto: thin
+            codegen_units: 16
+            # arm64 Linux links through GNU `ld` where x86_64 defaults to
+            # `rust-lld`, which is why only arm64 OOM'd. lld cut the largest
+            # linker process 7.0 -> 4.0 GiB (lancedb/sophon#7313).
+            linker: /tmp/aarch64-lld-clang
            pre_build: |-
              set -e &&
              apt-get update &&
@@ -112,9 +111,30 @@ jobs:
              # AT_HWCAP2 (added in Linux 3.17). Define it for aws-lc-sys.
              export CFLAGS="$CFLAGS -DAT_HWCAP2=26" &&
              rustup target add aarch64-unknown-linux-gnu
+              # Not `&&`-chained: in dash, errexit does not fire for a
+              # non-final command in an `&&` list, so failures were ignored.
+              #
+              # A wrapper rather than `-C link-arg` because the per-target
+              # rustflags variable does not reach every unit that links, while
+              # the linker variable does. `clang` because GCC silently ignores
+              # `-fuse-ld=lld` unless built with lld support. Two echoes
+              # because printf's newline escape gets rewritten to `;` between
+              # here and the container.
+              echo '#!/bin/sh' > /tmp/aarch64-lld-clang
+              echo 'exec clang --target=aarch64-unknown-linux-gnu --sysroot=/usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot --gcc-toolchain=/usr/aarch64-unknown-linux-gnu -fuse-ld=lld "$@"' >> /tmp/aarch64-lld-clang
+              chmod 0755 /tmp/aarch64-lld-clang
+              # Fail now, not at the cdylib link ~30 minutes later. Linking at
+              # all also proves lld resolved; clang errors out when it cannot.
+              echo 'int main(void){return 0;}' > /tmp/probe.c
+              /tmp/aarch64-lld-clang /tmp/probe.c -o /tmp/probe
+              readelf -h /tmp/probe | grep AArch64
          - target: aarch64-unknown-linux-musl
            host: ubuntu-2404-8x-x64
            features: ","
+            # Fat LTO took the whole runner down. lld cannot help: it died
+            # inside rustc's LLVM, before any linker was spawned.
+            lto: thin
+            codegen_units: 16
            pre_build: |-
              set -e &&
              sudo apt-get update &&
@@ -123,6 +143,19 @@ jobs:
              export EXTRA_ARGS="-x"
    name: build - ${{ matrix.settings.target }}
    runs-on: ${{ matrix.settings.host }}
+    # On the job, not exported from `pre_build`: `Swatinem/rust-cache` hashes
+    # `CARGO_*` into its cache key before any step runs, so a step-local export
+    # leaves the key unchanged while cargo still rebuilds cold. The ThinLTO
+    # legs had been doing that every run.
+    #
+    # Not `RUSTFLAGS`: setting it, even to "", discards every config-file
+    # rustflag, silently dropping .cargo/config.toml's `target-cpu` and
+    # `target-feature` from the published binaries.
+    env:
+      CARGO_PROFILE_RELEASE_LTO: ${{ matrix.settings.lto || 'fat' }}
+      CARGO_PROFILE_RELEASE_CODEGEN_UNITS: ${{ matrix.settings.codegen_units || '1' }}
+      # Empty elsewhere: a per-target variable is only read for that triple.
+      CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: ${{ matrix.settings.linker }}
    defaults:
      run:
        working-directory: nodejs
@@ -135,8 +168,7 @@ jobs:
      - name: Setup node
        uses: actions/setup-node@v6
        with:
-          # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
-          # in October.
+          # pnpm 11 requires Node >= 22.13.
          node-version: 24
          cache: pnpm
          cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -169,19 +201,15 @@ jobs:
          # creating ref). The nightly cadence also keeps entries inside
          # GitHub's 7-day eviction window, which a tag-only trigger would not.
          save-if: ${{ github.ref == 'refs/heads/main' }}
-      # Docker builds can use rust-cache too. `target/` already lives on the
-      # host because the whole workspace is bind-mounted into the container, and
-      # rust-cache's prune and save run host-side, so they can manage it -- which
-      # is what keeps the entry to dependency artifacts rather than a multi-GB
-      # copy of everything.
+      # Docker builds can use rust-cache too: the workspace is bind-mounted, so
+      # `target/` lives on the host and rust-cache's prune keeps the entry
+      # small.
      #
      # Two differences from the native builds. The container's CARGO_HOME is
-      # bind-mounted from `.cargo-cache` rather than the host's ~/.cargo, so that
-      # has to be cached explicitly. And the key is derived from the *host* rustc
-      # version, which is not the compiler that produced these artifacts; that is
-      # safe because cargo fingerprints the real compiler and rebuilds on a
-      # mismatch, it just means a base-image toolchain bump costs one cold build
-      # instead of invalidating the key.
+      # bind-mounted from `.cargo-cache` rather than ~/.cargo, so that is cached
+      # explicitly. And the key uses the *host* rustc version, not the compiler
+      # that built these artifacts -- safe, since cargo fingerprints the real
+      # one; a base-image bump just costs one cold build.
      - name: Cache cargo (docker builds)
        uses: Swatinem/rust-cache@v2
        if: ${{ matrix.settings.docker }}
@@ -210,14 +238,19 @@ jobs:
          # cache step above saves. Previously the registry mounts pointed at
          # `.cargo/...`, a path nothing cached, so the container re-downloaded
          # the whole crate registry on every run.
+          #
+          # `docker run` inherits nothing; `-e NAME` carries the job's `env:` in.
          options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
                    -v ${{ github.workspace }}/.cargo-cache/registry/cache:/usr/local/cargo/registry/cache \
                    -v ${{ github.workspace }}/.cargo-cache/registry/index:/usr/local/cargo/registry/index \
+                    -e CARGO_PROFILE_RELEASE_LTO \
+                    -e CARGO_PROFILE_RELEASE_CODEGEN_UNITS \
+                    -e CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER \
                    -v ${{ github.workspace }}:/build -w /build/nodejs"
          run: |
            set -e
            ${{ matrix.settings.pre_build }}
-            npx napi build --platform --release \
+            node_modules/.bin/napi build --platform --release \
              --features ${{ matrix.settings.features }} \
              --target ${{ matrix.settings.target }} \
              --dts ../lancedb/native.d.ts \
@@ -237,7 +270,7 @@ jobs:
      - name: Build
        run: |
          ${{ matrix.settings.pre_build }}
-          npx napi build --platform --release \
+          node_modules/.bin/napi build --platform --release \
              --features ${{ matrix.settings.features }} \
              --target ${{ matrix.settings.target }} \
              --dts ../lancedb/native.d.ts \
@@ -256,6 +289,18 @@ jobs:
        if: always()
        run: df -h
        shell: bash
+      - name: Report peak memory
+        if: always() && runner.os == 'Linux'
+        shell: bash
+        run: |
+          peak=$(find /sys/fs/cgroup -name memory.peak -readable \
+                 -exec cat {} + 2>/dev/null | sort -n | tail -1)
+          if [ -n "$peak" ]; then
+            echo "peak memory: $((peak / 1024 / 1024)) MiB"
+          else
+            echo "peak memory: unavailable (no readable cgroup v2 memory.peak)"
+          fi
+          free -g || true
      - name: Upload artifact
        uses: actions/upload-artifact@v7
        with:
@@ -293,7 +338,7 @@ jobs:
          - target: aarch64-unknown-linux-gnu
            host: ubuntu-2404-8x-arm64
        node:
-          - '20'
+          - '22'
    runs-on: ${{ matrix.settings.host }}
    defaults:
      run:
@@ -339,9 +384,9 @@ jobs:
      - name: Move built files
        run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
      - name: Test bindings
-        # Invoke jest directly because pnpm 11 itself requires Node 22+
-        # while the matrix tests on older Node versions.
-        run: npx jest --verbose
+        # Invoke the installed jest binary directly; the pnpm shim is set up
+        # against the install-phase Node, not the version selected above.
+        run: node_modules/.bin/jest --verbose
  publish:
    name: Publish
    runs-on: ubuntu-latest
@@ -232,7 +232,10 @@ jobs:
          ALL_FEATURES=`cargo metadata --format-version=1 --no-deps \
            | jq -r '.packages[] | .features | keys | .[]' \
            | grep -v s3-test | sort | uniq | paste -s -d "," -`
-          cargo test --profile ci --features $ALL_FEATURES --locked
+          # Run doctests before test binaries fill the runner disk. Examples are
+          # already built by the Linux job, so avoid retaining them here.
+          cargo test --profile ci --features $ALL_FEATURES --locked --doc
+          cargo test --profile ci --features $ALL_FEATURES --locked --lib --tests

  windows:
    strategy:
@@ -1,22 +0,0 @@
-name: Update package-lock.json
-
-on:
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-        with:
-          ref: main
-          persist-credentials: false
-          fetch-depth: 0
-          lfs: true
-      - uses: ./.github/workflows/update_package_lock
-        with:
-          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
@@ -1,22 +0,0 @@
-name: Update NodeJs package-lock.json
-
-on:
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-        with:
-          ref: main
-          persist-credentials: false
-          fetch-depth: 0
-          lfs: true
-      - uses: ./.github/workflows/update_package_lock_nodejs
-        with:
-          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
@@ -20,7 +20,10 @@ repos:
    hooks:
      - id: local-biome-check
        name: biome check
-        entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
+        # Use the biome from nodejs/package.json rather than a separately
+        # pinned one: the two drifted apart and disagreed on formatting, so
+        # this hook rejected code that `pnpm lint` accepted.
+        entry: nodejs/node_modules/.bin/biome check --config-path nodejs/biome.json nodejs/
        language: system
        types: [text]
        files: "nodejs/.*"
@@ -38,7 +38,7 @@ Before committing changes, run formatting for every language you touched. At min
 * Rust changes: run `cargo fmt --all`.
 * Python changes: run `ruff format .` and `ruff check .` from the repository root,
  and run targeted tests through `cd python && uv run ...`.
-* TypeScript changes: run the relevant `npm`/`pnpm` lint, format, build, and docs commands in `nodejs`.
+* TypeScript changes: run the relevant `pnpm` lint, format, build, and docs commands in `nodejs`.

 Before creating a PR, the exact value passed to `gh pr create --title` must follow
 Conventional Commits, such as `fix: support nested field paths in native index creation`
@@ -101,12 +101,12 @@ Python bindings changes:
 TypeScript bindings changes:

 1. Add napi-rs method binding on `Table` in `nodejs/src/table.rs`.
-2. Run `npm run build` to generate TypeScript definitions.
+2. Run `pnpm build` to generate TypeScript definitions.
 3. Add typescript method on abstract class `Table` in `nodejs/src/table.ts`.
 4. Add concrete method on `LocalTable` class in `nodejs/src/native_table.ts`.
    * Note: despite the name, this class is also used for remote tables.
 5. Add test in `nodejs/__test__/table.test.ts`.
-6. Run `npm run docs` to generate TypeScript documentation.
+6. Run `pnpm run docs` to generate TypeScript documentation.

 ## Python API reference

@@ -332,6 +332,34 @@ dependencies = [
 "num-traits",
 ]

+[[package]]
+name = "arrow-flight"
+version = "58.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2dbe34824c639e43136af8f106992792ab456540d54b880bc320a3192502d2e"
+dependencies = [
+ "arrow-arith",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-cast",
+ "arrow-data",
+ "arrow-ipc",
+ "arrow-ord",
+ "arrow-row",
+ "arrow-schema",
+ "arrow-select",
+ "arrow-string",
+ "base64 0.22.1",
+ "bytes",
+ "futures",
+ "once_cell",
+ "paste",
+ "prost",
+ "prost-types",
+ "tonic",
+ "tonic-prost",
+]
+
 [[package]]
 name = "arrow-ipc"
 version = "58.4.0"
@@ -535,9 +563,9 @@ dependencies = [

 [[package]]
 name = "async-trait"
-version = "0.1.91"
+version = "0.1.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
+checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -1129,7 +1157,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
 dependencies = [
 "async-trait",
- "axum-core",
+ "axum-core 0.4.5",
 "bytes",
 "futures-util",
 "http 1.5.0",
@@ -1138,7 +1166,7 @@ dependencies = [
 "hyper 1.9.0",
 "hyper-util",
 "itoa",
- "matchit",
+ "matchit 0.7.3",
 "memchr",
 "mime",
 "percent-encoding",
@@ -1156,6 +1184,31 @@ dependencies = [
 "tracing",
 ]

+[[package]]
+name = "axum"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"
+dependencies = [
+ "axum-core 0.5.6",
+ "bytes",
+ "futures-util",
+ "http 1.5.0",
+ "http-body 1.1.0",
+ "http-body-util",
+ "itoa",
+ "matchit 0.8.4",
+ "memchr",
+ "mime",
+ "percent-encoding",
+ "pin-project-lite",
+ "serde_core",
+ "sync_wrapper",
+ "tower",
+ "tower-layer",
+ "tower-service",
+]
+
 [[package]]
 name = "axum-core"
 version = "0.4.5"
@@ -1177,6 +1230,24 @@ dependencies = [
 "tracing",
 ]

+[[package]]
+name = "axum-core"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "http 1.5.0",
+ "http-body 1.1.0",
+ "http-body-util",
+ "mime",
+ "pin-project-lite",
+ "sync_wrapper",
+ "tower-layer",
+ "tower-service",
+]
+
 [[package]]
 name = "backoff"
 version = "0.4.0"
@@ -1443,9 +1514,9 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"

 [[package]]
 name = "bytemuck"
-version = "1.25.0"
+version = "1.25.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
+checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
 dependencies = [
 "bytemuck_derive",
 ]
@@ -1597,9 +1668,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"

 [[package]]
 name = "chacha20"
-version = "0.10.0"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601"
+checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
 dependencies = [
 "cfg-if 1.0.4",
 "cpufeatures 0.3.0",
@@ -3455,8 +3526,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"

 [[package]]
 name = "fsst"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-array",
 "rand 0.9.5",
@@ -4815,8 +4886,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"

 [[package]]
 name = "lance"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -4888,8 +4959,8 @@ dependencies = [

 [[package]]
 name = "lance-arrow"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4911,7 +4982,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4925,7 +4996,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -4934,8 +5005,8 @@ dependencies = [

 [[package]]
 name = "lance-bitpacking"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrayref",
 "crunchy",
@@ -4945,8 +5016,8 @@ dependencies = [

 [[package]]
 name = "lance-core"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4983,8 +5054,8 @@ dependencies = [

 [[package]]
 name = "lance-datafusion"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5000,6 +5071,7 @@ dependencies = [
 "datafusion-functions",
 "datafusion-physical-expr",
 "futures",
+ "half",
 "jsonb",
 "lance-arrow",
 "lance-core",
@@ -5013,8 +5085,8 @@ dependencies = [

 [[package]]
 name = "lance-datagen"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5031,8 +5103,8 @@ dependencies = [

 [[package]]
 name = "lance-derive"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -5041,8 +5113,8 @@ dependencies = [

 [[package]]
 name = "lance-encoding"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5075,8 +5147,8 @@ dependencies = [

 [[package]]
 name = "lance-file"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5107,8 +5179,8 @@ dependencies = [

 [[package]]
 name = "lance-index"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -5172,8 +5244,8 @@ dependencies = [

 [[package]]
 name = "lance-index-core"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5195,8 +5267,8 @@ dependencies = [

 [[package]]
 name = "lance-io"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5236,8 +5308,8 @@ dependencies = [

 [[package]]
 name = "lance-linalg"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5251,27 +5323,29 @@ dependencies = [

 [[package]]
 name = "lance-namespace"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow",
 "async-trait",
 "bytes",
 "lance-core",
 "lance-namespace-reqwest-client",
+ "serde",
+ "serde_json",
 "snafu 0.9.0",
 ]

 [[package]]
 name = "lance-namespace-impls"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow",
 "arrow-ipc",
 "arrow-schema",
 "async-trait",
- "axum",
+ "axum 0.7.9",
 "base64 0.22.1",
 "bytes",
 "chrono",
@@ -5304,9 +5378,9 @@ dependencies = [

 [[package]]
 name = "lance-namespace-reqwest-client"
-version = "0.11.0"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a030196da1c994b63a96a4f0bf5b0cfa459fe6dadc9e962320246ca328da22a"
+checksum = "d8d23e54b1634d5bbb434f8dd33dc3c05f6e58d876a9a27b3b4aef58ddbe11af"
 dependencies = [
 "reqwest 0.12.28",
 "serde",
@@ -5318,8 +5392,8 @@ dependencies = [

 [[package]]
 name = "lance-select"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5333,8 +5407,8 @@ dependencies = [

 [[package]]
 name = "lance-table"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5374,8 +5448,8 @@ dependencies = [

 [[package]]
 name = "lance-testing"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5388,8 +5462,8 @@ dependencies = [

 [[package]]
 name = "lance-tokenizer"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "frostem",
 "icu_segmenter",
@@ -5402,7 +5476,7 @@ dependencies = [

 [[package]]
 name = "lancedb"
-version = "0.38.0-beta.11"
+version = "0.39.0-beta.3"
 dependencies = [
 "ahash",
 "anyhow",
@@ -5411,6 +5485,7 @@ dependencies = [
 "arrow-buffer",
 "arrow-cast",
 "arrow-data",
+ "arrow-flight",
 "arrow-ipc",
 "arrow-ord",
 "arrow-schema",
@@ -5466,6 +5541,7 @@ dependencies = [
 "polars",
 "polars-arrow",
 "pprof 0.14.1",
+ "prost",
 "rand 0.9.5",
 "random_word",
 "regex",
@@ -5482,6 +5558,7 @@ dependencies = [
 "test-log",
 "tokenizers",
 "tokio",
+ "tonic",
 "url",
 "urlencoding",
 "uuid",
@@ -5490,7 +5567,7 @@ dependencies = [

 [[package]]
 name = "lancedb-nodejs"
-version = "0.38.0-beta.11"
+version = "0.39.0-beta.3"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5515,7 +5592,7 @@ dependencies = [

 [[package]]
 name = "lancedb-python"
-version = "0.38.0-beta.11"
+version = "0.39.0-beta.3"
 dependencies = [
 "arrow",
 "async-trait",
@@ -5539,6 +5616,7 @@ dependencies = [
 "serde_json",
 "snafu 0.8.9",
 "tokio",
+ "uuid",
 ]

 [[package]]
@@ -5748,9 +5826,9 @@ dependencies = [

 [[package]]
 name = "log"
-version = "0.4.33"
+version = "0.4.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
+checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"

 [[package]]
 name = "loom"
@@ -5859,6 +5937,12 @@ version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"

+[[package]]
+name = "matchit"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
+
 [[package]]
 name = "matrixmultiply"
 version = "0.3.10"
@@ -6001,9 +6085,9 @@ dependencies = [

 [[package]]
 name = "moka"
-version = "0.12.15"
+version = "0.12.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046"
+checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9"
 dependencies = [
 "async-lock",
 "crossbeam-channel",
@@ -6097,14 +6181,15 @@ dependencies = [

 [[package]]
 name = "napi"
-version = "3.11.0"
+version = "3.12.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de33522036981030a75c231829566bc63414e08101a6f5ff4ac6cef19c8e0941"
+checksum = "58c5f4d5375213fdb7be2655e152386e82f026f9a5ba36a75556e11359aafe09"
 dependencies = [
 "bitflags 2.11.1",
 "chrono",
 "ctor 1.0.12",
 "futures",
+ "libc",
 "napi-build",
 "napi-sys",
 "nohash-hasher",
@@ -6116,15 +6201,15 @@ dependencies = [

 [[package]]
 name = "napi-build"
-version = "2.4.0"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5282704fbe8d49b0cf8b08e3f33233416a528658f205c7e5ace63b582de0b11c"
+checksum = "60fdf9b392c50e7c4170fa633bd909490ed7835cea4c046776d1a4dd8d2ae0ab"

 [[package]]
 name = "napi-derive"
-version = "3.6.1"
+version = "3.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d5c9c02556ea6dc99dffd36c1ce60141411657438501a125b675776d011ce92"
+checksum = "0fa55ea69990c90b888e9e77044410e304ce7f35de599dc6d0b5c1923d2e59af"
 dependencies = [
 "convert_case",
 "ctor 1.0.12",
@@ -6136,9 +6221,9 @@ dependencies = [

 [[package]]
 name = "napi-derive-backend"
-version = "6.1.1"
+version = "6.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d60b5d773ad46c698c8cc2cd9fde0b283d39cbb7f71c04bee633c7bdba4423bd"
+checksum = "df4056ac7c18e4438ccf0edaed4340ca0d269278c8ec19284f7b23cb039fd0ae"
 dependencies = [
 "convert_case",
 "proc-macro2",
@@ -7733,6 +7818,7 @@ dependencies = [
 "pyo3-build-config",
 "pyo3-ffi",
 "pyo3-macros",
+ "uuid",
 ]

 [[package]]
@@ -8601,9 +8687,9 @@ dependencies = [

 [[package]]
 name = "roaring"
-version = "0.11.4"
+version = "0.11.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9"
+checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1"
 dependencies = [
 "bytemuck",
 "byteorder",
@@ -9063,9 +9149,9 @@ dependencies = [

 [[package]]
 name = "serde_with"
-version = "3.21.0"
+version = "3.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c"
+checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a"
 dependencies = [
 "base64 0.22.1",
 "bs58",
@@ -9073,6 +9159,7 @@ dependencies = [
 "hex",
 "indexmap 1.9.3",
 "indexmap 2.14.0",
+ "jiff",
 "schemars 0.9.0",
 "schemars 1.2.1",
 "serde_core",
@@ -9083,9 +9170,9 @@ dependencies = [

 [[package]]
 name = "serde_with_macros"
-version = "3.21.0"
+version = "3.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660"
+checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46"
 dependencies = [
 "darling 0.23.0",
 "proc-macro2",
@@ -10084,6 +10171,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef"
 dependencies = [
 "async-trait",
+ "axum 0.8.9",
 "base64 0.22.1",
 "bytes",
 "h2 0.4.16",
@@ -10095,9 +10183,11 @@ dependencies = [
 "hyper-util",
 "percent-encoding",
 "pin-project",
+ "rustls-native-certs",
 "socket2 0.6.3",
 "sync_wrapper",
 "tokio",
+ "tokio-rustls 0.26.4",
 "tokio-stream",
 "tower",
 "tower-layer",
@@ -10452,9 +10542,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"

 [[package]]
 name = "uuid"
-version = "1.24.0"
+version = "1.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
+checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812"
 dependencies = [
 "getrandom 0.4.2",
 "js-sys",
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"

 [workspace.dependencies]
-lance = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=12.0.0-beta.14", default-features = false, "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=12.0.0-beta.14", default-features = false, "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=12.0.0-beta.14", default-features = false, "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
 lancedb = { path = "rust/lancedb", default-features = false }
 ahash = "0.8"
 # Note that this one does not include pyarrow
@@ -39,6 +39,7 @@ arrow-ord = "58.0.0"
 arrow-schema = "58.0.0"
 arrow-select = "58.0.0"
 arrow-cast = "58.0.0"
+arrow-flight = { version = "58.0.0", features = ["flight-sql-experimental"] }
 async-trait = "0"
 bytes = "1"
 datafusion = { version = "54.0.0", default-features = false }
@@ -71,7 +72,8 @@ serde = "1"
 serde_json = "1"
 tempfile = "3.5.0"
 tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
-uuid = { version = "1.7.0", features = ["v4"] }
+tonic = { version = "0.14", features = ["tls-native-roots", "tls-ring"] }
+uuid = { version = "1.7.0", features = ["v4", "v7"] }
 chrono = { version = "0.4", default-features = false, features = ["clock"] }

 [profile.ci]
@@ -5,5 +5,5 @@ licenses:
 	cd python && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
 	cd python && uv sync --all-extras && uv tool run pip-licenses --python .venv/bin/python --format=markdown --with-urls --output-file=PYTHON_THIRD_PARTY_LICENSES.md
 	cd nodejs && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
-	cd nodejs && npx license-checker --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
+	cd nodejs && pnpm dlx license-checker@25 --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
 	cd java && ./mvnw license:aggregate-add-third-party -q
@@ -12,16 +12,12 @@ done
 # This updates the lockfile without building
 cargo metadata --quiet > /dev/null

-pushd nodejs || exit 1
-npm install --package-lock-only --silent
-popd
-
 if git diff --quiet --exit-code; then
  echo "No lockfile changes to commit; skipping amend."
 elif $AMEND; then
-  git add Cargo.lock nodejs/package-lock.json
+  git add Cargo.lock
  git commit --amend --no-edit
 else
-  git add Cargo.lock nodejs/package-lock.json
+  git add Cargo.lock
  git commit -m "Update lockfiles"
 fi
@@ -131,18 +131,13 @@ allow = [
    "BSD-3-Clause",
    "ISC",
    "Unicode-3.0",
-    "Unicode-DFS-2016",
    "Zlib",
    "CC0-1.0",
    "MPL-2.0",
    "BSL-1.0",
-    "OpenSSL",
    # 0BSD ("BSD Zero Clause") is effectively public domain — no attribution
    # required. Pulled in by `mock_instant`.
    "0BSD",
-    # bzip2-1.0.6 is the permissive upstream bzip2 license (BSD-like). Pulled
-    # in by `libbz2-rs-sys`, the pure-Rust bzip2 implementation.
-    "bzip2-1.0.6",
    # CDLA-Permissive-2.0 is a permissive data license used by `webpki-roots`
    # for the Mozilla CA root bundle. Data-only, distribution-compatible.
    "CDLA-Permissive-2.0",
@@ -150,12 +145,7 @@ allow = [
 confidence-threshold = 0.8
 # Per-crate license exceptions: allow a license for a specific crate only,
 # rather than globally via the `allow` list above.
-exceptions = [
-    # CDDL-1.0 (copyleft) is pulled in only as a dev/profiling dependency via
-    # `inferno` -> `pprof` -> `lance-testing`; it is a test dependency that we
-    # do not distribute, so scope the allowance to `inferno` alone.
-    { allow = ["CDDL-1.0"], crate = "inferno" },
-]
+exceptions = []
 # Crates whose license cannot be determined from Cargo metadata but whose
 # license we've manually confirmed from upstream. Keep this list minimal.
 [[licenses.clarify]]
@@ -47,22 +47,24 @@ pytest -vv python/tests/docs

 ### Checking typescript examples

-The `@lancedb/lancedb` package must be built before running the tests:
+The examples depend on `@lancedb/lancedb` at `file:../dist`, so the package must be
+built before running the tests. This uses pnpm; see the
+[Typescript contributing guide](../nodejs/CONTRIBUTING.md) for the toolchain setup.

 ```shell
 pushd nodejs
-npm ci
-npm run build
+pnpm install
+pnpm build
 popd
 ```

-Then you can run the examples by going to the `nodejs/examples` directory and
-running the tests like a normal npm package:
+Then you can run the examples by going to the `nodejs/examples` directory, which is a
+separate pnpm package with its own lockfile:

 ```shell
 pushd nodejs/examples
-npm ci
-npm test
+pnpm install
+pnpm test
 popd
 ```

@@ -84,6 +86,7 @@ The new files should be checked into the repository.

 ```shell
 pushd nodejs
-npm run docs
+# `pnpm docs` would invoke pnpm's built-in `docs` command, not the script.
+pnpm run docs
 popd
 ```
@@ -446,6 +446,15 @@ paths:
              properties:
                column:
                  type: string
+                name:
+                  type: string
+                  description: Optional name for the created index.
+                replace:
+                  type: boolean
+                  default: true
+                  description: |
+                    Whether to replace an existing index with the same resolved
+                    name. Defaults to true.
                metric_type:
                  type: string
                  nullable: false
@@ -1,135 +0,0 @@
-{
-  "name": "lancedb-docs-test",
-  "version": "1.0.0",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {
-    "": {
-      "name": "lancedb-docs-test",
-      "version": "1.0.0",
-      "license": "Apache 2",
-      "dependencies": {
-        "apache-arrow": "file:../node/node_modules/apache-arrow",
-        "vectordb": "file:../node"
-      },
-      "devDependencies": {
-        "@types/node": "^20.11.8",
-        "typescript": "^5.3.3"
-      }
-    },
-    "../node": {
-      "name": "vectordb",
-      "version": "0.21.2-beta.0",
-      "cpu": [
-        "x64",
-        "arm64"
-      ],
-      "license": "Apache-2.0",
-      "os": [
-        "darwin",
-        "linux",
-        "win32"
-      ],
-      "dependencies": {
-        "@neon-rs/load": "^0.0.74",
-        "axios": "^1.4.0"
-      },
-      "devDependencies": {
-        "@neon-rs/cli": "^0.0.160",
-        "@types/chai": "^4.3.4",
-        "@types/chai-as-promised": "^7.1.5",
-        "@types/mocha": "^10.0.1",
-        "@types/node": "^18.16.2",
-        "@types/sinon": "^10.0.15",
-        "@types/temp": "^0.9.1",
-        "@types/uuid": "^9.0.3",
-        "@typescript-eslint/eslint-plugin": "^5.59.1",
-        "apache-arrow-old": "npm:apache-arrow@13.0.0",
-        "cargo-cp-artifact": "^0.1",
-        "chai": "^4.3.7",
-        "chai-as-promised": "^7.1.1",
-        "eslint": "^8.39.0",
-        "eslint-config-standard-with-typescript": "^34.0.1",
-        "eslint-plugin-import": "^2.26.0",
-        "eslint-plugin-n": "^15.7.0",
-        "eslint-plugin-promise": "^6.1.1",
-        "mocha": "^10.2.0",
-        "openai": "^4.24.1",
-        "sinon": "^15.1.0",
-        "temp": "^0.9.4",
-        "ts-node": "^10.9.1",
-        "ts-node-dev": "^2.0.0",
-        "typedoc": "^0.24.7",
-        "typedoc-plugin-markdown": "^3.15.3",
-        "typescript": "^5.1.0",
-        "uuid": "^9.0.0"
-      },
-      "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.21.2-beta.0",
-        "@lancedb/vectordb-darwin-x64": "0.21.2-beta.0",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.21.2-beta.0",
-        "@lancedb/vectordb-linux-x64-gnu": "0.21.2-beta.0",
-        "@lancedb/vectordb-win32-x64-msvc": "0.21.2-beta.0"
-      },
-      "peerDependencies": {
-        "@apache-arrow/ts": "^14.0.2",
-        "apache-arrow": "^14.0.2"
-      }
-    },
-    "../node/node_modules/apache-arrow": {
-      "version": "14.0.2",
-      "license": "Apache-2.0",
-      "dependencies": {
-        "@types/command-line-args": "5.2.0",
-        "@types/command-line-usage": "5.0.2",
-        "@types/node": "20.3.0",
-        "@types/pad-left": "2.1.1",
-        "command-line-args": "5.2.1",
-        "command-line-usage": "7.0.1",
-        "flatbuffers": "23.5.26",
-        "json-bignum": "^0.0.3",
-        "pad-left": "^2.1.0",
-        "tslib": "^2.5.3"
-      },
-      "bin": {
-        "arrow2csv": "bin/arrow2csv.js"
-      }
-    },
-    "node_modules/@types/node": {
-      "version": "20.11.8",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.8.tgz",
-      "integrity": "sha512-i7omyekpPTNdv4Jb/Rgqg0RU8YqLcNsI12quKSDkRXNfx7Wxdm6HhK1awT3xTgEkgxPn3bvnSpiEAc7a7Lpyow==",
-      "dev": true,
-      "dependencies": {
-        "undici-types": "~5.26.4"
-      }
-    },
-    "node_modules/apache-arrow": {
-      "resolved": "../node/node_modules/apache-arrow",
-      "link": true
-    },
-    "node_modules/typescript": {
-      "version": "5.3.3",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz",
-      "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==",
-      "dev": true,
-      "bin": {
-        "tsc": "bin/tsc",
-        "tsserver": "bin/tsserver"
-      },
-      "engines": {
-        "node": ">=14.17"
-      }
-    },
-    "node_modules/undici-types": {
-      "version": "5.26.5",
-      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
-      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
-      "dev": true
-    },
-    "node_modules/vectordb": {
-      "resolved": "../node",
-      "link": true
-    }
-  }
-}
@@ -1,20 +0,0 @@
-{
-  "name": "lancedb-docs-test",
-  "version": "1.0.0",
-  "description": "auto-generated tests from doc",
-  "author": "dev@lancedb.com",
-  "license": "Apache 2",
-  "dependencies": {
-    "apache-arrow": "file:../node/node_modules/apache-arrow",
-    "vectordb": "file:../node"
-  },
-  "scripts": {
-    "build": "tsc -b && cd ../node && npm run build-release",
-    "example": "npm run build && node",
-    "test": "npm run build && ls dist/*.js | xargs -n 1 node"
-  },
-  "devDependencies": {
-    "@types/node": "^20.11.8",
-    "typescript": "^5.3.3"
-  }
-}
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-core</artifactId>
-    <version>0.38.0-beta.11</version>
+    <version>0.39.0-beta.4</version>
 </dependency>
 ```

@@ -448,26 +448,6 @@ on the returned job to know when cleanup has finished.

 ***

-### getJob()
-
-```ts
-abstract getJob(jobId): Promise<null | JobDescription>
-```
-
-Describe a single server-side job by id.
-
-Resolves to `null` when the server has no such job.
-
-#### Parameters
-
-* **jobId**: `string`
-
-#### Returns
-
-`Promise`&lt;`null` \| [`JobDescription`](../interfaces/JobDescription.md)&gt;
-
-***
-
 ### isOpen()

 ```ts
@@ -482,48 +462,6 @@ Return true if the connection has not been closed

 ***

-### job()
-
-```ts
-abstract job(jobId): Job
-```
-
-A [Job](Job.md) handle for a server-side job by id.
-
-The handle is constructed without a server round trip; an unknown id
-surfaces when the handle is used. Dropping the handle has no effect on
-the job itself.
-
-#### Parameters
-
-* **jobId**: `string`
-
-#### Returns
-
-[`Job`](Job.md)
-
-***
-
-### jobHistory()
-
-```ts
-abstract jobHistory(jobId?): Promise<Table<any>>
-```
-
-The lifecycle event history of a server-side job, as an Arrow table.
-
-Lists history across all jobs when `jobId` is omitted.
-
-#### Parameters
-
-* **jobId?**: `string`
-
-#### Returns
-
-`Promise`&lt;`Table`&lt;`any`&gt;&gt;
-
-***
-
 ### listJobs()

 ```ts
@@ -648,6 +586,30 @@ A page of table names and an

 ***

+### openJob()
+
+```ts
+abstract openJob(jobId): Promise<Job>
+```
+
+Open a server-side job by id, returning a handle with its record already
+populated. Rejects when the server has no such job, the way
+[Connection.openTable](Connection.md#opentable) does for a missing table.
+
+The returned [Job](Job.md) answers for its own state, specification,
+result, failure and event history, so there is no separate
+connection-level call for any of them.
+
+#### Parameters
+
+* **jobId**: `string`
+
+#### Returns
+
+`Promise`&lt;[`Job`](Job.md)&gt;
+
+***
+
 ### openMaterializedView()

 ```ts
@@ -8,19 +8,46 @@

 A handle to an operation that may still be running.

-## Constructors
+The operation may already be complete when the handle is created.

-### new Job()
+The detail getters read what the handle last observed. Submitting an
+operation returns only a job id, so populating them eagerly would cost an
+extra round trip on every call:
+
+- [Job.refresh](Job.md#refresh) and [Job.status](Job.md#status) fetch the whole record.
+- [Job.wait](Job.md#wait) records the terminal state it establishes, but not the
+  rest of the record.
+- Everything is null until one of those runs.
+
+## Accessors
+
+### creationMs

 ```ts
-new Job(): Job
+get creationMs(): null | number
 ```

+When the job was created, in milliseconds since the epoch.
+
 #### Returns

-[`Job`](Job.md)
+`null` \| `number`

-## Accessors
+***
+
+### failure
+
+```ts
+get failure(): null | JobFailureInfo
+```
+
+Why the job failed, when it failed and the server reports a reason.
+
+#### Returns
+
+`null` \| [`JobFailureInfo`](../interfaces/JobFailureInfo.md)
+
+***

 ### id

@@ -28,8 +55,69 @@ new Job(): Job
 get id(): null | string
 ```

-Identifies the operation on the server that is running it. Operations
-that run in this process have no server id. The value is opaque.
+Identifies the operation on the server that is running it.
+
+Operations that run in this process have no server id. The value is
+opaque: parsing it or storing it to resume the job later is not supported.
+
+#### Returns
+
+`null` \| `string`
+
+***
+
+### jobType
+
+```ts
+get jobType(): null | string
+```
+
+The job's type, as the server names it. Null for an in-process job, which
+has no server-side record.
+
+#### Returns
+
+`null` \| `string`
+
+***
+
+### result
+
+```ts
+get result(): any
+```
+
+The job-type-specific terminal result. Null until the job succeeds, so a
+job that never terminates reports its progress through [Job.events](Job.md#events)
+instead.
+
+#### Returns
+
+`any`
+
+***
+
+### spec
+
+```ts
+get spec(): any
+```
+
+The job-type-specific specification it was submitted with.
+
+#### Returns
+
+`any`
+
+***
+
+### state
+
+```ts
+get state(): null | string
+```
+
+The last observed lifecycle state, without contacting the backend.

 #### Returns

@@ -51,18 +139,61 @@ Request cancellation. Cancelling a finished operation is a no-op.

 ***

+### events()
+
+```ts
+events(options?): Promise<Table<any>>
+```
+
+This job's recorded lifecycle events.
+
+Where the getters above report a terminal result only once the job reaches
+one, events are written as the job runs and outlive the workers that
+produced them. A distributed job records a `claim`/`claim_complete` pair
+per unit of work, each carrying `rows_processed`, so a job that never
+finishes still accounts for what it did.
+
+The server caps results at 1000 rows by default and 10,000 at most, and
+truncates without saying so, so pass `limit` for a job that emits an event
+per fragment. `filter` is a SQL-like expression over the `state`,
+`updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
+
+#### Parameters
+
+* **options?**: [`JobEventsOptions`](../interfaces/JobEventsOptions.md)
+
+#### Returns
+
+`Promise`&lt;`Table`&lt;`any`&gt;&gt;
+
+***
+
+### refresh()
+
+```ts
+refresh(): Promise<void>
+```
+
+Ask the backend for this job's current state, and for a server-side job
+its full record, then cache it for the getters above.
+
+#### Returns
+
+`Promise`&lt;`void`&gt;
+
+***
+
 ### status()

 ```ts
 status(): Promise<string>
 ```

-The operation's current lifecycle state: "running", "finished",
-"failed", or "cancelled".
+The operation's current lifecycle state: "running", "finished", "failed",
+or "cancelled".

-A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject
-on a terminal failure state. States a newer server reports that this
-client version does not know pass through as-is.
+A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject on a
+terminal failure state. Also refreshes the getters above.

 #### Returns

@@ -70,6 +201,22 @@ client version does not know pass through as-is.

 ***

+### toString()
+
+```ts
+toString(): string
+```
+
+Every field the handle currently knows, one per line, with the JSON
+payloads indented -- a refresh job's spec and result are the point of
+printing it.
+
+#### Returns
+
+`string`
+
+***
+
 ### wait()

 ```ts
@@ -676,9 +676,17 @@ List all the versions of the table
 abstract mergeInsert(on): MergeInsertBuilder
 ```

+Create a [MergeInsertBuilder](MergeInsertBuilder.md), which combines new data with the
+existing table in a single transaction — inserting, updating and deleting
+rows depending on how they match.
+
 #### Parameters

 * **on**: `string` \| `string`[]
+    The column, or columns, to match source rows against target
+    rows on. Typically a key or id column. Several columns match on the
+    composite key: a source row updates a target row only when it agrees on
+    every one of them.

 #### Returns

@@ -96,7 +96,7 @@
 - [IvfFlatOptions](interfaces/IvfFlatOptions.md)
 - [IvfPqOptions](interfaces/IvfPqOptions.md)
 - [IvfRqOptions](interfaces/IvfRqOptions.md)
- [JobDescription](interfaces/JobDescription.md)
+- [JobEventsOptions](interfaces/JobEventsOptions.md)
 - [JobFailureInfo](interfaces/JobFailureInfo.md)
 - [JobInfo](interfaces/JobInfo.md)
 - [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
@@ -1,66 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / JobDescription
-
-# Interface: JobDescription
-
-A described job from `Connection.getJob`.
-
-## Properties
-
-### creationMs
-
-```ts
-creationMs: number;
-```
-
-When the job was created, in milliseconds since the epoch.
-
-***
-
-### failure?
-
-```ts
-optional failure: JobFailureInfo;
-```
-
-Why the job failed, when the job is failed and the server reports a
-reason.
-
-***
-
-### jobId
-
-```ts
-jobId: string;
-```
-
-***
-
-### jobType
-
-```ts
-jobType: string;
-```
-
-***
-
-### specJson?
-
-```ts
-optional specJson: string;
-```
-
-The job-type-specific specification as a JSON string, when present.
-
-***
-
-### state
-
-```ts
-state: string;
-```
-
-Lifecycle state: "running", "finished", "failed", or "cancelled".
@@ -0,0 +1,29 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / JobEventsOptions
+
+# Interface: JobEventsOptions
+
+Which of a job's events [Job.events](../classes/Job.md#events) returns.
+
+## Properties
+
+### filter?
+
+```ts
+optional filter: string;
+```
+
+SQL-like filter over the event columns.
+
+***
+
+### limit?
+
+```ts
+optional limit: number;
+```
+
+Maximum event rows to return, up to the server maximum of 10,000.
@@ -26,7 +26,7 @@ When the job was created, in milliseconds since the epoch.
 jobId: string;
 ```

-The job id -- what `Connection.getJob` and `Connection.cancelJob`
+The job id -- what `Connection.openJob` and `Connection.cancelJob`
 accept.

 ***
@@ -50,6 +50,16 @@ projections: [string, string][];

 ***

+### sourceNamespace
+
+```ts
+sourceNamespace: string[];
+```
+
+Namespace holding the source table; empty is the root namespace.
+
+***
+
 ### sourceTable

 ```ts
@@ -28,6 +28,59 @@ is also an [asynchronous API client](#connections-asynchronous).

 ::: lancedb.Session

+## Remote SQL
+
+Submit SQL against a remote LanceDB database through the connection.
+The connected database and `default_namespace_path=["public"]` are used for
+unqualified tables. Fully qualified references can still query other databases
+and namespaces available to the same deployment. `execute_query` returns a
+reader as soon as its initial result stream is available. `execute_query_async`
+returns a query handle immediately; use it to inspect progress, open a reader,
+or cancel the query. The SQL client is initialized by the first query and
+retained for the lifetime of the remote connection. Query ids are random,
+connection-scoped references rather than encoded SQL or durable resume tokens:
+
+```python
+import lancedb
+
+db = lancedb.connect(
+    "db://analytics",
+    api_key="ldb_...",
+    host_override="https://api.example.com",
+    sql_host_override="grpc+tls://sql.example.com:10026",
+)
+reader = db.execute_query(
+    """
+    SELECT events.id, accounts.name
+    FROM analytics.public.events AS events
+    JOIN users.public.accounts AS accounts ON events.user_id = accounts.id
+    """,
+    default_namespace_path=["public"],
+)
+for batch in reader:
+    print(batch.num_rows)
+
+query = db.execute_query_async("SELECT * FROM events")
+print(query.id)
+print(query.describe().status)
+for batch in query.reader():
+    print(batch.num_rows)
+
+# The async connection exposes the same lifecycle without blocking:
+# async_db = await lancedb.connect_async(
+#     "db://analytics",
+#     api_key="ldb_...",
+#     host_override="https://api.example.com",
+#     sql_host_override="grpc+tls://sql.example.com:10026",
+# )
+# reader = await async_db.execute_query("SELECT * FROM events")
+# query = await async_db.execute_query_async("SELECT * FROM events")
+# description = await async_db.describe_query(query.id)
+# async for batch in await query.reader():
+#     print(batch.num_rows)
+# await query.cancel()
+```
+
 ## Namespaces (Synchronous)

 A namespace-backed connection resolves tables through a
@@ -94,6 +147,8 @@ listing a storage directory.

 ::: lancedb.functions.OutputMapping

+::: lancedb.functions.AssignmentMapping
+
 ::: lancedb.functions.FunctionBinding

 ::: lancedb.functions.RefreshColumnResult
@@ -102,6 +157,18 @@ listing a storage directory.

 ::: lancedb.job.AsyncJob

+::: lancedb.job.JobInfo
+
+::: lancedb.job.JobDescription
+
+::: lancedb.job.JobFailureInfo
+
+::: lancedb.sql.Query
+
+::: lancedb.sql.AsyncQuery
+
+::: lancedb.sql.QueryDescription
+
 ## Materialized Views (Synchronous)

 ::: lancedb.materialized_view.MaterializedView
@@ -223,9 +290,13 @@ tokens = list(
 Blob columns store large binary values out of line so they can be read lazily
 instead of being materialized with the rest of the row.

-::: lancedb.blob
+`lancedb.BlobType` is `lance.blob.BlobType` when pylance is installed. Without
+pylance, LanceDB uses a matching `lance.blob.v2` extension type so blob columns
+still work. Queries return descriptors. Call
+[`fetch_blob_files`][lancedb.table.Table.fetch_blob_files] for lazy reads or
+[`fetch_blobs`][lancedb.table.Table.fetch_blobs] for eager bytes.

-::: lancedb.BlobType
+::: lancedb.blob

 ::: lancedb._blob.BlobFile
    options:
@@ -245,6 +316,12 @@ instead of being materialized with the rest of the row.

 ::: lancedb.exceptions.MissingColumnError

+::: lancedb.exceptions.JobNotFoundError
+
+::: lancedb.exceptions.JobFailedError
+
+::: lancedb.exceptions.JobCancelledError
+
 ## Integrations

 ## Pydantic
@@ -1,17 +0,0 @@
-{
-  "include": [
-    "src/*.ts",
-  ],
-  "compilerOptions": {
-    "target": "es2022",
-    "module": "nodenext",
-    "declaration": true,
-    "outDir": "./dist",
-    "strict": true,
-    "allowJs": true,
-    "resolveJsonModule": true,
-  },
-  "exclude": [
-    "./dist/*",
-  ]
-}
@@ -8,7 +8,7 @@
    <parent>
      <groupId>com.lancedb</groupId>
      <artifactId>lancedb-parent</artifactId>
-      <version>0.38.0-beta.11</version>
+      <version>0.39.0-beta.4</version>
      <relativePath>../pom.xml</relativePath>
    </parent>

@@ -6,7 +6,7 @@

    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-parent</artifactId>
-    <version>0.38.0-beta.11</version>
+    <version>0.39.0-beta.4</version>
    <packaging>pom</packaging>
    <name>${project.artifactId}</name>
    <description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>12.0.0-beta.2</lance-core.version>
+        <lance-core.version>12.0.0-beta.14</lance-core.version>
        <spotless.skip>false</spotless.skip>
        <spotless.version>2.30.0</spotless.version>
        <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.38.0-beta.11"
+version = "0.39.0-beta.4"
 publish = false
 license.workspace = true
 description.workspace = true
@@ -1,5 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import * as fs from "node:fs";
+import * as vm from "node:vm";
 import * as arrow15 from "apache-arrow-15";
 import * as arrow16 from "apache-arrow-16";
 import * as arrow17 from "apache-arrow-17";
@@ -40,6 +42,41 @@ function sampleRecords(): Array<Record<string, any>> {
  ];
 }

+it("serializes an Arrow Table created in another JavaScript realm", async () => {
+  const context = vm.createContext({
+    TextDecoder,
+    TextEncoder,
+    console,
+    setTimeout,
+    clearTimeout,
+  });
+  vm.runInContext(
+    fs.readFileSync(
+      require.resolve("apache-arrow-15/Arrow.es2015.min"),
+      "utf8",
+    ),
+    context,
+  );
+  const foreignTable: unknown = vm.runInContext(
+    "Arrow.tableFromArrays({ id: new Int32Array([1, 2, 3]), text: ['foo', 'bar', 'baz'] })",
+    context,
+  );
+
+  const foreignMetadata = (
+    foreignTable as { schema: { metadata: Map<string, string> } }
+  ).schema.metadata;
+  expect(foreignMetadata).not.toBeInstanceOf(Map);
+
+  const buf = await fromDataToBuffer(
+    foreignTable as Parameters<typeof fromDataToBuffer>[0],
+  );
+  const actual = currentTableFromIPC(buf);
+
+  expect(actual.numRows).toBe(3);
+  expect(actual.getChild("id")?.toJSON()).toEqual([1, 2, 3]);
+  expect(actual.getChild("text")?.toJSON()).toEqual(["foo", "bar", "baz"]);
+});
+
 it("preserves field metadata from a provided schema", async function () {
  const jsonMetadata = new Map([["ARROW:extension:name", "lance.json"]]);
  const schema = new CurrentSchema([
@@ -48,6 +48,28 @@ describe("materialized views", () => {
    expect(definitionFromMetadata(safe, "v").limit).toBe(42);
  });

+  it("reads the namespaced select kind and refuses unknown kinds", () => {
+    // "namespaced_select" is the namespaced form of "select": same shape, a
+    // separate kind so readers that predate it refuse instead of resolving
+    // the source at the root.
+    const namespaced = new Map([
+      [
+        DEFINITION_META_KEY,
+        '{"kind":"namespaced_select","source_table":"people","source_namespace":["ns"]}',
+      ],
+    ]);
+    const definition = definitionFromMetadata(namespaced, "v");
+    expect(definition.sourceTable).toBe("people");
+    expect(definition.sourceNamespace).toEqual(["ns"]);
+
+    const unknown = new Map([
+      [DEFINITION_META_KEY, '{"kind":"select_v3","source_table":"people"}'],
+    ]);
+    expect(() => definitionFromMetadata(unknown, "v")).toThrow(
+      /cannot refresh/,
+    );
+  });
+
  it("creates, refreshes and queries a view", async () => {
    const view = await db.createMaterializedView("adults", "people", {
      select: ["name", ["shout", "upper(name)"]],
@@ -5,8 +5,8 @@ import packageJson = require("../package.json");

 describe("package metadata", () => {
  it("requires Node.js type declarations compatible with the runtime", () => {
-    expect(packageJson.engines.node).toBe(">= 18");
-    expect(packageJson.peerDependencies["@types/node"]).toBe(">=18");
+    expect(packageJson.engines.node).toBe(">= 22");
+    expect(packageJson.peerDependencies["@types/node"]).toBe(">=22");
    expect(packageJson.peerDependenciesMeta["@types/node"]).toEqual({
      optional: true,
    });
@@ -3,6 +3,7 @@

 import * as http from "http";
 import { RequestListener } from "http";
+import packageJson = require("../package.json");
 import {
  ClientConfig,
  Connection,
@@ -70,7 +71,13 @@ async function withMockDatabase(
  try {
    await callback(db);
  } finally {
-    server.close();
+    // `close()` alone leaves the port bound until keep-alive sockets drain, so
+    // a single failing test would cascade into EADDRINUSE for every test after
+    // it. Destroy the connections and wait for the port to actually be free.
+    await new Promise<void>((resolve) => {
+      server.closeAllConnections();
+      server.close(() => resolve());
+    });
  }
 }

@@ -131,7 +138,7 @@ describe("remote connection", () => {
      (req, res) => {
        expect(req.headers["x-api-key"]).toEqual("fake");
        expect(req.headers["user-agent"]).toEqual(
-          `LanceDB-Node-Client/${process.env.npm_package_version}`,
+          `LanceDB-Node-Client/${packageJson.version}`,
        );

        const body = JSON.stringify({ tables: [] });
@@ -932,6 +939,7 @@ describe("remote connection jobs surface", () => {
    const { tableFromArrays, tableToIPC } = await import("apache-arrow");
    const eventsTable = tableFromArrays({ state: ["created", "succeeded"] });
    const eventsBody = Buffer.from(tableToIPC(eventsTable, "stream"));
+    const queryEventsPayloads: Record<string, unknown>[] = [];

    await withMockDatabase(
      (req, res) => {
@@ -960,6 +968,16 @@ describe("remote connection jobs surface", () => {
                );
            }
          } else if (req.url === "/v1/jobs/describe") {
+            if (payload["job_id"] === "job-2") {
+              res
+                .writeHead(200, { "Content-Type": "application/json" })
+                .end(
+                  '{"job_id": "job-2", "job_type": "refresh_column", ' +
+                    '"job_state": "DONE", "creation_ms": 2000, ' +
+                    '"result": {"rows_assigned": 1000000}}',
+                );
+              return;
+            }
            if (payload["job_id"] !== "job-1") {
              res.writeHead(404).end("no such job");
              return;
@@ -981,6 +999,7 @@ describe("remote connection jobs surface", () => {
              .writeHead(200, { "Content-Type": "application/json" })
              .end('{"job_id": "job-1"}');
          } else if (req.url === "/v1/jobs/query_events") {
+            queryEventsPayloads.push(payload);
            res
              .writeHead(200, {
                "Content-Type": "application/vnd.apache.arrow.stream",
@@ -997,22 +1016,65 @@ describe("remote connection jobs surface", () => {
        expect(jobs[0].state).toEqual("running");
        expect(jobs[1].state).toEqual("finished");

-        const description = await db.getJob("job-1");
-        expect(description?.state).toEqual("failed");
-        expect(JSON.parse(description?.specJson ?? "")).toEqual({
-          column: "vec",
-        });
-        expect(description?.failure?.message).toEqual("worker died");
-        expect(await db.getJob("missing")).toBeNull();
-
        expect(await db.cancelJob("job-1")).toBe(true);
        expect(await db.cancelJob("missing")).toBe(false);

-        const history = await db.jobHistory("job-1");
-        expect(history.numRows).toEqual(2);
+        // Opening a job hands back a populated handle; a missing one rejects.
+        await expect(db.openJob("missing")).rejects.toThrow("not found");
+        const finished = await db.openJob("job-2");
+        expect(finished.state).toEqual("finished");
+        expect(finished.result).toEqual({
+          // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
+          rows_assigned: 1000000,
+        });

-        const job = db.job("job-1");
+        const job = await db.openJob("job-1");
        expect(job.id).toEqual("job-1");
+
+        // openJob already populated the handle; refresh() re-reads it.
+        expect(job.state).toEqual("failed");
+        await job.refresh();
+        expect(job.state).toEqual("failed");
+        expect(job.jobType).toEqual("create_index");
+        expect(job.creationMs).toEqual(1000);
+        expect(job.spec).toEqual({ column: "vec" });
+        expect(job.result).toBeNull();
+        expect(job.failure?.message).toEqual("worker died");
+
+        // The handle reaches its own events, supplying its job id.
+        const jobEvents = await job.events({
+          limit: 500,
+          filter: "state = 'claim_complete'",
+        });
+        expect(jobEvents.numRows).toEqual(2);
+        expect(queryEventsPayloads.pop()).toEqual({
+          // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
+          job_id: "job-1",
+          limit: 500,
+          filter: "state = 'claim_complete'",
+        });
+
+        // Printing lays every known field out on its own line, with the JSON
+        // payloads indented rather than crammed onto one line.
+        expect(`${job}`).toEqual(
+          [
+            "Job(",
+            '    id="job-1",',
+            '    state="failed",',
+            '    jobType="create_index",',
+            "    creationMs=1000,",
+            "    spec={",
+            '        "column": "vec"',
+            "    },",
+            "    failure={",
+            '        "phase": "execute",',
+            '        "message": "worker died",',
+            '        "retryable": true',
+            "    },",
+            ")",
+          ].join("\n"),
+        );
+
        expect(await job.status()).toEqual("failed");
        await expect(job.wait()).rejects.toThrow("worker died");
      },
@@ -737,11 +737,12 @@ it("should query documents with LangChain PDF metadata", async () => {

 describe("merge insert", () => {
  let tmpDir: tmp.DirResult;
+  let conn: Connection;
  let table: Table;

  beforeEach(async () => {
    tmpDir = tmp.dirSync({ unsafeCleanup: true });
-    const conn = await connect(tmpDir.name);
+    conn = await connect(tmpDir.name);

    table = await conn.createTable("some_table", [
      { a: 1, b: "a" },
@@ -779,6 +780,38 @@ describe("merge insert", () => {

    expect(result.map((row) => ({ ...row }))).toEqual(expected);
  });
+  test("upsert on a composite key", async () => {
+    const composite = await conn.createTable("composite", [
+      { shard: "a", id: 1, val: "x" },
+      { shard: "a", id: 2, val: "y" },
+      { shard: "b", id: 1, val: "z" },
+    ]);
+
+    // ("a", 1) matches an existing row and updates it. ("b", 2) agrees with an
+    // existing row on each key column separately but on neither pair, so it is
+    // an insert.
+    const mergeInsertRes = await composite
+      .mergeInsert(["shard", "id"])
+      .whenMatchedUpdateAll()
+      .whenNotMatchedInsertAll()
+      .execute([
+        { shard: "a", id: 1, val: "X" },
+        { shard: "b", id: 2, val: "W" },
+      ]);
+    expect(mergeInsertRes.numUpdatedRows).toBe(1);
+    expect(mergeInsertRes.numInsertedRows).toBe(1);
+
+    const result = (await composite.toArrow())
+      .toArray()
+      .sort((a, b) => a.shard.localeCompare(b.shard) || a.id - b.id);
+
+    expect(result.map((row) => ({ ...row }))).toEqual([
+      { shard: "a", id: 1, val: "X" },
+      { shard: "a", id: 2, val: "y" },
+      { shard: "b", id: 1, val: "z" },
+      { shard: "b", id: 2, val: "W" },
+    ]);
+  });
  test("conditional update", async () => {
    const newData = [
      { a: 2, b: "x" },
@@ -8,7 +8,8 @@
    "//1": "--experimental-vm-modules is needed to run jest with sentence-transformers",
    "//2": "--testEnvironment is needed to run jest with sentence-transformers",
    "//3": "See: https://github.com/huggingface/transformers.js/issues/57",
-    "test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
+    "//4": "jest is invoked by its JS entry, not node_modules/.bin/jest: under pnpm that path is a shell shim, which `node` cannot execute",
+    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testEnvironment jest-environment-node-single-context --verbose",
    "lint": "biome check *.ts && biome format *.ts",
    "lint-ci": "biome ci .",
    "lint-fix": "biome check --write *.ts && pnpm format",
@@ -72,8 +72,7 @@ export type FieldLike =
    };

 export type DataLike =
-  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-  | import("apache-arrow").Data<Struct<any>>
+  | import("apache-arrow").Data
  | {
      // biome-ignore lint/suspicious/noExplicitAny: <explanation>
      type: any;
@@ -82,6 +81,7 @@ export type DataLike =
      stride: number;
      nullable: boolean;
      children: DataLike[];
+      dictionary?: { data: readonly DataLike[] };
      get nullCount(): number;
      // biome-ignore lint/suspicious/noExplicitAny: <explanation>
      values: Buffers<any>[BufferType.DATA];
@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors

-import { tableFromIPC } from "apache-arrow";
 import {
  Data,
  SchemaLike,
@@ -16,6 +15,7 @@ import {
  makeEmptyTable,
 } from "./arrow";
 import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
+import { Job } from "./job";
 import {
  MaterializedView,
  MaterializedViewSelect,
@@ -27,8 +27,6 @@ import type {
  CreateNamespaceResponse,
  DescribeNamespaceResponse,
  DropNamespaceResponse,
-  Job,
-  JobDescription,
  JobInfo,
  ListNamespacesResponse,
  ListTablesResponse,
@@ -557,24 +555,19 @@ export abstract class Connection {
  ): Promise<void>;

  /**
-   * A {@link Job} handle for a server-side job by id.
+   * Open a server-side job by id, returning a handle with its record already
+   * populated. Rejects when the server has no such job, the way
+   * {@link Connection.openTable} does for a missing table.
   *
-   * The handle is constructed without a server round trip; an unknown id
-   * surfaces when the handle is used. Dropping the handle has no effect on
-   * the job itself.
+   * The returned {@link Job} answers for its own state, specification,
+   * result, failure and event history, so there is no separate
+   * connection-level call for any of them.
   */
-  abstract job(jobId: string): Job;
+  abstract openJob(jobId: string): Promise<Job>;

  /** List server-side jobs across the database's tables. */
  abstract listJobs(): Promise<JobInfo[]>;

-  /**
-   * Describe a single server-side job by id.
-   *
-   * Resolves to `null` when the server has no such job.
-   */
-  abstract getJob(jobId: string): Promise<JobDescription | null>;
-
  /**
   * Request cancellation of a server-side job by id.
   *
@@ -582,13 +575,6 @@ export abstract class Connection {
   * such job exists. Cancelling an already-terminal job is a no-op success.
   */
  abstract cancelJob(jobId: string): Promise<boolean>;
-
-  /**
-   * The lifecycle event history of a server-side job, as an Arrow table.
-   *
-   * Lists history across all jobs when `jobId` is omitted.
-   */
-  abstract jobHistory(jobId?: string): Promise<ArrowTable>;
 }

 /** @hideconstructor */
@@ -869,7 +855,7 @@ export class LocalConnection extends Connection {
  }

  async dropTableAsync(name: string, namespacePath?: string[]): Promise<Job> {
-    return this.inner.dropTableAsync(name, namespacePath ?? []);
+    return new Job(await this.inner.dropTableAsync(name, namespacePath ?? []));
  }

  async dropAllTables(namespacePath?: string[]): Promise<void> {
@@ -928,29 +914,17 @@ export class LocalConnection extends Connection {
    );
  }

-  job(jobId: string): Job {
-    return this.inner.job(jobId);
+  async openJob(jobId: string): Promise<Job> {
+    return new Job(await this.inner.openJob(jobId));
  }

  async listJobs(): Promise<JobInfo[]> {
    return this.inner.listJobs();
  }

-  async getJob(jobId: string): Promise<JobDescription | null> {
-    return this.inner.getJob(jobId);
-  }
-
  async cancelJob(jobId: string): Promise<boolean> {
    return this.inner.cancelJob(jobId);
  }
-
-  async jobHistory(jobId?: string): Promise<ArrowTable> {
-    const buf = await this.inner.jobHistory(jobId);
-    if (buf.length === 0) {
-      return new ArrowTable();
-    }
-    return tableFromIPC(buf);
-  }
 }

 /**
@@ -94,13 +94,9 @@ export {
  RenameTableOptions,
 } from "./connection";

-export {
-  Job,
-  JobDescription,
-  JobFailureInfo,
-  JobInfo,
-  Session,
-} from "./native.js";
+export { JobFailureInfo, JobInfo, Session } from "./native.js";
+
+export { Job, JobEventsOptions } from "./job";

 export {
  AutoQuery,
@@ -0,0 +1,188 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import { Table as ArrowTable, tableFromIPC } from "apache-arrow";
+import { JobFailureInfo, Job as NativeJob } from "./native";
+
+/** Which of a job's events {@link Job.events} returns. */
+export interface JobEventsOptions {
+  /** Maximum event rows to return, up to the server maximum of 10,000. */
+  limit?: number;
+  /** SQL-like filter over the event columns. */
+  filter?: string;
+}
+
+/**
+ * A handle to an operation that may still be running.
+ *
+ * The operation may already be complete when the handle is created.
+ *
+ * The detail getters read what the handle last observed. Submitting an
+ * operation returns only a job id, so populating them eagerly would cost an
+ * extra round trip on every call:
+ *
+ * - {@link Job.refresh} and {@link Job.status} fetch the whole record.
+ * - {@link Job.wait} records the terminal state it establishes, but not the
+ *   rest of the record.
+ * - Everything is null until one of those runs.
+ *
+ * @hideconstructor
+ */
+export class Job {
+  private readonly inner: NativeJob;
+
+  constructor(inner: NativeJob) {
+    this.inner = inner;
+  }
+
+  /**
+   * Identifies the operation on the server that is running it.
+   *
+   * Operations that run in this process have no server id. The value is
+   * opaque: parsing it or storing it to resume the job later is not supported.
+   */
+  get id(): string | null {
+    return this.inner.id ?? null;
+  }
+
+  /** The last observed lifecycle state, without contacting the backend. */
+  get state(): string | null {
+    return this.inner.state ?? null;
+  }
+
+  /**
+   * The job's type, as the server names it. Null for an in-process job, which
+   * has no server-side record.
+   */
+  get jobType(): string | null {
+    return this.inner.jobType ?? null;
+  }
+
+  /** When the job was created, in milliseconds since the epoch. */
+  get creationMs(): number | null {
+    return this.inner.creationMs ?? null;
+  }
+
+  /** The job-type-specific specification it was submitted with. */
+  // biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
+  get spec(): any | null {
+    return parseJson(this.inner.specJson);
+  }
+
+  /**
+   * The job-type-specific terminal result. Null until the job succeeds, so a
+   * job that never terminates reports its progress through {@link Job.events}
+   * instead.
+   */
+  // biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
+  get result(): any | null {
+    return parseJson(this.inner.resultJson);
+  }
+
+  /** Why the job failed, when it failed and the server reports a reason. */
+  get failure(): JobFailureInfo | null {
+    return this.inner.failure ?? null;
+  }
+
+  /**
+   * The operation's current lifecycle state: "running", "finished", "failed",
+   * or "cancelled".
+   *
+   * A point snapshot; unlike {@link Job.wait} it does not block or reject on a
+   * terminal failure state. Also refreshes the getters above.
+   */
+  async status(): Promise<string> {
+    return this.inner.status();
+  }
+
+  /** Wait until the operation reaches a terminal state. */
+  async wait(): Promise<void> {
+    return this.inner.wait();
+  }
+
+  /** Request cancellation. Cancelling a finished operation is a no-op. */
+  async cancel(): Promise<void> {
+    return this.inner.cancel();
+  }
+
+  /**
+   * Ask the backend for this job's current state, and for a server-side job
+   * its full record, then cache it for the getters above.
+   */
+  async refresh(): Promise<void> {
+    return this.inner.refresh();
+  }
+
+  /**
+   * This job's recorded lifecycle events.
+   *
+   * Where the getters above report a terminal result only once the job reaches
+   * one, events are written as the job runs and outlive the workers that
+   * produced them. A distributed job records a `claim`/`claim_complete` pair
+   * per unit of work, each carrying `rows_processed`, so a job that never
+   * finishes still accounts for what it did.
+   *
+   * The server caps results at 1000 rows by default and 10,000 at most, and
+   * truncates without saying so, so pass `limit` for a job that emits an event
+   * per fragment. `filter` is a SQL-like expression over the `state`,
+   * `updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
+   */
+  async events(options?: JobEventsOptions): Promise<ArrowTable> {
+    const buf = await this.inner.events(options?.limit, options?.filter);
+    if (buf.length === 0) {
+      return new ArrowTable();
+    }
+    return tableFromIPC(buf);
+  }
+
+  /**
+   * Every field the handle currently knows, one per line, with the JSON
+   * payloads indented -- a refresh job's spec and result are the point of
+   * printing it.
+   */
+  toString(): string {
+    if (this.state === null) {
+      const known = this.id === null ? "" : `id=${JSON.stringify(this.id)}, `;
+      return `Job(${known}not refreshed)`;
+    }
+    const fields: string[] = [];
+    if (this.id !== null) {
+      fields.push(`id=${JSON.stringify(this.id)}`);
+    }
+    fields.push(`state=${JSON.stringify(this.state)}`);
+    if (this.jobType !== null) {
+      fields.push(`jobType=${JSON.stringify(this.jobType)}`);
+    }
+    if (this.creationMs !== null) {
+      fields.push(`creationMs=${this.creationMs}`);
+    }
+    for (const [name, value] of [
+      ["spec", this.spec],
+      ["result", this.result],
+    ] as const) {
+      if (value !== null) {
+        fields.push(`${name}=${indentJson(value)}`);
+      }
+    }
+    if (this.failure !== null) {
+      fields.push(`failure=${indentJson(this.failure)}`);
+    }
+    return `Job(${fields.map((field) => `\n${REPR_INDENT}${field},`).join("")}\n)`;
+  }
+
+  [Symbol.for("nodejs.util.inspect.custom")](): string {
+    return this.toString();
+  }
+}
+
+const REPR_INDENT = "    ";
+
+// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
+function indentJson(value: any): string {
+  return JSON.stringify(value, null, 4).replace(/\n/g, `\n${REPR_INDENT}`);
+}
+
+// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
+function parseJson(raw: string | null | undefined): any | null {
+  return raw === null || raw === undefined ? null : JSON.parse(raw);
+}
@@ -19,6 +19,8 @@ export interface MaterializedViewDefinition {
  limit?: number;
  /** Source columns the projections and filter read. */
  inputs: string[];
+  /** Namespace holding the source table; empty is the root namespace. */
+  sourceNamespace: string[];
 }

 /**
@@ -78,7 +80,8 @@ export function definitionFromMetadata(
  }
  // biome-ignore lint/suspicious/noExplicitAny: raw JSON
  const value: any = JSON.parse(raw);
-  if (value.kind !== "select") {
+  // "namespaced_select" keeps older readers from resolving the source at root.
+  if (value.kind !== "select" && value.kind !== "namespaced_select") {
    throw new Error(
      `materialized view '${name}' is defined by '${value.kind}', which this ` +
        "version of lancedb cannot refresh",
@@ -103,6 +106,7 @@ export function definitionFromMetadata(
    filter: value.filter ?? undefined,
    limit,
    inputs: value.inputs ?? [],
+    sourceNamespace: value.source_namespace ?? [],
  };
 }

@@ -94,17 +94,24 @@ export function sanitizeMetadata(
  if (metadataLike === undefined || metadataLike === null) {
    return undefined;
  }
-  if (!(metadataLike instanceof Map)) {
+
+  let entries: IterableIterator<[unknown, unknown]>;
+  try {
+    entries = Map.prototype.entries.call(metadataLike);
+  } catch {
    throw Error("Expected metadata, if present, to be a Map<string, string>");
  }
-  for (const item of metadataLike) {
-    if (typeof item[0] !== "string" || typeof item[1] !== "string") {
+
+  const metadata = new Map<string, string>();
+  for (const [key, value] of entries) {
+    if (typeof key !== "string" || typeof value !== "string") {
      throw Error(
        "Expected metadata, if present, to be a Map<string, string> but it had non-string keys or values",
      );
    }
+    metadata.set(key, value);
  }
-  return metadataLike as Map<string, string>;
+  return metadata;
 }

 export function sanitizeInt(typeLike: object) {
@@ -19,6 +19,7 @@ import {

 import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
 import { IndexOptions } from "./indices";
+import { Job } from "./job";
 import { MergeInsertBuilder } from "./merge";
 import {
  AddColumnsResult,
@@ -30,7 +31,6 @@ import {
  DropColumnsResult,
  IndexConfig,
  IndexStatistics,
-  Job,
  LsmStats,
  Branches as NativeBranches,
  OptimizeStats,
@@ -919,6 +919,16 @@ export abstract class Table {
  /** Return the table as an arrow table */
  abstract toArrow(): Promise<ArrowTable>;

+  /**
+   * Create a {@link MergeInsertBuilder}, which combines new data with the
+   * existing table in a single transaction — inserting, updating and deleting
+   * rows depending on how they match.
+   *
+   * @param on - The column, or columns, to match source rows against target
+   * rows on. Typically a key or id column. Several columns match on the
+   * composite key: a source row updates a target row only when it agrees on
+   * every one of them.
+   */
  abstract mergeInsert(on: string | string[]): MergeInsertBuilder;

  /** List all the stats of a specified index
@@ -1114,13 +1124,15 @@ export class LocalTable extends Table {
  ): Promise<Job> {
    // biome-ignore lint/suspicious/noExplicitAny: skip
    const nativeIndex = (options?.config as any)?.inner;
-    return await this.inner.createIndexAsync(
-      nativeIndex,
-      column,
-      options?.replace,
-      options?.waitTimeoutSeconds,
-      options?.name,
-      options?.train,
+    return new Job(
+      await this.inner.createIndexAsync(
+        nativeIndex,
+        column,
+        options?.replace,
+        options?.waitTimeoutSeconds,
+        options?.name,
+        options?.train,
+      ),
    );
  }

@@ -1303,7 +1315,7 @@ export class LocalTable extends Table {
  }

  async refreshColumnAsync(column: string): Promise<Job> {
-    return await this.inner.refreshColumnAsync(column);
+    return new Job(await this.inner.refreshColumnAsync(column));
  }

  async refreshMaterializedView(
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.38.0-beta.11",
+	"version": "0.39.0-beta.4",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.38.0-beta.11",
+	"version": "0.39.0-beta.4",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.38.0-beta.11",
+	"version": "0.39.0-beta.4",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.38.0-beta.11",
+	"version": "0.39.0-beta.4",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.38.0-beta.11",
+	"version": "0.39.0-beta.4",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",
@@ -1,6 +1,6 @@
 {
  "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.38.0-beta.11",
+  "version": "0.39.0-beta.4",
  "os": [
    "win32"
  ],
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.38.0-beta.11",
+	"version": "0.39.0-beta.4",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",
@@ -11,7 +11,7 @@
    "ann"
  ],
  "private": false,
-  "version": "0.38.0-beta.11",
+  "version": "0.39.0-beta.4",
  "main": "dist/index.js",
  "exports": {
    ".": "./dist/index.js",
@@ -44,7 +44,7 @@
    "@biomejs/biome": "^1.7.3",
    "@jest/globals": "^29.7.0",
    "@napi-rs/cli": "3.7.0",
-    "@opentelemetry/sdk-metrics": "^1.30.0",
+    "@opentelemetry/sdk-metrics": "^2.10.0",
    "@types/axios": "^0.14.0",
    "@types/jest": "^29.1.2",
    "@types/node": "22.7.4",
@@ -56,7 +56,7 @@
    "eslint": "^8.57.0",
    "jest": "^29.7.0",
    "shx": "^0.3.4",
-    "tmp": "^0.2.3",
+    "tmp": "^0.2.7",
    "ts-jest": "^29.1.2",
    "typedoc": "0.26.4",
    "typedoc-plugin-markdown": "4.2.1",
@@ -67,7 +67,7 @@
    "timeout": "3m"
  },
  "engines": {
-    "node": ">= 18"
+    "node": ">= 22"
  },
  "packageManager": "pnpm@11.1.1",
  "cpu": ["x64", "arm64"],
@@ -101,7 +101,7 @@
    "openai": "4.29.2"
  },
  "peerDependencies": {
-    "@types/node": ">=18",
+    "@types/node": ">=22",
    "apache-arrow": ">=15.0.0 <=18.1.0"
  },
  "peerDependenciesMeta": {
@@ -16,3 +16,41 @@ allowBuilds:
  onnxruntime-node: true
  protobufjs: true
  sharp: true
+
+minimumReleaseAgeExclude:
+  - protobufjs@7.5.8
+  - tmp@0.2.6
+  - form-data@4.0.6
+  - tar@7.5.16
+  - markdown-it@14.1.2
+  - linkify-it@5.0.1
+  - js-yaml@3.15.0
+  - js-yaml@4.1.2
+  - protobufjs@7.6.1
+  - protobufjs@7.6.3
+  - '@babel/core@7.29.1'
+  - axios@1.18.0
+  - brace-expansion@2.1.2
+  - brace-expansion@1.1.16
+  - js-yaml@4.3.0
+  - tar@7.5.18
+  - tar@7.5.19
+  - tar@7.5.17
+  - protobufjs@7.6.5
+  - linkify-it@5.0.2
+  - sharp@0.35.0
+  - brace-expansion@1.1.17
+  - brace-expansion@2.1.3
+  - brace-expansion@2.1.4
+  - brace-expansion@1.1.18
+  - js-yaml@3.15.1
+  - js-yaml@4.3.1
+  - tar@7.5.21
+  - '@opentelemetry/core@2.8.0'
+
+# @huggingface/transformers pins sharp ^0.33.5 and no released version has moved
+# past ^0.34.5, all of which inherit the libvips CVEs in GHSA-f88m-g3jw-g9cj.
+# Force the patched line. sharp is only reached by transformers' image pipeline,
+# which LanceDB's text embedding function never uses.
+overrides:
+  sharp: ^0.35.4
@@ -442,13 +442,15 @@ impl Connection {
        self.get_inner()?.drop_all_tables(&ns).await.default_error()
    }

-    /// A `Job` handle for a server-side job by id.
+    /// Open a server-side job by id, returning a handle with its record
+    /// already populated. Rejects when the server has no such job.
    ///
-    /// The handle is constructed without a server round trip; an unknown id
-    /// surfaces when the handle is used.
-    #[napi]
-    pub fn job(&self, job_id: String) -> napi::Result<crate::job::Job> {
-        let job = self.get_inner()?.job(job_id).default_error()?;
+    /// The returned handle answers for its own state, specification, result,
+    /// failure and event history, so there is no separate connection-level
+    /// call for any of them.
+    #[napi(catch_unwind)]
+    pub async fn open_job(&self, job_id: String) -> napi::Result<crate::job::Job> {
+        let job = self.get_inner()?.open_job(&job_id).await.default_error()?;
        Ok(crate::job::Job::new(job))
    }

@@ -459,17 +461,6 @@ impl Connection {
        Ok(jobs.into_iter().map(Into::into).collect())
    }

-    /// Describe a single server-side job by id. `null` when the server has
-    /// no such job.
-    #[napi(catch_unwind)]
-    pub async fn get_job(
-        &self,
-        job_id: String,
-    ) -> napi::Result<Option<crate::job::JobDescription>> {
-        let description = self.get_inner()?.get_job(&job_id).await.default_error()?;
-        Ok(description.map(Into::into))
-    }
-
    /// Request cancellation of a server-side job by id. Returns true if the
    /// server accepted the cancellation, false if no such job exists.
    #[napi(catch_unwind)]
@@ -477,34 +468,6 @@ impl Connection {
        self.get_inner()?.cancel_job(&job_id).await.default_error()
    }

-    /// The lifecycle event history of a server-side job (all jobs when
-    /// `job_id` is null), as an Arrow IPC stream buffer. Empty when there is
-    /// no history.
-    #[napi(catch_unwind)]
-    pub async fn job_history(&self, job_id: Option<String>) -> napi::Result<Buffer> {
-        let batches = self
-            .get_inner()?
-            .job_history(job_id.as_deref())
-            .await
-            .default_error()?;
-        let Some(first) = batches.first() else {
-            return Ok(Buffer::from(Vec::<u8>::new()));
-        };
-        let mut out = Vec::new();
-        let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut out, &first.schema())
-            .map_err(|e| napi::Error::from_reason(e.to_string()))?;
-        for batch in &batches {
-            writer
-                .write(batch)
-                .map_err(|e| napi::Error::from_reason(e.to_string()))?;
-        }
-        writer
-            .finish()
-            .map_err(|e| napi::Error::from_reason(e.to_string()))?;
-        drop(writer);
-        Ok(Buffer::from(out))
-    }
-
    #[napi(catch_unwind)]
    /// Describe a namespace and return its properties.
    pub async fn describe_namespace(
@@ -3,6 +3,9 @@

 use std::sync::Arc;

+use arrow_array::RecordBatch;
+use lancedb::job::JobEventsRequest;
+use napi::bindgen_prelude::Buffer;
 use napi_derive::napi;

 use crate::error::NapiErrorExt;
@@ -55,12 +58,98 @@ impl Job {
    pub async fn cancel(&self) -> napi::Result<()> {
        self.inner.cancel().await.default_error()
    }
+
+    /// Ask the backend for this job's current state, and for a server-side job
+    /// its full record, then cache it for the getters below.
+    ///
+    /// They are all null until this runs, because submitting an operation
+    /// returns only a job id. {@link Job.status} fetches the whole record too;
+    /// {@link Job.wait} records only the terminal state it establishes.
+    #[napi(catch_unwind)]
+    pub async fn refresh(&self) -> napi::Result<()> {
+        self.inner.refresh().await.default_error()
+    }
+
+    /// The last observed lifecycle state, without contacting the backend.
+    #[napi(getter)]
+    pub fn state(&self) -> Option<String> {
+        self.inner.state()
+    }
+
+    /// The job's type, as the server names it. Null for an in-process job,
+    /// which has no server-side record.
+    #[napi(getter)]
+    pub fn job_type(&self) -> Option<String> {
+        self.inner.job_type()
+    }
+
+    /// When the job was created, in milliseconds since the epoch.
+    #[napi(getter)]
+    pub fn creation_ms(&self) -> Option<i64> {
+        self.inner.creation_ms()
+    }
+
+    /// The job-type-specific specification as a JSON string, when present.
+    #[napi(getter)]
+    pub fn spec_json(&self) -> Option<String> {
+        self.inner.spec().map(|spec| spec.to_string())
+    }
+
+    /// The job-type-specific terminal result as a JSON string. Null until the
+    /// job succeeds, so a job that never terminates reports its progress
+    /// through {@link Job.events} instead.
+    #[napi(getter)]
+    pub fn result_json(&self) -> Option<String> {
+        self.inner.result().map(|result| result.to_string())
+    }
+
+    /// Why the job failed, when it failed and the server reports a reason.
+    #[napi(getter)]
+    pub fn failure(&self) -> Option<JobFailureInfo> {
+        self.inner.failure().map(|failure| JobFailureInfo {
+            phase: failure.phase,
+            message: failure.message,
+            retryable: failure.retryable,
+        })
+    }
+
+    /// This job's recorded lifecycle events, as an Arrow IPC stream buffer.
+    /// The TypeScript wrapper turns it into an Arrow table.
+    #[napi(catch_unwind)]
+    pub async fn events(&self, limit: Option<u32>, filter: Option<String>) -> napi::Result<Buffer> {
+        let batches = self
+            .inner
+            .events(JobEventsRequest { limit, filter })
+            .await
+            .default_error()?;
+        batches_to_ipc_buffer(&batches)
+    }
+}
+
+/// Serialise Arrow batches as a single IPC stream for the TypeScript layer.
+fn batches_to_ipc_buffer(batches: &[RecordBatch]) -> napi::Result<Buffer> {
+    let Some(first) = batches.first() else {
+        return Ok(Buffer::from(Vec::<u8>::new()));
+    };
+    let mut out = Vec::new();
+    let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut out, &first.schema())
+        .map_err(|e| napi::Error::from_reason(e.to_string()))?;
+    for batch in batches {
+        writer
+            .write(batch)
+            .map_err(|e| napi::Error::from_reason(e.to_string()))?;
+    }
+    writer
+        .finish()
+        .map_err(|e| napi::Error::from_reason(e.to_string()))?;
+    drop(writer);
+    Ok(Buffer::from(out))
 }

 /// A row from `Connection.listJobs`: one server-side job.
 #[napi(object)]
 pub struct JobInfo {
-    /// The job id -- what `Connection.getJob` and `Connection.cancelJob`
+    /// The job id -- what `Connection.openJob` and `Connection.cancelJob`
    /// accept.
    pub job_id: String,
    /// The table the job runs against, without URI or namespace.
@@ -91,36 +180,3 @@ pub struct JobFailureInfo {
    pub message: Option<String>,
    pub retryable: Option<bool>,
 }
-
-/// A described job from `Connection.getJob`.
-#[napi(object)]
-pub struct JobDescription {
-    pub job_id: String,
-    pub job_type: String,
-    /// Lifecycle state: "running", "finished", "failed", or "cancelled".
-    pub state: String,
-    /// When the job was created, in milliseconds since the epoch.
-    pub creation_ms: i64,
-    /// The job-type-specific specification as a JSON string, when present.
-    pub spec_json: Option<String>,
-    /// Why the job failed, when the job is failed and the server reports a
-    /// reason.
-    pub failure: Option<JobFailureInfo>,
-}
-
-impl From<lancedb::database::JobDescription> for JobDescription {
-    fn from(description: lancedb::database::JobDescription) -> Self {
-        Self {
-            job_id: description.job_id,
-            job_type: description.job_type,
-            state: description.state,
-            creation_ms: description.creation_ms,
-            spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
-            failure: description.failure.map(|failure| JobFailureInfo {
-                phase: failure.phase,
-                message: failure.message,
-                retryable: failure.retryable,
-            }),
-        }
-    }
-}
@@ -664,7 +664,11 @@ impl JsFullTextQuery {
 }

 fn parse_fts_query(query: Object) -> napi::Result<FullTextSearchQuery> {
-    if let Ok(Some(query)) = query.get::<&JsFullTextQuery>("query") {
+    // `&JsFullTextQuery` recovers a native class reference through napi's borrow-tracked
+    // path, which is only usable from generated `#[napi]` argument conversion. This is a
+    // manual lookup on a nested `Object` property instead, so use `ClassInstance`, which
+    // unwraps the class without requiring a borrow scope.
+    if let Ok(Some(query)) = query.get::<ClassInstance<JsFullTextQuery>>("query") {
        Ok(FullTextSearchQuery::new_query(query.inner.clone()))
    } else if let Ok(Some(query_text)) = query.get::<String>("query") {
        let mut query_text = query_text;
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.38.0-beta.11"
+version = "0.39.0-beta.4"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"
@@ -28,7 +28,7 @@ env_logger.workspace = true
 log.workspace = true
 # Maturin enables extension-module mode for Python builds. Keeping it out of
 # Cargo features lets Rust unit tests link against libpython.
-pyo3 = { version = "0.28", features = ["abi3-py310", "chrono"] }
+pyo3 = { version = "0.28", features = ["abi3-py310", "chrono", "uuid"] }
 chrono.workspace = true
 pyo3-async-runtimes = { version = "0.28", features = [
    "attributes",
@@ -40,6 +40,7 @@ serde.workspace = true
 serde_json.workspace = true
 snafu.workspace = true
 tokio.workspace = true
+uuid.workspace = true
 libc = "0.2"

 [build-dependencies]
@@ -139,6 +139,7 @@ include = [
    "python/lancedb/exceptions.py",
    "python/lancedb/background_loop.py",
    "python/lancedb/schema.py",
+    "python/lancedb/sql.py",
    "python/lancedb/remote/__init__.py",
    "python/lancedb/remote/errors.py",
    "python/lancedb/embeddings/__init__.py",
@@ -6,7 +6,7 @@ import importlib.metadata
 import os
 from concurrent.futures import ThreadPoolExecutor
 from datetime import timedelta
-from typing import Dict, Optional, Union, Any, List, Iterable
+from typing import Dict, Optional, Union, Any, List, Iterable, TYPE_CHECKING

 __version__ = importlib.metadata.version("lancedb")

@@ -20,9 +20,13 @@ from .db import AsyncConnection, DBConnection, LanceDBConnection
 from .remote import ClientConfig
 from .remote.db import RemoteDBConnection
 from .expr import Expr, col, lit, func
-from .schema import blob, vector, BlobType
+from .schema import blob, vector
 from .job import AsyncJob, Job
+from .sql import AsyncQuery as AsyncSqlQuery
+from .sql import Query as SqlQuery
+from .sql import QueryDescription
 from .functions import (
+    AssignmentMapping as AssignmentMapping,
    FunctionArtifactRequest as FunctionArtifactRequest,
    FunctionApplication as FunctionApplication,
    FunctionBinding as FunctionBinding,
@@ -49,6 +53,19 @@ from .namespace import (
 )


+if TYPE_CHECKING:
+    from lance.blob import BlobType as BlobType
+
+
+def __getattr__(name: str):
+    if name == "BlobType":
+        from .schema import BlobType
+
+        globals()["BlobType"] = BlobType
+        return BlobType
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
 def _check_s3_bucket_with_dots(
    uri: str, storage_options: Optional[Dict[str, str]]
 ) -> None:
@@ -88,6 +105,7 @@ def connect(
    api_key: Optional[str] = None,
    region: str = "us-east-1",
    host_override: Optional[str] = None,
+    sql_host_override: Optional[str] = None,
    read_consistency_interval: Optional[timedelta] = None,
    request_thread_pool: Optional[Union[int, ThreadPoolExecutor]] = None,
    client_config: Union[ClientConfig, Dict[str, Any], None] = None,
@@ -116,6 +134,9 @@ def connect(
        The region to use for LanceDB Cloud.
    host_override: str, optional
        The override url for LanceDB Cloud.
+    sql_host_override: str, optional
+        The remote SQL service endpoint override. The client connects lazily when SQL
+        is first executed and retains that connection.
    read_consistency_interval: timedelta, default None
        The interval at which to check for updates to the table from other
        processes. If None, then consistency is not checked. For performance
@@ -257,6 +278,7 @@ def connect(
            api_key,
            region,
            host_override,
+            sql_host_override=sql_host_override,
            # TODO: remove this (deprecation warning downstream)
            request_thread_pool=request_thread_pool,
            client_config=client_config,
@@ -399,6 +421,7 @@ def deserialize_conn(
            parsed["api_key"],
            parsed.get("region", "us-east-1"),
            host_override=parsed.get("host_override"),
+            sql_host_override=parsed.get("sql_host_override"),
            client_config=parsed.get("client_config"),
            storage_options=storage_options,
        )
@@ -412,6 +435,7 @@ async def connect_async(
    api_key: Optional[str] = None,
    region: str = "us-east-1",
    host_override: Optional[str] = None,
+    sql_host_override: Optional[str] = None,
    read_consistency_interval: Optional[timedelta] = None,
    client_config: Optional[Union[ClientConfig, Dict[str, Any]]] = None,
    storage_options: Optional[Dict[str, str]] = None,
@@ -434,6 +458,9 @@ async def connect_async(
        The region to use for LanceDB Cloud.
    host_override: str, optional
        The override url for LanceDB Cloud.
+    sql_host_override: str, optional
+        The remote SQL service endpoint override. The client connects lazily when SQL
+        is first executed and retains that connection.
    read_consistency_interval: timedelta, default None
        The interval at which to check for updates to the table from other
        processes. If None, then consistency is not checked. For performance
@@ -521,6 +548,7 @@ async def connect_async(
            api_key,
            region,
            host_override,
+            sql_host_override,
            read_consistency_interval_secs,
            client_config,
            storage_options,
@@ -543,6 +571,7 @@ __all__ = [
    "connect_namespace_async",
    "AsyncConnection",
    "AsyncJob",
+    "AsyncSqlQuery",
    "AsyncLanceNamespaceDBConnection",
    "AsyncTable",
    "FtsToken",
@@ -557,6 +586,8 @@ __all__ = [
    "vector",
    "DBConnection",
    "Job",
+    "QueryDescription",
+    "SqlQuery",
    "LanceDBConnection",
    "LanceNamespaceDBConnection",
    "LsmWriteSpec",
@@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Optional, Union
 import pyarrow as pa

 from .expr import Expr
-from .schema import blob_v2_column_paths
+from .schema import row_addressable_blob_v2_paths
 from .types import BlobMode, QueryProjection, QueryProjectionSpec

 if TYPE_CHECKING:
@@ -119,7 +119,7 @@ def blob_v2_projection_sources(
    schema: pa.Schema,
    projection: QueryProjection,
 ) -> dict[str, str]:
-    blob_columns = blob_v2_column_paths(schema)
+    blob_columns = row_addressable_blob_v2_paths(schema)
    if not blob_columns:
        return {}
    columns = set(blob_columns)
@@ -140,7 +140,9 @@ def v2_projection_needs_row_id(
 ) -> bool:
    if with_row_id:
        return False
-    return projection_includes_blob_column(projection, blob_v2_column_paths(schema))
+    return projection_includes_blob_column(
+        projection, row_addressable_blob_v2_paths(schema)
+    )


 def blob_auto_row_id_for_scan(
@@ -270,7 +272,8 @@ def _iter_projection_pairs(
            if isinstance(expr, str):
                yield name, expr
            elif isinstance(expr, Expr):
-                yield name, expr.to_sql()
+                source = expr._column_name()
+                yield name, source if source is not None else expr.to_sql()
        return
    for column in projection:
        if isinstance(column, str):
@@ -280,7 +283,8 @@ def _iter_projection_pairs(
            if isinstance(expr, str):
                yield name, expr
            elif isinstance(expr, Expr):
-                yield name, expr.to_sql()
+                source = expr._column_name()
+                yield name, source if source is not None else expr.to_sql()


 def _set_blob_column(tbl: pa.Table, output_name: str, blobs: pa.Array) -> pa.Table:
@@ -1,6 +1,7 @@
 from datetime import date, datetime, timedelta
 from decimal import Decimal
 from typing import Dict, List, Optional, Tuple, Any, TypedDict, Union, Literal
+from uuid import UUID

 import pyarrow as pa

@@ -87,6 +88,7 @@ class PyExpr:
    def contains(self, substr: "PyExpr") -> "PyExpr": ...
    def isin(self, values: List["PyExpr"]) -> "PyExpr": ...
    def cast(self, data_type: pa.DataType) -> "PyExpr": ...
+    def column_name(self) -> Optional[str]: ...
    def to_sql(self) -> str: ...

 def expr_col(name: str) -> PyExpr: ...
@@ -146,15 +148,20 @@ class Connection(object):
        start_after: Optional[str],
        limit: Optional[int],
    ) -> list[str]: ...  # Deprecated: Use list_tables instead
-    def job(self, job_id: str) -> Job: ...
+    async def open_job(self, job_id: str) -> Job: ...
    async def create_function_async(self, request_json: str) -> Job: ...
    async def get_function(self, name: str, version: str) -> str: ...
+    async def list_functions(self) -> List[str]: ...
+    async def drop_function(self, name: str, version: str) -> bool: ...
    async def list_jobs(self) -> List[JobInfo]: ...
-    async def get_job(self, job_id: str) -> Optional[JobDescription]: ...
    async def cancel_job(self, job_id: str) -> bool: ...
-    async def job_history(
-        self, job_id: Optional[str] = None
-    ) -> List[pa.RecordBatch]: ...
+    async def execute_query_async(
+        self,
+        query: str,
+        *,
+        default_namespace_path: Optional[List[str]] = None,
+    ) -> SqlQuery: ...
+    async def describe_query(self, query_id: UUID) -> QueryDescription: ...
    async def create_table(
        self,
        name: str,
@@ -233,9 +240,20 @@ class BlobFile:
 class Job:
    @property
    def id(self) -> Optional[str]: ...
+    @property
+    def _state(self) -> Optional[str]: ...
+    @property
+    def _description(self) -> Optional[JobDescription]: ...
    async def status(self) -> str: ...
    async def wait(self) -> Optional[str]: ...
    async def cancel(self) -> None: ...
+    async def refresh(self) -> None: ...
+    async def events(
+        self,
+        *,
+        limit: Optional[int] = None,
+        filter: Optional[str] = None,
+    ) -> pa.Table: ...

 class JobInfo:
    @property
@@ -267,10 +285,33 @@ class JobDescription:
    @property
    def creation_ms(self) -> int: ...
    @property
-    def spec_json(self) -> Optional[str]: ...
+    def _spec_json(self) -> Optional[str]: ...
+    @property
+    def _result_json(self) -> Optional[str]: ...
+    @property
+    def spec(self) -> Optional[Any]: ...
+    @property
+    def result(self) -> Optional[Any]: ...
    @property
    def failure(self) -> Optional[JobFailureInfo]: ...

+class SqlQuery:
+    @property
+    def id(self) -> UUID: ...
+    async def describe(self) -> QueryDescription: ...
+    async def reader(self) -> RecordBatchStream: ...
+    async def cancel(self) -> None: ...
+
+class QueryDescription:
+    @property
+    def id(self) -> UUID: ...
+    @property
+    def status(self) -> str: ...
+    @property
+    def progress(self) -> Optional[float]: ...
+    @property
+    def expires_at(self) -> Optional[datetime]: ...
+
 class Table:
    def name(self) -> str: ...
    def __repr__(self) -> str: ...
@@ -449,6 +490,7 @@ async def connect(
    api_key: Optional[str],
    region: Optional[str],
    host_override: Optional[str],
+    sql_host_override: Optional[str],
    read_consistency_interval: Optional[float],
    client_config: Optional[Union[ClientConfig, Dict[str, Any]]],
    storage_options: Optional[Dict[str, str]],
@@ -605,9 +647,11 @@ class FullTextQuery:
 class PyQueryRequest:
    limit: Optional[int]
    offset: Optional[int]
+    take_offsets: Optional[List[int]]
    filter: Optional[Union[str, bytes]]
    full_text_search: Optional[FullTextQuery]
    select: Optional[Union[str, List[str]]]
+    select_source_columns: Optional[Dict[str, str]]
    fast_search: Optional[bool]
    with_row_id: Optional[bool]
    use_lsm: Optional[bool]
@@ -19,6 +19,7 @@ from typing import (
    Optional,
    Union,
 )
+from uuid import UUID

 if sys.version_info >= (3, 12):
    from typing import override
@@ -47,6 +48,9 @@ from . import __version__
 from ._lancedb import connect as lancedb_connect  # type: ignore
 from .functions import FunctionVersion, UdfDefinition
 from .job import AsyncJob, Job, _typed_job
+from .sql import AsyncQuery as AsyncSqlQuery
+from .sql import Query as SqlQuery
+from .sql import QueryDescription
 from .materialized_view import (
    AsyncMaterializedView,
    MaterializedView,
@@ -68,10 +72,11 @@ import deprecation

 if TYPE_CHECKING:
    import pyarrow as pa
+    from .arrow import AsyncRecordBatchReader
    from .pydantic import LanceModel

    from ._lancedb import Connection as LanceDbConnection
-    from ._lancedb import JobDescription, JobInfo
+    from ._lancedb import JobInfo
    from .common import DATA, URI
    from .embeddings import EmbeddingFunctionConfig
    from ._lancedb import Session
@@ -712,26 +717,51 @@ class DBConnection(EnforceOverrides):
            "Function catalog operations are not supported for this connection type"
        )

-    def job(self, job_id: str) -> Job:
-        """A [Job][lancedb.job.Job] handle for a server-side job by id.
+    def list_functions(self) -> List[FunctionVersion]:
+        """List every published immutable Function version.

-        The handle is constructed without a server round trip; an unknown id
-        surfaces when the handle is used. Dropping the handle has no effect
-        on the job itself.
+        Results are ordered by Function name then version. Local connections
+        raise ``NotImplementedError``.
+
+        Examples
+        --------
+        List the identities available to use in Function-backed columns:
+
+        ```python
+        [(function.name, function.version) for function in db.list_functions()]
+        ```
        """
-        raise NotImplementedError("job is not supported for this connection type")
+        raise NotImplementedError(
+            "Function catalog operations are not supported for this connection type"
+        )
+
+    def drop_function(self, name: str, *, version: str) -> bool:
+        """Drop one exact immutable Function version from the remote catalog.
+
+        Returns True when the version changed to Dropped and False for an
+        idempotent replay. Local connections raise NotImplementedError.
+        """
+        raise NotImplementedError(
+            "Function catalog operations are not supported for this connection type"
+        )
+
+    def open_job(self, job_id: str) -> Job:
+        """Open a server-side job by id, returning a handle with its record
+        already populated.
+
+        The returned [Job][lancedb.job.Job] answers for its own state,
+        specification, result, failure and event history, so there is no
+        separate connection-level call for any of them.
+
+        Raises `JobNotFoundError` when the server has no such job, the way
+        `open_table` does for a missing table.
+        """
+        raise NotImplementedError("open_job is not supported for this connection type")

    def list_jobs(self) -> List[JobInfo]:
        """List server-side jobs across the database's tables."""
        raise NotImplementedError("list_jobs is not supported for this connection type")

-    def get_job(self, job_id: str) -> Optional[JobDescription]:
-        """Describe a single server-side job by id.
-
-        Returns None when the server has no such job.
-        """
-        raise NotImplementedError("get_job is not supported for this connection type")
-
    def cancel_job(self, job_id: str) -> bool:
        """Request cancellation of a server-side job by id.

@@ -743,14 +773,38 @@ class DBConnection(EnforceOverrides):
            "cancel_job is not supported for this connection type"
        )

-    def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
-        """The lifecycle event history of a server-side job, as Arrow batches.
+    def execute_query(
+        self,
+        query: str,
+        *,
+        default_namespace_path: Optional[List[str]] = None,
+    ) -> pa.RecordBatchReader:
+        """Execute SQL and return a blocking Arrow reader.

-        Lists history across all jobs when `job_id` is None.
+        This submits through :meth:`execute_query_async` and waits until the
+        initial result stream is readable. It does not wait for the full query
+        to finish.
        """
-        raise NotImplementedError(
-            "job_history is not supported for this connection type"
-        )
+        return self.execute_query_async(
+            query,
+            default_namespace_path=default_namespace_path,
+        ).reader()
+
+    def execute_query_async(
+        self,
+        query: str,
+        *,
+        default_namespace_path: Optional[List[str]] = None,
+    ) -> SqlQuery:
+        """Start executing SQL and return its query handle.
+
+        Local connections do not support SQL.
+        """
+        raise NotImplementedError("SQL is not supported for this connection type")
+
+    def describe_query(self, query_id: UUID) -> QueryDescription:
+        """Describe a submitted SQL query by its connection-scoped id."""
+        raise NotImplementedError("SQL is not supported for this connection type")


 class LanceDBConnection(DBConnection):
@@ -847,6 +901,7 @@ class LanceDBConnection(DBConnection):
                None,
                None,
                None,
+                None,
                read_consistency_interval_secs,
                None,
                storage_options,
@@ -1395,14 +1450,11 @@ class LanceDBConnection(DBConnection):
        )

    @override
-    def job(self, job_id: str) -> Job:
-        """A [Job][lancedb.job.Job] handle for a server-side job by id.
-
-        The handle is constructed without a server round trip; an unknown id
-        surfaces when the handle is used. Dropping the handle has no effect
-        on the job itself.
+    def open_job(self, job_id: str) -> Job:
+        """Open a server-side job by id. See
+        [DBConnection.open_job][lancedb.db.DBConnection.open_job].
        """
-        return Job(self._conn.job(job_id))
+        return Job(LOOP.run(self._conn.open_job(job_id)))

    @override
    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
@@ -1413,19 +1465,19 @@ class LanceDBConnection(DBConnection):
    def get_function(self, name: str, *, version: str) -> FunctionVersion:
        return LOOP.run(self._conn.get_function(name, version=version))

+    @override
+    def list_functions(self) -> List[FunctionVersion]:
+        return LOOP.run(self._conn.list_functions())
+
+    @override
+    def drop_function(self, name: str, *, version: str) -> bool:
+        return LOOP.run(self._conn.drop_function(name, version=version))
+
    @override
    def list_jobs(self) -> List[JobInfo]:
        """List server-side jobs across the database's tables."""
        return LOOP.run(self._conn.list_jobs())

-    @override
-    def get_job(self, job_id: str) -> Optional[JobDescription]:
-        """Describe a single server-side job by id.
-
-        Returns None when the server has no such job.
-        """
-        return LOOP.run(self._conn.get_job(job_id))
-
    @override
    def cancel_job(self, job_id: str) -> bool:
        """Request cancellation of a server-side job by id.
@@ -1436,14 +1488,6 @@ class LanceDBConnection(DBConnection):
        """
        return LOOP.run(self._conn.cancel_job(job_id))

-    @override
-    def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
-        """The lifecycle event history of a server-side job, as Arrow batches.
-
-        Lists history across all jobs when `job_id` is None.
-        """
-        return LOOP.run(self._conn.job_history(job_id))
-
    @override
    def namespace_client(self) -> LanceNamespace:
        """Get the equivalent namespace client for this connection.
@@ -2214,15 +2258,11 @@ class AsyncConnection(object):
            namespace_path = []
        await self._inner.drop_all_tables(namespace_path=namespace_path)

-    def job(self, job_id: str) -> AsyncJob:
-        """An [AsyncJob][lancedb.job.AsyncJob] handle for a server-side job
-        by id.
-
-        The handle is constructed without a server round trip; an unknown id
-        surfaces when the handle is used. Dropping the handle has no effect
-        on the job itself.
+    async def open_job(self, job_id: str) -> AsyncJob:
+        """Open a server-side job by id. See
+        [DBConnection.open_job][lancedb.db.DBConnection.open_job].
        """
-        return AsyncJob(self._inner.job(job_id))
+        return AsyncJob(await self._inner.open_job(job_id))

    async def create_function_async(
        self, definition: UdfDefinition
@@ -2243,17 +2283,25 @@ class AsyncConnection(object):
        """Open one exact immutable Function version from the remote catalog."""
        return FunctionVersion.from_json(await self._inner.get_function(name, version))

+    async def list_functions(self) -> List[FunctionVersion]:
+        """List every published immutable Function version.
+
+        Results are ordered by Function name then version. Local connections
+        raise ``NotImplementedError``.
+        """
+        return [
+            FunctionVersion.from_json(value)
+            for value in await self._inner.list_functions()
+        ]
+
+    async def drop_function(self, name: str, *, version: str) -> bool:
+        """Drop one exact immutable Function version from the remote catalog."""
+        return await self._inner.drop_function(name, version)
+
    async def list_jobs(self) -> List[JobInfo]:
        """List server-side jobs across the database's tables."""
        return await self._inner.list_jobs()

-    async def get_job(self, job_id: str) -> Optional[JobDescription]:
-        """Describe a single server-side job by id.
-
-        Returns None when the server has no such job.
-        """
-        return await self._inner.get_job(job_id)
-
    async def cancel_job(self, job_id: str) -> bool:
        """Request cancellation of a server-side job by id.

@@ -2263,12 +2311,46 @@ class AsyncConnection(object):
        """
        return await self._inner.cancel_job(job_id)

-    async def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
-        """The lifecycle event history of a server-side job, as Arrow batches.
+    async def execute_query(
+        self,
+        query: str,
+        *,
+        default_namespace_path: Optional[List[str]] = None,
+    ) -> AsyncRecordBatchReader:
+        """Execute SQL and return an asynchronous Arrow reader.

-        Lists history across all jobs when `job_id` is None.
+        This submits through :meth:`execute_query_async` and waits until the
+        initial result stream is readable. It does not wait for the full query
+        to finish.
        """
-        return await self._inner.job_history(job_id)
+        submitted = await self.execute_query_async(
+            query,
+            default_namespace_path=default_namespace_path,
+        )
+        return await submitted.reader()
+
+    async def execute_query_async(
+        self,
+        query: str,
+        *,
+        default_namespace_path: Optional[List[str]] = None,
+    ) -> AsyncSqlQuery:
+        """Start executing SQL and return its query handle.
+
+        The database from ``connect_async`` is used for unqualified database
+        references. The namespace defaults to ``["public"]``. Local
+        connections raise ``NotImplementedError``.
+        """
+        return AsyncSqlQuery(
+            await self._inner.execute_query_async(
+                query,
+                default_namespace_path=default_namespace_path,
+            )
+        )
+
+    async def describe_query(self, query_id: UUID) -> QueryDescription:
+        """Describe a submitted SQL query by its connection-scoped id."""
+        return await self._inner.describe_query(query_id)

    async def namespace_client(self) -> LanceNamespace:
        """Get the equivalent namespace client for this connection.
@@ -35,3 +35,9 @@ class JobCancelledError(RuntimeError):
    """Exception raised when an asynchronous job was cancelled."""

    pass
+
+
+class JobNotFoundError(ValueError):
+    """Exception raised when opening a job the server does not have."""
+
+    pass
@@ -249,6 +249,10 @@ class Expr:

    # ── utilities ────────────────────────────────────────────────────────────

+    def _column_name(self) -> str | None:
+        """Return the source name when this is a bare column expression."""
+        return self._inner.column_name()
+
    def to_sql(self) -> str:
        """Render the expression as a SQL string (useful for debugging)."""
        return self._inner.to_sql()
@@ -312,7 +316,7 @@ def func(name: str, *args: ExprLike) -> Expr:
    --------
    >>> from lancedb.expr import col, func
    >>> func("lower", col("name"))
-    Expr(lower(name))
+    Expr(lower(`name`))
    """
    inner_args = [_coerce(a)._inner for a in args]
    return Expr(expr_func(name, inner_args))
@@ -49,11 +49,25 @@ from pydantic import (
    model_validator,
 )

+from .schema import is_blob_v2_field as _is_blob_v2_field
+
 _Int32 = conint(strict=True, ge=-(2**31), le=2**31 - 1)
 _UInt32 = conint(strict=True, ge=0, le=2**32 - 1)
 _UInt64 = conint(strict=True, ge=0, le=2**64 - 1)


+def _validate_gpu_wire_marker(value: Any) -> bool:
+    if value is not True:
+        raise ValueError("runtime.gpu must be true")
+    return True
+
+
+def _normalize_gpu_marker(value: bool) -> Optional[bool]:
+    if not isinstance(value, bool):
+        raise ValueError("gpu must be a boolean")
+    return True if value else None
+
+
 class _FrozenDict(dict):
    def _immutable(self, *args, **kwargs):
        raise TypeError("remote canonical values are immutable")
@@ -239,6 +253,23 @@ class PythonRuntimeSpec(_RemoteValue):
    python_version: Optional[str] = None
    environment: Optional[PythonEnvironmentSpec] = None
    env: Optional[Mapping[str, str]] = None
+    gpu: Optional[bool] = None
+
+    @model_validator(mode="before")
+    @classmethod
+    def _discard_unknown_runtime_payload(cls, value):
+        if isinstance(value, Mapping):
+            kind = value.get("kind")
+            if isinstance(kind, str) and kind not in {"python", "python_v2"}:
+                return {"kind": kind}
+        return value
+
+    @field_validator("gpu", mode="before")
+    @classmethod
+    def _validate_gpu_marker(cls, value):
+        if value is None:
+            return None
+        return _validate_gpu_wire_marker(value)

    @model_validator(mode="after")
    def _validate_runtime_kind(self):
@@ -247,18 +278,28 @@ class PythonRuntimeSpec(_RemoteValue):
                raise ValueError("python runtime requires python_version")
            if self.environment is None:
                raise ValueError("python runtime requires environment")
+            if self.gpu is not None:
+                raise ValueError("python runtime with gpu requires kind='python_v2'")
+        elif self.kind == "python_v2":
+            if self.python_version is None:
+                raise ValueError("python_v2 runtime requires python_version")
+            if self.environment is None:
+                raise ValueError("python_v2 runtime requires environment")
+            if self.gpu is None:
+                raise ValueError("python_v2 runtime requires gpu")
        else:
            object.__setattr__(self, "python_version", None)
            object.__setattr__(self, "environment", None)
            object.__setattr__(self, "env", None)
+            object.__setattr__(self, "gpu", None)
        return self


 class FunctionVersion(_RemoteValue):
    """An exact immutable Function version returned by Enterprise.

-    Scheduling resources, priority, concurrency, and retry policy belong to
-    the submitting Job and are not part of this identity.
+    The GPU execution requirement is part of this identity. CPU and memory sizing,
+    priority, concurrency, and retry policy belong to the execution platform.
    """

    name: str
@@ -429,11 +470,7 @@ class InputBinding(_RemoteValue):


 class OutputMapping(_RemoteValue):
-    """One stable result-field mapping.
-
-    Assignment state is outside the Slice 1 client contract. During the NULL
-    transition Lance exposes no public cell-flag identifier to persist here.
-    """
+    """One stable result-field mapping."""

    result_field: str
    output_name: str
@@ -443,6 +480,13 @@ class OutputMapping(_RemoteValue):
    nullable: bool


+class AssignmentMapping(_RemoteValue):
+    """Internal physical column preserving flattened struct validity."""
+
+    output_name: str
+    output_field_id: _Int32
+
+
 class FunctionBinding(_RemoteValue):
    """Immutable Function binding persisted by the Enterprise table service."""

@@ -450,6 +494,7 @@ class FunctionBinding(_RemoteValue):
    function: FunctionVersionRef
    inputs: tuple[InputBinding, ...]
    outputs: tuple[OutputMapping, ...]
+    assignment: Optional[AssignmentMapping] = None
    input_schema: Optional[Mapping[str, Any]] = None
    output_schema: Optional[Mapping[str, Any]] = None

@@ -479,6 +524,13 @@ class RefreshColumnResult(_RemoteValue):


 _FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$")
+_FUNCTION_BLOB_V2_TYPE = "blob_v2"
+_ARROW_EXTENSION_NAME_KEY = "ARROW:extension:name"
+_BLOB_V2_EXTENSION_NAME = "lance.blob.v2"
+_NESTED_BLOB_COLLECTION_ERROR = (
+    "unsupported Arrow type for Function signature: Blob v2 fields nested under "
+    "collection types are not supported"
+)


 _GRAMMAR_PRIMITIVES = (
@@ -495,6 +547,7 @@ _GRAMMAR_PRIMITIVES = (
    (pa.float32(), "float32"),
    (pa.float64(), "float64"),
    (pa.string(), "utf8"),
+    (pa.large_string(), "large_utf8"),
    (pa.binary(), "binary"),
    (pa.date32(), "date32"),
    (pa.date64(), "date64"),
@@ -502,31 +555,258 @@ _GRAMMAR_PRIMITIVES = (


 def _canonical_arrow_type(data_type: pa.DataType) -> str:
-    """The server's V1 Function type grammar. Anything outside it is rejected
-    here rather than at registration."""
+    """The compact Function grammar, or canonical exact JSON for nested types."""
+    grammar = _grammar_arrow_type(data_type)
+    if grammar is not None:
+        return grammar
+    exact = _exact_arrow_type(data_type)
+    return json.dumps(exact, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+
+
+def _grammar_arrow_type(data_type: pa.DataType) -> Optional[str]:
    for candidate, name in _GRAMMAR_PRIMITIVES:
        if data_type == candidate:
            return name
    if pa.types.is_list(data_type) or pa.types.is_large_list(data_type):
+        item = _grammar_list_item(data_type)
+        if item is None:
+            return None
        prefix = "list" if pa.types.is_list(data_type) else "large_list"
-        return f"{prefix}<{_canonical_list_item(data_type)}>"
+        return f"{prefix}<{item}>"
    if pa.types.is_fixed_size_list(data_type) and data_type.list_size > 0:
-        return (
-            f"fixed_size_list<{_canonical_list_item(data_type)}, {data_type.list_size}>"
-        )
-    raise TypeError(f"unsupported Arrow type for Function signature: {data_type}")
+        item = _grammar_list_item(data_type)
+        if item is not None:
+            return f"fixed_size_list<{item}, {data_type.list_size}>"
+    return None


-def _canonical_list_item(data_type: pa.DataType) -> str:
+def _grammar_list_item(data_type: pa.DataType) -> Optional[str]:
    """The grammar names only the item type; it always means a non-nullable
-    child called `item`, so any other child metadata cannot be represented."""
+    child called `item`, so other child properties require exact JSON."""
    child = data_type.value_field
    if child.name != "item" or child.nullable or child.metadata:
+        return None
+    return _grammar_arrow_type(child.type)
+
+
+def _validate_exact_arrow_field(field: pa.Field) -> None:
+    if not field.name:
        raise TypeError(
-            "unsupported Arrow type for Function signature: list items must be a "
-            f"non-nullable field named 'item', got {child}"
+            "unsupported Arrow type for Function signature: field names "
+            "must not be empty"
        )
-    return _canonical_arrow_type(child.type)
+    if _is_blob_v2_field(field):
+        if not _has_supported_blob_v2_layout(field):
+            raise TypeError(
+                "unsupported Arrow type for Function signature: lance.blob.v2 "
+                f"requires a supported Blob storage layout, got {field}"
+            )
+        metadata = {
+            (key.decode() if isinstance(key, bytes) else key): (
+                value.decode() if isinstance(value, bytes) else value
+            )
+            for key, value in (field.metadata or {}).items()
+        }
+        if metadata and metadata != {
+            _ARROW_EXTENSION_NAME_KEY: _BLOB_V2_EXTENSION_NAME
+        }:
+            raise TypeError(
+                "unsupported Arrow type for Function signature: lance.blob.v2 "
+                "field metadata must contain only its canonical extension marker"
+            )
+    elif field.metadata:
+        raise TypeError(
+            "unsupported Arrow type for Function signature: field metadata "
+            f"is not supported, got {field}"
+        )
+
+
+def _has_supported_blob_v2_layout(field: pa.Field) -> bool:
+    data_type = field.type
+    if isinstance(data_type, pa.ExtensionType):
+        data_type = data_type.storage_type
+    if not pa.types.is_struct(data_type):
+        return False
+
+    fields = tuple(data_type)
+
+    def matches(spec, compare_nullable) -> bool:
+        return len(fields) == len(spec) and all(
+            actual.name == name
+            and actual.type == expected_type
+            and (not check_nullable or actual.nullable == nullable)
+            for actual, (name, expected_type, nullable), check_nullable in zip(
+                fields, spec, compare_nullable
+            )
+        )
+
+    logical_minimal = (
+        ("data", pa.large_binary(), True),
+        ("uri", pa.utf8(), True),
+    )
+    logical_full = logical_minimal + (
+        ("position", pa.uint64(), True),
+        ("size", pa.uint64(), True),
+    )
+    prepared = (
+        ("kind", pa.uint8(), True),
+        ("data", pa.large_binary(), True),
+        ("uri", pa.utf8(), True),
+        ("blob_id", pa.uint32(), True),
+        ("blob_size", pa.uint64(), True),
+        ("position", pa.uint64(), True),
+    )
+    descriptor = (
+        ("kind", pa.uint8(), False),
+        ("position", pa.uint64(), False),
+        ("size", pa.uint64(), False),
+        ("blob_id", pa.uint32(), False),
+        ("blob_uri", pa.utf8(), False),
+    )
+    return (
+        matches(logical_minimal, (True, True))
+        or matches(logical_full, (True, True, False, False))
+        or matches(prepared, (True,) * len(prepared))
+        or matches(descriptor, (False,) * len(descriptor))
+    )
+
+
+def _canonical_arrow_field(field: pa.Field) -> str:
+    _validate_exact_arrow_field(field)
+    if _is_blob_v2_field(field):
+        return _FUNCTION_BLOB_V2_TYPE
+    return _canonical_arrow_type(field.type)
+
+
+def _blob_storage_type(field: pa.Field) -> pa.DataType:
+    data_type = field.type
+    if isinstance(data_type, pa.ExtensionType):
+        return data_type.storage_type
+    return data_type
+
+
+def _exact_blob_storage_type(field: pa.Field) -> dict[str, Any]:
+    storage = _blob_storage_type(field)
+    if not pa.types.is_struct(storage):
+        raise TypeError(
+            "unsupported Arrow type for Function signature: lance.blob.v2 "
+            "requires struct storage"
+        )
+    return {
+        "type": "struct",
+        "fields": [
+            {
+                "name": child.name,
+                "nullable": child.nullable,
+                "type": (
+                    {"type": "large_binary"}
+                    if pa.types.is_large_binary(child.type)
+                    else _exact_arrow_type(child.type)
+                ),
+            }
+            for child in storage
+        ],
+    }
+
+
+def _data_type_has_blob_v2(data_type: pa.DataType) -> bool:
+    if pa.types.is_struct(data_type):
+        return any(
+            _is_blob_v2_field(field) or _data_type_has_blob_v2(field.type)
+            for field in data_type
+        )
+    if (
+        pa.types.is_list(data_type)
+        or pa.types.is_large_list(data_type)
+        or pa.types.is_fixed_size_list(data_type)
+    ):
+        field = data_type.value_field
+        return _is_blob_v2_field(field) or _data_type_has_blob_v2(field.type)
+    if pa.types.is_map(data_type):
+        return any(
+            _is_blob_v2_field(field) or _data_type_has_blob_v2(field.type)
+            for field in (data_type.key_field, data_type.item_field)
+        )
+    return False
+
+
+def _exact_arrow_field(
+    field: pa.Field, *, inside_collection: bool = False
+) -> dict[str, Any]:
+    _validate_exact_arrow_field(field)
+    if _is_blob_v2_field(field):
+        if inside_collection:
+            raise TypeError(_NESTED_BLOB_COLLECTION_ERROR)
+        return {
+            "name": field.name,
+            "nullable": field.nullable,
+            "type": _exact_blob_storage_type(field),
+            "metadata": {
+                _ARROW_EXTENSION_NAME_KEY: _BLOB_V2_EXTENSION_NAME,
+            },
+        }
+    value = {
+        "name": field.name,
+        "nullable": field.nullable,
+        "type": _exact_arrow_type(field.type, inside_collection=inside_collection),
+    }
+    return value
+
+
+def _exact_arrow_type(
+    data_type: pa.DataType, *, inside_collection: bool = False
+) -> dict[str, Any]:
+    for candidate, name in _GRAMMAR_PRIMITIVES:
+        if data_type == candidate:
+            return {"type": name}
+    if pa.types.is_struct(data_type):
+        fields = list(data_type)
+        names = [field.name for field in fields]
+        if not fields or len(set(names)) != len(names):
+            raise TypeError(
+                "unsupported Arrow type for Function signature: structs must have "
+                "non-empty, uniquely named fields"
+            )
+        return {
+            "type": "struct",
+            "fields": [
+                _exact_arrow_field(field, inside_collection=inside_collection)
+                for field in fields
+            ],
+        }
+    if (
+        pa.types.is_list(data_type)
+        or pa.types.is_large_list(data_type)
+        or pa.types.is_fixed_size_list(data_type)
+    ):
+        if pa.types.is_fixed_size_list(data_type):
+            if data_type.value_field.name != "item":
+                raise TypeError(
+                    "unsupported Arrow type for Function signature: fixed-size list "
+                    "items must be named 'item'"
+                )
+            if data_type.list_size <= 0:
+                raise TypeError(
+                    f"unsupported Arrow type for Function signature: {data_type}"
+                )
+        value: dict[str, Any] = {
+            "type": (
+                "list"
+                if pa.types.is_list(data_type)
+                else "large_list"
+                if pa.types.is_large_list(data_type)
+                else "fixed_size_list"
+            ),
+            "fields": [
+                _exact_arrow_field(data_type.value_field, inside_collection=True)
+            ],
+        }
+        if pa.types.is_fixed_size_list(data_type):
+            value["length"] = data_type.list_size
+        return value
+    if pa.types.is_map(data_type) and _data_type_has_blob_v2(data_type):
+        raise TypeError(_NESTED_BLOB_COLLECTION_ERROR)
+    raise TypeError(f"unsupported Arrow type for Function signature: {data_type}")


 def _list_of(item: pa.DataType) -> pa.DataType:
@@ -600,8 +880,15 @@ def _callable_parameters(function: Callable[..., Any]) -> tuple[inspect.Paramete

 def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutput:
    if isinstance(output, pa.Schema):
+        if output.metadata:
+            raise TypeError("Function output schema metadata is not supported")
        fields = tuple(output)
-    elif isinstance(output, pa.Field) and pa.types.is_struct(output.type):
+    elif (
+        isinstance(output, pa.Field)
+        and not _is_blob_v2_field(output)
+        and pa.types.is_struct(output.type)
+    ):
+        _validate_exact_arrow_field(output)
        if output.nullable:
            raise ValueError("Function output must be non-nullable")
        fields = tuple(output.type)
@@ -617,18 +904,19 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
            raise TypeError(
                "output_schema must be a PyArrow DataType, Field, or Schema"
            )
+        _validate_exact_arrow_field(field)
        if field.nullable:
            raise ValueError("Function output must be non-nullable")
        return FunctionOutput(
            kind="scalar",
-            arrow_type=_canonical_arrow_type(field.type),
+            arrow_type=_canonical_arrow_field(field),
            nullable=False,
        )

    if not fields:
        raise ValueError("named-struct Function output must contain at least one field")
-    if any(field.nullable for field in fields):
-        raise ValueError("Function output fields must be non-nullable")
+    for field in fields:
+        _validate_exact_arrow_field(field)
    names = [field.name for field in fields]
    if len(set(names)) != len(names):
        raise ValueError("Function output field names must be unique")
@@ -637,8 +925,8 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
        fields=tuple(
            FunctionResultField(
                name=field.name,
-                arrow_type=_canonical_arrow_type(field.type),
-                nullable=False,
+                arrow_type=_canonical_arrow_field(field),
+                nullable=field.nullable,
            )
            for field in fields
        ),
@@ -657,6 +945,10 @@ def _infer_signature(
    if input_schema is not None:
        if not isinstance(input_schema, pa.Schema):
            raise TypeError("input_schema must be a PyArrow Schema")
+        if input_schema.metadata:
+            raise TypeError("Function input schema metadata is not supported")
+        for field in input_schema:
+            _validate_exact_arrow_field(field)
        expected = tuple(parameter.name for parameter in parameters)
        actual = tuple(input_schema.names)
        if actual != expected:
@@ -667,7 +959,7 @@ def _infer_signature(
        inputs = tuple(
            FunctionParameter(
                name=field.name,
-                arrow_type=_canonical_arrow_type(field.type),
+                arrow_type=_canonical_arrow_field(field),
                nullable=field.nullable,
            )
            for field in input_schema
@@ -690,7 +982,9 @@ def _infer_signature(
        inputs.append(
            FunctionParameter(
                name=parameter.name,
-                arrow_type=_canonical_arrow_type(data_type),
+                arrow_type=_canonical_arrow_field(
+                    pa.field(parameter.name, data_type, nullable=nullable)
+                ),
                nullable=nullable,
            )
        )
@@ -910,6 +1204,7 @@ class UdfDefinition:
        pip: tuple[str, ...],
        env: Mapping[str, str],
        python_version: Optional[str],
+        gpu: bool = False,
        conda: tuple[str, ...] = (),
        conda_channels: tuple[str, ...] = (),
    ):
@@ -938,12 +1233,14 @@ class UdfDefinition:
        signature = _infer_signature(function, input_schema, output_schema)
        source = _package_source(function)
        digest = f"sha256:{hashlib.sha256(source).hexdigest()}"
+        gpu_marker = _normalize_gpu_marker(gpu)
        runtime = PythonRuntimeSpec(
-            kind="python",
+            kind="python_v2" if gpu_marker is not None else "python",
            python_version=python_version
            or f"{sys.version_info.major}.{sys.version_info.minor}",
            environment=environment_spec,
            env=environment,
+            gpu=gpu_marker,
        )
        self._function = function
        self._request = FunctionRegistrationRequest(
@@ -989,6 +1286,7 @@ def udf(
    pip: tuple[str, ...] | list[str] = (),
    env: Optional[Mapping[str, str]] = None,
    python_version: Optional[str] = None,
+    gpu: bool = False,
    conda: tuple[str, ...] | list[str] = (),
    conda_channels: tuple[str, ...] | list[str] = (),
 ) -> Callable[[Callable[..., Any]], UdfDefinition]: ...
@@ -1003,6 +1301,7 @@ def udf(
    pip: tuple[str, ...] | list[str] = (),
    env: Optional[Mapping[str, str]] = None,
    python_version: Optional[str] = None,
+    gpu: bool = False,
    conda: tuple[str, ...] | list[str] = (),
    conda_channels: tuple[str, ...] | list[str] = (),
 ):
@@ -1010,8 +1309,9 @@ def udf(

    Input and output signatures are inferred from supported annotations. For
    Arrow types annotations cannot express precisely, pass ``input_schema``
-    and ``output_schema`` together. Nullable outputs are rejected because V1
-    uses physical NULL to represent unassigned computed-column rows.
+    and ``output_schema`` together. Scalar outputs must be non-nullable. Every
+    named-struct field may be nullable; Enterprise preserves the struct's
+    validity when the result is expanded into sibling columns.

    Parameters
    ----------
@@ -1023,8 +1323,8 @@ def udf(
        Explicit input fields in the exact order of the callable parameters.
        Must be provided together with ``output_schema``.
    output_schema : pyarrow.DataType, pyarrow.Field, or pyarrow.Schema, optional
-        Explicit scalar or named-struct output. Must be non-nullable and be
-        provided together with ``input_schema``.
+        Explicit scalar or named-struct output. Scalar outputs must be
+        non-nullable. Must be provided together with ``input_schema``.
    pip : sequence of str, optional
        Pip requirements for the remote environment.
    conda : sequence of str, optional
@@ -1035,6 +1335,10 @@ def udf(
        Environment variables included in the Function definition.
    python_version : str, optional
        Remote Python major/minor version. Defaults to the client version.
+    gpu : bool, default False
+        Whether every remote execution requires a GPU. The execution platform
+        selects one compatible GPU for each worker. The requirement is part of
+        the immutable Function version.

    The packaged artifact is a snapshot: the function source plus exactly
    the module-level names it references (modules as imports, importable
@@ -1059,6 +1363,11 @@ def udf(
    ...     return value * 2
    >>> score(1.5)
    3.0
+    >>> @udf(pip=["cupy-cuda12x"], gpu=True)
+    ... def gpu_score(value: int) -> int:
+    ...     return value * 2
+    >>> gpu_score.registration_request.runtime.gpu
+    True
    """

    def decorate(target: Callable[..., Any]) -> UdfDefinition:
@@ -1070,6 +1379,7 @@ def udf(
            pip=tuple(pip),
            env={} if env is None else env,
            python_version=python_version,
+            gpu=gpu,
            conda=tuple(conda),
            conda_channels=tuple(conda_channels),
        )
@@ -1080,6 +1390,7 @@ def udf(


 __all__ = [
+    "AssignmentMapping",
    "ApplicationInput",
    "FunctionApplication",
    "FunctionArtifact",
@@ -4,15 +4,27 @@
 """Handles to operations a server may run asynchronously."""

 import asyncio
+import json
 from datetime import timedelta
 from typing import Any, Callable, Generic, Optional, TypeVar, cast

+import pyarrow as pa
+
 from lancedb.background_loop import LOOP

 from . import _lancedb
+from ._lancedb import JobDescription, JobFailureInfo, JobInfo

 T = TypeVar("T")

+__all__ = [
+    "AsyncJob",
+    "Job",
+    "JobDescription",
+    "JobFailureInfo",
+    "JobInfo",
+]
+

 class AsyncJob(Generic[T]):
    """A handle to an operation that may still be running.
@@ -78,6 +90,149 @@ class AsyncJob(Generic[T]):
            return
        await self._inner.cancel()

+    async def refresh(self) -> None:
+        """Ask the backend for this job's current state, and for a server-side
+        job its full record, then cache it for the properties below.
+
+        The properties are all `None` until this runs, because submitting an
+        operation returns only a job id. `status` fetches the whole record too;
+        `wait` records only the terminal state it establishes.
+        """
+        if self._inner is None:
+            return
+        await self._inner.refresh()
+
+    @property
+    def state(self) -> Optional[str]:
+        """The last observed lifecycle state, without contacting the backend.
+
+        `None` until the handle has talked to it. See :meth:`AsyncJob.refresh`.
+        """
+        if self._inner is None:
+            return "finished"
+        return self._inner._state
+
+    @property
+    def job_type(self) -> Optional[str]:
+        """The job's type, as the server names it.
+
+        `None` for an in-process job, which has no server-side record.
+        """
+        return self._field("job_type")
+
+    @property
+    def creation_ms(self) -> Optional[int]:
+        """When the job was created, in milliseconds since the epoch."""
+        return self._field("creation_ms")
+
+    @property
+    def spec(self) -> Optional[Any]:
+        """The job-type-specific specification it was submitted with."""
+        return self._field("spec")
+
+    @property
+    def result(self) -> Optional[Any]:
+        """The job-type-specific terminal result, as reported data rather than
+        the typed model :meth:`AsyncJob.wait` returns.
+
+        `None` until the job succeeds, so a job that never terminates reports
+        its progress through :meth:`AsyncJob.events` instead.
+        """
+        return self._field("result")
+
+    @property
+    def failure(self) -> Optional[JobFailureInfo]:
+        """Why the job failed, when it failed and the server reports a reason."""
+        return self._field("failure")
+
+    @property
+    def _spec_json(self) -> Optional[str]:
+        return self._field("_spec_json")
+
+    @property
+    def _result_json(self) -> Optional[str]:
+        return self._field("_result_json")
+
+    def _field(self, name: str) -> Optional[Any]:
+        description = self._inner._description if self._inner is not None else None
+        return getattr(description, name) if description is not None else None
+
+    async def events(
+        self,
+        *,
+        limit: Optional[int] = None,
+        filter: Optional[str] = None,
+    ) -> "pa.Table":
+        """This job's recorded lifecycle events.
+
+        Where the properties above report a terminal result only once the job
+        reaches one, events are written as the job runs and outlive the workers
+        that produced them. A distributed job records a `claim`/`claim_complete`
+        pair per unit of work, each carrying `rows_processed`, so a job that
+        never finishes still accounts for what it did.
+
+        Parameters
+        ----------
+        limit: int, optional
+            Maximum event rows to return. The server caps results at 1000 by
+            default and 10,000 at most, and truncates without saying so, so
+            pass this for a job that emits an event per fragment.
+        filter: str, optional
+            SQL-like expression over the `state`, `updated_by`, `emitted_from`,
+            `emitted_by`, and `claim_entity` columns, such as
+            ``state = 'claim_complete'``.
+        """
+        if self._inner is None:
+            raise NotImplementedError(
+                "job event history is only available for server-side jobs"
+            )
+        return await self._inner.events(limit=limit, filter=filter)
+
+    def __repr__(self) -> str:
+        return _job_repr("AsyncJob", self)
+
+
+_REPR_INDENT = " " * 4
+
+
+def _repr_payload(value: Any) -> str:
+    """Render a job payload as indented JSON, aligned under its field."""
+    try:
+        rendered = json.dumps(value, indent=4)
+    except TypeError:
+        return repr(value)
+    return rendered.replace("\n", "\n" + _REPR_INDENT)
+
+
+def _job_repr(kind: str, job: Any) -> str:
+    """Render every field the handle currently knows, omitting the rest.
+
+    One field per line, with the JSON payloads indented, because a refresh
+    job's spec and result are the point of printing it.
+    """
+    state = job.state
+    if state is None:
+        # Nothing has been fetched yet, so there is nothing to lay out.
+        known = f"id={job.id!r}, " if job.id is not None else ""
+        return f"{kind}({known}not refreshed)"
+
+    fields = []
+    if job.id is not None:
+        fields.append(f"id={job.id!r}")
+    fields.append(f"state={state!r}")
+    for name in ("job_type", "creation_ms"):
+        value = getattr(job, name)
+        if value is not None:
+            fields.append(f"{name}={value!r}")
+    for name in ("spec", "result"):
+        value = getattr(job, name)
+        if value is not None:
+            fields.append(f"{name}={_repr_payload(value)}")
+    if job.failure is not None:
+        fields.append(f"failure={job.failure!r}")
+    body = "".join(f"\n{_REPR_INDENT}{field}," for field in fields)
+    return f"{kind}({body}\n)"
+

 class Job(Generic[T]):
    """Synchronous counterpart of `AsyncJob` with the same result type."""
@@ -122,6 +277,75 @@ class Job(Generic[T]):
            return
        LOOP.run(self._inner.cancel())

+    def refresh(self) -> None:
+        """Ask the backend for this job's current state and record.
+
+        See :meth:`AsyncJob.refresh`.
+        """
+        if self._inner is None:
+            return
+        LOOP.run(self._inner.refresh())
+
+    @property
+    def state(self) -> Optional[str]:
+        """The last observed lifecycle state. See :attr:`AsyncJob.state`."""
+        return self._inner.state if self._inner is not None else "finished"
+
+    @property
+    def job_type(self) -> Optional[str]:
+        """The job's type. See :attr:`AsyncJob.job_type`."""
+        return self._field("job_type")
+
+    @property
+    def creation_ms(self) -> Optional[int]:
+        """When the job was created. See :attr:`AsyncJob.creation_ms`."""
+        return self._field("creation_ms")
+
+    @property
+    def spec(self) -> Optional[Any]:
+        """The job's specification. See :attr:`AsyncJob.spec`."""
+        return self._field("spec")
+
+    @property
+    def result(self) -> Optional[Any]:
+        """The job's terminal result. See :attr:`AsyncJob.result`."""
+        return self._field("result")
+
+    @property
+    def failure(self) -> Optional[JobFailureInfo]:
+        """Why the job failed. See :attr:`AsyncJob.failure`."""
+        return self._field("failure")
+
+    @property
+    def _spec_json(self) -> Optional[str]:
+        return self._field("_spec_json")
+
+    @property
+    def _result_json(self) -> Optional[str]:
+        return self._field("_result_json")
+
+    def _field(self, name: str) -> Optional[Any]:
+        return getattr(self._inner, name) if self._inner is not None else None
+
+    def events(
+        self,
+        *,
+        limit: Optional[int] = None,
+        filter: Optional[str] = None,
+    ) -> "pa.Table":
+        """This job's recorded lifecycle events.
+
+        See :meth:`AsyncJob.events`.
+        """
+        if self._inner is None:
+            raise NotImplementedError(
+                "job event history is only available for server-side jobs"
+            )
+        return LOOP.run(self._inner.events(limit=limit, filter=filter))
+
+    def __repr__(self) -> str:
+        return _job_repr("Job", self)
+

 def _typed_job(
    inner: "_lancedb.Job", result_decoder: Callable[[str], T]
@@ -42,6 +42,8 @@ class MaterializedViewDefinition:
    """Cap on the number of rows the view holds."""
    inputs: List[str] = field(default_factory=list)
    """Source columns the projections and filter read."""
+    source_namespace: List[str] = field(default_factory=list)
+    """Namespace holding the source table; empty is the root namespace."""


 def _definition_from_schema(
@@ -53,7 +55,8 @@ def _definition_from_schema(
        raise ValueError(f"Table '{name}' is not a materialized view")
    value = json.loads(raw)
    kind = value.get("kind")
-    if kind != "select":
+    # "namespaced_select" keeps older readers from resolving the source at root.
+    if kind not in ("select", "namespaced_select"):
        raise NotImplementedError(
            f"materialized view '{name}' is defined by '{kind}', which this "
            "version of lancedb cannot refresh"
@@ -66,6 +69,7 @@ def _definition_from_schema(
        filter=value.get("filter"),
        limit=value.get("limit"),
        inputs=value.get("inputs", []),
+        source_namespace=value.get("source_namespace", []),
    )


@@ -12,6 +12,7 @@ from __future__ import annotations

 import sys
 from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
+from uuid import UUID

 if sys.version_info >= (3, 12):
    from typing import override
@@ -48,8 +49,11 @@ from lancedb._lancedb import (
    connect_namespace_client as _connect_namespace_client,
 )
 from lancedb.background_loop import LOOP
+from lancedb.arrow import AsyncRecordBatchReader
 from lancedb.db import AsyncConnection, DBConnection
 from lancedb.job import AsyncJob, Job
+from lancedb.sql import AsyncQuery as AsyncSqlQuery
+from lancedb.sql import QueryDescription
 from lance_namespace import (
    LanceNamespace,
    connect as namespace_connect,
@@ -1447,6 +1451,37 @@ class AsyncLanceNamespaceDBConnection:
            namespace_path=namespace_path, page_token=page_token, limit=limit
        )

+    async def execute_query(
+        self,
+        query: str,
+        *,
+        default_namespace_path: Optional[List[str]] = None,
+    ) -> AsyncRecordBatchReader:
+        """Execute SQL when supported by the underlying connection."""
+        return await self._inner.execute_query(
+            query,
+            default_namespace_path=default_namespace_path,
+        )
+
+    async def execute_query_async(
+        self,
+        query: str,
+        *,
+        default_namespace_path: Optional[List[str]] = None,
+    ) -> AsyncSqlQuery:
+        """Start executing SQL when supported by the underlying connection.
+
+        Namespace-backed local connections do not support SQL.
+        """
+        return await self._inner.execute_query_async(
+            query,
+            default_namespace_path=default_namespace_path,
+        )
+
+    async def describe_query(self, query_id: UUID) -> QueryDescription:
+        """Describe a submitted SQL query when supported."""
+        return await self._inner.describe_query(query_id)
+
    async def namespace_client(self) -> LanceNamespace:
        """Get the namespace client for this connection.

@@ -109,6 +109,7 @@ def _query_is_plain_scan(query: Query) -> bool:
    return (
        query.vector is None
        and query.full_text_query is None
+        and query.take_offsets is None
        and not query.postfilter
        and not query.order_by
    )
@@ -167,6 +168,12 @@ def _projection_to_scanner_kwargs(columns: QueryProjection) -> Dict[str, Any]:
    return {"columns": projection}


+def _query_request_projection(req: "PyQueryRequest") -> QueryProjection:
+    if req.select_source_columns is not None:
+        return req.select_source_columns
+    return req.select
+
+
 def _scanner_kwargs_for_query(
    query: Query,
    blob_mode: BlobMode,
@@ -798,6 +805,10 @@ class Query(pydantic.BaseModel):
    # offset to start fetching results from
    offset: Optional[int] = None

+    # Dataset offsets whose duplicate occurrences must be restored after lookup.
+    # This is populated when a take query is converted to this serializable form.
+    take_offsets: Optional[List[int]] = None
+
    # if true, will only search the indexed data
    fast_search: Optional[bool] = None

@@ -819,6 +830,7 @@ class Query(pydantic.BaseModel):
        query = cls()
        query.limit = req.limit
        query.offset = req.offset
+        query.take_offsets = req.take_offsets
        query.filter = req.filter
        query.full_text_query = req.full_text_search
        query.columns = req.select
@@ -2799,15 +2811,16 @@ class AsyncQueryBase(object):

        req = self._inner.to_query_request()
        schema = await self._table.schema()
+        projection = _query_request_projection(req)
        self._blob_auto_row_id = blob_auto_row_id_for_scan(
            schema,
-            req.select,
+            projection,
            with_row_id=self._with_row_id,
        )
        if not self._blob_auto_row_id:
            self._blob_paths = ()
            return
-        self._blob_paths = tuple(blob_v2_projection_sources(schema, req.select).keys())
+        self._blob_paths = tuple(blob_v2_projection_sources(schema, projection).keys())
        self._inner.with_row_id()

    def select(self, columns: Union[List[str], dict[str, str]]) -> Self:
@@ -3894,14 +3907,15 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
        blob_paths: tuple[str, ...] = ()
        if self._table is not None:
            schema = await self._table.schema()
+            projection = _query_request_projection(req)
            blob_auto_row_id = blob_auto_row_id_for_scan(
                schema,
-                req.select,
+                projection,
                with_row_id=self._with_row_id,
            )
            if blob_auto_row_id:
                blob_paths = tuple(
-                    blob_v2_projection_sources(schema, req.select).keys()
+                    blob_v2_projection_sources(schema, projection).keys()
                )
        self._blob_auto_row_id = blob_auto_row_id
        self._blob_paths = blob_paths
@@ -9,6 +9,7 @@ from concurrent.futures import ThreadPoolExecutor
 import sys
 from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
 from urllib.parse import urlparse
+from uuid import UUID
 import warnings

 if sys.version_info >= (3, 12):
@@ -25,10 +26,12 @@ from ..common import DATA
 from ..db import DBConnection, LOOP
 from ..functions import FunctionVersion, UdfDefinition
 from ..job import AsyncJob, Job
+from ..sql import Query as SqlQuery
+from ..sql import QueryDescription
 from ..materialized_view import MaterializedView, SelectArg

 if TYPE_CHECKING:
-    from .._lancedb import JobDescription, JobInfo
+    from .._lancedb import JobInfo
 from ..embeddings import EmbeddingFunctionConfig
 from lance_namespace import (
    LanceNamespace,
@@ -116,6 +119,7 @@ class RemoteDBConnection(DBConnection):
        read_timeout: Optional[float] = None,
        storage_options: Optional[Dict[str, str]] = None,
        read_consistency_interval: Optional[timedelta] = None,
+        sql_host_override: Optional[str] = None,
    ):
        """Connect to a remote LanceDB database."""
        if isinstance(client_config, dict):
@@ -161,6 +165,7 @@ class RemoteDBConnection(DBConnection):
        self.api_key = api_key
        self.region = region
        self.host_override = host_override
+        self.sql_host_override = sql_host_override
        self.storage_options = storage_options
        self.db_name = parsed.netloc

@@ -175,6 +180,7 @@ class RemoteDBConnection(DBConnection):
                api_key=api_key,
                region=region,
                host_override=host_override,
+                sql_host_override=sql_host_override,
                client_config=client_config,
                storage_options=storage_options,
                read_consistency_interval=read_consistency_interval,
@@ -193,6 +199,7 @@ class RemoteDBConnection(DBConnection):
                "api_key": self.api_key,
                "region": self.region,
                "host_override": self.host_override,
+                "sql_host_override": self.sql_host_override,
                "client_config": _client_config_to_dict(self.client_config),
                "storage_options": self.storage_options,
            }
@@ -732,14 +739,11 @@ class RemoteDBConnection(DBConnection):
        )

    @override
-    def job(self, job_id: str) -> Job:
-        """A [Job][lancedb.job.Job] handle for a server-side job by id.
-
-        The handle is constructed without a server round trip; an unknown id
-        surfaces when the handle is used. Dropping the handle has no effect
-        on the job itself.
+    def open_job(self, job_id: str) -> Job:
+        """Open a server-side job by id. See
+        [DBConnection.open_job][lancedb.db.DBConnection.open_job].
        """
-        return Job(self._conn.job(job_id))
+        return Job(LOOP.run(self._conn.open_job(job_id)))

    @override
    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
@@ -749,19 +753,19 @@ class RemoteDBConnection(DBConnection):
    def get_function(self, name: str, *, version: str) -> FunctionVersion:
        return LOOP.run(self._conn.get_function(name, version=version))

+    @override
+    def list_functions(self) -> List[FunctionVersion]:
+        return LOOP.run(self._conn.list_functions())
+
+    @override
+    def drop_function(self, name: str, *, version: str) -> bool:
+        return LOOP.run(self._conn.drop_function(name, version=version))
+
    @override
    def list_jobs(self) -> List["JobInfo"]:
        """List server-side jobs across the database's tables."""
        return LOOP.run(self._conn.list_jobs())

-    @override
-    def get_job(self, job_id: str) -> Optional["JobDescription"]:
-        """Describe a single server-side job by id.
-
-        Returns None when the server has no such job.
-        """
-        return LOOP.run(self._conn.get_job(job_id))
-
    @override
    def cancel_job(self, job_id: str) -> bool:
        """Request cancellation of a server-side job by id.
@@ -773,12 +777,35 @@ class RemoteDBConnection(DBConnection):
        return LOOP.run(self._conn.cancel_job(job_id))

    @override
-    def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
-        """The lifecycle event history of a server-side job, as Arrow batches.
+    def execute_query_async(
+        self,
+        query: str,
+        *,
+        default_namespace_path: Optional[List[str]] = None,
+    ) -> SqlQuery:
+        """Start executing SQL through this remote connection.

-        Lists history across all jobs when `job_id` is None.
+        Unqualified tables use this connection's database and the
+        ``["public"]`` namespace by default. Fully qualified table names may
+        reference other databases available to the same deployment.
        """
-        return LOOP.run(self._conn.job_history(job_id))
+        return SqlQuery(
+            LOOP.run(
+                self._conn.execute_query_async(
+                    query,
+                    default_namespace_path=default_namespace_path,
+                )
+            )
+        )
+
+    @override
+    def describe_query(self, query_id: UUID) -> QueryDescription:
+        """Describe a submitted SQL query by its connection-scoped id."""
+        return LOOP.run(
+            self._conn.describe_query(
+                query_id,
+            )
+        )

    @override
    def namespace_client(self) -> LanceNamespace:
@@ -177,4 +177,7 @@ class OAuthProvider(HeaderProvider):
        if not self._current_token:
            raise RuntimeError("Failed to obtain OAuth token")

-        return {"Authorization": f"Bearer {self._current_token}"}
+        return {
+            "Authorization": f"Bearer {self._current_token}",
+            "x-lancedb-credential-type": "oidc",
+        }
@@ -36,6 +36,7 @@ from lancedb._lancedb import (
    UpdateResult,
 )
 from lancedb.embeddings.base import EmbeddingFunctionConfig
+from lancedb.expr import Expr
 from lancedb.index import (
    FTS,
    BTree,
@@ -66,7 +67,15 @@ from ..query import (
    LanceTakeQueryBuilder,
    LanceVectorQueryBuilder,
 )
-from ..table import AsyncTable, BlobMode, Branches, IndexStatistics, Query, Table, Tags
+from ..table import (
+    AsyncTable,
+    BlobMode,
+    Branches,
+    IndexStatistics,
+    Query,
+    Table,
+    Tags,
+)
 from ..types import BaseTokenizerType


@@ -539,6 +548,7 @@ class RemoteTable(Table):
        LOOP.run(
            self._table.create_index(
                column,
+                replace=replace,
                config=config,
                wait_timeout=wait_timeout,
                name=name,
@@ -863,7 +873,7 @@ class RemoteTable(Table):

    def update(
        self,
-        where: Optional[str] = None,
+        where: Optional[Union[str, Expr]] = None,
        values: Optional[dict] = None,
        *,
        values_sql: Optional[Dict[str, str]] = None,
@@ -874,9 +884,11 @@ class RemoteTable(Table):

        Parameters
        ----------
-        where: str, optional
-            The SQL where clause to use when updating rows. For example, 'x = 2'
-            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
+        where: str or [Expr][lancedb.expr.Expr], optional
+            The filter condition. Can be a SQL string or a type-safe
+            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
+            [lit][lancedb.expr.lit]. The filter must not be empty, or it will
+            error.
        values: dict, optional
            The values to update. The keys are the column names and the values
            are the values to set.
@@ -4,30 +4,34 @@

 """Schema helpers for Lance blob columns."""

+import importlib
+from typing import TYPE_CHECKING
+
 import pyarrow as pa
+import pyarrow.ipc
+
+if TYPE_CHECKING:
+    from lance.blob import BlobType as BlobType

 _BLOB_EXTENSION_NAME = "lance.blob.v2"
 _BLOB_V1_KEY = "lance-encoding:blob"
 _ARROW_EXT_NAME_KEY = "ARROW:extension:name"
+_BLOB_V2_STORAGE_TYPE = pa.struct(
+    [
+        pa.field("data", pa.large_binary(), nullable=True),
+        pa.field("uri", pa.utf8(), nullable=True),
+        pa.field("position", pa.uint64(), nullable=True),
+        pa.field("size", pa.uint64(), nullable=True),
+    ]
+)
+_resolved_blob_type = None


-class BlobType(pa.ExtensionType):
-    """PyArrow extension type for a Lance blob v2 column.
-
-    Queries return descriptors; call :meth:`~lancedb.table.Table.fetch_blob_files`
-    for lazy reads or :meth:`~lancedb.table.Table.fetch_blobs` for eager bytes.
-    """
+class _FallbackBlobType(pa.ExtensionType):
+    """lance.blob.v2 extension type used when pylance is not installed."""

    def __init__(self) -> None:
-        storage_type = pa.struct(
-            [
-                pa.field("data", pa.large_binary(), nullable=True),
-                pa.field("uri", pa.utf8(), nullable=True),
-                pa.field("position", pa.uint64(), nullable=True),
-                pa.field("size", pa.uint64(), nullable=True),
-            ]
-        )
-        super().__init__(storage_type, _BLOB_EXTENSION_NAME)
+        pa.ExtensionType.__init__(self, _BLOB_V2_STORAGE_TYPE, _BLOB_EXTENSION_NAME)

    def __arrow_ext_serialize__(self) -> bytes:
        return b""
@@ -35,23 +39,16 @@ class BlobType(pa.ExtensionType):
    @classmethod
    def __arrow_ext_deserialize__(
        cls, storage_type: pa.DataType, serialized: bytes
-    ) -> "BlobType":
+    ) -> "_FallbackBlobType":
        return cls()

    def __reduce__(self):
-        # Ensure pickle round-trips on older pyarrow (apache/arrow#35599).
        return type(self).__arrow_ext_deserialize__, (
            self.storage_type,
            self.__arrow_ext_serialize__(),
        )


-try:
-    pa.register_extension_type(BlobType())  # type: ignore[arg-type]
-except pa.ArrowKeyError:
-    pass
-
-
 def _metadata_value(metadata: dict, key: str):
    return metadata.get(key.encode()) or metadata.get(key)

@@ -92,43 +89,105 @@ def is_blob_like_field(field: pa.Field) -> bool:
    return is_blob_v2_field(field) or _metadata_marks_legacy_blob(field.metadata or {})


-def _collect_blob_paths(schema: pa.Schema, is_blob) -> list[str]:
-    paths: list[str] = []
+def _collect_blob_paths(schema: pa.Schema, is_blob) -> list[tuple[str, bool]]:
+    """Walk the schema and return (path, has_list_ancestor) for each blob field."""
+    paths: list[tuple[str, bool]] = []

-    def walk(fields, prefix: str) -> None:
+    def walk(fields, prefix: str, has_list_ancestor: bool) -> None:
        for field in fields:
            path = f"{prefix}.{field.name}" if prefix else field.name
            if is_blob(field):
-                paths.append(path)
+                paths.append((path, has_list_ancestor))
            elif pa.types.is_struct(field.type):
-                walk(field.type, path)
+                walk(field.type, path, has_list_ancestor)
            elif (
                pa.types.is_list(field.type)
                or pa.types.is_large_list(field.type)
                or pa.types.is_fixed_size_list(field.type)
            ):
-                walk([field.type.value_field], path)
+                walk([field.type.value_field], path, True)

-    walk(schema, "")
+    walk(schema, "", False)
    return paths


 def blob_column_paths(schema: pa.Schema) -> list[str]:
    """Dotted paths of blob-like columns (v2 extension or legacy metadata)."""
-    return _collect_blob_paths(schema, is_blob_like_field)
+    return [path for path, _ in _collect_blob_paths(schema, is_blob_like_field)]


 def blob_v2_column_paths(schema: pa.Schema) -> list[str]:
-    return _collect_blob_paths(schema, is_blob_v2_field)
+    return [path for path, _ in _collect_blob_paths(schema, is_blob_v2_field)]
+
+
+def row_addressable_blob_v2_paths(schema: pa.Schema) -> list[str]:
+    """Blob v2 paths with one blob addressable by table row id.
+
+    ``fetch_blobs`` and the descriptor row-id ride-along address one blob per
+    row, so a blob inside a list container has no row-id slot and no fetch
+    path. Those columns still store and query as raw descriptors.
+    """
+    return [
+        path
+        for path, has_list_ancestor in _collect_blob_paths(schema, is_blob_v2_field)
+        if not has_list_ancestor
+    ]


 def schema_has_blob_field(schema: pa.Schema) -> bool:
    return bool(blob_column_paths(schema))


+def _deserialize_registered_type(extension_type: pa.ExtensionType) -> pa.DataType:
+    """Return the type Arrow reconstructs for this extension name."""
+    schema = pa.schema([pa.field("value", extension_type)])
+    restored = pa.ipc.read_schema(schema.serialize())
+    return restored.field("value").type
+
+
+def _resolve_blob_type():
+    """Return the BlobType class this process should use.
+
+    pylance's class when it owns the lance.blob.v2 registry entry,
+    otherwise LanceDB's fallback. A different registered class is an error.
+    """
+    global _resolved_blob_type
+    if _resolved_blob_type is not None:
+        return _resolved_blob_type
+    try:
+        blob_module = importlib.import_module("lance.blob")
+    except ModuleNotFoundError as err:
+        if err.name not in ("lance", "lance.blob"):
+            raise
+    else:
+        blob_type = getattr(blob_module, "BlobType", None)
+        if blob_type is not None:
+            registered_type = _deserialize_registered_type(blob_type())
+            if type(registered_type) is not blob_type:
+                registered_cls = type(registered_type)
+                raise ValueError(
+                    "lance.blob.v2 is already registered by "
+                    f"{registered_cls.__module__}.{registered_cls.__qualname__}"
+                )
+            _resolved_blob_type = blob_type
+            return blob_type
+    try:
+        pa.register_extension_type(_FallbackBlobType())  # type: ignore[arg-type]
+    except pa.ArrowKeyError as err:
+        raise ValueError(
+            "lance.blob.v2 is already registered by another extension class"
+        ) from err
+    _resolved_blob_type = _FallbackBlobType
+    return _resolved_blob_type
+
+
 def blob(name: str, nullable: bool = True) -> pa.Field:
-    """Create a Lance blob v2 column field."""
-    return pa.field(name, BlobType(), nullable=nullable)
+    """Create a Lance blob v2 column field.
+
+    When pylance is installed this is ``lance.blob.BlobType``.
+    """
+    blob_type = _resolve_blob_type()
+    return pa.field(name, blob_type(), nullable=nullable)


 def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataType:
@@ -155,3 +214,11 @@ def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataTyp
    ... ])
    """
    return pa.list_(value_type, dimension)
+
+
+def __getattr__(name: str):
+    if name == "BlobType":
+        blob_type = _resolve_blob_type()
+        globals()["BlobType"] = blob_type
+        return blob_type
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+"""Handles to SQL queries running on a remote database."""
+
+from uuid import UUID
+
+import pyarrow as pa
+
+from lancedb.background_loop import LOOP
+
+from . import _lancedb
+from .arrow import AsyncRecordBatchReader
+
+QueryDescription = _lancedb.QueryDescription
+
+
+class AsyncQuery:
+    """A handle to a submitted SQL query on an asynchronous connection."""
+
+    def __init__(self, inner: "_lancedb.SqlQuery"):
+        self._inner = inner
+
+    @property
+    def id(self) -> UUID:
+        """The stable identifier scoped to the connection that submitted it."""
+        return self._inner.id
+
+    async def describe(self) -> QueryDescription:
+        """Get a point-in-time description of the query."""
+        return await self._inner.describe()
+
+    async def reader(self) -> AsyncRecordBatchReader:
+        """Wait for the initial result stream and return its Arrow reader.
+
+        Results are single-consumer. Calling this method more than once on the
+        same query raises an error. Later batches are streamed as they become
+        available without waiting for the full query to finish.
+        """
+        return AsyncRecordBatchReader(await self._inner.reader())
+
+    async def cancel(self) -> None:
+        """Request cancellation of the query."""
+        await self._inner.cancel()
+
+
+class Query:
+    """Synchronous counterpart of :class:`AsyncQuery`."""
+
+    def __init__(self, inner: AsyncQuery):
+        self._inner = inner
+
+    @property
+    def id(self) -> UUID:
+        """The stable identifier scoped to the connection that submitted it."""
+        return self._inner.id
+
+    def describe(self) -> QueryDescription:
+        """Get a point-in-time description of the query."""
+        return LOOP.run(self._inner.describe())
+
+    def reader(self) -> pa.RecordBatchReader:
+        """Wait for the initial result stream and return a blocking reader.
+
+        Results are single-consumer. Calling this method more than once on the
+        same query raises an error. Later batches block only until they become
+        available, without waiting for the full query to finish.
+        """
+        reader = LOOP.run(self._inner.reader())
+
+        def next_batch():
+            try:
+                return LOOP.run(reader.__anext__())
+            except StopAsyncIteration:
+                return None
+
+        def batches():
+            while (batch := next_batch()) is not None:
+                yield batch
+
+        return pa.RecordBatchReader.from_batches(reader.schema, batches())
+
+    def cancel(self) -> None:
+        """Request cancellation of the query."""
+        LOOP.run(self._inner.cancel())
+
+
+__all__ = ["AsyncQuery", "Query", "QueryDescription"]
@@ -104,7 +104,12 @@ from .util import (
    value_to_sql,
 )
 from .index import lang_mapping
-from .schema import blob_v2_column_paths, schema_has_blob_field
+from .schema import (
+    blob_v2_column_paths,
+    is_blob_v2_field,
+    row_addressable_blob_v2_paths,
+    schema_has_blob_field,
+)


 def _should_push_down_query_table(
@@ -426,6 +431,7 @@ def _cast_to_target_schema(

    def gen():
        for batch in reader:
+            batch = _coerce_blob_write_columns(batch, reordered_schema)
            # Table but not RecordBatch has cast.
            cast_batches = (
                pa.Table.from_batches([batch]).cast(reordered_schema).to_batches()
@@ -438,6 +444,166 @@ def _cast_to_target_schema(
    return pa.RecordBatchReader.from_batches(reordered_schema, gen())


+def _coerce_blob_write_columns(
+    batch: pa.RecordBatch, target_schema: pa.Schema
+) -> pa.RecordBatch:
+    """Materialize blob storage structs before the stream leaves Python.
+
+    merge_insert requires its source reader to already match the table's
+    physical schema. Unlike add and insert, it does not pass through
+    LanceDB's Rust blob coercion, so preserving binary input here would
+    reach Lance as binary and fail the schema check.
+    """
+    columns = []
+    fields = []
+    changed = False
+    for field, column in zip(batch.schema, batch.columns):
+        target_field = target_schema.field(field.name)
+        coerced = _coerce_blob_value(column, target_field)
+        if coerced is not column:
+            column = coerced
+            field = pa.field(
+                field.name,
+                coerced.type,
+                field.nullable,
+                target_field.metadata,
+            )
+            changed = True
+        columns.append(column)
+        fields.append(field)
+    if not changed:
+        return batch
+    return pa.RecordBatch.from_arrays(
+        columns, schema=pa.schema(fields, metadata=batch.schema.metadata)
+    )
+
+
+def _coerce_blob_value(column: pa.Array, target_field: pa.Field) -> pa.Array:
+    if is_blob_v2_field(target_field) and _can_coerce_to_blob(column.type):
+        return _coerce_value_to_blob(column, target_field)
+
+    target_type = target_field.type
+    if pa.types.is_struct(target_type) and pa.types.is_struct(column.type):
+        children = []
+        fields = []
+        changed = False
+        for source_field in column.type:
+            source_column = column.field(source_field.name)
+            nested_target = next(
+                (field for field in target_type if field.name == source_field.name),
+                None,
+            )
+            if nested_target is None:
+                children.append(source_column)
+                fields.append(source_field)
+                continue
+            coerced = _coerce_blob_value(source_column, nested_target)
+            if coerced is not source_column:
+                changed = True
+            child_array, child_type = _physical_array_and_type(coerced)
+            children.append(child_array)
+            fields.append(
+                pa.field(
+                    source_field.name,
+                    child_type,
+                    source_field.nullable,
+                    nested_target.metadata,
+                )
+            )
+        if not changed:
+            return column
+        return pa.StructArray.from_arrays(
+            children,
+            fields=fields,
+            mask=column.is_null() if column.null_count else None,
+        )
+
+    if _is_list_like(target_type) and _is_list_like(column.type):
+        return _coerce_blob_list_values(column, target_type.value_field)
+
+    return column
+
+
+def _coerce_blob_list_values(
+    column: pa.Array, target_value_field: pa.Field
+) -> pa.Array:
+    """Coerce blob values inside a list column, preserving offsets and nulls.
+
+    Works on the raw child values window instead of ``pc.list_flatten`` because
+    flatten drops values spanned by null slots, which would misalign offsets.
+    """
+    mask = column.is_null() if column.null_count else None
+    if pa.types.is_fixed_size_list(column.type):
+        list_size = column.type.list_size
+        values = column.values.slice(column.offset * list_size, len(column) * list_size)
+        coerced = _coerce_blob_value(values, target_value_field)
+        if coerced is values:
+            return column
+        physical_values, _ = _physical_array_and_type(coerced)
+        return pa.FixedSizeListArray.from_arrays(physical_values, list_size, mask=mask)
+    offsets = column.offsets
+    first_offset = offsets[0].as_py()
+    values = column.values.slice(
+        first_offset,
+        offsets[-1].as_py() - first_offset,
+    )
+    coerced = _coerce_blob_value(values, target_value_field)
+    if coerced is values:
+        return column
+    physical_values, _ = _physical_array_and_type(coerced)
+    if first_offset:
+        offsets = pc.subtract(offsets, pa.scalar(first_offset, offsets.type))
+    if pa.types.is_large_list(column.type):
+        return pa.LargeListArray.from_arrays(offsets, physical_values, mask=mask)
+    return pa.ListArray.from_arrays(offsets, physical_values, mask=mask)
+
+
+def _coerce_value_to_blob(values: pa.Array, target_field: pa.Field) -> pa.Array:
+    if pa.types.is_null(values.type):
+        data = pa.nulls(len(values), type=pa.large_binary())
+    elif pa.types.is_large_binary(values.type):
+        data = values
+    else:
+        data = values.cast(pa.large_binary())
+    length = len(values)
+    storage_type = target_field.type
+    if isinstance(storage_type, pa.ExtensionType):
+        storage_type = storage_type.storage_type
+    storage_fields = list(storage_type)
+    children = []
+    for storage_field in storage_fields:
+        if storage_field.name == "data":
+            children.append(data)
+        else:
+            children.append(pa.nulls(length, type=storage_field.type))
+    storage = pa.StructArray.from_arrays(
+        children,
+        fields=storage_fields,
+        mask=values.is_null() if values.null_count else None,
+    )
+    if isinstance(target_field.type, pa.ExtensionType):
+        return pa.ExtensionArray.from_storage(target_field.type, storage)
+    return storage
+
+
+def _physical_array_and_type(array: pa.Array) -> tuple[pa.Array, pa.DataType]:
+    if isinstance(array.type, pa.ExtensionType):
+        return array.storage, array.type.storage_type
+    return array, array.type
+
+
+def _can_coerce_to_blob(data_type: pa.DataType) -> bool:
+    return _is_binary_like(data_type) or pa.types.is_null(data_type)
+
+
+def _is_binary_like(data_type: pa.DataType) -> bool:
+    return (
+        pa.types.is_binary(data_type)
+        or pa.types.is_large_binary(data_type)
+        or pa.types.is_binary_view(data_type)
+    )
+
+
 def _field_extension_name(field: pa.Field) -> Optional[str]:
    extension_name = getattr(field.type, "extension_name", None)
    if extension_name is not None:
@@ -464,63 +630,71 @@ def _align_field_types(
        target_field = next((f for f in target_fields if f.name == field.name), None)
        if target_field is None:
            raise ValueError(f"Field '{field.name}' not found in target schema")
-        # Preserve arrow.json input until it reaches Lance. LanceDB exposes stored
-        # JSON columns as lance.json (JSONB-backed LargeBinary), but casting the
-        # input to that storage type here merely relabels the raw JSON bytes as
-        # JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
-        if (
-            _field_extension_name(field) == "arrow.json"
-            and _field_extension_name(target_field) == "lance.json"
-        ):
-            new_fields.append(field)
-            continue
-        if pa.types.is_struct(target_field.type):
-            if pa.types.is_struct(field.type):
-                new_type = pa.struct(
-                    _align_field_types(
-                        field.type.fields,
-                        target_field.type.fields,
-                    )
+        new_fields.append(_align_field(field, target_field))
+    return new_fields
+
+
+def _align_list_value_field(
+    value_field: pa.Field, target_value_field: pa.Field
+) -> pa.Field:
+    # A list has exactly one child, so the inferred child name ("item") aligns
+    # positionally and adopts the table's child name; pa.Table.cast renames it.
+    return _align_field(value_field, target_value_field).with_name(
+        target_value_field.name
+    )
+
+
+def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field:
+    # Preserve arrow.json input until it reaches Lance. LanceDB exposes stored
+    # JSON columns as lance.json (JSONB-backed LargeBinary), but casting the
+    # input to that storage type here merely relabels the raw JSON bytes as
+    # JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
+    if (
+        _field_extension_name(field) == "arrow.json"
+        and _field_extension_name(target_field) == "lance.json"
+    ):
+        return field
+    if pa.types.is_struct(target_field.type):
+        if pa.types.is_struct(field.type):
+            new_type = pa.struct(
+                _align_field_types(
+                    field.type.fields,
+                    target_field.type.fields,
                )
-            else:
-                new_type = target_field.type
-        elif pa.types.is_list(target_field.type):
-            if _is_list_like(field.type):
-                new_type = pa.list_(
-                    _align_field_types(
-                        [field.type.value_field],
-                        [target_field.type.value_field],
-                    )[0]
-                )
-            else:
-                new_type = target_field.type
-        elif pa.types.is_large_list(target_field.type):
-            if _is_list_like(field.type):
-                new_type = pa.large_list(
-                    _align_field_types(
-                        [field.type.value_field],
-                        [target_field.type.value_field],
-                    )[0]
-                )
-            else:
-                new_type = target_field.type
-        elif pa.types.is_fixed_size_list(target_field.type):
-            if _is_list_like(field.type):
-                new_type = pa.list_(
-                    _align_field_types(
-                        [field.type.value_field],
-                        [target_field.type.value_field],
-                    )[0],
-                    target_field.type.list_size,
-                )
-            else:
-                new_type = target_field.type
+            )
        else:
            new_type = target_field.type
-        new_fields.append(
-            pa.field(field.name, new_type, field.nullable, target_field.metadata)
-        )
-    return new_fields
+    elif pa.types.is_list(target_field.type):
+        if _is_list_like(field.type):
+            new_type = pa.list_(
+                _align_list_value_field(
+                    field.type.value_field, target_field.type.value_field
+                )
+            )
+        else:
+            new_type = target_field.type
+    elif pa.types.is_large_list(target_field.type):
+        if _is_list_like(field.type):
+            new_type = pa.large_list(
+                _align_list_value_field(
+                    field.type.value_field, target_field.type.value_field
+                )
+            )
+        else:
+            new_type = target_field.type
+    elif pa.types.is_fixed_size_list(target_field.type):
+        if _is_list_like(field.type):
+            new_type = pa.list_(
+                _align_list_value_field(
+                    field.type.value_field, target_field.type.value_field
+                ),
+                target_field.type.list_size,
+            )
+        else:
+            new_type = target_field.type
+    else:
+        new_type = target_field.type
+    return pa.field(field.name, new_type, field.nullable, target_field.metadata)


 def _infer_subschema(
@@ -589,7 +763,7 @@ def sanitize_create_table(
        schema = data.schema
    else:
        if schema is not None:
-            data = pa.Table.from_pylist([], schema)
+            data = pa.Table.from_batches([], schema=schema)
    if schema is None:
        if data is None:
            raise ValueError("Either data or schema must be provided")
@@ -1373,7 +1547,9 @@ class Table(ABC):
        on: Union[str, Iterable[str]]
            A column (or columns) to join on.  This is how records from the
            source table and target table are matched.  Typically this is some
-            kind of key or id column.
+            kind of key or id column.  Passing several columns matches on the
+            composite key: a source row updates a target row only when it
+            agrees on every one of them.

        Examples
        --------
@@ -1504,9 +1680,9 @@ class Table(ABC):
        Offsets are mostly useful for sampling as the set of all valid offsets is easily
        known in advance to be [0, len(table)).

-        No guarantees are made regarding the order in which results are returned.  If
-        you desire an output order that matches the order of the given offsets, you will
-        need to add the row offset column to the output and align it yourself.
+        No guarantees are made regarding the order in which results are returned.
+        Repeated offsets produce repeated rows, which makes this method suitable for
+        sampling with replacement.

        Parameters
        ----------
@@ -1617,6 +1793,9 @@ class Table(ABC):
        The result has the same length and order as ``row_ids``. Null blobs
        produce null slots; valid empty blobs produce ``b""``.

+        ``_rowid`` values stay valid after compaction when the table has stable
+        row ids.
+
        Convenience for small payloads. For large values use
        :meth:`fetch_blob_files`.
        """
@@ -1634,6 +1813,9 @@ class Table(ABC):
        The result has the same length and order as ``requests``; null blobs
        produce null slots and empty ranges on non-null blobs produce ``b""``.

+        ``_rowid`` values stay valid after compaction when the table has stable
+        row ids.
+
        Row IDs can be obtained from a query with ``with_row_id(True)``. This
        API is currently supported only by local tables.
        """
@@ -1649,6 +1831,9 @@ class Table(ABC):
        ``_rowid`` or a ``_lance_row_id`` field on the blob descriptor. Null
        rows are ``None``. Remote tables require LanceDB Cloud server 0.5.0 or
        newer.
+
+        ``_rowid`` values stay valid after compaction when the table has stable
+        row ids.
        """

    @abstractmethod
@@ -1744,7 +1929,7 @@ class Table(ABC):
    @abstractmethod
    def update(
        self,
-        where: Optional[str] = None,
+        where: Optional[Union[str, Expr]] = None,
        values: Optional[dict] = None,
        *,
        values_sql: Optional[Dict[str, str]] = None,
@@ -1759,9 +1944,11 @@ class Table(ABC):

        Parameters
        ----------
-        where: str, optional
-            The SQL where clause to use when updating rows. For example, 'x = 2'
-            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
+        where: str or [Expr][lancedb.expr.Expr], optional
+            The filter condition. Can be a SQL string or a type-safe
+            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
+            [lit][lancedb.expr.lit]. The filter must not be empty, or it will
+            error.
        values: dict, optional
            The values to update. The keys are the column names and the values
            are the values to set.
@@ -1779,6 +1966,7 @@ class Table(ABC):
        Examples
        --------
        >>> import lancedb
+        >>> from lancedb.expr import col
        >>> import pandas as pd
        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1.0, 2], [3, 4], [5, 6]]})
        >>> db = lancedb.connect("./.lancedb")
@@ -1788,7 +1976,7 @@ class Table(ABC):
        0  1  [1.0, 2.0]
        1  2  [3.0, 4.0]
        2  3  [5.0, 6.0]
-        >>> table.update(where="x = 2", values={"vector": [10.0, 10]})
+        >>> table.update(where=col("x") == 2, values={"vector": [10.0, 10]})
        UpdateResult(rows_updated=1, version=2)
        >>> table.to_pandas()
           x        vector
@@ -1988,9 +2176,11 @@ class Table(ABC):
            Function columns are supported only on LanceDB Cloud and
            Enterprise.
        computed: Dict[str, str], optional
-            A map of column name to a SQL expression defining the column. The
-            column's type and inputs are derived from the expression, so no
-            data type is supplied.
+            A mapping from output column names to SQL expressions derives each
+            output field from its expression. A direct projection of a Blob v2
+            field inherits Blob v2 semantics; other expressions derive their
+            ordinary Arrow type. Mapping order is declaration and dependency
+            order.

            Unlike ``transforms``, the expression is stored rather than
            evaluated now: the column is committed with no values, and rows get
@@ -2695,7 +2885,7 @@ class LanceTable(Table):
            arrow_tbl = self.to_arrow()
            if blob_mode == "descriptions":
                arrow_tbl = strip_auto_row_ids(
-                    arrow_tbl, blob_v2_column_paths(self.schema)
+                    arrow_tbl, row_addressable_blob_v2_paths(self.schema)
                )
            return arrow_tbl.to_pandas(**kwargs)

@@ -3841,7 +4031,7 @@ class LanceTable(Table):

    def update(
        self,
-        where: Optional[str] = None,
+        where: Optional[Union[str, Expr]] = None,
        values: Optional[dict] = None,
        *,
        values_sql: Optional[Dict[str, str]] = None,
@@ -3852,9 +4042,11 @@ class LanceTable(Table):

        Parameters
        ----------
-        where: str, optional
-            The SQL where clause to use when updating rows. For example, 'x = 2'
-            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
+        where: str or [Expr][lancedb.expr.Expr], optional
+            The filter condition. Can be a SQL string or a type-safe
+            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
+            [lit][lancedb.expr.lit]. The filter must not be empty, or it will
+            error.
        values: dict, optional
            The values to update. The keys are the column names and the values
            are the values to set.
@@ -3872,6 +4064,7 @@ class LanceTable(Table):
        Examples
        --------
        >>> import lancedb
+        >>> from lancedb.expr import col
        >>> import pandas as pd
        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1.0, 2], [3, 4], [5, 6]]})
        >>> db = lancedb.connect("./.lancedb")
@@ -3881,7 +4074,7 @@ class LanceTable(Table):
        0  1  [1.0, 2.0]
        1  2  [3.0, 4.0]
        2  3  [5.0, 6.0]
-        >>> table.update(where="x = 2", values={"vector": [10.0, 10]})
+        >>> table.update(where=col("x") == 2, values={"vector": [10.0, 10]})
        UpdateResult(rows_updated=1, version=2)
        >>> table.to_pandas()
           x        vector
@@ -3908,6 +4101,7 @@ class LanceTable(Table):
            )
            and not self._route_pushdown_to_rust
            and self.current_branch() is None
+            and query.take_offsets is None
        ):
            from lancedb.namespace import _execute_server_side_query

@@ -5096,7 +5290,9 @@ class AsyncTable:
        if blob_mode == "descriptions" or not schema_has_blob_field(schema):
            arrow_tbl = await self.to_arrow()
            if blob_mode == "descriptions":
-                arrow_tbl = strip_auto_row_ids(arrow_tbl, blob_v2_column_paths(schema))
+                arrow_tbl = strip_auto_row_ids(
+                    arrow_tbl, row_addressable_blob_v2_paths(schema)
+                )
            return arrow_tbl.to_pandas(**kwargs)

        if blob_mode == "lazy" and get_uri_scheme(await self.uri()) == "memory":
@@ -5516,7 +5712,9 @@ class AsyncTable:
        on: Union[str, Iterable[str]]
            A column (or columns) to join on.  This is how records from the
            source table and target table are matched.  Typically this is some
-            kind of key or id column.
+            kind of key or id column.  Passing several columns matches on the
+            composite key: a source row updates a target row only when it
+            agrees on every one of them.

        Examples
        --------
@@ -5799,7 +5997,23 @@ class AsyncTable:

    def _sync_query_to_async(
        self, query: Query
-    ) -> AsyncHybridQuery | AsyncFTSQuery | AsyncVectorQuery | AsyncQuery:
+    ) -> (
+        AsyncHybridQuery
+        | AsyncFTSQuery
+        | AsyncVectorQuery
+        | AsyncQuery
+        | AsyncTakeQuery
+    ):
+        if query.take_offsets is not None:
+            take_query = self.take_offsets(query.take_offsets)
+            if query.columns:
+                take_query = take_query.select(query.columns)
+            if query.use_lsm is not None:
+                take_query = take_query.use_lsm(query.use_lsm)
+            if query.with_row_id:
+                take_query = take_query.with_row_id()
+            return take_query
+
        async_query = self.query()
        if query.limit is not None:
            async_query = async_query.limit(query.limit)
@@ -5864,6 +6078,7 @@ class AsyncTable:
                self._namespace_client, self._pushdown_operations
            )
            and not self._route_pushdown_to_rust
+            and query.take_offsets is None
        ):
            from lancedb.namespace import _execute_server_side_query

@@ -5995,7 +6210,7 @@ class AsyncTable:
        self,
        updates: Optional[Dict[str, Any]] = None,
        *,
-        where: Optional[str] = None,
+        where: Optional[Union[str, Expr]] = None,
        updates_sql: Optional[Dict[str, str]] = None,
    ) -> UpdateResult:
        """
@@ -6010,9 +6225,11 @@ class AsyncTable:
            The updates to apply.  The keys should be the name of the column to
            update.  The values should be the new values to assign.  This is
            required unless updates_sql is supplied.
-        where: str, optional
-            An SQL filter that controls which rows are updated. For example, 'x = 2'
-            or 'x IN (1, 2, 3)'.  Only rows that satisfy this filter will be udpated.
+        where: str or [Expr][lancedb.expr.Expr], optional
+            The filter condition. Can be a SQL string or a type-safe
+            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
+            [lit][lancedb.expr.lit]. Only rows that satisfy this filter will
+            be updated.
        updates_sql: dict, optional
            The updates to apply, expressed as SQL expression strings.  The keys should
            be column names. The values should be SQL expressions.  These can be SQL
@@ -6030,13 +6247,14 @@ class AsyncTable:
        --------
        >>> import asyncio
        >>> import lancedb
+        >>> from lancedb.expr import col
        >>> import pandas as pd
        >>> async def demo_update():
        ...     data = pd.DataFrame({"x": [1, 2], "vector": [[1, 2], [3, 4]]})
        ...     db = await lancedb.connect_async("./.lancedb")
        ...     table = await db.create_table("my_table", data)
        ...     # x is [1, 2], vector is [[1, 2], [3, 4]]
-        ...     await table.update({"vector": [10, 10]}, where="x = 2")
+        ...     await table.update({"vector": [10, 10]}, where=col("x") == 2)
        ...     # x is [1, 2], vector is [[1, 2], [10, 10]]
        ...     await table.update(updates_sql={"x": "x + 1"})
        ...     # x is [2, 3], vector is [[1, 2], [10, 10]]
@@ -6050,7 +6268,8 @@ class AsyncTable:
        if updates is not None:
            updates_sql = {k: value_to_sql(v) for k, v in updates.items()}

-        return await self._inner.update(updates_sql, where)
+        predicate = where.to_sql() if isinstance(where, Expr) else where
+        return await self._inner.update(updates_sql, predicate)

    async def add_columns(
        self,
@@ -6082,8 +6301,11 @@ class AsyncTable:
            Function columns are supported only on LanceDB Cloud and
            Enterprise.
        computed: Dict[str, str], optional
-            A map of column name to a SQL expression defining the column. The
-            column's type and inputs are derived from the expression.
+            A mapping from output column names to SQL expressions derives each
+            output field from its expression. A direct projection of a Blob v2
+            field inherits Blob v2 semantics; other expressions derive their
+            ordinary Arrow type. Mapping order is declaration and dependency
+            order.

            Unlike ``transforms``, the expression is stored rather than
            evaluated now: the column is committed with no values, and rows get
@@ -6354,6 +6576,9 @@ class AsyncTable:
        Offsets are mostly useful for sampling as the set of all valid offsets is easily
        known in advance to be [0, len(table)).

+        No guarantees are made regarding the order in which results are returned.
+        Repeated offsets produce repeated rows.
+
        Parameters
        ----------
        offsets: list[int]
@@ -2,17 +2,41 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors

 import io
+import subprocess
+import sys
+import textwrap

+import lance
 import pyarrow as pa
 import pyarrow.compute as pc
 import pytest
+from lance.blob import BlobType as LanceBlobType

 import lancedb
-from lancedb._blob import read_row_ids_from_hits, stash_auto_row_ids
+from lancedb._blob import (
+    blob_v2_projection_sources,
+    read_row_ids_from_hits,
+    stash_auto_row_ids,
+)
+from lancedb.expr import col
 from lancedb.index import FTS
 from lancedb.schema import blob_column_paths, blob_v2_column_paths


+_HIDE_LANCE_BLOB = """\
+import importlib.abc
+import sys
+
+class _MissingLanceBlob(importlib.abc.MetaPathFinder):
+    def find_spec(self, fullname, path, target=None):
+        if fullname == "lance.blob" or fullname.startswith("lance.blob."):
+            raise ModuleNotFoundError(fullname, name="lance.blob")
+
+sys.modules.pop("lance.blob", None)
+sys.meta_path.insert(0, _MissingLanceBlob())
+"""
+
+
 def _blob_table(name, rows):
    db = lancedb.connect("memory:///")
    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
@@ -42,10 +66,204 @@ def _row_ids_by_id(table):
    return dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))


+def _assert_missing_blob_row_ids(exc_info):
+    message = str(exc_info.value)
+    assert "row ids" in message
+    assert "rowaddr" not in message
+    assert "fragment" not in message
+
+
+def _assert_fetch_apis_reject_missing_row_ids(table, row_ids):
+    with pytest.raises(ValueError) as exc_info:
+        table.fetch_blobs("image", row_ids)
+    _assert_missing_blob_row_ids(exc_info)
+    with pytest.raises(ValueError) as exc_info:
+        table.fetch_blob_files("image", row_ids)
+    _assert_missing_blob_row_ids(exc_info)
+    with pytest.raises(ValueError) as exc_info:
+        table.fetch_blob_ranges("image", [(row_id, 0, 1) for row_id in row_ids])
+    _assert_missing_blob_row_ids(exc_info)
+
+
 def test_blob_factory_declares_v2_field():
    field = lancedb.blob("image")
    assert isinstance(field.type, pa.ExtensionType)
    assert field.type.extension_name == "lance.blob.v2"
+    assert lancedb.BlobType is LanceBlobType
+    assert type(field.type) is LanceBlobType
+
+
+def test_blob_type_works_without_pylance():
+    script = _HIDE_LANCE_BLOB + textwrap.dedent(
+        """\
+        import lancedb
+        import pyarrow as pa
+
+        field = lancedb.blob("image")
+        if not isinstance(field.type, pa.ExtensionType):
+            raise SystemExit("expected an extension type")
+        if field.type.extension_name != "lance.blob.v2":
+            raise SystemExit(field.type.extension_name)
+        if lancedb.BlobType is not type(field.type):
+            raise SystemExit("BlobType is not the field type class")
+        if lancedb.BlobType.__module__ != "lancedb.schema":
+            raise SystemExit(lancedb.BlobType.__module__)
+
+        db = lancedb.connect("memory:///")
+        table = db.create_table(
+            "images",
+            schema=pa.schema([pa.field("id", pa.int64()), field]),
+        )
+        table.add([{"id": 1, "image": b"hello"}])
+        result = (
+            table.merge_insert("id")
+            .when_matched_update_all()
+            .when_not_matched_insert_all()
+            .execute([{"id": 1, "image": b"updated"}, {"id": 2, "image": b"inserted"}])
+        )
+        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
+            raise SystemExit(
+                f"merge_insert rows updated={result.num_updated_rows} "
+                f"inserted={result.num_inserted_rows}"
+            )
+        """
+    )
+    result = subprocess.run(
+        [sys.executable, "-c", script],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr
+
+
+def test_blob_resolves_pylance_type_without_eager_import():
+    script = textwrap.dedent(
+        """\
+        import sys
+        import lancedb
+
+        if "lance.blob" in sys.modules:
+            raise SystemExit("import lancedb imported lance.blob")
+        field = lancedb.blob("image")
+        from lance.blob import BlobType
+
+        if type(field.type) is not BlobType:
+            raise SystemExit(f"{type(field.type)} is not {BlobType}")
+        import lance
+
+        image = lance.blob_array([b"x"])
+        if type(image.type) is not BlobType:
+            raise SystemExit("blob_array used a different class")
+        if type(image.type) is not type(field.type):
+            raise SystemExit("field and array classes differ")
+        """
+    )
+    result = subprocess.run(
+        [sys.executable, "-c", script],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr
+
+
+def test_blob_fallback_fails_if_name_already_registered():
+    script = _HIDE_LANCE_BLOB + textwrap.dedent(
+        """\
+        import pyarrow as pa
+
+        class OtherBlobType(pa.ExtensionType):
+            def __init__(self):
+                super().__init__(
+                    pa.struct([pa.field("data", pa.large_binary())]),
+                    "lance.blob.v2",
+                )
+
+            def __arrow_ext_serialize__(self):
+                return b""
+
+            @classmethod
+            def __arrow_ext_deserialize__(cls, storage_type, serialized):
+                return cls()
+
+        pa.register_extension_type(OtherBlobType())
+        import lancedb
+
+        try:
+            lancedb.blob("image")
+        except ValueError as err:
+            if "already registered" not in str(err):
+                raise SystemExit(err)
+        else:
+            raise SystemExit("expected ValueError")
+        """
+    )
+    result = subprocess.run(
+        [sys.executable, "-c", script],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr
+
+
+def test_blob_type_rejects_competing_registration_with_pylance():
+    script = textwrap.dedent(
+        """\
+        import pyarrow as pa
+        import pyarrow.ipc
+
+        class OtherBlobType(pa.ExtensionType):
+            def __init__(self):
+                super().__init__(
+                    pa.struct(
+                        [
+                            pa.field("data", pa.large_binary()),
+                            pa.field("uri", pa.utf8()),
+                            pa.field("position", pa.uint64()),
+                            pa.field("size", pa.uint64()),
+                        ]
+                    ),
+                    "lance.blob.v2",
+                )
+
+            def __arrow_ext_serialize__(self):
+                return b""
+
+            @classmethod
+            def __arrow_ext_deserialize__(cls, storage_type, serialized):
+                return cls()
+
+        pa.register_extension_type(OtherBlobType())
+
+        from lance.blob import BlobType
+
+        if BlobType is OtherBlobType:
+            raise SystemExit("pylance BlobType was replaced")
+        schema = pa.schema([pa.field("value", BlobType())])
+        restored = pa.ipc.read_schema(schema.serialize())
+        if type(restored.field("value").type) is not OtherBlobType:
+            raise SystemExit(type(restored.field("value").type))
+
+        import lancedb
+
+        try:
+            lancedb.blob("image")
+        except ValueError as err:
+            if "__main__.OtherBlobType" not in str(err):
+                raise SystemExit(err)
+        else:
+            raise SystemExit("expected ValueError")
+        """
+    )
+    result = subprocess.run(
+        [sys.executable, "-c", script],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr


 def test_blob_v2_column_paths_include_list_children():
@@ -70,6 +288,14 @@ def test_blob_v2_column_paths_include_list_children():
    ]


+def test_blob_v2_projection_sources_use_typed_column_name():
+    schema = pa.schema([lancedb.blob("blob")])
+
+    assert blob_v2_projection_sources(schema, {"blob_alias": col("blob")}) == {
+        "blob_alias": "blob"
+    }
+
+
 def _legacy_v1_table(name):
    db = lancedb.connect("memory:///")
    schema = pa.schema(
@@ -166,6 +392,20 @@ async def test_async_table_to_pandas_descriptions_mode_omits_row_id():
    assert set(descriptor.keys()) == {"kind", "position", "size", "blob_id", "blob_uri"}


+@pytest.mark.asyncio
+async def test_async_typed_blob_projection_preserves_source_column():
+    db = await lancedb.connect_async("memory:///typed_blob_projection")
+    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("blob")])
+    table = await db.create_table("typed_blob_projection", schema=schema)
+    await table.add([{"id": 1, "blob": b"alpha"}])
+
+    hits = await table.query().select({"blob_alias": col("blob")}).to_arrow()
+
+    assert "_lance_row_id" in hits.schema.field("blob_alias").type.names
+    blobs = await table.fetch_blobs("blob", hits)
+    assert blobs.to_pylist() == [b"alpha"]
+
+
 def test_fetch_blobs_round_trip():
    table = _blob_table(
        "round_trip",
@@ -176,6 +416,292 @@ def test_fetch_blobs_round_trip():
    assert [blobs[0].as_py(), blobs[1].as_py()] == [b"alpha", b"beta"]


+def test_merge_insert_writes_python_bytes():
+    table = _blob_table("merge_bytes", [{"id": 1, "image": b"before"}])
+    result = (
+        table.merge_insert("id")
+        .when_matched_update_all()
+        .when_not_matched_insert_all()
+        .execute([{"id": 1, "image": b"updated"}, {"id": 2, "image": b"inserted"}])
+    )
+    assert result.num_updated_rows == 1
+    assert result.num_inserted_rows == 1
+    by_id = _row_ids_by_id(table)
+    blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
+    assert blobs.to_pylist() == [b"updated", b"inserted"]
+
+
+def test_merge_insert_bytes_after_reopen_without_touching_blob_type(tmp_path):
+    db = lancedb.connect(tmp_path)
+    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
+    table = db.create_table("images", schema=schema)
+    table.add([{"id": 1, "image": b"hello"}])
+
+    script = textwrap.dedent(
+        f"""\
+        import lancedb
+
+        db = lancedb.connect({str(tmp_path)!r})
+        table = db.open_table("images")
+        image_type = table.schema.field("image").type
+        if type(image_type).__name__ != "StructType":
+            raise SystemExit(f"expected StructType, got {{type(image_type)}}")
+        result = (
+            table.merge_insert("id")
+            .when_matched_update_all()
+            .when_not_matched_insert_all()
+            .execute(
+                [{{"id": 1, "image": b"updated"}}, {{"id": 2, "image": b"inserted"}}]
+            )
+        )
+        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
+            raise SystemExit(
+                f"rows updated={{result.num_updated_rows}} "
+                f"inserted={{result.num_inserted_rows}}"
+            )
+        hits = table.search().with_row_id(True).limit(10).to_arrow()
+        by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
+        blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
+        if blobs.to_pylist() != [b"updated", b"inserted"]:
+            raise SystemExit(blobs.to_pylist())
+        """
+    )
+    result = subprocess.run(
+        [sys.executable, "-c", script],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr
+
+
+def test_merge_insert_bytes_after_reopen_without_pylance(tmp_path):
+    db = lancedb.connect(tmp_path)
+    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
+    table = db.create_table("images", schema=schema)
+    table.add([{"id": 1, "image": b"hello"}])
+
+    script = _HIDE_LANCE_BLOB + textwrap.dedent(
+        f"""\
+        import lancedb
+
+        db = lancedb.connect({str(tmp_path)!r})
+        table = db.open_table("images")
+        image_type = table.schema.field("image").type
+        if type(image_type).__name__ != "StructType":
+            raise SystemExit(f"expected StructType, got {{type(image_type)}}")
+        result = (
+            table.merge_insert("id")
+            .when_matched_update_all()
+            .when_not_matched_insert_all()
+            .execute(
+                [{{"id": 1, "image": b"updated"}}, {{"id": 2, "image": b"inserted"}}]
+            )
+        )
+        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
+            raise SystemExit(
+                f"rows updated={{result.num_updated_rows}} "
+                f"inserted={{result.num_inserted_rows}}"
+            )
+        hits = table.search().with_row_id(True).limit(10).to_arrow()
+        by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
+        blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
+        if blobs.to_pylist() != [b"updated", b"inserted"]:
+            raise SystemExit(blobs.to_pylist())
+        """
+    )
+    result = subprocess.run(
+        [sys.executable, "-c", script],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr
+
+
+def test_merge_insert_blob_array_into_reopened_unregistered_table(tmp_path):
+    db = lancedb.connect(tmp_path)
+    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
+    table = db.create_table("images", schema=schema)
+    table.add([{"id": 1, "image": b"before"}])
+
+    script = textwrap.dedent(
+        f"""\
+        import pyarrow as pa
+        import lancedb
+
+        db = lancedb.connect({str(tmp_path)!r})
+        table = db.open_table("images")
+        image_type = table.schema.field("image").type
+        if type(image_type).__name__ != "StructType":
+            raise SystemExit(
+                f"expected StructType before lance import, got {{type(image_type)}}"
+            )
+
+        import lance
+
+        updates = pa.Table.from_arrays(
+            [
+                pa.array([1, 2], type=pa.int64()),
+                lance.blob_array([b"updated", b"inserted"]),
+            ],
+            names=["id", "image"],
+        )
+        result = (
+            table.merge_insert("id")
+            .when_matched_update_all()
+            .when_not_matched_insert_all()
+            .execute(updates)
+        )
+        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
+            raise SystemExit(
+                f"rows updated={{result.num_updated_rows}} "
+                f"inserted={{result.num_inserted_rows}}"
+            )
+        hits = table.search().with_row_id(True).limit(10).to_arrow()
+        by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
+        blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
+        if blobs.to_pylist() != [b"updated", b"inserted"]:
+            raise SystemExit(blobs.to_pylist())
+        """
+    )
+    result = subprocess.run(
+        [sys.executable, "-c", script],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr
+
+
+def test_add_all_null_blob_column():
+    db = lancedb.connect("memory:///")
+    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
+    table = db.create_table("all_null", schema=schema)
+    table.add([{"id": 1, "image": None}, {"id": 2, "image": None}])
+    by_id = _row_ids_by_id(table)
+    blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
+    assert blobs.to_pylist() == [None, None]
+
+
+def test_create_table_nested_blob_schema_without_rows():
+    db = lancedb.connect("memory:///")
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field("info", pa.struct([lancedb.blob("blob")])),
+            pa.field("images", pa.list_(lancedb.blob("image"))),
+        ]
+    )
+    table = db.create_table("nested_empty", schema=schema)
+    assert table.count_rows() == 0
+
+
+def test_merge_insert_nested_blob_dicts():
+    db = lancedb.connect("memory:///")
+    info = pa.StructArray.from_arrays(
+        [
+            pa.array(["first"], type=pa.string()),
+            _blob_array("blob", [b"before"]),
+        ],
+        names=["name", "blob"],
+    )
+    data = pa.Table.from_arrays(
+        [pa.array([1], type=pa.int64()), info],
+        names=["id", "info"],
+    )
+    table = db.create_table("nested_merge", data=data)
+    result = (
+        table.merge_insert("id")
+        .when_matched_update_all()
+        .execute([{"id": 1, "info": {"name": "first", "blob": b"after"}}])
+    )
+    assert result.num_updated_rows == 1
+    by_id = _row_ids_by_id(table)
+    blobs = table.fetch_blobs("info.blob", [by_id[1]])
+    assert blobs.to_pylist() == [b"after"]
+
+
+def _list_blob_table(name):
+    db = lancedb.connect("memory:///")
+    blob_field = lancedb.blob("image")
+    images = pa.ListArray.from_arrays(
+        pa.array([0, 1], type=pa.int32()), _blob_array("image", [b"before"])
+    )
+    data = pa.Table.from_arrays(
+        [pa.array([1], type=pa.int64()), images],
+        schema=pa.schema(
+            [pa.field("id", pa.int64()), pa.field("images", pa.list_(blob_field))]
+        ),
+    )
+    return db.create_table(name, data=data)
+
+
+def test_merge_insert_list_blob_dicts():
+    table = _list_blob_table("list_merge")
+    result = (
+        table.merge_insert("id")
+        .when_matched_update_all()
+        .when_not_matched_insert_all()
+        .execute([{"id": 1, "images": [b"one", b"two"]}, {"id": 2, "images": None}])
+    )
+    assert result.num_updated_rows == 1
+    assert result.num_inserted_rows == 1
+    hits = table.search().limit(10).to_arrow()
+    sizes = {
+        row["id"]: None if row["images"] is None else [d["size"] for d in row["images"]]
+        for row in hits.to_pylist()
+    }
+    assert sizes == {1: [3, 3], 2: None}
+
+
+def test_list_blob_column_queries_as_raw_descriptors():
+    table = _list_blob_table("list_query")
+    hits = table.search().limit(10).to_arrow()
+    element = hits.schema.field("images").type.value_type
+    assert pa.types.is_struct(element)
+    assert "_lance_row_id" not in element.names
+    with pytest.raises(ValueError, match="expected struct before segment"):
+        table.fetch_blobs("images.image", [0])
+
+
+def test_row_addressable_paths_exclude_list_children():
+    from lancedb.schema import row_addressable_blob_v2_paths
+
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field("info", pa.struct([lancedb.blob("blob")])),
+            pa.field("images", pa.list_(lancedb.blob("image"))),
+        ]
+    )
+    assert blob_v2_column_paths(schema) == ["info.blob", "images.image"]
+    assert row_addressable_blob_v2_paths(schema) == ["info.blob"]
+
+
+def test_merge_insert_writes_pylance_blob_array():
+    table = _blob_table("merge_pylance", [{"id": 1, "image": b"before"}])
+    image = lance.blob_array([b"updated", b"inserted"])
+    assert type(image.type) is LanceBlobType
+    assert type(image.type) is type(lancedb.BlobType())
+    updates = pa.Table.from_arrays(
+        [pa.array([1, 2], type=pa.int64()), image], names=["id", "image"]
+    )
+
+    result = (
+        table.merge_insert("id")
+        .when_matched_update_all()
+        .when_not_matched_insert_all()
+        .execute(updates)
+    )
+
+    assert result.num_updated_rows == 1
+    assert result.num_inserted_rows == 1
+    by_id = _row_ids_by_id(table)
+    blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
+    assert blobs.to_pylist() == [b"updated", b"inserted"]
+
+
 def test_fetch_blobs_accepts_query_result():
    table = _blob_table("from_result", [{"id": 1, "image": b"gamma"}])
    hits = table.search().limit(10).to_arrow()
@@ -184,6 +710,25 @@ def test_fetch_blobs_accepts_query_result():
    assert {blobs[i].as_py() for i in range(len(blobs))} == {b"gamma"}


+def test_fetch_blobs_after_compact_with_stable_row_ids(tmp_path):
+    db = lancedb.connect(
+        tmp_path, storage_options={"new_table_enable_stable_row_ids": "true"}
+    )
+    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
+    table = db.create_table("t", schema=schema)
+    table.add([{"id": 1, "image": b"frag-one"}])
+    table.add([{"id": 2, "image": b"frag-two"}])
+    by_id = _row_ids_by_id(table)
+    ids = [by_id[1], by_id[2]]
+
+    table.optimize()
+
+    blobs = table.fetch_blobs("image", ids)
+    assert blobs.to_pylist() == [b"frag-one", b"frag-two"]
+    ranges = table.fetch_blob_ranges("image", [(ids[0], 5, 3), (ids[1], 5, 3)])
+    assert ranges.to_pylist() == [b"one", b"two"]
+
+
 def test_fetch_blobs_preserves_null_and_empty_values():
    table = _blob_table(
        "nulls",
@@ -232,8 +777,25 @@ def test_fetch_blob_ranges_validates_requests():
    with pytest.raises(ValueError, match="offset \\+ length overflowed"):
        table.fetch_blob_ranges("image", [(row_id, 2**64 - 1, 1)])

-    with pytest.raises(ValueError, match="row IDs"):
+    with pytest.raises(ValueError) as exc_info:
        table.fetch_blob_ranges("image", [(2**64 - 1, 0, 1)])
+    _assert_missing_blob_row_ids(exc_info)
+
+
+def test_fetch_blob_apis_reject_missing_fragment_row_addr():
+    table = _blob_table("missing_frag", [{"id": 1, "image": b"x"}])
+    live = _row_ids_by_id(table)[1]
+    _assert_fetch_apis_reject_missing_row_ids(table, [1 << 32, live])
+
+
+def test_fetch_blob_apis_reject_deleted_row_ids():
+    table = _blob_table(
+        "deleted_rows",
+        [{"id": 1, "image": b"one"}, {"id": 2, "image": b"two"}],
+    )
+    by_id = _row_ids_by_id(table)
+    table.delete("id = 2")
+    _assert_fetch_apis_reject_missing_row_ids(table, [by_id[2], by_id[1]])


 def test_fetch_blob_ranges_empty_requests_returns_empty_array():
@@ -403,6 +965,50 @@ async def test_blob_v2_hybrid_fetch_blobs_async():
    assert {blobs[i].as_py() for i in range(len(blobs))} == {b"alpha", b"beta"}


+@pytest.mark.asyncio
+async def test_async_hybrid_typed_blob_projection_preserves_source_column():
+    db = await lancedb.connect_async("memory:///hybrid_typed_blob")
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field("text", pa.utf8()),
+            pa.field("vector", pa.list_(pa.float32(), list_size=2)),
+            lancedb.blob("blob"),
+        ]
+    )
+    table = await db.create_table("hybrid_typed_blob", schema=schema)
+    await table.add(
+        [
+            {
+                "id": 1,
+                "text": "hello alpha",
+                "vector": [1.0, 0.0],
+                "blob": b"alpha",
+            },
+            {
+                "id": 2,
+                "text": "hello beta",
+                "vector": [0.9, 0.1],
+                "blob": b"beta",
+            },
+        ]
+    )
+    await table.create_index("text", config=FTS(with_position=False))
+
+    hits = await (
+        table.query()
+        .nearest_to([1.0, 0.0])
+        .nearest_to_text("hello")
+        .select({"blob_alias": col("blob")})
+        .limit(2)
+        .to_arrow()
+    )
+
+    assert "_lance_row_id" in hits.schema.field("blob_alias").type.names
+    blobs = await table.fetch_blobs("blob", hits)
+    assert {blobs[i].as_py() for i in range(len(blobs))} == {b"alpha", b"beta"}
+
+
 def test_blob_file_seek_read_and_read_range():
    payload = _identifiable_payload(1024)
    table = _blob_table("seek_read", [{"id": 1, "image": payload}])
@@ -52,7 +52,7 @@ class TestExprConstruction:
    def test_func(self):
        e = func("lower", col("name"))
        assert isinstance(e, Expr)
-        assert e.to_sql() == "lower(name)"
+        assert e.to_sql() == "lower(`name`)"

    def test_func_unknown_raises(self):
        with pytest.raises(Exception):
@@ -115,7 +115,7 @@ class TestExprOperators:
    def test_and_operator(self):
        e = (col("age") > lit(18)) & (col("status") == lit("active"))
        assert isinstance(e, Expr)
-        assert e.to_sql() == "((age > 18) AND (status = 'active'))"
+        assert e.to_sql() == "((age > 18) AND (`status` = 'active'))"

    def test_or_operator(self):
        e = (col("a") == lit(1)) | (col("b") == lit(2))
@@ -166,7 +166,7 @@ class TestExprOperators:
    def test_coerce_plain_str(self):
        e = col("name") == "alice"
        assert isinstance(e, Expr)
-        assert e.to_sql() == "(name = 'alice')"
+        assert e.to_sql() == "(`name` = 'alice')"

    def test_reflexive_comparisons(self):
        # 10 < col("age") swaps to col("age") > 10
@@ -198,85 +198,85 @@ class TestExprBytesLiteral:

    def test_bytes_equality_expr_sql(self):
        e = col("data") == lit(b"\xca\xfe")
-        assert e.to_sql() == "(data = X'CAFE')"
+        assert e.to_sql() == "(`data` = X'CAFE')"

    def test_bytes_ne_expr_sql(self):
        e = col("data") != lit(b"\xff")
-        assert e.to_sql() == "(data <> X'FF')"
+        assert e.to_sql() == "(`data` <> X'FF')"

    def test_bytes_compound_expr_sql(self):
        e = (col("data") == lit(b"\x01")) & (col("id") > lit(5))
-        assert e.to_sql() == "((data = X'01') AND (id > 5))"
+        assert e.to_sql() == "((`data` = X'01') AND (id > 5))"

    def test_bytes_in_function_call(self):
        # Regression test: binary literals inside scalar function calls
        # used to fail because DataFusion's unparser does not support Binary
        # scalars.  Now handled via a placeholder-substitution rewrite.
        e = func("contains", col("data"), lit(b"\xff"))
-        assert e.to_sql() == "contains(data, X'FF')"
+        assert e.to_sql() == "contains(`data`, X'FF')"

    def test_bytes_in_not(self):
        e = ~(col("data") == lit(b"\xff"))
-        assert e.to_sql() == "NOT (data = X'FF')"
+        assert e.to_sql() == "NOT (`data` = X'FF')"


 class TestExprStringMethods:
    def test_lower(self):
        e = col("name").lower()
        assert isinstance(e, Expr)
-        assert e.to_sql() == "lower(name)"
+        assert e.to_sql() == "lower(`name`)"

    def test_upper(self):
        e = col("name").upper()
        assert isinstance(e, Expr)
-        assert e.to_sql() == "upper(name)"
+        assert e.to_sql() == "upper(`name`)"

    def test_contains(self):
        e = col("text").contains(lit("hello"))
        assert isinstance(e, Expr)
-        assert e.to_sql() == "contains(text, 'hello')"
+        assert e.to_sql() == "contains(`text`, 'hello')"

    def test_contains_with_str_coerce(self):
        e = col("text").contains("hello")
        assert isinstance(e, Expr)
-        assert e.to_sql() == "contains(text, 'hello')"
+        assert e.to_sql() == "contains(`text`, 'hello')"

    def test_chained_lower_eq(self):
        e = col("name").lower() == lit("alice")
        assert isinstance(e, Expr)
-        assert e.to_sql() == "(lower(name) = 'alice')"
+        assert e.to_sql() == "(lower(`name`) = 'alice')"


 class TestExprCast:
    def test_cast_string(self):
        e = col("id").cast("string")
        assert isinstance(e, Expr)
-        assert e.to_sql() == "CAST(id AS VARCHAR)"
+        assert e.to_sql() == "arrow_cast(id, 'Utf8')"

    def test_cast_int32(self):
        e = col("score").cast("int32")
        assert isinstance(e, Expr)
-        assert e.to_sql() == "CAST(score AS INTEGER)"
+        assert e.to_sql() == "arrow_cast(score, 'Int32')"

    def test_cast_float64(self):
        e = col("val").cast("float64")
        assert isinstance(e, Expr)
-        assert e.to_sql() == "CAST(val AS DOUBLE)"
+        assert e.to_sql() == "arrow_cast(val, 'Float64')"

    def test_cast_pyarrow_type(self):
        e = col("score").cast(pa.int32())
        assert isinstance(e, Expr)
-        assert e.to_sql() == "CAST(score AS INTEGER)"
+        assert e.to_sql() == "arrow_cast(score, 'Int32')"

    def test_cast_pyarrow_float64(self):
        e = col("val").cast(pa.float64())
        assert isinstance(e, Expr)
-        assert e.to_sql() == "CAST(val AS DOUBLE)"
+        assert e.to_sql() == "arrow_cast(val, 'Float64')"

    def test_cast_pyarrow_string(self):
        e = col("id").cast(pa.string())
        assert isinstance(e, Expr)
-        assert e.to_sql() == "CAST(id AS VARCHAR)"
+        assert e.to_sql() == "arrow_cast(id, 'Utf8')"

    def test_cast_pyarrow_and_string_equivalent(self):
        # pa.int32() and "int32" should produce equivalent SQL
@@ -597,14 +597,14 @@ class TestExprIsin:
    def test_isin_strs(self):
        assert (
            col("status").isin(["active", "pending"]).to_sql()
-            == "status IN ('active', 'pending')"
+            == "`status` IN ('active', 'pending')"
        )

    def test_isin_coerces_and_mixes(self):
        assert col("id").isin([lit(1), 2]).to_sql() == "id IN (1, 2)"

    def test_isin_empty(self):
-        assert col("id").isin([]).to_sql() == "id IN ()"
+        assert col("id").isin([]).to_sql() == "false"

    def test_isin_filter(self, simple_table):
        result = simple_table.search().where(col("id").isin([1, 3, 5])).to_arrow()
@@ -12,6 +12,8 @@ from datetime import date
 import http.server
 import json
 from pathlib import Path
+import subprocess
+import sys
 import threading
 from typing import Optional

@@ -19,7 +21,13 @@ import pyarrow as pa
 import pytest

 import lancedb
-from lancedb.functions import UdfDefinition, udf
+from lancedb.functions import (
+    PythonRuntimeSpec,
+    UdfDefinition,
+    _canonical_arrow_type,
+    _GRAMMAR_PRIMITIVES,
+    udf,
+)

 THRESHOLD = 20
 _CACHE = None
@@ -61,6 +69,80 @@ def test_scalar_udf_matches_shared_registration_golden_and_remains_callable():
    }


+def _main_udf_source(
+    *, threshold: int = 20, input_annotation: str = "int", comparison: str = ">="
+) -> str:
+    return (
+        "from __future__ import annotations\n"
+        "from lancedb.functions import udf\n"
+        f"THRESHOLD = {threshold}\n"
+        "\n"
+        "@udf\n"
+        f"def label(value: {input_annotation}) -> str:\n"
+        f"    return 'big' if value {comparison} THRESHOLD else 'small'\n"
+        "\n"
+        "assert label.__module__ == '__main__'\n"
+        "print(label.registration_request.to_canonical_json())\n"
+    )
+
+
+def _run_main_udf(path: Path, source: str) -> dict:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(source)
+    result = subprocess.run(
+        [sys.executable, str(path)],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return json.loads(result.stdout)
+
+
+def test_main_udf_registration_identity_is_stable_across_processes_and_paths(
+    tmp_path,
+):
+    source = _main_udf_source()
+    original_path = tmp_path / "original" / "job.py"
+    moved_path = tmp_path / "moved" / "renamed_job.py"
+
+    original_runs = [_run_main_udf(original_path, source) for _ in range(2)]
+    moved_run = _run_main_udf(moved_path, source)
+
+    assert len({run["artifact"]["digest"] for run in [*original_runs, moved_run]}) == 1
+    assert all(
+        run["signature"] == original_runs[0]["signature"]
+        for run in [original_runs[1], moved_run]
+    )
+    assert original_runs[0] == original_runs[1] == moved_run
+
+    body_change = _run_main_udf(
+        tmp_path / "changes" / "body.py", _main_udf_source(comparison=">")
+    )
+    global_change = _run_main_udf(
+        tmp_path / "changes" / "global.py", _main_udf_source(threshold=21)
+    )
+    annotation_change = _run_main_udf(
+        tmp_path / "changes" / "annotation.py",
+        _main_udf_source(input_annotation="float"),
+    )
+
+    baseline = original_runs[0]
+    assert baseline["signature"] == body_change["signature"]
+    assert baseline["signature"] == global_change["signature"]
+    assert baseline["signature"] != annotation_change["signature"]
+    assert (
+        len(
+            {
+                baseline["artifact"]["digest"],
+                body_change["artifact"]["digest"],
+                global_change["artifact"]["digest"],
+                annotation_change["artifact"]["digest"],
+            }
+        )
+        == 4
+    )
+
+
 def _run_packaged(definition, *args):
    """Execute the shipped artifact in a fresh namespace, as a worker would."""
    source = base64.b64decode(definition.registration_request.artifact.content.data)
@@ -89,6 +171,58 @@ def test_udf_conda_environment():
        udf(name="channels", conda_channels=["conda-forge"])(lambda value: value)


+def test_udf_gpu_marker_uses_gpu_runtime():
+    @udf(pip=["cupy-cuda12x"], gpu=True)
+    def double_on_gpu(value: int) -> int:
+        return value * 2
+
+    request = json.loads(double_on_gpu.registration_request.to_canonical_json())
+    assert request["runtime"]["kind"] == "python_v2"
+    assert request["runtime"]["gpu"] is True
+
+    @udf(pip=["pyarrow"])
+    def cpu_function(value: int) -> int:
+        return value
+
+    cpu_runtime = json.loads(cpu_function.registration_request.to_canonical_json())[
+        "runtime"
+    ]
+    assert cpu_runtime["kind"] == "python"
+    assert "gpu" not in cpu_runtime
+
+    def identity(value: int) -> int:
+        return value
+
+    for invalid in [None, 0, 1, -1, 1.5, "", "true", "1", "H100"]:
+        with pytest.raises(ValueError, match="gpu must be a boolean"):
+            udf(name="invalid_gpu", gpu=invalid)(identity)
+
+    base_runtime = {
+        "kind": "python_v2",
+        "python_version": "3.12",
+        "environment": {"kind": "pip"},
+    }
+    runtime = PythonRuntimeSpec.model_validate({**base_runtime, "gpu": True})
+    assert runtime.gpu is True
+    for invalid in [False, 1, 0, "", "true", "1", "H100"]:
+        with pytest.raises(ValueError, match="runtime.gpu must be true"):
+            PythonRuntimeSpec.model_validate({**base_runtime, "gpu": invalid})
+
+
+def test_unknown_runtime_discards_payload_before_known_field_validation():
+    for payload in [
+        {"kind": "python_v3", "gpu": {"model": "H100"}},
+        {"kind": "python_v3", "resources": []},
+        {
+            "kind": "python_v3",
+            "environment": {"kind": []},
+            "python_version": 3.15,
+        },
+    ]:
+        runtime = PythonRuntimeSpec.model_validate(payload)
+        assert runtime.to_canonical_json() == '{"kind":"python_v3"}'
+
+
 def test_udf_packages_attribute_access_and_body_imports():
    @udf
    def word_norm(body: str) -> float:
@@ -168,9 +302,7 @@ def test_udf_resolves_module_globals_before_builtins(tmp_path):
        udf(module.uses_callable_shadow)


-def test_canonical_arrow_type_is_exactly_the_grammar():
-    from lancedb.functions import _GRAMMAR_PRIMITIVES, _canonical_arrow_type
-
+def test_canonical_arrow_type_prefers_the_compact_grammar():
    golden = json.loads(
        (
            Path(__file__).parents[3]
@@ -181,14 +313,19 @@ def test_canonical_arrow_type_is_exactly_the_grammar():
        case["arrow_type"] for case in golden["valid"] if "<" not in case["arrow_type"]
    ]
    assert [name for _, name in _GRAMMAR_PRIMITIVES] == primitives
+    assert _canonical_arrow_type(pa.list_(pa.field("item", pa.float32(), False))) == (
+        "list<float32>"
+    )
+    assert (
+        _canonical_arrow_type(pa.large_list(pa.field("item", pa.float32(), False)))
+        == "large_list<float32>"
+    )
    for outside in [
        pa.timestamp("us"),
        pa.decimal128(10, 2),
-        pa.large_string(),
        pa.large_binary(),
        pa.binary(4),
        pa.duration("s"),
-        pa.struct([pa.field("a", pa.int32())]),
        pa.list_(pa.float32(), 0),
        pa.list_(pa.timestamp("us")),
    ]:
@@ -378,14 +515,27 @@ def test_udf_recursion_versus_a_rebound_module_name(tmp_path):
        udf(raw_fact)


-def test_canonical_arrow_type_rejects_unrepresentable_list_children():
-    from lancedb.functions import _canonical_arrow_type
-
+def test_canonical_arrow_type_uses_exact_json_for_list_child_properties():
+    nullable = pa.list_(pa.float32())
+    assert json.loads(_canonical_arrow_type(nullable)) == {
+        "type": "list",
+        "fields": [
+            {
+                "name": "item",
+                "nullable": True,
+                "type": {"type": "float32"},
+            }
+        ],
+    }
+    named = pa.list_(pa.field("custom", pa.float32(), nullable=False))
+    assert json.loads(_canonical_arrow_type(named))["fields"][0]["name"] == "custom"
    for outside in [
-        pa.list_(pa.float32()),  # pyarrow default: nullable child
-        pa.list_(pa.field("custom", pa.float32(), nullable=False)),
        pa.list_(pa.field("item", pa.float32(), nullable=False, metadata={"k": "v"})),
        pa.list_(pa.field("item", pa.float32(), nullable=False), 0),
+        pa.list_(
+            pa.field("item", pa.float32(), nullable=False, metadata={"k": "v"}), 3
+        ),
+        pa.list_(pa.field("custom", pa.float32(), nullable=False), 3),
    ]:
        with pytest.raises(TypeError, match="unsupported Arrow type"):
            _canonical_arrow_type(outside)
@@ -395,6 +545,29 @@ def test_canonical_arrow_type_rejects_unrepresentable_list_children():
        )
        == "fixed_size_list<float32, 3>"
    )
+    fixed = json.loads(_canonical_arrow_type(pa.list_(pa.float32(), 3)))
+    assert fixed == {
+        "type": "fixed_size_list",
+        "fields": [
+            {
+                "name": "item",
+                "nullable": True,
+                "type": {"type": "float32"},
+            }
+        ],
+        "length": 3,
+    }
+    large = json.loads(_canonical_arrow_type(pa.large_list(pa.float32())))
+    assert large["type"] == "large_list"
+    assert large["fields"][0]["nullable"] is True
+
+    for invalid_struct in [
+        pa.struct([]),
+        pa.struct([pa.field("a", pa.int32()), pa.field("a", pa.int64())]),
+        pa.struct([pa.field("", pa.int32())]),
+    ]:
+        with pytest.raises(TypeError, match="unsupported Arrow type"):
+            _canonical_arrow_type(invalid_struct)


 def _calls_missing(value: int) -> int:
@@ -432,6 +605,7 @@ def _arrow_type_from_golden(spec: dict) -> pa.DataType:
        "null": pa.null(),
        "bool": pa.bool_(),
        "utf8": pa.string(),
+        "large_utf8": pa.large_string(),
        "binary": pa.binary(),
        "float16": pa.float16(),
        "float32": pa.float32(),
@@ -448,8 +622,6 @@ def test_arrow_type_grammar_matches_the_shared_golden():
            / "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json"
        ).read_text()
    )
-    from lancedb.functions import _canonical_arrow_type
-
    emitted = {
        case["arrow_type"]: _canonical_arrow_type(_arrow_type_from_golden(case["json"]))
        for case in golden["valid"]
@@ -482,6 +654,432 @@ def test_explicit_arrow_schema_is_deterministic():
    assert signature.output.nullable is False


+def test_blob_fields_use_the_scalar_function_semantic_type():
+    @udf(
+        input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
+        output_schema=lancedb.blob("result", nullable=False),
+    )
+    def copy_blob(image):
+        return image
+
+    signature = copy_blob.registration_request.signature
+    assert signature.inputs[0].arrow_type == "blob_v2"
+    assert signature.output.kind == "scalar"
+    assert signature.output.arrow_type == "blob_v2"
+
+
+def test_whole_named_struct_function_can_include_a_blob_result_field():
+    @udf(
+        input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
+        output_schema=pa.field(
+            "payload",
+            pa.struct(
+                [
+                    pa.field("mime_type", pa.string(), nullable=False),
+                    lancedb.blob("image", nullable=False),
+                ]
+            ),
+            nullable=False,
+        ),
+    )
+    def inspect_blob(image):
+        return {"mime_type": "image/png", "image": image}
+
+    output = inspect_blob.registration_request.signature.output
+    assert output.kind == "named_struct"
+    assert [(field.name, field.arrow_type) for field in output.fields] == [
+        ("mime_type", "utf8"),
+        ("image", "blob_v2"),
+    ]
+
+
+def test_struct_blob_signature_fields_preserve_exact_metadata_and_nullability():
+    nested_input = pa.field(
+        "payload",
+        pa.struct(
+            [
+                pa.field("mime_type", pa.string(), nullable=False),
+                pa.field(
+                    "nested",
+                    pa.struct([lancedb.blob("image", nullable=True)]),
+                    nullable=True,
+                ),
+            ]
+        ),
+        nullable=True,
+    )
+    nested_output = pa.field(
+        "result",
+        pa.struct(
+            [
+                pa.field("mime_type", pa.string(), nullable=False),
+                pa.field(
+                    "nested",
+                    pa.struct([lancedb.blob("image", nullable=True)]),
+                    nullable=False,
+                ),
+            ]
+        ),
+        nullable=False,
+    )
+
+    @udf(input_schema=pa.schema([nested_input]), output_schema=nested_output)
+    def copy_payload(payload):
+        return payload
+
+    signature = copy_payload.registration_request.signature
+    input_type = json.loads(signature.inputs[0].arrow_type)
+    assert input_type["fields"][1]["nullable"] is True
+    input_blob = input_type["fields"][1]["type"]["fields"][0]
+    assert input_blob["nullable"] is True
+    assert input_blob["metadata"] == {"ARROW:extension:name": "lance.blob.v2"}
+
+    assert signature.output.kind == "named_struct"
+    nested_result = next(
+        field for field in signature.output.fields if field.name == "nested"
+    )
+    output_type = json.loads(nested_result.arrow_type)
+    output_blob = output_type["fields"][0]
+    assert output_blob["nullable"] is True
+    assert output_blob["metadata"] == {"ARROW:extension:name": "lance.blob.v2"}
+
+
+def test_struct_blob_signature_supports_multiple_struct_levels():
+    recursive = pa.field(
+        "value",
+        pa.struct(
+            [
+                pa.field(
+                    "level_1",
+                    pa.struct(
+                        [
+                            pa.field(
+                                "level_2",
+                                pa.struct([lancedb.blob("image", nullable=False)]),
+                                nullable=False,
+                            )
+                        ]
+                    ),
+                    nullable=False,
+                )
+            ]
+        ),
+        nullable=False,
+    )
+
+    @udf(
+        input_schema=pa.schema([recursive]),
+        output_schema=pa.field("size", pa.int64(), nullable=False),
+    )
+    def blob_size(value):
+        return len(value["level_1"]["level_2"]["image"])
+
+    encoded = json.loads(blob_size.registration_request.signature.inputs[0].arrow_type)
+    blob = encoded["fields"][0]["type"]["fields"][0]["type"]["fields"][0]
+    assert blob["metadata"]["ARROW:extension:name"] == "lance.blob.v2"
+
+
+@pytest.mark.parametrize(
+    "data_type",
+    [
+        pa.list_(lancedb.blob("item", nullable=False)),
+        pa.large_list(lancedb.blob("item", nullable=False)),
+        pa.list_(lancedb.blob("item", nullable=False), 2),
+        pa.map_(pa.string(), lancedb.blob("value", nullable=False).type),
+    ],
+)
+def test_blob_signature_rejects_collection_ancestors(data_type):
+    with pytest.raises(
+        TypeError,
+        match="Blob v2 fields nested under collection types are not supported",
+    ):
+
+        @udf(
+            input_schema=pa.schema([pa.field("value", data_type, nullable=False)]),
+            output_schema=pa.field("size", pa.int64(), nullable=False),
+        )
+        def blob_size(value):
+            return len(value)
+
+
+def test_blob_signature_rejects_collection_below_a_struct():
+    nested = pa.field(
+        "value",
+        pa.struct(
+            [
+                pa.field(
+                    "images",
+                    pa.list_(lancedb.blob("item", nullable=False)),
+                    nullable=False,
+                )
+            ]
+        ),
+        nullable=False,
+    )
+    with pytest.raises(
+        TypeError,
+        match="Blob v2 fields nested under collection types are not supported",
+    ):
+
+        @udf(
+            input_schema=pa.schema([nested]),
+            output_schema=pa.field("size", pa.int64(), nullable=False),
+        )
+        def blob_size(value):
+            return len(value["images"])
+
+
+def test_named_struct_function_can_include_a_blob_result_field():
+    @udf(
+        input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
+        output_schema=pa.schema(
+            [
+                lancedb.blob("thumbnail", nullable=False),
+                pa.field("width", pa.int32(), nullable=False),
+            ]
+        ),
+    )
+    def inspect_blob(image):
+        return {"thumbnail": image, "width": 1}
+
+    output = inspect_blob.registration_request.signature.output
+    assert output.kind == "named_struct"
+    assert [(field.name, field.arrow_type) for field in output.fields] == [
+        ("thumbnail", "blob_v2"),
+        ("width", "int32"),
+    ]
+
+
+def test_named_struct_function_preserves_nullable_result_fields():
+    @udf(
+        input_schema=pa.schema([pa.field("value", pa.int64(), nullable=False)]),
+        output_schema=pa.schema(
+            [
+                pa.field("result", pa.int64(), nullable=True),
+                pa.field("failure_code", pa.int32(), nullable=False),
+            ]
+        ),
+    )
+    def nullable_result(value):
+        return {"result": value, "failure_code": 0}
+
+    output = nullable_result.registration_request.signature.output
+    assert [(field.name, field.nullable) for field in output.fields] == [
+        ("result", True),
+        ("failure_code", False),
+    ]
+
+    @udf(
+        input_schema=pa.schema([pa.field("value", pa.int64(), nullable=False)]),
+        output_schema=pa.schema(
+            [
+                pa.field("result", pa.int64(), nullable=True),
+                pa.field("failure_code", pa.int32(), nullable=True),
+            ]
+        ),
+    )
+    def all_nullable(value):
+        return {"result": value, "failure_code": None}
+
+    assert all(
+        field.nullable
+        for field in all_nullable.registration_request.signature.output.fields
+    )
+
+
+def test_metadata_marked_blob_field_uses_the_semantic_type():
+    extension = lancedb.blob("image", nullable=False).type
+    storage = (
+        extension.storage_type if isinstance(extension, pa.ExtensionType) else extension
+    )
+    metadata_blob = pa.field(
+        "image",
+        storage,
+        nullable=False,
+        metadata={"ARROW:extension:name": "lance.blob.v2"},
+    )
+
+    @udf(
+        input_schema=pa.schema([metadata_blob]),
+        output_schema=pa.field("size", pa.int64(), nullable=False),
+    )
+    def blob_size(image):
+        return len(image)
+
+    assert blob_size.registration_request.signature.inputs[0].arrow_type == "blob_v2"
+
+
+def test_blob_marker_rejects_invalid_storage_layout():
+    malformed = pa.field(
+        "image",
+        pa.int64(),
+        nullable=False,
+        metadata={"ARROW:extension:name": "lance.blob.v2"},
+    )
+
+    with pytest.raises(TypeError, match="requires a supported Blob storage layout"):
+
+        @udf(
+            input_schema=pa.schema([malformed]),
+            output_schema=pa.field("size", pa.int64(), nullable=False),
+        )
+        def blob_size(image):
+            return len(image)
+
+
+def test_nested_non_blob_extension_is_not_silently_unwrapped():
+    class TestExtension(pa.ExtensionType):
+        def __init__(self):
+            super().__init__(pa.int64(), "test.function.extension")
+
+        def __arrow_ext_serialize__(self):
+            return b""
+
+        @classmethod
+        def __arrow_ext_deserialize__(cls, storage_type, serialized):
+            return cls()
+
+    nested = pa.field(
+        "value",
+        pa.struct([pa.field("extended", TestExtension(), nullable=False)]),
+        nullable=False,
+    )
+    with pytest.raises(TypeError, match="unsupported Arrow type"):
+
+        @udf(
+            input_schema=pa.schema([nested]),
+            output_schema=pa.field("result", pa.int64(), nullable=False),
+        )
+        def extension_value(value):
+            return value["extended"]
+
+
+def test_explicit_large_utf8_schemas_use_the_canonical_function_name():
+    input_schema = pa.schema([pa.field("text", pa.large_string(), nullable=True)])
+    output_schema = pa.field("result", pa.large_string(), nullable=False)
+
+    @udf(input_schema=input_schema, output_schema=output_schema)
+    def preserve(text):
+        return text
+
+    signature = preserve.registration_request.signature
+    assert signature.inputs[0].arrow_type == "large_utf8"
+    assert signature.inputs[0].nullable is True
+    assert signature.output.arrow_type == "large_utf8"
+    assert signature.output.nullable is False
+
+    nested = pa.struct([pa.field("text", pa.large_string(), nullable=True)])
+    assert json.loads(_canonical_arrow_type(nested)) == {
+        "type": "struct",
+        "fields": [
+            {
+                "name": "text",
+                "nullable": True,
+                "type": {"type": "large_utf8"},
+            }
+        ],
+    }
+
+
+def test_nested_struct_output_uses_canonical_exact_json():
+    token = pa.struct(
+        [
+            pa.field("position", pa.int32(), nullable=False),
+            pa.field("value", pa.string(), nullable=False),
+            pa.field("length", pa.int32(), nullable=False),
+        ]
+    )
+    analysis = pa.struct(
+        [
+            pa.field("normalized_text", pa.string(), nullable=False),
+            pa.field("has_content", pa.bool_(), nullable=False),
+            pa.field(
+                "metrics",
+                pa.struct(
+                    [
+                        pa.field("character_count", pa.int64(), nullable=False),
+                        pa.field("word_count", pa.int32(), nullable=False),
+                        pa.field("average_word_length", pa.float64(), nullable=False),
+                    ]
+                ),
+                nullable=False,
+            ),
+            pa.field(
+                "diagnostics",
+                pa.struct(
+                    [
+                        pa.field("status", pa.string(), nullable=False),
+                        pa.field(
+                            "normalization",
+                            pa.struct(
+                                [
+                                    pa.field("changed", pa.bool_(), nullable=False),
+                                    pa.field(
+                                        "original_length", pa.int64(), nullable=False
+                                    ),
+                                ]
+                            ),
+                            nullable=False,
+                        ),
+                    ]
+                ),
+                nullable=False,
+            ),
+            pa.field(
+                "token_preview",
+                pa.list_(pa.field("item", token, nullable=False)),
+                nullable=False,
+            ),
+        ]
+    )
+
+    @udf(
+        input_schema=pa.schema([pa.field("text", pa.string(), nullable=False)]),
+        output_schema=pa.field("analysis", analysis, nullable=False),
+    )
+    def analyze(text):
+        return {"normalized_text": text}
+
+    output = analyze.registration_request.signature.output
+    assert output.kind == "named_struct"
+    assert [field.name for field in output.fields] == [
+        "normalized_text",
+        "has_content",
+        "metrics",
+        "diagnostics",
+        "token_preview",
+    ]
+    metrics = json.loads(output.fields[2].arrow_type)
+    assert metrics == {
+        "type": "struct",
+        "fields": [
+            {
+                "name": "character_count",
+                "nullable": False,
+                "type": {"type": "int64"},
+            },
+            {
+                "name": "word_count",
+                "nullable": False,
+                "type": {"type": "int32"},
+            },
+            {
+                "name": "average_word_length",
+                "nullable": False,
+                "type": {"type": "float64"},
+            },
+        ],
+    }
+    preview = json.loads(output.fields[4].arrow_type)
+    assert preview["type"] == "list"
+    assert preview["fields"][0]["type"]["type"] == "struct"
+    assert [field["name"] for field in preview["fields"][0]["type"]["fields"]] == [
+        "position",
+        "value",
+        "length",
+    ]
+
+
 def test_annotation_and_explicit_schema_validation_fail_closed():
    with pytest.raises(TypeError, match="missing Function annotations"):

@@ -525,6 +1123,72 @@ def test_annotation_and_explicit_schema_validation_fail_closed():
        def nullable_explicit(value):
            return value

+    for invalid_field in [
+        pa.field("", pa.int32(), nullable=False),
+        pa.field("result", pa.int32(), nullable=False, metadata={"k": "v"}),
+    ]:
+        with pytest.raises(TypeError, match="unsupported Arrow type"):
+
+            @udf(
+                input_schema=pa.schema([pa.field("value", pa.int64())]),
+                output_schema=pa.schema([invalid_field]),
+            )
+            def invalid_explicit_field(value):
+                return value
+
+    with pytest.raises(TypeError, match="unsupported Arrow type"):
+
+        @udf(
+            input_schema=pa.schema(
+                [pa.field("value", pa.int64(), metadata={"k": "v"})]
+            ),
+            output_schema=pa.int64(),
+        )
+        def input_field_metadata(value):
+            return value
+
+    with pytest.raises(TypeError, match="unsupported Arrow type"):
+
+        @udf(
+            input_schema=pa.schema([pa.field("value", pa.int64())]),
+            output_schema=pa.field(
+                "result", pa.int64(), nullable=False, metadata={"k": "v"}
+            ),
+        )
+        def scalar_output_field_metadata(value):
+            return value
+
+    struct_type = pa.struct([pa.field("value", pa.int64(), nullable=False)])
+    with pytest.raises(TypeError, match="unsupported Arrow type"):
+
+        @udf(
+            input_schema=pa.schema([pa.field("value", pa.int64())]),
+            output_schema=pa.field(
+                "result", struct_type, nullable=False, metadata={"k": "v"}
+            ),
+        )
+        def struct_output_field_metadata(value):
+            return {"value": value}
+
+    for input_schema, output_schema in [
+        (
+            pa.schema([pa.field("value", pa.int64())], metadata={"k": "v"}),
+            pa.int64(),
+        ),
+        (
+            pa.schema([pa.field("value", pa.int64())]),
+            pa.schema(
+                [pa.field("result", pa.int64(), nullable=False)],
+                metadata={"k": "v"},
+            ),
+        ),
+    ]:
+        with pytest.raises(TypeError, match="schema metadata"):
+
+            @udf(input_schema=input_schema, output_schema=output_schema)
+            def schema_metadata(value):
+                return value
+

 def test_local_function_catalog_operations_are_not_supported(tmp_path):
    db = lancedb.connect(tmp_path)
@@ -535,6 +1199,10 @@ def test_local_function_catalog_operations_are_not_supported(tmp_path):
        db.create_function_async(normalize_score)
    with pytest.raises(NotImplementedError, match=message):
        db.get_function("normalize_score", version="fv_exact")
+    with pytest.raises(NotImplementedError, match=message):
+        db.list_functions()
+    with pytest.raises(NotImplementedError, match=message):
+        db.drop_function("normalize_score", version="fv_exact")


@contextlib.contextmanager
@@ -580,6 +1248,28 @@ def _mock_remote_function_catalog():
                    "version": "fv_exact",
                }
                response = state["version"]
+            elif self.path == "/v1/functions/list":
+                assert body["include_definition"] is True
+                if "page_token" not in body:
+                    response = {
+                        "functions": [
+                            {
+                                "name": "normalize_score",
+                                "version": "fv_exact",
+                                "definition": state["version"],
+                            }
+                        ],
+                        "page_token": "next",
+                    }
+                else:
+                    assert body["page_token"] == "next"
+                    response = {"functions": []}
+            elif self.path == "/v1/functions/drop":
+                assert body == {
+                    "name": "normalize_score",
+                    "version": "fv_exact",
+                }
+                response = {"dropped": True}
            else:
                status = 404
                response = {"error": "not found"}
@@ -638,3 +1328,83 @@ def test_blocking_remote_registration_returns_function_version():
        "/v1/functions/create",
        "/v1/jobs/describe",
    ]
+
+
+def test_remote_list_functions_paginates_and_returns_typed_versions():
+    with _mock_remote_function_catalog() as (host, state):
+        db = lancedb.connect(
+            "db://dev",
+            api_key="fake",
+            host_override=host,
+            client_config={"retry_config": {"retries": 0}},
+        )
+        created = db.create_function(normalize_score)
+        state["requests"].clear()
+        functions = db.list_functions()
+
+    assert functions == [created]
+    assert state["requests"] == [
+        ("/v1/functions/list", {"include_definition": True}),
+        (
+            "/v1/functions/list",
+            {"include_definition": True, "page_token": "next"},
+        ),
+    ]
+
+
+@pytest.mark.asyncio
+async def test_async_remote_list_functions_returns_typed_versions():
+    with _mock_remote_function_catalog() as (host, state):
+        db = await lancedb.connect_async(
+            "db://dev",
+            api_key="fake",
+            host_override=host,
+            client_config={"retry_config": {"retries": 0}},
+        )
+        registration = await db.create_function_async(normalize_score)
+        created = await registration.wait()
+        state["requests"].clear()
+        functions = await db.list_functions()
+
+    assert functions == [created]
+    assert [path for path, _ in state["requests"]] == [
+        "/v1/functions/list",
+        "/v1/functions/list",
+    ]
+
+
+def test_remote_drop_function_sends_exact_version():
+    with _mock_remote_function_catalog() as (host, state):
+        db = lancedb.connect(
+            "db://dev",
+            api_key="fake",
+            host_override=host,
+            client_config={"retry_config": {"retries": 0}},
+        )
+        assert db.drop_function("normalize_score", version="fv_exact") is True
+
+    assert state["requests"] == [
+        (
+            "/v1/functions/drop",
+            {"name": "normalize_score", "version": "fv_exact"},
+        )
+    ]
+
+
+@pytest.mark.asyncio
+async def test_async_remote_drop_function_sends_exact_version():
+    with _mock_remote_function_catalog() as (host, state):
+        db = await lancedb.connect_async(
+            "db://dev",
+            api_key="fake",
+            host_override=host,
+            client_config={"retry_config": {"retries": 0}},
+        )
+        assert await db.drop_function("normalize_score", version="fv_exact") is True
+
+    assert state["requests"] == [
+        (
+            "/v1/functions/drop",
+            {"name": "normalize_score", "version": "fv_exact"},
+        )
+    ]
@@ -54,7 +54,10 @@ class TestOAuthProvider:
        provider = OAuthProvider(fetcher)
        headers = provider.get_headers()

-        assert headers == {"Authorization": "Bearer token123"}
+        assert headers == {
+            "Authorization": "Bearer token123",
+            "x-lancedb-credential-type": "oidc",
+        }
        assert provider._current_token == "token123"
        assert provider._token_expires_at is not None

@@ -73,14 +76,20 @@ class TestOAuthProvider:

        # First call
        headers1 = provider.get_headers()
-        assert headers1 == {"Authorization": "Bearer token1"}
+        assert headers1 == {
+            "Authorization": "Bearer token1",
+            "x-lancedb-credential-type": "oidc",
+        }

        # Wait for token to expire
        time.sleep(1.1)

        # Second call should refresh
        headers2 = provider.get_headers()
-        assert headers2 == {"Authorization": "Bearer token2"}
+        assert headers2 == {
+            "Authorization": "Bearer token2",
+            "x-lancedb-credential-type": "oidc",
+        }
        assert call_count == 2

    def test_no_expiry_info(self):
@@ -92,12 +101,18 @@ class TestOAuthProvider:
        provider = OAuthProvider(fetcher)
        headers = provider.get_headers()

-        assert headers == {"Authorization": "Bearer permanent_token"}
+        assert headers == {
+            "Authorization": "Bearer permanent_token",
+            "x-lancedb-credential-type": "oidc",
+        }
        assert provider._token_expires_at is None

        # Should not refresh on second call
        headers2 = provider.get_headers()
-        assert headers2 == {"Authorization": "Bearer permanent_token"}
+        assert headers2 == {
+            "Authorization": "Bearer permanent_token",
+            "x-lancedb-credential-type": "oidc",
+        }

    def test_missing_access_token(self):
        """Test error handling when access_token is missing."""
@@ -121,7 +136,10 @@ class TestOAuthProvider:
        provider = OAuthProvider(fetcher)
        headers = provider.get_headers()

-        assert headers == {"Authorization": "Bearer sync_token"}
+        assert headers == {
+            "Authorization": "Bearer sync_token",
+            "x-lancedb-credential-type": "oidc",
+        }


 class TestClientConfigIntegration:
@@ -266,3 +266,38 @@ async def test_async_namespace_connection_materialized_views(tmp_path):
            handle._route_pushdown_to_rust == through_namespace._route_pushdown_to_rust
        )
        assert handle._namespace_path == through_namespace._namespace_path
+
+
+def test_namespaced_select_kind_is_read_and_unknown_kinds_are_refused():
+    import json
+
+    import pyarrow as pa
+
+    from lancedb.materialized_view import _definition_from_schema
+
+    def schema_with(definition: dict) -> pa.Schema:
+        return pa.schema([pa.field("id", pa.int32())]).with_metadata(
+            {b"mv.definition": json.dumps(definition).encode()}
+        )
+
+    # "namespaced_select" is the namespaced form of "select": same shape,
+    # a separate kind so readers that predate it refuse instead of
+    # resolving the source at the root.
+    definition = _definition_from_schema(
+        schema_with(
+            {
+                "kind": "namespaced_select",
+                "source_table": "people",
+                "source_namespace": ["ns"],
+                "projections": [{"output": "name", "expression": "name"}],
+            }
+        ),
+        "v",
+    )
+    assert definition.source_table == "people"
+    assert definition.source_namespace == ["ns"]
+
+    with pytest.raises(NotImplementedError, match="cannot refresh"):
+        _definition_from_schema(
+            schema_with({"kind": "select_v3", "source_table": "people"}), "v"
+        )
@@ -675,6 +675,21 @@ def test_distance_range(table: lancedb.table.Table):
    assert res["_distance"].to_pylist() == [min_dist, max_dist]


+@pytest.mark.parametrize("expression", ["1 - _distance", "1.0 - _distance"])
+def test_select_arithmetic_with_distance(table, expression):
+    result = (
+        table.search([10, 10])
+        .select({"similarity": expression, "_distance": "_distance"})
+        .distance_type("cosine")
+        .to_arrow()
+    )
+
+    assert result.schema.field("similarity").type == pa.float32()
+    assert result["similarity"].to_pylist() == pytest.approx(
+        [1 - distance for distance in result["_distance"].to_pylist()]
+    )
+
+
@pytest.mark.asyncio
 async def test_distance_range_async(table_async: AsyncTable):
    q = [0, 0]
@@ -1908,6 +1923,21 @@ def test_take_queries(tmp_path):
        17,
    ]

+    # Duplicate offsets are occurrences, not set members. Ordering is unspecified.
+    assert sorted(table.take_offsets([5, 2, 5, 17]).to_pandas()["idx"].to_list()) == [
+        2,
+        5,
+        5,
+        17,
+    ]
+
+    # Converting a take builder to its serializable query representation must
+    # retain occurrence metadata and execute with the same multiplicity.
+    query = table.take_offsets([5, 2, 5, 17]).select(["idx"]).to_query_object()
+    assert query.take_offsets == [5, 2, 5, 17]
+    converted = table._execute_query(query).read_all()
+    assert sorted(converted["idx"].to_pylist()) == [2, 5, 5, 17]
+
    # Take by row id
    assert list(
        sorted(table.take_row_ids([5, 2, 17]).to_pandas()["idx"].to_list())
@@ -479,24 +479,49 @@ def test_remote_permutation_is_picklable():
                match = re.search(
                    r"_rowoffset\s+in\s+\((.*?)\)", body["filter"], re.IGNORECASE
                )
-                offsets = [int(o.strip()) for o in match.group(1).split(",")]
+                offsets = list(
+                    dict.fromkeys(int(o.strip()) for o in match.group(1).split(","))
+                )
            else:
                offsets = list(range(len(rows)))
-            table = pa.table({"a": [rows[offset] for offset in offsets]})
+            columns = body.get("columns") or ["a"]
+            table = pa.table(
+                {
+                    column: (
+                        [rows[offset] for offset in offsets]
+                        if column == "a"
+                        else offsets
+                    )
+                    for column in columns
+                }
+            )

            request.send_response(200)
            request.send_header("Content-Type", "application/vnd.apache.arrow.file")
            request.end_headers()
            with pa.ipc.new_file(request.wfile, schema=table.schema) as writer:
-                writer.write_table(table)
+                writer.write_table(table, max_chunksize=2)
        else:
            request.send_response(404)
            request.end_headers()

    with mock_lancedb_connection(handler) as db:
-        permutation = Permutation.identity(db.open_table("test"))
+        table = db.open_table("test")
+        assert table.take_offsets([0, 2, 0, 4]).to_list() == [
+            {"a": 0},
+            {"a": 0},
+            {"a": 2},
+            {"a": 4},
+        ]
+
+        permutation = Permutation.identity(table)
        restored = pickle.loads(pickle.dumps(permutation))
-        assert restored.__getitems__([0, 2, 4]) == [{"a": 0}, {"a": 2}, {"a": 4}]
+        assert restored.__getitems__([0, 2, 0, 4]) == [
+            {"a": 0},
+            {"a": 2},
+            {"a": 0},
+            {"a": 4},
+        ]


 def test_create_table_exist_ok():
@@ -795,11 +820,13 @@ def test_table_create_indices():
        scalar_req = received_requests[0]
        assert "name" in scalar_req
        assert scalar_req["name"] == "custom_scalar_idx"
+        assert scalar_req["replace"] is False

        # Check FTS index request has custom name
        fts_req = received_requests[1]
        assert "name" in fts_req
        assert fts_req["name"] == "custom_fts_idx"
+        assert fts_req["replace"] is False
        assert fts_req["block_size"] == 256
        assert fts_req["custom_stop_words"] == ["cloud"]

@@ -807,6 +834,7 @@ def test_table_create_indices():
        vector_req = received_requests[2]
        assert "name" in vector_req
        assert vector_req["name"] == "custom_vector_idx"
+        assert "replace" not in vector_req

        table.wait_for_index(["custom_scalar_idx"], timedelta(seconds=2))
        table.wait_for_index(
@@ -1079,6 +1107,9 @@ def test_remote_create_index_new_api():
            table.create_index("text", config=FTS(block_size=256))
            # IvfRq via new API
            table.create_index("vector", config=IvfRq(distance_type="l2"))
+            table.create_index(
+                "vector", config=IvfPq(distance_type="l2"), replace=False
+            )

        # Legacy index_type="IVF_RQ" routes to IvfRq config under the hood.
        with pytest.warns(DeprecationWarning, match="create_index"):
@@ -1088,15 +1119,17 @@ def test_remote_create_index_new_api():
                num_partitions=8,
            )

-        assert len(received_requests) == 5
+        assert len(received_requests) == 6
        assert [req["column"] for req in received_requests] == [
            "vector",
            "category",
            "text",
            "vector",
            "vector",
+            "vector",
        ]
        assert received_requests[2]["block_size"] == 256
+        assert received_requests[4]["replace"] is False


 def test_table_wait_for_index_timeout():
@@ -2434,7 +2467,7 @@ def test_remote_blob_byte_apis_not_supported_on_old_server():


 def test_remote_connection_jobs_surface():
-    from lancedb.exceptions import JobFailedError
+    from lancedb.exceptions import JobFailedError, JobNotFoundError

    schema = pa.schema([("state", pa.string())])
    batch = pa.record_batch([pa.array(["created", "done"])], schema=schema)
@@ -2442,6 +2475,7 @@ def test_remote_connection_jobs_surface():
    with pa.ipc.new_stream(sink, schema) as writer:
        writer.write_batch(batch)
    events_body = sink.getvalue().to_pybytes()
+    query_events_payloads = []

    def handler(request):
        content_len = int(request.headers.get("Content-Length", 0))
@@ -2479,6 +2513,22 @@ def test_remote_connection_jobs_surface():
            request.end_headers()
            request.wfile.write(json.dumps(rsp).encode())
        elif request.path == "/v1/jobs/describe":
+            if payload["job_id"] == "job-2":
+                request.send_response(200)
+                request.send_header("Content-Type", "application/json")
+                request.end_headers()
+                request.wfile.write(
+                    json.dumps(
+                        dict(
+                            job_id="job-2",
+                            job_type="refresh_column",
+                            job_state="DONE",
+                            creation_ms=2000,
+                            result=dict(rows_assigned=1000000, rows_failed=0),
+                        )
+                    ).encode()
+                )
+                return
            if payload["job_id"] != "job-1":
                request.send_response(404)
                request.end_headers()
@@ -2510,7 +2560,7 @@ def test_remote_connection_jobs_surface():
            request.end_headers()
            request.wfile.write(b'{"job_id": "job-1"}')
        elif request.path == "/v1/jobs/query_events":
-            assert payload["job_id"] == "job-1"
+            query_events_payloads.append(payload)
            request.send_response(200)
            request.send_header("Content-Type", "application/vnd.apache.arrow.stream")
            request.end_headers()
@@ -2526,24 +2576,109 @@ def test_remote_connection_jobs_surface():
        assert jobs[0].table == "t1"
        assert jobs[1].state == "finished"

-        description = db.get_job("job-1")
-        assert description.job_type == "create_index"
-        assert description.state == "failed"
-        assert json.loads(description.spec_json) == {"column": "vec"}
-        assert description.failure.message == "worker died"
-        assert description.failure.retryable is True
-        assert db.get_job("missing") is None
-
        assert db.cancel_job("job-1") is True
        assert db.cancel_job("missing") is False

-        batches = db.job_history("job-1")
-        assert len(batches) == 1
-        assert batches[0].num_rows == 2
-        assert batches[0].column("state").to_pylist() == ["created", "done"]
+        # Opening a job hands back a populated handle; a missing one fails.
+        with pytest.raises(JobNotFoundError, match="missing"):
+            db.open_job("missing")
+        finished = db.open_job("job-2")
+        assert finished.state == "finished"
+        assert finished.result == {"rows_assigned": 1000000, "rows_failed": 0}

-        job = db.job("job-1")
+        job = db.open_job("job-1")
        assert job.id == "job-1"
+        # Opening already populated the handle.
+        assert job.state == "failed"
+        assert job.spec == {"column": "vec"}
+        assert job.failure.message == "worker died"
        assert job.status() == "failed"
        with pytest.raises(JobFailedError, match="worker died"):
            job.wait(timeout=timedelta(seconds=5))
+
+
+def test_remote_job_handle_reports_its_own_detail():
+    schema = pa.schema([("state", pa.string())])
+    batch = pa.record_batch([pa.array(["claim_complete"])], schema=schema)
+    sink = pa.BufferOutputStream()
+    with pa.ipc.new_stream(sink, schema) as writer:
+        writer.write_batch(batch)
+    events_body = sink.getvalue().to_pybytes()
+    event_payloads = []
+
+    def handler(request):
+        content_len = int(request.headers.get("Content-Length", 0))
+        body = request.rfile.read(content_len) if content_len > 0 else b""
+        payload = json.loads(body) if body else {}
+        if request.path == "/v1/jobs/describe":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(
+                json.dumps(
+                    dict(
+                        job_id="job-1",
+                        job_type="refresh_column",
+                        job_state="DONE",
+                        creation_ms=2000,
+                        spec=dict(column="vec"),
+                        result=dict(rows_assigned=1000000),
+                    )
+                ).encode()
+            )
+        elif request.path == "/v1/jobs/query_events":
+            event_payloads.append(payload)
+            request.send_response(200)
+            request.send_header("Content-Type", "application/vnd.apache.arrow.stream")
+            request.end_headers()
+            request.wfile.write(events_body)
+        else:
+            request.send_response(404)
+            request.end_headers()
+
+    with mock_lancedb_connection(handler) as db:
+        job = db.open_job("job-1")
+
+        # Opening populates the handle in the same round trip.
+        assert job.state == "finished"
+        job.refresh()
+        assert job.job_type == "refresh_column"
+        assert job.creation_ms == 2000
+        assert job.spec == {"column": "vec"}
+        assert job.result == {"rows_assigned": 1000000}
+        assert job.failure is None
+        # The JSON payloads stay reachable, but as internal APIs.
+        assert json.loads(job._spec_json) == {"column": "vec"}
+        assert json.loads(job._result_json) == {"rows_assigned": 1000000}
+
+        # print() shows everything the handle knows and nothing it does not.
+        # print() lays every known field out on its own line, with the JSON
+        # payloads indented rather than crammed onto one line.
+        assert repr(job) == "\n".join(
+            [
+                "Job(",
+                "    id='job-1',",
+                "    state='finished',",
+                "    job_type='refresh_column',",
+                "    creation_ms=2000,",
+                "    spec={",
+                '        "column": "vec"',
+                "    },",
+                "    result={",
+                '        "rows_assigned": 1000000',
+                "    },",
+                ")",
+            ]
+        )
+        # Nothing it does not know shows up.
+        assert "failure" not in repr(job)
+
+        events = job.events(filter="state = 'claim_complete'", limit=500)
+        assert isinstance(events, pa.Table)
+        assert events.column("state").to_pylist() == ["claim_complete"]
+        # The handle supplies job_id; the caller only narrows the query.
+        assert event_payloads[-1] == {
+            "job_id": "job-1",
+            "limit": 500,
+            "filter": "state = 'claim_complete'",
+        }
@@ -0,0 +1,162 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+from uuid import UUID
+
+import pytest
+import pyarrow as pa
+
+import lancedb
+from lancedb import _lancedb
+from lancedb.arrow import AsyncRecordBatchReader
+from lancedb.db import AsyncConnection
+from lancedb.remote.db import RemoteDBConnection
+from lancedb.sql import AsyncQuery, Query
+
+NIL_QUERY_ID = UUID(int=0)
+
+
+class FakeNativeQuery:
+    id = UUID("0198f1b2-c3d4-7e5f-8123-456789abcdef")
+
+    async def reader(self):
+        return pa.table({"value": [1, 2]})
+
+
+class FakeNativeConnection:
+    async def execute_query_async(self, query, *, default_namespace_path=None):
+        return FakeNativeQuery()
+
+
+class FakeAsyncConnection:
+    async def execute_query_async(self, query, *, default_namespace_path=None):
+        return AsyncQuery(FakeNativeQuery())
+
+
+def remote_connection(sql_host_override=None):
+    return lancedb.connect(
+        "db://analytics",
+        api_key="test-key",
+        host_override="http://localhost:10024",
+        sql_host_override=sql_host_override,
+    )
+
+
+def test_sql_is_connection_scoped():
+    assert hasattr(lancedb, "sql")
+    assert not callable(lancedb.sql)
+    assert not hasattr(_lancedb, "sql")
+    assert not hasattr(remote_connection(), "sql")
+    assert hasattr(remote_connection(), "execute_query")
+    assert hasattr(remote_connection(), "execute_query_async")
+    assert hasattr(remote_connection(), "describe_query")
+
+
+def test_query_id_is_uuid():
+    query = AsyncQuery(FakeNativeQuery())
+    assert isinstance(query.id, UUID)
+    assert Query(query).id == query.id
+
+
+def test_connection_serializes_sql_host_override():
+    endpoint = "grpc+tls://sql.example.com:10026"
+    restored = lancedb.deserialize_conn(
+        remote_connection(sql_host_override=endpoint).serialize()
+    )
+    assert restored.sql_host_override == endpoint
+
+
+@pytest.mark.asyncio
+async def test_async_sql_reader_is_record_batch_stream():
+    reader = await AsyncQuery(FakeNativeQuery()).reader()
+    assert isinstance(reader, AsyncRecordBatchReader)
+    assert (await reader.read_all())[0].column(0).to_pylist() == [1, 2]
+
+
+def test_sync_sql_reader_is_record_batch_reader():
+    reader = Query(AsyncQuery(FakeNativeQuery())).reader()
+    assert isinstance(reader, pa.RecordBatchReader)
+    assert reader.read_all().column(0).to_pylist() == [1, 2]
+
+
+def test_execute_query_returns_blocking_reader():
+    connection = RemoteDBConnection.__new__(RemoteDBConnection)
+    connection._conn = FakeAsyncConnection()
+    reader = connection.execute_query("SELECT 1")
+    assert isinstance(reader, pa.RecordBatchReader)
+    assert reader.read_all().column(0).to_pylist() == [1, 2]
+
+
+@pytest.mark.asyncio
+async def test_async_execute_query_returns_async_reader():
+    connection = AsyncConnection(FakeNativeConnection())
+    reader = await connection.execute_query("SELECT 1")
+    assert isinstance(reader, AsyncRecordBatchReader)
+    assert (await reader.read_all())[0].column(0).to_pylist() == [1, 2]
+
+
+def test_local_connection_rejects_sql(tmp_path):
+    connection = lancedb.connect(tmp_path)
+    with pytest.raises(NotImplementedError, match="SQL"):
+        connection.execute_query("SELECT 1")
+    with pytest.raises(NotImplementedError, match="SQL"):
+        connection.execute_query_async("SELECT 1")
+    with pytest.raises(NotImplementedError, match="SQL"):
+        connection.describe_query(NIL_QUERY_ID)
+
+
+@pytest.mark.asyncio
+async def test_local_async_connection_rejects_sql(tmp_path):
+    connection = await lancedb.connect_async(tmp_path)
+    with pytest.raises(NotImplementedError, match="SQL"):
+        await connection.execute_query("SELECT 1")
+    with pytest.raises(NotImplementedError, match="SQL"):
+        await connection.execute_query_async("SELECT 1")
+    with pytest.raises(NotImplementedError, match="SQL"):
+        await connection.describe_query(NIL_QUERY_ID)
+
+
+@pytest.mark.asyncio
+async def test_async_namespace_connection_rejects_sql(tmp_path):
+    connection = lancedb.connect_namespace_async("dir", {"root": str(tmp_path)})
+    with pytest.raises(NotImplementedError, match="SQL"):
+        await connection.execute_query("SELECT 1")
+    with pytest.raises(NotImplementedError, match="SQL"):
+        await connection.execute_query_async("SELECT 1")
+    with pytest.raises(NotImplementedError, match="SQL"):
+        await connection.describe_query(NIL_QUERY_ID)
+
+
+def test_describe_query_requires_uuid():
+    with pytest.raises(TypeError, match="UUID"):
+        remote_connection().describe_query(str(NIL_QUERY_ID))
+
+
+@pytest.mark.parametrize(
+    "default_namespace_path",
+    ["public", ("public",), [1]],
+)
+def test_execute_query_async_requires_namespace_path_list(default_namespace_path):
+    with pytest.raises(ValueError, match="default_namespace_path"):
+        remote_connection().execute_query_async(
+            "SELECT 1", default_namespace_path=default_namespace_path
+        )
+
+
+def test_execute_query_async_rejects_invalid_endpoint():
+    connection = remote_connection(sql_host_override="invalid://localhost")
+    with pytest.raises(ValueError, match="sql_host_override"):
+        connection.execute_query_async("SELECT 1")
+
+
+@pytest.mark.parametrize(
+    "default_namespace_path",
+    [[""], ["café"], ["pub\tlic"], ["events$raw"]],
+)
+def test_execute_query_async_rejects_invalid_namespace_components(
+    default_namespace_path,
+):
+    with pytest.raises(ValueError, match="default_namespace_path"):
+        remote_connection().execute_query_async(
+            "SELECT 1", default_namespace_path=default_namespace_path
+        )
@@ -11,6 +11,7 @@ import warnings
 import weakref
 from concurrent.futures import ThreadPoolExecutor
 from datetime import date, datetime, timedelta
+from decimal import Decimal
 from time import sleep
 from typing import List
 from unittest.mock import patch
@@ -336,6 +337,21 @@ async def test_update_async(mem_db_async: AsyncConnection):
    assert await table.count_rows("id == 10") == 1


+@pytest.mark.asyncio
+async def test_update_expr_filter_literals_async(mem_db_async: AsyncConnection):
+    values = ["5", "4.66e-84", "it's"]
+    table = await mem_db_async.create_table(
+        "update_expr_literals",
+        data=[{"field": value, "result": "original"} for value in values],
+    )
+
+    for value in values:
+        update_res = await table.update({"result": value}, where=col("field") == value)
+        assert update_res.rows_updated == 1
+
+    assert (await table.to_arrow())["result"].to_pylist() == values
+
+
 def test_create_table(mem_db: DBConnection):
    schema = pa.schema(
        {
@@ -2343,6 +2359,148 @@ def test_update(mem_db: DBConnection):
    assert np.allclose(v, np.array([[1.2, 1.9], [1.1, 1.1]]))


+def test_update_expr_filter_literals(mem_db: DBConnection):
+    values = ["5", "4.66e-84", "it's"]
+    table = mem_db.create_table(
+        "update_expr_literals",
+        data=[{"field": value, "result": "original"} for value in values],
+    )
+
+    for value in values:
+        update_res = table.update(where=col("field") == value, values={"result": value})
+        assert update_res.rows_updated == 1
+
+    assert table.to_arrow()["result"].to_pylist() == values
+
+
+def test_update_expr_filter_preserves_typed_semantics(mem_db: DBConnection):
+    low = Decimal("1.234567890123456789")
+    high = Decimal("1.234567890123456790")
+    decimal_schema = pa.schema(
+        [("val", pa.decimal128(19, 18)), ("result", pa.string())]
+    )
+    decimal_table = mem_db.create_table(
+        "update_expr_decimal",
+        pa.table(
+            {"val": [low, high], "result": ["old", "old"]},
+            schema=decimal_schema,
+        ),
+    )
+    predicate = col("val") < lit(high)
+    assert decimal_table.search().where(predicate).to_arrow().num_rows == 1
+    result = decimal_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 1
+
+    keyword_table = mem_db.create_table(
+        "update_expr_keyword", [{"null": 1, "result": "old"}]
+    )
+    predicate = col("null") == 1
+    assert keyword_table.search().where(predicate).to_arrow().num_rows == 1
+    result = keyword_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 1
+
+    empty_in_table = mem_db.create_table(
+        "update_expr_empty_in", [{"id": 1, "result": "old"}]
+    )
+    predicate = col("id").isin([])
+    assert empty_in_table.search().where(predicate).to_arrow().num_rows == 0
+    result = empty_in_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 0
+
+    marker = "__lancedb_binary_placeholder_0__"
+    binary_schema = pa.schema(
+        [("payload", pa.binary()), ("text", pa.string()), ("result", pa.string())]
+    )
+    binary_table = mem_db.create_table(
+        "update_expr_binary",
+        pa.table(
+            {
+                "payload": [b"\x01", b"\x02"],
+                "text": ["other", marker],
+                "result": ["old", "old"],
+            },
+            schema=binary_schema,
+        ),
+    )
+    predicate = (col("payload") == lit(b"\x01")) | (col("text") == marker)
+    assert binary_table.search().where(predicate).to_arrow().num_rows == 2
+    result = binary_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 2
+
+    nonfinite_table = mem_db.create_table(
+        "update_expr_nonfinite",
+        [{"x": 1.0, "result": "old"}, {"x": 2.0, "result": "old"}],
+    )
+    predicate = col("x") < float("inf")
+    assert nonfinite_table.search().where(predicate).to_arrow().num_rows == 2
+    result = nonfinite_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 2
+
+    float16_table = mem_db.create_table(
+        "update_expr_float16",
+        [{"x": 1.0, "result": "old"}, {"x": 3.0, "result": "old"}],
+    )
+    predicate = col("x").cast(pa.float16()) < 2.0
+    assert float16_table.search().where(predicate).to_arrow().num_rows == 1
+    result = float16_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 1
+
+    string_cast_table = mem_db.create_table(
+        "update_expr_string_cast",
+        [{"x": 1, "result": "old"}, {"x": 2, "result": "old"}],
+    )
+    predicate = col("x").cast("string") == "1"
+    assert string_cast_table.search().where(predicate).to_arrow().num_rows == 1
+    result = string_cast_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 1
+
+    quoted_identifier_schema = pa.schema(
+        [("payload", pa.binary()), ("odd'name", pa.int64()), ("result", pa.string())]
+    )
+    quoted_identifier_table = mem_db.create_table(
+        "update_expr_quoted_identifier",
+        pa.table(
+            {"payload": [b"\x01"], "odd'name": [1], "result": ["old"]},
+            schema=quoted_identifier_schema,
+        ),
+    )
+    predicate = (col("payload") == lit(b"\x01")) & (col("odd'name") == 1)
+    assert quoted_identifier_table.search().where(predicate).to_arrow().num_rows == 1
+    result = quoted_identifier_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 1
+
+    decimal256_schema = pa.schema(
+        [("val", pa.decimal256(40, 2)), ("result", pa.string())]
+    )
+    decimal256_table = mem_db.create_table(
+        "update_expr_decimal256",
+        pa.table(
+            {
+                "val": [Decimal("1.00"), Decimal("3.00")],
+                "result": ["old", "old"],
+            },
+            schema=decimal256_schema,
+        ),
+    )
+    predicate = col("val") < lit(Decimal("2.00")).cast(pa.decimal256(40, 2))
+    assert decimal256_table.search().where(predicate).to_arrow().num_rows == 1
+    result = decimal256_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 1
+
+    binary_empty_table = mem_db.create_table(
+        "update_expr_binary_empty",
+        pa.table(
+            {"payload": [b"\x01", b"\x02"], "result": ["old", "old"]},
+            schema=pa.schema([("payload", pa.binary()), ("result", pa.string())]),
+        ),
+    )
+    predicate = (col("payload") == lit(b"\x01")).isin([])
+    assert binary_empty_table.search().where(predicate).to_arrow().num_rows == 0
+    assert predicate.to_sql() == "false"
+    result = binary_empty_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 0
+
+
 def test_update_with_arrow_scalar(mem_db: DBConnection):
    schema = pa.schema({"id": pa.int64(), "vector": pa.list_(pa.float32(), 4)})
    table = mem_db.create_table("my_table", schema=schema)
@@ -2524,6 +2682,43 @@ def test_merge_insert(mem_db: DBConnection):
        )


+def test_merge_insert_composite_key(mem_db: DBConnection):
+    table = mem_db.create_table(
+        "my_table",
+        data=pa.table(
+            {
+                "shard": ["a", "a", "b"],
+                "id": [1, 2, 1],
+                "val": ["x", "y", "z"],
+            }
+        ),
+    )
+
+    # ("a", 1) matches an existing row and updates it. ("b", 2) agrees with an
+    # existing row on each key column separately but on neither pair, so it is
+    # an insert.
+    new_data = pa.table({"shard": ["a", "b"], "id": [1, 2], "val": ["X", "W"]})
+    res = (
+        table.merge_insert(["shard", "id"])
+        .when_matched_update_all()
+        .when_not_matched_insert_all()
+        .execute(new_data)
+    )
+    assert res.num_updated_rows == 1
+    assert res.num_inserted_rows == 1
+
+    expected = pa.table(
+        {
+            "shard": ["a", "a", "b", "b"],
+            "id": [1, 2, 1, 2],
+            "val": ["X", "y", "z", "W"],
+        }
+    )
+    assert table.to_arrow().sort_by([("shard", "ascending"), ("id", "ascending")]) == (
+        expected
+    )
+
+
 def test_merge_insert_nullable_pandas_into_pydantic_schema(mem_db: DBConnection):
    # Regression test for https://github.com/lancedb/lancedb/issues/2366
    pd = pytest.importorskip("pandas")
@@ -3929,6 +4124,29 @@ def test_computed_column_rejects_transforms_and_computed_together(tmp_path):
        table.add_columns({"a": "x + 1"}, computed={"b": "x * 2"})


+def test_computed_column_blob_projection_inherits_semantics(tmp_path):
+    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("computed_column_blob", schema=schema)
+    table.add(
+        [
+            {"id": 1, "image": b"hello"},
+            {"id": 2, "image": b""},
+            {"id": 3, "image": None},
+        ]
+    )
+
+    table.add_columns(computed={"image_copy": "image", "second_copy": "image_copy"})
+    assert table.refresh_column("image_copy").rows_filled == 2
+    assert table.refresh_column("second_copy").rows_filled == 2
+    assert table.blob_columns() == ["image", "image_copy", "second_copy"]
+
+    hits = table.search().with_row_id(True).limit(10).to_arrow()
+    rows = sorted(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
+    copied = table.fetch_blobs("second_copy", [row_id for _, row_id in rows])
+    assert copied.to_pylist() == [b"hello", b"", None]
+
+
@pytest.mark.asyncio
 async def test_computed_column_async(tmp_path):
    db = await lancedb.connect_async(tmp_path)
@@ -7,6 +7,7 @@ import pathlib
 from typing import Optional

 import lance
+from lance.blob import BlobType as LanceBlobType
 from lancedb.conftest import MockTextEmbeddingFunction
 from lancedb.embeddings.base import EmbeddingFunctionConfig
 from lancedb.embeddings.registry import EmbeddingFunctionRegistry
@@ -907,6 +908,165 @@ def test_cast_to_target_schema():
    assert output == expected


+def test_cast_to_target_schema_coerces_binary_to_blob_v2():
+    data = pa.table({"image": pa.array([b"hello", None], type=pa.binary())})
+    target = pa.schema([lancedb.blob("image")])
+
+    output = _cast_to_target_schema(data.to_reader(), target).read_all()
+
+    image = output["image"].chunk(0)
+    assert type(image.type) is lancedb.BlobType
+    assert image.storage.to_pylist() == [
+        {"data": b"hello", "uri": None, "position": None, "size": None},
+        None,
+    ]
+
+
+def test_cast_to_target_schema_coerces_binary_to_metadata_blob_struct():
+    storage = lancedb.blob("image").type.storage_type
+    target = pa.schema(
+        [
+            pa.field(
+                "image",
+                storage,
+                metadata={
+                    b"ARROW:extension:name": b"lance.blob.v2",
+                    b"ARROW:extension:metadata": b"",
+                },
+            )
+        ]
+    )
+    data = pa.table({"image": pa.array([b"hello", None], type=pa.binary())})
+
+    output = _cast_to_target_schema(data.to_reader(), target).read_all()
+
+    image = output["image"].chunk(0)
+    assert not isinstance(image.type, pa.ExtensionType)
+    assert image.to_pylist() == [
+        {"data": b"hello", "uri": None, "position": None, "size": None},
+        None,
+    ]
+
+
+def test_cast_to_target_schema_coerces_nested_binary_blob():
+    data = pa.table(
+        {
+            "info": pa.array(
+                [{"blob": b"hello"}, {"blob": None}],
+                type=pa.struct([pa.field("blob", pa.binary())]),
+            )
+        }
+    )
+    target = pa.schema([pa.field("info", pa.struct([lancedb.blob("blob")]))])
+
+    output = _cast_to_target_schema(data.to_reader(), target).read_all()
+
+    blob = output["info"].chunk(0).field("blob")
+    assert type(blob.type) is lancedb.BlobType
+    assert blob.storage.to_pylist() == [
+        {"data": b"hello", "uri": None, "position": None, "size": None},
+        None,
+    ]
+
+
+def test_cast_to_target_schema_coerces_list_binary_blob_with_inferred_child_name():
+    data = pa.table(
+        {"images": pa.array([[b"a", b"b"], None], type=pa.list_(pa.binary()))}
+    )
+    target = pa.schema([pa.field("images", pa.list_(lancedb.blob("image")))])
+
+    output = _cast_to_target_schema(data.to_reader(), target).read_all()
+
+    images = output["images"].chunk(0)
+    assert images.type.value_field.name == "image"
+    assert type(images.type.value_type) is lancedb.BlobType
+    assert images.to_pylist()[1] is None
+    assert images.values.storage.to_pylist() == [
+        {"data": b"a", "uri": None, "position": None, "size": None},
+        {"data": b"b", "uri": None, "position": None, "size": None},
+    ]
+
+
+def test_list_blob_coercion_preserves_null_slots_with_nonzero_extent():
+    child = pa.field("image", pa.binary())
+    source = pa.ListArray.from_arrays(
+        pa.array([0, 2, 4], type=pa.int32()),
+        pa.array([b"a", b"b", b"dead", b"beef"], type=pa.binary()),
+        mask=pa.array([False, True]),
+    ).cast(pa.list_(child))
+    target = pa.schema([pa.field("images", pa.list_(lancedb.blob("image")))])
+
+    output = _cast_to_target_schema(
+        pa.table({"images": source}).to_reader(), target
+    ).read_all()
+
+    images = output["images"].chunk(0)
+    assert images.to_pylist()[1] is None
+    assert [b["data"] for b in images.to_pylist()[0]] == [b"a", b"b"]
+
+
+def test_fixed_size_list_blob_coercion_keeps_null_rows():
+    child = pa.field("frame", pa.binary())
+    source = (
+        pa.FixedSizeListArray.from_arrays(
+            pa.array([b"a", b"b", b"c", b"d"], type=pa.binary()), 2
+        )
+        .take(pa.array([0, None], type=pa.int32()))
+        .cast(pa.list_(child, 2))
+    )
+    target = pa.schema([pa.field("frames", pa.list_(lancedb.blob("frame"), 2))])
+
+    output = _cast_to_target_schema(
+        pa.table({"frames": source}).to_reader(), target
+    ).read_all()
+
+    frames = output["frames"].chunk(0)
+    assert frames.to_pylist()[1] is None
+    assert [b["data"] for b in frames.to_pylist()[0]] == [b"a", b"b"]
+
+
+def test_cast_to_target_schema_accepts_pylance_blob_v2():
+    target_type = lancedb.BlobType()
+    source = lance.blob_array([b"hello", None])
+    assert type(source.type) is LanceBlobType
+    assert type(source.type) is type(target_type)
+    data = pa.table({"image": source})
+    target = pa.schema([pa.field("image", target_type)])
+
+    output = _cast_to_target_schema(data.to_reader(), target).read_all()
+
+    image = output["image"].chunk(0)
+    assert type(image.type) is LanceBlobType
+    assert image.type == target_type
+    assert image.storage.to_pylist() == [
+        {"data": b"hello", "uri": None, "position": None, "size": None},
+        None,
+    ]
+
+
+def test_cast_to_target_schema_rejects_different_blob_v2_class():
+    class OtherBlobType(pa.ExtensionType):
+        def __init__(self):
+            super().__init__(lancedb.BlobType().storage_type, "lance.blob.v2")
+
+        def __arrow_ext_serialize__(self) -> bytes:
+            return b""
+
+        @classmethod
+        def __arrow_ext_deserialize__(
+            cls, storage_type: pa.DataType, serialized: bytes
+        ) -> "OtherBlobType":
+            return cls()
+
+    storage = lance.blob_array([b"hello"]).storage
+    source = pa.ExtensionArray.from_storage(OtherBlobType(), storage)
+    data = pa.table({"image": source})
+    target = pa.schema([lancedb.blob("image")])
+
+    with pytest.raises(pa.ArrowTypeError, match="different extension type"):
+        _cast_to_target_schema(data.to_reader(), target).read_all()
+
+
 def test_sanitize_data_stream():
    # Make sure we don't collect the whole stream when running sanitize_data
    schema = pa.schema({"a": pa.int32()})
@@ -13,11 +13,7 @@ use crate::{
    runtime::future_into_py,
    table::Table,
 };
-use arrow::{
-    datatypes::Schema,
-    ffi_stream::ArrowArrayStreamReader,
-    pyarrow::{FromPyArrow, ToPyArrow},
-};
+use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::FromPyArrow};
 use lancedb::{
    connection::Connection as LanceConnection,
    connection::NamespaceClientPushdownOperation,
@@ -28,7 +24,7 @@ use pyo3::{
    Bound, FromPyObject, Py, PyAny, PyRef, PyResult, Python,
    exceptions::{PyRuntimeError, PyValueError},
    pyclass, pyfunction, pymethods,
-    types::{PyDict, PyDictMethods, PyList, PyListMethods},
+    types::{PyAnyMethods, PyDict, PyDictMethods, PyList},
 };

 #[pyclass]
@@ -86,6 +82,24 @@ impl Connection {
    }
 }

+fn parse_default_namespace_path(path: Option<Bound<'_, PyAny>>) -> PyResult<Vec<String>> {
+    match path {
+        Some(path) => {
+            if !path.is_instance_of::<PyList>() {
+                return Err(PyValueError::new_err(
+                    "Connection.execute_query_async default_namespace_path must be a list",
+                ));
+            }
+            path.extract::<Vec<String>>().map_err(|_| {
+                PyValueError::new_err(
+                    "Connection.execute_query_async default_namespace_path components must be strings",
+                )
+            })
+        }
+        None => Ok(vec!["public".to_string()]),
+    }
+}
+
 #[pymethods]
 impl Connection {
    fn __repr__(&self) -> String {
@@ -108,6 +122,40 @@ impl Connection {
        self.get_inner().map(|inner| inner.uri().to_string())
    }

+    #[pyo3(signature = (query, *, default_namespace_path=None))]
+    pub fn execute_query_async<'a>(
+        self_: PyRef<'a, Self>,
+        query: String,
+        default_namespace_path: Option<Bound<'_, PyAny>>,
+    ) -> PyResult<Bound<'a, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        let default_namespace_path = parse_default_namespace_path(default_namespace_path)?;
+        future_into_py(self_.py(), async move {
+            let operation = inner
+                .execute_query_async(query)
+                .default_namespace_path(default_namespace_path);
+            operation
+                .execute()
+                .await
+                .map(crate::sql::Query::new)
+                .infer_error()
+        })
+    }
+
+    pub fn describe_query<'a>(
+        self_: PyRef<'a, Self>,
+        query_id: uuid::Uuid,
+    ) -> PyResult<Bound<'a, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        future_into_py(self_.py(), async move {
+            inner
+                .describe_query(query_id)
+                .await
+                .map(crate::sql::QueryDescription::from)
+                .infer_error()
+        })
+    }
+
    #[pyo3(signature = ())]
    pub fn get_read_consistency_interval(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
        let inner = self_.get_inner()?.clone();
@@ -592,9 +640,12 @@ impl Connection {
        })
    }

-    pub fn job(&self, job_id: String) -> PyResult<crate::job::Job> {
-        let inner = self.get_inner()?.clone();
-        Ok(crate::job::Job::new(inner.job(job_id).infer_error()?))
+    pub fn open_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        future_into_py(self_.py(), async move {
+            let job = inner.open_job(&job_id).await.infer_error()?;
+            Ok(crate::job::Job::new(job))
+        })
    }

    pub fn create_function_async(
@@ -629,6 +680,30 @@ impl Connection {
        })
    }

+    pub fn list_functions(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        future_into_py(self_.py(), async move {
+            inner
+                .list_functions()
+                .await
+                .infer_error()?
+                .into_iter()
+                .map(|function| function.to_canonical_json().infer_error())
+                .collect::<PyResult<Vec<_>>>()
+        })
+    }
+
+    pub fn drop_function(
+        self_: PyRef<'_, Self>,
+        name: String,
+        version: String,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        future_into_py(self_.py(), async move {
+            inner.drop_function(name, version).await.infer_error()
+        })
+    }
+
    pub fn list_jobs(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
        let inner = self_.get_inner()?.clone();
        future_into_py(self_.py(), async move {
@@ -640,42 +715,16 @@ impl Connection {
        })
    }

-    pub fn get_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.get_inner()?.clone();
-        future_into_py(self_.py(), async move {
-            let description = inner.get_job(&job_id).await.infer_error()?;
-            Ok(description.map(crate::job::JobDescription::from))
-        })
-    }
-
    pub fn cancel_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
        let inner = self_.get_inner()?.clone();
        future_into_py(self_.py(), async move {
            inner.cancel_job(&job_id).await.infer_error()
        })
    }
-
-    #[pyo3(signature = (job_id=None))]
-    pub fn job_history(
-        self_: PyRef<'_, Self>,
-        job_id: Option<String>,
-    ) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.get_inner()?.clone();
-        future_into_py(self_.py(), async move {
-            let batches = inner.job_history(job_id.as_deref()).await.infer_error()?;
-            Python::attach(|py| {
-                let list = PyList::empty(py);
-                for batch in batches {
-                    list.append(batch.to_pyarrow(py)?)?;
-                }
-                Ok(list.unbind())
-            })
-        })
-    }
 }

 #[pyfunction]
-#[pyo3(signature = (uri, api_key=None, region=None, host_override=None, read_consistency_interval=None, client_config=None, storage_options=None, session=None, manifest_enabled=false, namespace_client_properties=None, oauth_config=None))]
+#[pyo3(signature = (uri, api_key=None, region=None, host_override=None, sql_host_override=None, read_consistency_interval=None, client_config=None, storage_options=None, session=None, manifest_enabled=false, namespace_client_properties=None, oauth_config=None))]
 #[allow(clippy::too_many_arguments)]
 pub fn connect(
    py: Python<'_>,
@@ -683,6 +732,7 @@ pub fn connect(
    api_key: Option<String>,
    region: Option<String>,
    host_override: Option<String>,
+    sql_host_override: Option<String>,
    read_consistency_interval: Option<f64>,
    client_config: Option<PyClientConfig>,
    storage_options: Option<HashMap<String, String>>,
@@ -702,6 +752,12 @@ pub fn connect(
        if let Some(host_override) = host_override {
            builder = builder.host_override(&host_override);
        }
+        #[cfg(feature = "remote")]
+        if let Some(sql_host_override) = sql_host_override {
+            builder = builder.sql_host_override(&sql_host_override);
+        }
+        #[cfg(not(feature = "remote"))]
+        let _ = sql_host_override;
        if let Some(read_consistency_interval) = read_consistency_interval {
            let read_consistency_interval = Duration::from_secs_f64(read_consistency_interval);
            builder = builder.read_consistency_interval(read_consistency_interval);
@@ -114,6 +114,12 @@ impl<T> PythonErrorExt<T> for std::result::Result<T, LanceError> {
                        .getattr(intern!(py, "JobCancelledError"))?;
                    Err(PyErr::from_value(cls.call1((err.to_string(),))?))
                }),
+                LanceError::JobNotFound { .. } => Python::attach(|py| {
+                    let cls = py
+                        .import(intern!(py, "lancedb.exceptions"))?
+                        .getattr(intern!(py, "JobNotFoundError"))?;
+                    Err(PyErr::from_value(cls.call1((err.to_string(),))?))
+                }),
                _ => self.runtime_error(),
            },
        }
@@ -130,6 +130,14 @@ impl PyExpr {

    // ── utilities ────────────────────────────────────────────────────────────

+    /// Return the referenced column name for a bare column expression.
+    fn column_name(&self) -> Option<String> {
+        match &self.0 {
+            DfExpr::Column(column) if column.relation.is_none() => Some(column.name.clone()),
+            _ => None,
+        }
+    }
+
    /// Render the expression as a SQL string (useful for debugging).
    fn to_sql(&self) -> PyResult<String> {
        lancedb::expr::expr_to_sql_string(&self.0).map_err(|e| PyValueError::new_err(e.to_string()))
@@ -4,11 +4,50 @@
 use std::sync::Arc;

 use crate::runtime::future_into_py;
-use pyo3::{Bound, PyAny, PyRef, PyResult, pyclass, pymethods};
+use arrow::{
+    datatypes::Schema,
+    pyarrow::{IntoPyArrow, Table as PyArrowTable},
+};
+use lancedb::job::JobEventsRequest;
+use pyo3::{
+    Bound, PyAny, PyRef, PyResult, Python,
+    exceptions::PyValueError,
+    pyclass, pymethods,
+    types::{PyAnyMethods, PyDict, PyDictMethods},
+};
 use serde::Serialize;

 use crate::error::PythonErrorExt;

+const REPR_INDENT: &str = "    ";
+
+/// Parse a stored JSON payload into Python data. The bindings carry these as
+/// strings because that is what crosses the boundary cheaply; the public
+/// Python surface is the parsed form.
+fn parse_json_payload<'py>(
+    py: Python<'py>,
+    raw: Option<&str>,
+) -> PyResult<Option<Bound<'py, PyAny>>> {
+    match raw {
+        None => Ok(None),
+        Some(raw) => Ok(Some(py.import("json")?.call_method1("loads", (raw,))?)),
+    }
+}
+
+/// A payload rendered as indented JSON, aligned under the field that holds it.
+fn pretty_json_payload(py: Python<'_>, raw: Option<&str>) -> PyResult<Option<String>> {
+    let Some(parsed) = parse_json_payload(py, raw)? else {
+        return Ok(None);
+    };
+    let kwargs = PyDict::new(py);
+    kwargs.set_item("indent", 4)?;
+    let rendered: String = py
+        .import("json")?
+        .call_method("dumps", (parsed,), Some(&kwargs))?
+        .extract()?;
+    Ok(Some(rendered.replace('\n', &format!("\n{REPR_INDENT}"))))
+}
+
 #[pyclass]
 pub struct Job {
    inner: Arc<lancedb::Job<std::result::Result<Option<String>, String>>>,
@@ -67,6 +106,48 @@ impl Job {
            Ok(())
        })
    }
+
+    pub fn refresh(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        future_into_py(self_.py(), async move {
+            inner.refresh().await.infer_error()?;
+            Ok(())
+        })
+    }
+
+    /// The last observed lifecycle state, without contacting the backend.
+    #[getter]
+    pub fn _state(&self) -> Option<String> {
+        self.inner.state()
+    }
+
+    /// The last observed server-side record. `None` for an in-process job.
+    #[getter]
+    pub fn _description(&self) -> Option<JobDescription> {
+        self.inner.description().map(JobDescription::from)
+    }
+
+    #[pyo3(signature = (*, limit=None, filter=None))]
+    pub fn events(
+        self_: PyRef<'_, Self>,
+        limit: Option<u32>,
+        filter: Option<String>,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        let request = JobEventsRequest { limit, filter };
+        future_into_py(self_.py(), async move {
+            let batches = inner.events(request).await.infer_error()?;
+            Python::attach(|py| {
+                let schema = batches
+                    .first()
+                    .map(|batch| batch.schema())
+                    .unwrap_or_else(|| Arc::new(Schema::empty()));
+                let table = PyArrowTable::try_new(batches, schema)
+                    .map_err(|err| PyValueError::new_err(err.to_string()))?;
+                table.into_pyarrow(py).map(|table| table.unbind())
+            })
+        })
+    }
 }

 /// A row from `Connection.list_jobs`: one server-side job.
@@ -121,7 +202,7 @@ impl JobFailureInfo {
    }
 }

-/// A described job from `Connection.get_job`.
+/// The server-side record behind a `Job` handle.
 #[pyclass(get_all, skip_from_py_object)]
 #[derive(Clone)]
 pub struct JobDescription {
@@ -129,17 +210,49 @@ pub struct JobDescription {
    job_type: String,
    state: String,
    creation_ms: i64,
-    spec_json: Option<String>,
+    /// Internal: the wire form behind the `spec` property.
+    _spec_json: Option<String>,
+    /// Internal: the wire form behind the `result` property.
+    _result_json: Option<String>,
    failure: Option<JobFailureInfo>,
 }

 #[pymethods]
 impl JobDescription {
-    fn __repr__(&self) -> String {
-        format!(
-            "JobDescription(job_id={:?}, job_type={:?}, state={:?}, creation_ms={})",
-            self.job_id, self.job_type, self.state, self.creation_ms
-        )
+    /// The job-type-specific specification it was submitted with.
+    #[getter]
+    fn spec<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyAny>>> {
+        parse_json_payload(py, self._spec_json.as_deref())
+    }
+
+    /// The job-type-specific terminal result. `None` until the job succeeds.
+    #[getter]
+    fn result<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyAny>>> {
+        parse_json_payload(py, self._result_json.as_deref())
+    }
+
+    fn __repr__(&self, py: Python<'_>) -> PyResult<String> {
+        let mut fields = vec![
+            format!("job_id={:?}", self.job_id),
+            format!("job_type={:?}", self.job_type),
+            format!("state={:?}", self.state),
+            format!("creation_ms={}", self.creation_ms),
+        ];
+        // Lay the payloads out as indented JSON, the same way the `Job` repr
+        // does, so the two agree on how the same data looks.
+        for (name, payload) in [("spec", &self._spec_json), ("result", &self._result_json)] {
+            if let Some(rendered) = pretty_json_payload(py, payload.as_deref())? {
+                fields.push(format!("{name}={rendered}"));
+            }
+        }
+        if let Some(failure) = &self.failure {
+            fields.push(format!("failure={}", failure.__repr__()));
+        }
+        let body = fields
+            .iter()
+            .map(|field| format!("\n{REPR_INDENT}{field},"))
+            .collect::<String>();
+        Ok(format!("JobDescription({body}\n)"))
    }
 }

@@ -150,7 +263,11 @@ impl From<lancedb::database::JobDescription> for JobDescription {
            job_type: description.job_type,
            state: description.state,
            creation_ms: description.creation_ms,
-            spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
+            _spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
+            _result_json: description
+                .result
+                .filter(|result| !result.is_null())
+                .map(|result| result.to_string()),
            failure: description.failure.map(|failure| JobFailureInfo {
                phase: failure.phase,
                message: failure.message,
@@ -34,6 +34,7 @@ pub mod permutation;
 pub mod query;
 pub mod runtime;
 pub mod session;
+pub mod sql;
 pub mod table;
 pub mod util;

@@ -50,6 +51,8 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
    m.add_class::<crate::job::JobInfo>()?;
    m.add_class::<crate::job::JobDescription>()?;
    m.add_class::<crate::job::JobFailureInfo>()?;
+    m.add_class::<crate::sql::Query>()?;
+    m.add_class::<crate::sql::QueryDescription>()?;
    m.add_class::<PyBlobFile>()?;
    m.add_class::<IndexConfig>()?;
    m.add_class::<Query>()?;
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors

+use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Duration;

@@ -322,9 +323,11 @@ impl<'py> IntoPyObject<'py> for PyQueryVectors {
 pub struct PyQueryRequest {
    pub limit: Option<usize>,
    pub offset: Option<usize>,
+    pub take_offsets: Option<Vec<u64>>,
    pub filter: Option<PyQueryFilter>,
    pub full_text_search: Option<PyLanceDB<FtsQuery>>,
    pub select: PySelect,
+    pub select_source_columns: Option<HashMap<String, String>>,
    pub fast_search: Option<bool>,
    pub with_row_id: Option<bool>,
    pub use_lsm: Option<bool>,
@@ -351,10 +354,12 @@ impl From<AnyQuery> for PyQueryRequest {
            AnyQuery::Query(query_request) => Self {
                limit: query_request.limit,
                offset: query_request.offset,
+                take_offsets: query_request.take_offsets,
                filter: query_request.filter.map(PyQueryFilter),
                full_text_search: query_request
                    .full_text_search
                    .map(|fts| PyLanceDB(fts.query)),
+                select_source_columns: PySelect::source_columns(&query_request.select),
                select: PySelect(query_request.select),
                fast_search: Some(query_request.fast_search),
                with_row_id: Some(query_request.with_row_id),
@@ -378,8 +383,10 @@ impl From<AnyQuery> for PyQueryRequest {
            AnyQuery::VectorQuery(vector_query) => Self {
                limit: vector_query.base.limit,
                offset: vector_query.base.offset,
+                take_offsets: vector_query.base.take_offsets,
                filter: vector_query.base.filter.map(PyQueryFilter),
                full_text_search: None,
+                select_source_columns: PySelect::source_columns(&vector_query.base.select),
                select: PySelect(vector_query.base.select),
                fast_search: Some(vector_query.base.fast_search),
                with_row_id: Some(vector_query.base.with_row_id),
@@ -412,6 +419,25 @@ impl From<AnyQuery> for PyQueryRequest {
 #[derive(Clone)]
 pub struct PySelect(Select);

+impl PySelect {
+    fn source_columns(select: &Select) -> Option<HashMap<String, String>> {
+        match select {
+            Select::Expr(pairs) => Some(
+                pairs
+                    .iter()
+                    .filter_map(|(output, expr)| match expr {
+                        lancedb::expr::DfExpr::Column(column) if column.relation.is_none() => {
+                            Some((output.clone(), column.name.clone()))
+                        }
+                        _ => None,
+                    })
+                    .collect(),
+            ),
+            _ => None,
+        }
+    }
+}
+
 impl<'py> IntoPyObject<'py> for PySelect {
    type Target = PyAny;
    type Output = Bound<'py, Self::Target>;
@@ -0,0 +1,90 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+use std::sync::Arc;
+
+use chrono::{DateTime, Utc};
+use pyo3::{Bound, PyAny, PyRef, PyResult, pyclass, pymethods};
+use uuid::Uuid;
+
+use crate::arrow::RecordBatchStream;
+use crate::error::PythonErrorExt;
+use crate::runtime::future_into_py;
+
+#[pyclass(name = "SqlQuery")]
+pub struct Query {
+    inner: Arc<lancedb::sql::Query>,
+}
+
+impl Query {
+    pub(crate) fn new(inner: lancedb::sql::Query) -> Self {
+        Self {
+            inner: Arc::new(inner),
+        }
+    }
+}
+
+#[pymethods]
+impl Query {
+    #[getter]
+    pub fn id(&self) -> Uuid {
+        self.inner.id()
+    }
+
+    pub fn describe(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        future_into_py(self_.py(), async move {
+            inner
+                .describe()
+                .await
+                .map(QueryDescription::from)
+                .infer_error()
+        })
+    }
+
+    pub fn reader(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        future_into_py(self_.py(), async move {
+            let stream = inner.reader().await.infer_error()?;
+            Ok(RecordBatchStream::new(stream))
+        })
+    }
+
+    pub fn cancel(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        future_into_py(self_.py(), async move {
+            inner.cancel().await.infer_error()?;
+            Ok(())
+        })
+    }
+}
+
+#[pyclass(get_all, skip_from_py_object)]
+#[derive(Clone)]
+pub struct QueryDescription {
+    id: Uuid,
+    status: String,
+    progress: Option<f64>,
+    expires_at: Option<DateTime<Utc>>,
+}
+
+#[pymethods]
+impl QueryDescription {
+    fn __repr__(&self) -> String {
+        format!(
+            "QueryDescription(id={:?}, status={:?}, progress={:?}, expires_at={:?})",
+            self.id, self.status, self.progress, self.expires_at
+        )
+    }
+}
+
+impl From<lancedb::sql::QueryDescription> for QueryDescription {
+    fn from(description: lancedb::sql::QueryDescription) -> Self {
+        Self {
+            id: description.id,
+            status: description.status.to_string(),
+            progress: description.progress,
+            expires_at: description.expires_at,
+        }
+    }
+}
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.38.0-beta.11"
+version = "0.39.0-beta.4"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
@@ -21,6 +21,8 @@ arrow-select = { workspace = true }
 arrow-ord = { workspace = true }
 arrow-cast = { workspace = true }
 arrow-ipc.workspace = true
+arrow-flight = { workspace = true, optional = true }
+prost = { version = "0.14", optional = true }
 chrono = { workspace = true }
 datafusion-catalog.workspace = true
 datafusion-common.workspace = true
@@ -77,6 +79,7 @@ reqwest = { version = "0.12.0", default-features = false, features = [
    "rustls-tls-native-roots",
    "stream",
 ], optional = true }
+tonic = { workspace = true, optional = true }
 http = { version = "1", optional = true } # Matching what is in reqwest
 urlencoding = { version = "2", optional = true }
 uuid = { workspace = true, features = ["v5"] }
@@ -145,8 +148,11 @@ huggingface = [
 ]
 dynamodb = ["lance/dynamodb", "aws"]
 remote = [
+    "dep:arrow-flight",
+    "dep:prost",
    "dep:reqwest",
    "dep:http",
+    "dep:tonic",
    "dep:urlencoding",
    "lance-namespace-impls/rest",
    "lance-namespace-impls/rest-adapter",
@@ -7,7 +7,9 @@
 //! raw `Binary` / `LargeBinary` into the blob struct layout. Queries return
 //! small descriptors, not bytes.
 //!
-//! Blob tables require Lance file format >= 2.2 and stable row ids at create.
+//! Blob tables require Lance file format >= 2.2. `_rowid` values stay valid
+//! after compaction when the table has stable row ids. Overwrite is a new
+//! create and does not keep the previous table's stable row id setting.

 use std::ops::Range;
 use std::sync::Arc;
@@ -324,6 +326,7 @@ pub(crate) fn blob_column_names(schema: &Schema) -> Vec<String> {
 }

 /// Bumps storage format to at least [`LanceFileVersion::V2_2`] for blob schemas.
+/// Leaves `enable_stable_row_ids` unchanged.
 pub(crate) fn ensure_blob_storage_version(schema: &Schema, params: &mut WriteParams) {
    if !has_blob_columns(schema) {
        return;
@@ -385,6 +388,30 @@ fn ensure_all_row_ids_resolved(column: &str, requested: usize, resolved: usize)
    }
 }

+/// Lance take reports a missing physical row address as NotSupported or InvalidInput.
+fn map_blob_take_error(column: &str, requested: usize, err: lance::Error) -> Error {
+    let missing_row_addr = match &err {
+        lance::Error::NotSupported { source, .. } => {
+            source.to_string().contains("must not target deleted rows")
+        }
+        lance::Error::InvalidInput { source, .. } => source
+            .to_string()
+            .contains("belongs to non-existent fragment"),
+        _ => false,
+    };
+
+    if missing_row_addr {
+        Error::InvalidInput {
+            message: format!(
+                "blob read for column '{column}' requested {requested} row ids but some \
+                 do not exist in the table; pass row ids collected from this table"
+            ),
+        }
+    } else {
+        err.into()
+    }
+}
+
 /// Materialize blob-local ranges (same length and order as `requests`, nulls preserved).
 pub(crate) async fn take_blob_ranges_aligned(
    dataset: &Arc<Dataset>,
@@ -405,7 +432,8 @@ pub(crate) async fn take_blob_ranges_aligned(
        .with_row_ids(lance_requests)
        .preserve_order(true)
        .execute()
-        .await?;
+        .await
+        .map_err(|err| map_blob_take_error(column, requests.len(), err))?;
    ensure_all_row_ids_resolved(column, requests.len(), payloads.len())?;

    let mut builder = LargeBinaryBuilder::new();
@@ -434,7 +462,8 @@ pub(crate) async fn take_blobs_aligned(
        .with_row_ids(row_ids.to_vec())
        .preserve_order(true)
        .execute()
-        .await?;
+        .await
+        .map_err(|err| map_blob_take_error(column, row_ids.len(), err))?;
    ensure_all_row_ids_resolved(column, row_ids.len(), payloads.len())?;

    let mut builder = LargeBinaryBuilder::new();
@@ -458,7 +487,10 @@ pub(crate) async fn take_blob_files_aligned(
        return Ok(Vec::new());
    }

-    let handles = dataset.take_blobs(row_ids, column).await?;
+    let handles = dataset
+        .take_blobs(row_ids, column)
+        .await
+        .map_err(|err| map_blob_take_error(column, row_ids.len(), err))?;
    ensure_all_row_ids_resolved(column, row_ids.len(), handles.len())?;
    Ok(handles
        .into_iter()
@@ -504,6 +536,21 @@ mod tests {
            params.data_storage_version.unwrap().resolve(),
            ConcreteFileVersion::V2_2
        );
+        assert!(!params.enable_stable_row_ids);
+    }
+
+    #[test]
+    fn storage_version_leaves_stable_row_ids_enabled() {
+        let mut params = WriteParams {
+            enable_stable_row_ids: true,
+            ..Default::default()
+        };
+        ensure_blob_storage_version(&blob_schema(), &mut params);
+        assert!(params.enable_stable_row_ids);
+        assert_eq!(
+            params.data_storage_version.unwrap().resolve(),
+            ConcreteFileVersion::V2_2
+        );
    }

    #[test]
@@ -576,5 +623,6 @@ mod tests {
        let mut params = WriteParams::default();
        ensure_blob_storage_version(&schema, &mut params);
        assert!(params.data_storage_version.is_none());
+        assert!(!params.enable_stable_row_ids);
    }
 }
--- a/Show More
+++ b/Show More