fix(secrets): admit periods in Secret names and namespace segments

LanceDB namespace and table names are `[A-Za-z0-9_.-]`, so excluding periods here put Secrets out of reach inside any namespace a user already has one in -- unreachable to create, alter, describe, drop or bind, with no way to recover but renaming the namespace. The rule is that set and nothing further. No positional constraint rides along: the established grammar says nothing about which character comes first, so a segment may begin with `_`, `-` or `.` today, and a Secret has to be nameable wherever a namespace already is. A narrower rule would reintroduce the same unaddressability it is here to remove, one character class over. Names and path segments follow one rule, and the client matches the service. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UfmeJ533rQDnPBkMtjerV6
fix(secrets): complete the binding and value contract at the Rust boundary
2026-09-11 15:52:17 +00:00 · 2026-09-11 15:32:05 +00:00 · 2026-09-09 17:33:59 +00:00 · 2026-09-09 16:59:40 +00:00 · 2026-09-09 16:59:40 +00:00 · 2026-09-09 00:12:57 +00:00
174 changed files with 21667 additions and 13925 deletions
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.38.0-beta.9"
+current_version = "0.39.0-beta.4"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
@@ -44,3 +44,27 @@ updates:
      python-deps:
        patterns:
          - "*"
  # The npm ecosystem covers pnpm lockfiles. There are two separate installs:
  # the bindings themselves and the examples, which have their own lockfile.
  # As with cargo and pip above, only bump the lockfile — the version ranges
  # in package.json are our consumers' constraints, not ours.
  - package-ecosystem: npm
    directory: /nodejs
    schedule:
      interval: weekly
    versioning-strategy: lockfile-only
    groups:
      nodejs-deps:
        patterns:
          - "*"
  - package-ecosystem: npm
    directory: /nodejs/examples
    schedule:
      interval: weekly
    versioning-strategy: lockfile-only
    groups:
      nodejs-examples-deps:
        patterns:
          - "*"
@@ -29,12 +29,14 @@ jobs:
    steps:
      - uses: actions/setup-node@v6
        with:
-          node-version: "18"
+          node-version: "24"
      - uses: pnpm/action-setup@v6
        with:
          version: 11.1.1
      # These rules are disabled because Github will always ensure there
      # is a blank line between the title and the body and Github will
      # word wrap the description field to ensure a reasonable max line
      # length.
      - run: npm install @commitlint/config-conventional
      - run: >
          echo 'module.exports = {
            "rules": {
@@ -43,7 +45,11 @@ jobs:
              "body-leading-blank": [0, "always"]
            }
          }' > .commitlintrc.js
-      - run: npx commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
+      - run: >
          pnpm dlx
          --package @commitlint/cli@21.2.2
          --package @commitlint/config-conventional@21.2.2
          commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
        env:
          COMMIT_MSG: >
            ${{ github.event.pull_request.title }}
@@ -54,7 +60,7 @@ jobs:
        with:
          script: |
            const message = `**ACTION NEEDED**
-              
+
              Lance follows the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) for release automation.
              The PR title and description are used as the merge commit message.\
@@ -56,7 +56,7 @@ jobs:
        uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0
        with:
          # Restricted to http(s) on purpose. Much of docs/src is generated
-          # API reference (the js/ tree comes from `npm run docs` in nodejs)
+          # API reference (the js/ tree comes from `pnpm run docs` in nodejs)
          # and the hand-written pages use mkdocstrings cross-references and
          # nav-relative paths that only resolve in the site mkdocs builds,
          # not in this checkout, so relative links would be reported as
@@ -55,9 +55,7 @@ jobs:
      - name: Set up node
        uses: actions/setup-node@v6
        with:
-          node-version: 20
+          node-version: 24
          cache: 'npm'
          cache-dependency-path: docs/package-lock.json
      - name: Install node dependencies
        working-directory: nodejs
        run: |
@@ -47,9 +47,8 @@ jobs:
        version: 11.1.1
    - uses: actions/setup-node@v6
      with:
-        # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+        # Build on a supported LTS; the matrix job below covers every
-        # in October. The library itself still supports Node >= 18
+        # Node version the library claims to support.
        # (see test matrix below).
        node-version: 24
        cache: 'pnpm'
        cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -84,7 +83,7 @@ jobs:
    timeout-minutes: 30
    strategy:
      matrix:
-        node-version: [ "18", "20" ]
+        node-version: [ "22", "24", "26" ]
    runs-on: "ubuntu-22.04"
    defaults:
      run:
@@ -101,9 +100,9 @@ jobs:
    - uses: actions/setup-node@v6
      name: Setup Node.js 24 for build
      with:
-        # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+        # Build and install once on a fixed version so the generated docs
-        # in October. Build/install runs on Node 24; tests run on the
+        # are identical across matrix legs; the tests below then run on each
-        # matrix version below using direct jest invocation.
+        # supported Node version.
        node-version: 24
        cache: 'pnpm'
        cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -152,9 +151,9 @@ jobs:
        S3_TEST: "1"
        # Newer @smithy/core uses dynamic ESM imports.
        NODE_OPTIONS: "--experimental-vm-modules"
-      # Invoke jest directly because pnpm 11 itself requires Node 22+
+      # Invoke the installed jest binary directly; the pnpm shim is set up
-      # while the matrix tests on older Node versions.
+      # against the build-phase Node, not the version selected above.
-      run: npx jest --verbose
+      run: node_modules/.bin/jest --verbose
    - name: Test examples
      working-directory: ./
      env:
@@ -164,7 +163,7 @@ jobs:
      run: |
        python ci/mock_openai.py &
        cd nodejs/examples
-        npx jest --testEnvironment jest-environment-node-single-context --verbose
+        node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose
  macos:
    timeout-minutes: 30
    # macos-15 ships a newer linker; the older macos-14 linker fails to insert
@@ -185,8 +184,7 @@ jobs:
        version: 11.1.1
    - uses: actions/setup-node@v6
      with:
-        # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+        # pnpm 11 requires Node >= 22.13.
        # in October.
        node-version: 24
        cache: 'pnpm'
        cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -40,40 +40,31 @@ jobs:
          - target: aarch64-apple-darwin
            host: macos-latest
            features: fp16kernels
            # Fat LTO was ~111 of this job's ~113 minutes.
            lto: thin
            codegen_units: 16
            pre_build: |-
              brew install protobuf
              # Fat LTO (the workspace default in .cargo/config.toml) is
              # single-threaded and is the peak-memory step of the build. On
              # this runner it accounted for ~111 of the job's ~113 minutes,
              # making it the critical path of the entire publish pipeline.
              # ThinLTO parallelizes it across the runner's cores, for a few
              # percent of runtime performance.
              export CARGO_PROFILE_RELEASE_LTO=thin
              export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
          - target: x86_64-pc-windows-msvc
            host: windows-2025
            features: ","
            # The lower peak also keeps this on the standard 4-core runner.
            lto: thin
            codegen_units: 16
            pre_build: |-
              choco install --no-progress protoc ninja nasm
              tail -n 1000 /c/ProgramData/chocolatey/logs/chocolatey.log
              # There is an issue where choco doesn't add nasm to the path
              export PATH="$PATH:/c/Program Files/NASM"
              nasm -v
              # See the ThinLTO note on aarch64-apple-darwin above. Keeping
              # peak memory down is also what lets this run on the standard
              # 4-core runner: the 8-core larger runner was only needed to
              # stop fat LTO from OOMing rustc-LLVM.
              export CARGO_PROFILE_RELEASE_LTO=thin
              export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
          - target: aarch64-pc-windows-msvc
            host: windows-2025
            features: ","
            lto: thin
            codegen_units: 16
            pre_build: |-
                choco install --no-progress protoc
                rustup target add aarch64-pc-windows-msvc
                # See the ThinLTO note on aarch64-apple-darwin above.
                export CARGO_PROFILE_RELEASE_LTO=thin
                export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
          - target: x86_64-unknown-linux-gnu
            host: ubuntu-latest
            features: fp16kernels
@@ -103,6 +94,14 @@ jobs:
            # https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
            features: "fp16kernels"
            # Fat LTO OOM-killed rustc every nightly; even with lld it peaked
            # at 31391 MiB of the runner's 32 GiB.
            lto: thin
            codegen_units: 16
            # arm64 Linux links through GNU `ld` where x86_64 defaults to
            # `rust-lld`, which is why only arm64 OOM'd. lld cut the largest
            # linker process 7.0 -> 4.0 GiB (lancedb/sophon#7313).
            linker: /tmp/aarch64-lld-clang
            pre_build: |-
              set -e &&
              apt-get update &&
@@ -112,9 +111,30 @@ jobs:
              # AT_HWCAP2 (added in Linux 3.17). Define it for aws-lc-sys.
              export CFLAGS="$CFLAGS -DAT_HWCAP2=26" &&
              rustup target add aarch64-unknown-linux-gnu
              # Not `&&`-chained: in dash, errexit does not fire for a
              # non-final command in an `&&` list, so failures were ignored.
              #
              # A wrapper rather than `-C link-arg` because the per-target
              # rustflags variable does not reach every unit that links, while
              # the linker variable does. `clang` because GCC silently ignores
              # `-fuse-ld=lld` unless built with lld support. Two echoes
              # because printf's newline escape gets rewritten to `;` between
              # here and the container.
              echo '#!/bin/sh' > /tmp/aarch64-lld-clang
              echo 'exec clang --target=aarch64-unknown-linux-gnu --sysroot=/usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot --gcc-toolchain=/usr/aarch64-unknown-linux-gnu -fuse-ld=lld "$@"' >> /tmp/aarch64-lld-clang
              chmod 0755 /tmp/aarch64-lld-clang
              # Fail now, not at the cdylib link ~30 minutes later. Linking at
              # all also proves lld resolved; clang errors out when it cannot.
              echo 'int main(void){return 0;}' > /tmp/probe.c
              /tmp/aarch64-lld-clang /tmp/probe.c -o /tmp/probe
              readelf -h /tmp/probe | grep AArch64
          - target: aarch64-unknown-linux-musl
            host: ubuntu-2404-8x-x64
            features: ","
            # Fat LTO took the whole runner down. lld cannot help: it died
            # inside rustc's LLVM, before any linker was spawned.
            lto: thin
            codegen_units: 16
            pre_build: |-
              set -e &&
              sudo apt-get update &&
@@ -123,6 +143,19 @@ jobs:
              export EXTRA_ARGS="-x"
    name: build - ${{ matrix.settings.target }}
    runs-on: ${{ matrix.settings.host }}
    # On the job, not exported from `pre_build`: `Swatinem/rust-cache` hashes
    # `CARGO_*` into its cache key before any step runs, so a step-local export
    # leaves the key unchanged while cargo still rebuilds cold. The ThinLTO
    # legs had been doing that every run.
    #
    # Not `RUSTFLAGS`: setting it, even to "", discards every config-file
    # rustflag, silently dropping .cargo/config.toml's `target-cpu` and
    # `target-feature` from the published binaries.
    env:
      CARGO_PROFILE_RELEASE_LTO: ${{ matrix.settings.lto || 'fat' }}
      CARGO_PROFILE_RELEASE_CODEGEN_UNITS: ${{ matrix.settings.codegen_units || '1' }}
      # Empty elsewhere: a per-target variable is only read for that triple.
      CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: ${{ matrix.settings.linker }}
    defaults:
      run:
        working-directory: nodejs
@@ -135,8 +168,7 @@ jobs:
      - name: Setup node
        uses: actions/setup-node@v6
        with:
-          # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+          # pnpm 11 requires Node >= 22.13.
          # in October.
          node-version: 24
          cache: pnpm
          cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -169,19 +201,15 @@ jobs:
          # creating ref). The nightly cadence also keeps entries inside
          # GitHub's 7-day eviction window, which a tag-only trigger would not.
          save-if: ${{ github.ref == 'refs/heads/main' }}
-      # Docker builds can use rust-cache too. `target/` already lives on the
+      # Docker builds can use rust-cache too: the workspace is bind-mounted, so
-      # host because the whole workspace is bind-mounted into the container, and
+      # `target/` lives on the host and rust-cache's prune keeps the entry
-      # rust-cache's prune and save run host-side, so they can manage it -- which
+      # small.
      # is what keeps the entry to dependency artifacts rather than a multi-GB
      # copy of everything.
      #
      # Two differences from the native builds. The container's CARGO_HOME is
-      # bind-mounted from `.cargo-cache` rather than the host's ~/.cargo, so that
+      # bind-mounted from `.cargo-cache` rather than ~/.cargo, so that is cached
-      # has to be cached explicitly. And the key is derived from the *host* rustc
+      # explicitly. And the key uses the *host* rustc version, not the compiler
-      # version, which is not the compiler that produced these artifacts; that is
+      # that built these artifacts -- safe, since cargo fingerprints the real
-      # safe because cargo fingerprints the real compiler and rebuilds on a
+      # one; a base-image bump just costs one cold build.
      # mismatch, it just means a base-image toolchain bump costs one cold build
      # instead of invalidating the key.
      - name: Cache cargo (docker builds)
        uses: Swatinem/rust-cache@v2
        if: ${{ matrix.settings.docker }}
@@ -210,14 +238,19 @@ jobs:
          # cache step above saves. Previously the registry mounts pointed at
          # `.cargo/...`, a path nothing cached, so the container re-downloaded
          # the whole crate registry on every run.
          #
          # `docker run` inherits nothing; `-e NAME` carries the job's `env:` in.
          options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
                    -v ${{ github.workspace }}/.cargo-cache/registry/cache:/usr/local/cargo/registry/cache \
                    -v ${{ github.workspace }}/.cargo-cache/registry/index:/usr/local/cargo/registry/index \
                    -e CARGO_PROFILE_RELEASE_LTO \
                    -e CARGO_PROFILE_RELEASE_CODEGEN_UNITS \
                    -e CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER \
                    -v ${{ github.workspace }}:/build -w /build/nodejs"
          run: |
            set -e
            ${{ matrix.settings.pre_build }}
-            npx napi build --platform --release \
+            node_modules/.bin/napi build --platform --release \
              --features ${{ matrix.settings.features }} \
              --target ${{ matrix.settings.target }} \
              --dts ../lancedb/native.d.ts \
@@ -237,7 +270,7 @@ jobs:
      - name: Build
        run: |
          ${{ matrix.settings.pre_build }}
-          npx napi build --platform --release \
+          node_modules/.bin/napi build --platform --release \
              --features ${{ matrix.settings.features }} \
              --target ${{ matrix.settings.target }} \
              --dts ../lancedb/native.d.ts \
@@ -256,6 +289,18 @@ jobs:
        if: always()
        run: df -h
        shell: bash
      - name: Report peak memory
        if: always() && runner.os == 'Linux'
        shell: bash
        run: |
          peak=$(find /sys/fs/cgroup -name memory.peak -readable \
                 -exec cat {} + 2>/dev/null | sort -n | tail -1)
          if [ -n "$peak" ]; then
            echo "peak memory: $((peak / 1024 / 1024)) MiB"
          else
            echo "peak memory: unavailable (no readable cgroup v2 memory.peak)"
          fi
          free -g || true
      - name: Upload artifact
        uses: actions/upload-artifact@v7
        with:
@@ -293,7 +338,7 @@ jobs:
          - target: aarch64-unknown-linux-gnu
            host: ubuntu-2404-8x-arm64
        node:
-          - '20'
+          - '22'
    runs-on: ${{ matrix.settings.host }}
    defaults:
      run:
@@ -339,9 +384,9 @@ jobs:
      - name: Move built files
        run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
      - name: Test bindings
-        # Invoke jest directly because pnpm 11 itself requires Node 22+
+        # Invoke the installed jest binary directly; the pnpm shim is set up
-        # while the matrix tests on older Node versions.
+        # against the install-phase Node, not the version selected above.
-        run: npx jest --verbose
+        run: node_modules/.bin/jest --verbose
  publish:
    name: Publish
    runs-on: ubuntu-latest
@@ -232,7 +232,10 @@ jobs:
          ALL_FEATURES=`cargo metadata --format-version=1 --no-deps \
            | jq -r '.packages[] | .features | keys | .[]' \
            | grep -v s3-test | sort | uniq | paste -s -d "," -`
-          cargo test --profile ci --features $ALL_FEATURES --locked
+          # Run doctests before test binaries fill the runner disk. Examples are
          # already built by the Linux job, so avoid retaining them here.
          cargo test --profile ci --features $ALL_FEATURES --locked --doc
          cargo test --profile ci --features $ALL_FEATURES --locked --lib --tests
  windows:
    strategy:
@@ -1,22 +0,0 @@
 name: Update package-lock.json
 on:
  workflow_dispatch:
 permissions:
  contents: read
 jobs:
  publish:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v6
        with:
          ref: main
          persist-credentials: false
          fetch-depth: 0
          lfs: true
      - uses: ./.github/workflows/update_package_lock
        with:
          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
@@ -1,22 +0,0 @@
 name: Update NodeJs package-lock.json
 on:
  workflow_dispatch:
 permissions:
  contents: read
 jobs:
  publish:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v6
        with:
          ref: main
          persist-credentials: false
          fetch-depth: 0
          lfs: true
      - uses: ./.github/workflows/update_package_lock_nodejs
        with:
          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
@@ -20,7 +20,10 @@ repos:
    hooks:
      - id: local-biome-check
        name: biome check
-        entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
+        # Use the biome from nodejs/package.json rather than a separately
        # pinned one: the two drifted apart and disagreed on formatting, so
        # this hook rejected code that `pnpm lint` accepted.
        entry: nodejs/node_modules/.bin/biome check --config-path nodejs/biome.json nodejs/
        language: system
        types: [text]
        files: "nodejs/.*"
@@ -38,7 +38,7 @@ Before committing changes, run formatting for every language you touched. At min
 * Rust changes: run `cargo fmt --all`.
 * Python changes: run `ruff format .` and `ruff check .` from the repository root,
  and run targeted tests through `cd python && uv run ...`.
-* TypeScript changes: run the relevant `npm`/`pnpm` lint, format, build, and docs commands in `nodejs`.
+* TypeScript changes: run the relevant `pnpm` lint, format, build, and docs commands in `nodejs`.
 Before creating a PR, the exact value passed to `gh pr create --title` must follow
 Conventional Commits, such as `fix: support nested field paths in native index creation`
@@ -101,12 +101,12 @@ Python bindings changes:
 TypeScript bindings changes:
 1. Add napi-rs method binding on `Table` in `nodejs/src/table.rs`.
-2. Run `npm run build` to generate TypeScript definitions.
+2. Run `pnpm build` to generate TypeScript definitions.
 3. Add typescript method on abstract class `Table` in `nodejs/src/table.ts`.
 4. Add concrete method on `LocalTable` class in `nodejs/src/native_table.ts`.
    * Note: despite the name, this class is also used for remote tables.
 5. Add test in `nodejs/__test__/table.test.ts`.
-6. Run `npm run docs` to generate TypeScript documentation.
+6. Run `pnpm run docs` to generate TypeScript documentation.
 ## Python API reference
@@ -332,6 +332,34 @@ dependencies = [
 "num-traits",
 ]
 [[package]]
 name = "arrow-flight"
 version = "58.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b2dbe34824c639e43136af8f106992792ab456540d54b880bc320a3192502d2e"
 dependencies = [
 "arrow-arith",
 "arrow-array",
 "arrow-buffer",
 "arrow-cast",
 "arrow-data",
 "arrow-ipc",
 "arrow-ord",
 "arrow-row",
 "arrow-schema",
 "arrow-select",
 "arrow-string",
 "base64 0.22.1",
 "bytes",
 "futures",
 "once_cell",
 "paste",
 "prost",
 "prost-types",
 "tonic",
 "tonic-prost",
 ]
 [[package]]
 name = "arrow-ipc"
 version = "58.4.0"
@@ -535,9 +563,9 @@ dependencies = [
 [[package]]
 name = "async-trait"
-version = "0.1.91"
+version = "0.1.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
+checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -1129,7 +1157,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
 dependencies = [
 "async-trait",
- "axum-core",
+ "axum-core 0.4.5",
 "bytes",
 "futures-util",
 "http 1.5.0",
@@ -1138,7 +1166,7 @@ dependencies = [
 "hyper 1.9.0",
 "hyper-util",
 "itoa",
- "matchit",
+ "matchit 0.7.3",
 "memchr",
 "mime",
 "percent-encoding",
@@ -1156,6 +1184,31 @@ dependencies = [
 "tracing",
 ]
 [[package]]
 name = "axum"
 version = "0.8.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"
 dependencies = [
 "axum-core 0.5.6",
 "bytes",
 "futures-util",
 "http 1.5.0",
 "http-body 1.1.0",
 "http-body-util",
 "itoa",
 "matchit 0.8.4",
 "memchr",
 "mime",
 "percent-encoding",
 "pin-project-lite",
 "serde_core",
 "sync_wrapper",
 "tower",
 "tower-layer",
 "tower-service",
 ]
 [[package]]
 name = "axum-core"
 version = "0.4.5"
@@ -1177,6 +1230,24 @@ dependencies = [
 "tracing",
 ]
 [[package]]
 name = "axum-core"
 version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1"
 dependencies = [
 "bytes",
 "futures-core",
 "http 1.5.0",
 "http-body 1.1.0",
 "http-body-util",
 "mime",
 "pin-project-lite",
 "sync_wrapper",
 "tower-layer",
 "tower-service",
 ]
 [[package]]
 name = "backoff"
 version = "0.4.0"
@@ -1443,9 +1514,9 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
 [[package]]
 name = "bytemuck"
-version = "1.25.0"
+version = "1.25.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
+checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
 dependencies = [
 "bytemuck_derive",
 ]
@@ -1597,9 +1668,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
 [[package]]
 name = "chacha20"
-version = "0.10.0"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601"
+checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
 dependencies = [
 "cfg-if 1.0.4",
 "cpufeatures 0.3.0",
@@ -3455,8 +3526,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 [[package]]
 name = "fsst"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-array",
 "rand 0.9.5",
@@ -4815,8 +4886,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 [[package]]
 name = "lance"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -4888,8 +4959,8 @@ dependencies = [
 [[package]]
 name = "lance-arrow"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4911,7 +4982,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4925,7 +4996,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -4934,8 +5005,8 @@ dependencies = [
 [[package]]
 name = "lance-bitpacking"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrayref",
 "crunchy",
@@ -4945,8 +5016,8 @@ dependencies = [
 [[package]]
 name = "lance-core"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4983,8 +5054,8 @@ dependencies = [
 [[package]]
 name = "lance-datafusion"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5000,6 +5071,7 @@ dependencies = [
 "datafusion-functions",
 "datafusion-physical-expr",
 "futures",
 "half",
 "jsonb",
 "lance-arrow",
 "lance-core",
@@ -5013,8 +5085,8 @@ dependencies = [
 [[package]]
 name = "lance-datagen"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5031,8 +5103,8 @@ dependencies = [
 [[package]]
 name = "lance-derive"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -5041,8 +5113,8 @@ dependencies = [
 [[package]]
 name = "lance-encoding"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5075,8 +5147,8 @@ dependencies = [
 [[package]]
 name = "lance-file"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5107,8 +5179,8 @@ dependencies = [
 [[package]]
 name = "lance-index"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -5172,8 +5244,8 @@ dependencies = [
 [[package]]
 name = "lance-index-core"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5195,8 +5267,8 @@ dependencies = [
 [[package]]
 name = "lance-io"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5236,8 +5308,8 @@ dependencies = [
 [[package]]
 name = "lance-linalg"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5251,27 +5323,29 @@ dependencies = [
 [[package]]
 name = "lance-namespace"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow",
 "async-trait",
 "bytes",
 "lance-core",
 "lance-namespace-reqwest-client",
 "serde",
 "serde_json",
 "snafu 0.9.0",
 ]
 [[package]]
 name = "lance-namespace-impls"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow",
 "arrow-ipc",
 "arrow-schema",
 "async-trait",
- "axum",
+ "axum 0.7.9",
 "base64 0.22.1",
 "bytes",
 "chrono",
@@ -5304,9 +5378,9 @@ dependencies = [
 [[package]]
 name = "lance-namespace-reqwest-client"
-version = "0.11.0"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a030196da1c994b63a96a4f0bf5b0cfa459fe6dadc9e962320246ca328da22a"
+checksum = "d8d23e54b1634d5bbb434f8dd33dc3c05f6e58d876a9a27b3b4aef58ddbe11af"
 dependencies = [
 "reqwest 0.12.28",
 "serde",
@@ -5318,8 +5392,8 @@ dependencies = [
 [[package]]
 name = "lance-select"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5333,8 +5407,8 @@ dependencies = [
 [[package]]
 name = "lance-table"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5374,8 +5448,8 @@ dependencies = [
 [[package]]
 name = "lance-testing"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5388,8 +5462,8 @@ dependencies = [
 [[package]]
 name = "lance-tokenizer"
-version = "11.0.0-beta.22"
+version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
 dependencies = [
 "frostem",
 "icu_segmenter",
@@ -5402,7 +5476,7 @@ dependencies = [
 [[package]]
 name = "lancedb"
-version = "0.38.0-beta.9"
+version = "0.39.0-beta.4"
 dependencies = [
 "ahash",
 "anyhow",
@@ -5411,6 +5485,7 @@ dependencies = [
 "arrow-buffer",
 "arrow-cast",
 "arrow-data",
 "arrow-flight",
 "arrow-ipc",
 "arrow-ord",
 "arrow-schema",
@@ -5466,6 +5541,7 @@ dependencies = [
 "polars",
 "polars-arrow",
 "pprof 0.14.1",
 "prost",
 "rand 0.9.5",
 "random_word",
 "regex",
@@ -5482,6 +5558,7 @@ dependencies = [
 "test-log",
 "tokenizers",
 "tokio",
 "tonic",
 "url",
 "urlencoding",
 "uuid",
@@ -5490,7 +5567,7 @@ dependencies = [
 [[package]]
 name = "lancedb-nodejs"
-version = "0.38.0-beta.9"
+version = "0.39.0-beta.4"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5515,7 +5592,7 @@ dependencies = [
 [[package]]
 name = "lancedb-python"
-version = "0.38.0-beta.9"
+version = "0.39.0-beta.4"
 dependencies = [
 "arrow",
 "async-trait",
@@ -5539,6 +5616,7 @@ dependencies = [
 "serde_json",
 "snafu 0.8.9",
 "tokio",
 "uuid",
 ]
 [[package]]
@@ -5748,9 +5826,9 @@ dependencies = [
 [[package]]
 name = "log"
-version = "0.4.33"
+version = "0.4.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
+checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
 [[package]]
 name = "loom"
@@ -5859,6 +5937,12 @@ version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
 [[package]]
 name = "matchit"
 version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
 [[package]]
 name = "matrixmultiply"
 version = "0.3.10"
@@ -6001,9 +6085,9 @@ dependencies = [
 [[package]]
 name = "moka"
-version = "0.12.15"
+version = "0.12.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046"
+checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9"
 dependencies = [
 "async-lock",
 "crossbeam-channel",
@@ -6097,14 +6181,15 @@ dependencies = [
 [[package]]
 name = "napi"
-version = "3.11.0"
+version = "3.12.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de33522036981030a75c231829566bc63414e08101a6f5ff4ac6cef19c8e0941"
+checksum = "58c5f4d5375213fdb7be2655e152386e82f026f9a5ba36a75556e11359aafe09"
 dependencies = [
 "bitflags 2.11.1",
 "chrono",
 "ctor 1.0.12",
 "futures",
 "libc",
 "napi-build",
 "napi-sys",
 "nohash-hasher",
@@ -6116,15 +6201,15 @@ dependencies = [
 [[package]]
 name = "napi-build"
-version = "2.4.0"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5282704fbe8d49b0cf8b08e3f33233416a528658f205c7e5ace63b582de0b11c"
+checksum = "60fdf9b392c50e7c4170fa633bd909490ed7835cea4c046776d1a4dd8d2ae0ab"
 [[package]]
 name = "napi-derive"
-version = "3.6.1"
+version = "3.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d5c9c02556ea6dc99dffd36c1ce60141411657438501a125b675776d011ce92"
+checksum = "0fa55ea69990c90b888e9e77044410e304ce7f35de599dc6d0b5c1923d2e59af"
 dependencies = [
 "convert_case",
 "ctor 1.0.12",
@@ -6136,9 +6221,9 @@ dependencies = [
 [[package]]
 name = "napi-derive-backend"
-version = "6.1.1"
+version = "6.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d60b5d773ad46c698c8cc2cd9fde0b283d39cbb7f71c04bee633c7bdba4423bd"
+checksum = "df4056ac7c18e4438ccf0edaed4340ca0d269278c8ec19284f7b23cb039fd0ae"
 dependencies = [
 "convert_case",
 "proc-macro2",
@@ -7733,6 +7818,7 @@ dependencies = [
 "pyo3-build-config",
 "pyo3-ffi",
 "pyo3-macros",
 "uuid",
 ]
 [[package]]
@@ -8601,9 +8687,9 @@ dependencies = [
 [[package]]
 name = "roaring"
-version = "0.11.4"
+version = "0.11.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9"
+checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1"
 dependencies = [
 "bytemuck",
 "byteorder",
@@ -9063,9 +9149,9 @@ dependencies = [
 [[package]]
 name = "serde_with"
-version = "3.21.0"
+version = "3.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c"
+checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a"
 dependencies = [
 "base64 0.22.1",
 "bs58",
@@ -9073,6 +9159,7 @@ dependencies = [
 "hex",
 "indexmap 1.9.3",
 "indexmap 2.14.0",
 "jiff",
 "schemars 0.9.0",
 "schemars 1.2.1",
 "serde_core",
@@ -9083,9 +9170,9 @@ dependencies = [
 [[package]]
 name = "serde_with_macros"
-version = "3.21.0"
+version = "3.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660"
+checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46"
 dependencies = [
 "darling 0.23.0",
 "proc-macro2",
@@ -10084,6 +10171,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef"
 dependencies = [
 "async-trait",
 "axum 0.8.9",
 "base64 0.22.1",
 "bytes",
 "h2 0.4.16",
@@ -10095,9 +10183,11 @@ dependencies = [
 "hyper-util",
 "percent-encoding",
 "pin-project",
 "rustls-native-certs",
 "socket2 0.6.3",
 "sync_wrapper",
 "tokio",
 "tokio-rustls 0.26.4",
 "tokio-stream",
 "tower",
 "tower-layer",
@@ -10452,9 +10542,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 [[package]]
 name = "uuid"
-version = "1.24.0"
+version = "1.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
+checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812"
 dependencies = [
 "getrandom 0.4.2",
 "js-sys",
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"
 [workspace.dependencies]
-lance = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=12.0.0-beta.14", default-features = false, "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=12.0.0-beta.14", default-features = false, "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=12.0.0-beta.14", default-features = false, "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
 lancedb = { path = "rust/lancedb", default-features = false }
 ahash = "0.8"
 # Note that this one does not include pyarrow
@@ -39,6 +39,7 @@ arrow-ord = "58.0.0"
 arrow-schema = "58.0.0"
 arrow-select = "58.0.0"
 arrow-cast = "58.0.0"
 arrow-flight = { version = "58.0.0", features = ["flight-sql-experimental"] }
 async-trait = "0"
 bytes = "1"
 datafusion = { version = "54.0.0", default-features = false }
@@ -71,7 +72,8 @@ serde = "1"
 serde_json = "1"
 tempfile = "3.5.0"
 tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
-uuid = { version = "1.7.0", features = ["v4"] }
+tonic = { version = "0.14", features = ["tls-native-roots", "tls-ring"] }
 uuid = { version = "1.7.0", features = ["v4", "v7"] }
 chrono = { version = "0.4", default-features = false, features = ["clock"] }
 [profile.ci]
@@ -5,5 +5,5 @@ licenses:
 	cd python && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
 	cd python && uv sync --all-extras && uv tool run pip-licenses --python .venv/bin/python --format=markdown --with-urls --output-file=PYTHON_THIRD_PARTY_LICENSES.md
 	cd nodejs && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
-	cd nodejs && npx license-checker --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
+	cd nodejs && pnpm dlx license-checker@25 --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
 	cd java && ./mvnw license:aggregate-add-third-party -q
@@ -12,16 +12,12 @@ done
 # This updates the lockfile without building
 cargo metadata --quiet > /dev/null
 pushd nodejs || exit 1
 npm install --package-lock-only --silent
 popd
 if git diff --quiet --exit-code; then
  echo "No lockfile changes to commit; skipping amend."
 elif $AMEND; then
-  git add Cargo.lock nodejs/package-lock.json
+  git add Cargo.lock
  git commit --amend --no-edit
 else
-  git add Cargo.lock nodejs/package-lock.json
+  git add Cargo.lock
  git commit -m "Update lockfiles"
 fi
@@ -131,18 +131,13 @@ allow = [
    "BSD-3-Clause",
    "ISC",
    "Unicode-3.0",
    "Unicode-DFS-2016",
    "Zlib",
    "CC0-1.0",
    "MPL-2.0",
    "BSL-1.0",
    "OpenSSL",
    # 0BSD ("BSD Zero Clause") is effectively public domain — no attribution
    # required. Pulled in by `mock_instant`.
    "0BSD",
    # bzip2-1.0.6 is the permissive upstream bzip2 license (BSD-like). Pulled
    # in by `libbz2-rs-sys`, the pure-Rust bzip2 implementation.
    "bzip2-1.0.6",
    # CDLA-Permissive-2.0 is a permissive data license used by `webpki-roots`
    # for the Mozilla CA root bundle. Data-only, distribution-compatible.
    "CDLA-Permissive-2.0",
@@ -150,12 +145,7 @@ allow = [
 confidence-threshold = 0.8
 # Per-crate license exceptions: allow a license for a specific crate only,
 # rather than globally via the `allow` list above.
-exceptions = [
+exceptions = []
    # CDDL-1.0 (copyleft) is pulled in only as a dev/profiling dependency via
    # `inferno` -> `pprof` -> `lance-testing`; it is a test dependency that we
    # do not distribute, so scope the allowance to `inferno` alone.
    { allow = ["CDDL-1.0"], crate = "inferno" },
 ]
 # Crates whose license cannot be determined from Cargo metadata but whose
 # license we've manually confirmed from upstream. Keep this list minimal.
 [[licenses.clarify]]
@@ -47,22 +47,24 @@ pytest -vv python/tests/docs
 ### Checking typescript examples
-The `@lancedb/lancedb` package must be built before running the tests:
+The examples depend on `@lancedb/lancedb` at `file:../dist`, so the package must be
 built before running the tests. This uses pnpm; see the
 [Typescript contributing guide](../nodejs/CONTRIBUTING.md) for the toolchain setup.
 ```shell
 pushd nodejs
-npm ci
+pnpm install
-npm run build
+pnpm build
 popd
 ```
-Then you can run the examples by going to the `nodejs/examples` directory and
+Then you can run the examples by going to the `nodejs/examples` directory, which is a
-running the tests like a normal npm package:
+separate pnpm package with its own lockfile:
 ```shell
 pushd nodejs/examples
-npm ci
+pnpm install
-npm test
+pnpm test
 popd
 ```
@@ -84,6 +86,7 @@ The new files should be checked into the repository.
 ```shell
 pushd nodejs
-npm run docs
+# `pnpm docs` would invoke pnpm's built-in `docs` command, not the script.
 pnpm run docs
 popd
 ```
@@ -446,6 +446,15 @@ paths:
              properties:
                column:
                  type: string
                name:
                  type: string
                  description: Optional name for the created index.
                replace:
                  type: boolean
                  default: true
                  description: |
                    Whether to replace an existing index with the same resolved
                    name. Defaults to true.
                metric_type:
                  type: string
                  nullable: false
@@ -1,135 +0,0 @@
 {
  "name": "lancedb-docs-test",
  "version": "1.0.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "lancedb-docs-test",
      "version": "1.0.0",
      "license": "Apache 2",
      "dependencies": {
        "apache-arrow": "file:../node/node_modules/apache-arrow",
        "vectordb": "file:../node"
      },
      "devDependencies": {
        "@types/node": "^20.11.8",
        "typescript": "^5.3.3"
      }
    },
    "../node": {
      "name": "vectordb",
      "version": "0.21.2-beta.0",
      "cpu": [
        "x64",
        "arm64"
      ],
      "license": "Apache-2.0",
      "os": [
        "darwin",
        "linux",
        "win32"
      ],
      "dependencies": {
        "@neon-rs/load": "^0.0.74",
        "axios": "^1.4.0"
      },
      "devDependencies": {
        "@neon-rs/cli": "^0.0.160",
        "@types/chai": "^4.3.4",
        "@types/chai-as-promised": "^7.1.5",
        "@types/mocha": "^10.0.1",
        "@types/node": "^18.16.2",
        "@types/sinon": "^10.0.15",
        "@types/temp": "^0.9.1",
        "@types/uuid": "^9.0.3",
        "@typescript-eslint/eslint-plugin": "^5.59.1",
        "apache-arrow-old": "npm:apache-arrow@13.0.0",
        "cargo-cp-artifact": "^0.1",
        "chai": "^4.3.7",
        "chai-as-promised": "^7.1.1",
        "eslint": "^8.39.0",
        "eslint-config-standard-with-typescript": "^34.0.1",
        "eslint-plugin-import": "^2.26.0",
        "eslint-plugin-n": "^15.7.0",
        "eslint-plugin-promise": "^6.1.1",
        "mocha": "^10.2.0",
        "openai": "^4.24.1",
        "sinon": "^15.1.0",
        "temp": "^0.9.4",
        "ts-node": "^10.9.1",
        "ts-node-dev": "^2.0.0",
        "typedoc": "^0.24.7",
        "typedoc-plugin-markdown": "^3.15.3",
        "typescript": "^5.1.0",
        "uuid": "^9.0.0"
      },
      "optionalDependencies": {
        "@lancedb/vectordb-darwin-arm64": "0.21.2-beta.0",
        "@lancedb/vectordb-darwin-x64": "0.21.2-beta.0",
        "@lancedb/vectordb-linux-arm64-gnu": "0.21.2-beta.0",
        "@lancedb/vectordb-linux-x64-gnu": "0.21.2-beta.0",
        "@lancedb/vectordb-win32-x64-msvc": "0.21.2-beta.0"
      },
      "peerDependencies": {
        "@apache-arrow/ts": "^14.0.2",
        "apache-arrow": "^14.0.2"
      }
    },
    "../node/node_modules/apache-arrow": {
      "version": "14.0.2",
      "license": "Apache-2.0",
      "dependencies": {
        "@types/command-line-args": "5.2.0",
        "@types/command-line-usage": "5.0.2",
        "@types/node": "20.3.0",
        "@types/pad-left": "2.1.1",
        "command-line-args": "5.2.1",
        "command-line-usage": "7.0.1",
        "flatbuffers": "23.5.26",
        "json-bignum": "^0.0.3",
        "pad-left": "^2.1.0",
        "tslib": "^2.5.3"
      },
      "bin": {
        "arrow2csv": "bin/arrow2csv.js"
      }
    },
    "node_modules/@types/node": {
      "version": "20.11.8",
      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.8.tgz",
      "integrity": "sha512-i7omyekpPTNdv4Jb/Rgqg0RU8YqLcNsI12quKSDkRXNfx7Wxdm6HhK1awT3xTgEkgxPn3bvnSpiEAc7a7Lpyow==",
      "dev": true,
      "dependencies": {
        "undici-types": "~5.26.4"
      }
    },
    "node_modules/apache-arrow": {
      "resolved": "../node/node_modules/apache-arrow",
      "link": true
    },
    "node_modules/typescript": {
      "version": "5.3.3",
      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz",
      "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==",
      "dev": true,
      "bin": {
        "tsc": "bin/tsc",
        "tsserver": "bin/tsserver"
      },
      "engines": {
        "node": ">=14.17"
      }
    },
    "node_modules/undici-types": {
      "version": "5.26.5",
      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
      "dev": true
    },
    "node_modules/vectordb": {
      "resolved": "../node",
      "link": true
    }
  }
 }
@@ -1,20 +0,0 @@
 {
  "name": "lancedb-docs-test",
  "version": "1.0.0",
  "description": "auto-generated tests from doc",
  "author": "dev@lancedb.com",
  "license": "Apache 2",
  "dependencies": {
    "apache-arrow": "file:../node/node_modules/apache-arrow",
    "vectordb": "file:../node"
  },
  "scripts": {
    "build": "tsc -b && cd ../node && npm run build-release",
    "example": "npm run build && node",
    "test": "npm run build && ls dist/*.js | xargs -n 1 node"
  },
  "devDependencies": {
    "@types/node": "^20.11.8",
    "typescript": "^5.3.3"
  }
 }
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-core</artifactId>
-    <version>0.38.0-beta.9</version>
+    <version>0.39.0-beta.4</version>
 </dependency>
 ```
@@ -0,0 +1,518 @@
 [**@lancedb/lancedb**](../README.md) • **Docs**
 ***
 [@lancedb/lancedb](../globals.md) / AutoQuery
 # Class: AutoQuery
 A builder for automatic string searches.
 Automatic search determines whether to use full-text or vector search from
 the table revision selected for each execution. This builder exposes the
 common operations supported by both query families.
 ## Extends
 - `StandardQueryBase`&lt;`NativeQuery` \| `NativeVectorQuery`&gt;
 ## Properties
 ### inner
 ```ts
 protected inner: Query | VectorQuery | Promise<Query | VectorQuery>;
 ```
 #### Inherited from
 `StandardQueryBase.inner`
 ## Methods
 ### analyzePlan()
 ```ts
 analyzePlan(distributedMetrics?): Promise<string>
 ```
 Executes the query and returns the physical query plan annotated with runtime metrics.
 This is useful for debugging and performance analysis, as it shows how the query was executed
 and includes metrics such as elapsed time, rows processed, and I/O statistics.
 #### Parameters
 * **distributedMetrics?**: [`AnalyzePlanDistributedMetrics`](../type-aliases/AnalyzePlanDistributedMetrics.md)
    How distributed worker metrics are displayed for remote query plans.
    Defaults to `"aggregate"`.
 #### Returns
 `Promise`&lt;`string`&gt;
 A query execution plan with runtime metrics for each step.
 #### Example
 ```ts
 import * as lancedb from "@lancedb/lancedb"
 const db = await lancedb.connect("./.lancedb");
 const table = await db.createTable("my_table", [
  { vector: [1.1, 0.9], id: "1" },
 ]);
 const plan = await table.query().nearestTo([0.5, 0.2]).analyzePlan();
 Example output (with runtime metrics inlined):
 AnalyzeExec verbose=true, metrics=[]
 ProjectionExec: expr=[id@3 as id, vector@0 as vector, _distance@2 as _distance], metrics=[output_rows=1, elapsed_compute=3.292µs]
  Take: columns="vector, _rowid, _distance, (id)", metrics=[output_rows=1, elapsed_compute=66.001µs, batches_processed=1, bytes_read=8, iops=1, requests=1]
   CoalesceBatchesExec: target_batch_size=1024, metrics=[output_rows=1, elapsed_compute=3.333µs]
    GlobalLimitExec: skip=0, fetch=10, metrics=[output_rows=1, elapsed_compute=167ns]
     FilterExec: _distance@2 IS NOT NULL, metrics=[output_rows=1, elapsed_compute=8.542µs]
      SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=63.25µs, row_replacements=1]
       KNNVectorDistance: metric=l2, metrics=[output_rows=1, elapsed_compute=114.333µs, output_batches=1]
        LanceScan: uri=/path/to/data, projection=[vector], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1, elapsed_compute=103.626µs, bytes_read=549, iops=2, requests=2]
 ```
 #### Inherited from
 `StandardQueryBase.analyzePlan`
 ***
 ### execute()
 ```ts
 protected execute(options?): AsyncGenerator<RecordBatch<any>, void, unknown>
 ```
 Execute the query and return the results as an
 #### Parameters
 * **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
 #### Returns
 `AsyncGenerator`&lt;`RecordBatch`&lt;`any`&gt;, `void`, `unknown`&gt;
 #### See
 - AsyncIterator
 of
 - RecordBatch.
 By default, LanceDb will use many threads to calculate results and, when
 the result set is large, multiple batches will be processed at one time.
 This readahead is limited however and backpressure will be applied if this
 stream is consumed slowly (this constrains the maximum memory used by a
 single query)
 #### Inherited from
 `StandardQueryBase.execute`
 ***
 ### explainPlan()
 ```ts
 explainPlan(verbose): Promise<string>
 ```
 Generates an explanation of the query execution plan.
 #### Parameters
 * **verbose**: `boolean` = `false`
    If true, provides a more detailed explanation. Defaults to false.
 #### Returns
 `Promise`&lt;`string`&gt;
 A Promise that resolves to a string containing the query execution plan explanation.
 #### Example
 ```ts
 import * as lancedb from "@lancedb/lancedb"
 const db = await lancedb.connect("./.lancedb");
 const table = await db.createTable("my_table", [
  { vector: [1.1, 0.9], id: "1" },
 ]);
 const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
 ```
 #### Inherited from
 `StandardQueryBase.explainPlan`
 ***
 ### fastSearch()
 ```ts
 fastSearch(): this
 ```
 Skip searching un-indexed data. This can make search faster, but will miss
 any data that is not yet indexed.
 Use [Table#optimize](Table.md#optimize) to index all un-indexed data.
 #### Returns
 `this`
 #### Inherited from
 `StandardQueryBase.fastSearch`
 ***
 ### ~~filter()~~
 ```ts
 filter(predicate): this
 ```
 A filter statement to be applied to this query.
 #### Parameters
 * **predicate**: `string`
 #### Returns
 `this`
 #### See
 where
 #### Deprecated
 Use `where` instead
 #### Inherited from
 `StandardQueryBase.filter`
 ***
 ### fullTextSearch()
 ```ts
 fullTextSearch(query, options?): this
 ```
 #### Parameters
 * **query**: `string` \| [`FullTextQuery`](../interfaces/FullTextQuery.md)
 * **options?**: `Partial`&lt;[`FullTextSearchOptions`](../interfaces/FullTextSearchOptions.md)&gt;
 #### Returns
 `this`
 #### Inherited from
 `StandardQueryBase.fullTextSearch`
 ***
 ### limit()
 ```ts
 limit(limit): this
 ```
 Set the maximum number of results to return.
 By default, a plain search has no limit.  If this method is not
 called then every valid row from the table will be returned.
 #### Parameters
 * **limit**: `number`
 #### Returns
 `this`
 #### Inherited from
 `StandardQueryBase.limit`
 ***
 ### offset()
 ```ts
 offset(offset): this
 ```
 Set the number of rows to skip before returning results.
 This is useful for pagination.
 #### Parameters
 * **offset**: `number`
 #### Returns
 `this`
 #### Inherited from
 `StandardQueryBase.offset`
 ***
 ### orderBy()
 ```ts
 orderBy(ordering): this
 ```
 Sort the results by the specified column(s).
 #### Parameters
 * **ordering**: [`ColumnOrdering`](../interfaces/ColumnOrdering.md) \| [`ColumnOrdering`](../interfaces/ColumnOrdering.md)[]
 #### Returns
 `this`
 This query builder.
 #### Inherited from
 `StandardQueryBase.orderBy`
 ***
 ### outputSchema()
 ```ts
 outputSchema(): Promise<Schema<any>>
 ```
 Returns the schema of the output that will be returned by this query.
 This can be used to inspect the types and names of the columns that will be
 returned by the query before executing it.
 #### Returns
 `Promise`&lt;`Schema`&lt;`any`&gt;&gt;
 An Arrow Schema describing the output columns.
 #### Inherited from
 `StandardQueryBase.outputSchema`
 ***
 ### select()
 ```ts
 select(columns): this
 ```
 Return only the specified columns.
 By default a query will return all columns from the table.  However, this can have
 a very significant impact on latency.  LanceDb stores data in a columnar fashion.  This
 means we can finely tune our I/O to select exactly the columns we need.
 As a best practice you should always limit queries to the columns that you need.  If you
 pass in an array of column names then only those columns will be returned.
 You can also use this method to create new "dynamic" columns based on your existing columns.
 For example, you may not care about "a" or "b" but instead simply want "a + b".  This is often
 seen in the SELECT clause of an SQL query (e.g. `SELECT a+b FROM my_table`).
 To create dynamic columns you can pass in a Map<string, string>.  A column will be returned
 for each entry in the map.  The key provides the name of the column.  The value is
 an SQL string used to specify how the column is calculated.
 For example, an SQL query might state `SELECT a + b AS combined, c`.  The equivalent
 input to this method would be:
 #### Parameters
 * **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
 #### Returns
 `this`
 #### Example
 ```ts
 new Map([["combined", "a + b"], ["c", "c"]])
 Columns will always be returned in the order given, even if that order is different than
 the order used when adding the data.
 Note that you can pass in a `Record<string, string>` (e.g. an object literal). This method
 uses `Object.entries` which should preserve the insertion order of the object.  However,
 object insertion order is easy to get wrong and `Map` is more foolproof.
 ```
 #### Inherited from
 `StandardQueryBase.select`
 ***
 ### toArray()
 ```ts
 toArray(options?): Promise<any[]>
 ```
 Collect the results as an array of objects.
 #### Parameters
 * **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
 #### Returns
 `Promise`&lt;`any`[]&gt;
 #### Inherited from
 `StandardQueryBase.toArray`
 ***
 ### toArrow()
 ```ts
 toArrow(options?): Promise<Table<any>>
 ```
 Collect the results as an Arrow
 #### Parameters
 * **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
 #### Returns
 `Promise`&lt;`Table`&lt;`any`&gt;&gt;
 #### See
 ArrowTable.
 #### Inherited from
 `StandardQueryBase.toArrow`
 ***
 ### useLsm()
 ```ts
 useLsm(enable): this
 ```
 Control MemWAL read routing for this query.
 By default (unset), when the table carries a MemWAL write spec (see
 [Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so
 they also return data written via the `mergeInsert` LSM path that has not yet
 been compacted into the base table (the active/frozen in-memory memtables and
 the flushed generations), deduplicated by primary key; a table without a spec
 reads the base table.
 #### Parameters
 * **enable**: `boolean`
    `true` forces the LSM scanner and errors if the table has no
    MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
    even when a spec is present.
    Note: the LSM scanner does not support every query shape (e.g. reranking,
    hybrid search, `orderBy`). On a MemWAL table those shapes error unless
    `useLsm(false)` is set, because a base-only read would silently exclude
    un-compacted MemWAL data.
 #### Returns
 `this`
 #### Inherited from
 `StandardQueryBase.useLsm`
 ***
 ### where()
 ```ts
 where(predicate): this
 ```
 A filter statement to be applied to this query.
 The filter should be supplied as an SQL query string.  For example:
 #### Parameters
 * **predicate**: `string`
 #### Returns
 `this`
 #### Example
 ```ts
 x > 10
 y > 0 AND y < 100
 x > 5 OR y = 'test'
 Filtering performance can often be improved by creating a scalar index
 on the filter column(s).
 Calling this multiple times combines the filters with a logical AND rather
 than replacing the previous filter.
 ```
 #### Inherited from
 `StandardQueryBase.where`
 ***
 ### withRowId()
 ```ts
 withRowId(): this
 ```
 Whether to return the row id in the results.
 This column can be used to match results between different queries. For
 example, to match results from a full text search and a vector search in
 order to perform hybrid search.
 #### Returns
 `this`
 #### Inherited from
 `StandardQueryBase.withRowId`
@@ -448,26 +448,6 @@ on the returned job to know when cleanup has finished.
 ***
 ### getJob()
 ```ts
 abstract getJob(jobId): Promise<null | JobDescription>
 ```
 Describe a single server-side job by id.
 Resolves to `null` when the server has no such job.
 #### Parameters
 * **jobId**: `string`
 #### Returns
 `Promise`&lt;`null` \| [`JobDescription`](../interfaces/JobDescription.md)&gt;
 ***
 ### isOpen()
 ```ts
@@ -482,48 +462,6 @@ Return true if the connection has not been closed
 ***
 ### job()
 ```ts
 abstract job(jobId): Job
 ```
 A [Job](Job.md) handle for a server-side job by id.
 The handle is constructed without a server round trip; an unknown id
 surfaces when the handle is used. Dropping the handle has no effect on
 the job itself.
 #### Parameters
 * **jobId**: `string`
 #### Returns
 [`Job`](Job.md)
 ***
 ### jobHistory()
 ```ts
 abstract jobHistory(jobId?): Promise<Table<any>>
 ```
 The lifecycle event history of a server-side job, as an Arrow table.
 Lists history across all jobs when `jobId` is omitted.
 #### Parameters
 * **jobId?**: `string`
 #### Returns
 `Promise`&lt;`Table`&lt;`any`&gt;&gt;
 ***
 ### listJobs()
 ```ts
@@ -648,6 +586,30 @@ A page of table names and an
 ***
 ### openJob()
 ```ts
 abstract openJob(jobId): Promise<Job>
 ```
 Open a server-side job by id, returning a handle with its record already
 populated. Rejects when the server has no such job, the way
 [Connection.openTable](Connection.md#opentable) does for a missing table.
 The returned [Job](Job.md) answers for its own state, specification,
 result, failure and event history, so there is no separate
 connection-level call for any of them.
 #### Parameters
 * **jobId**: `string`
 #### Returns
 `Promise`&lt;[`Job`](Job.md)&gt;
 ***
 ### openMaterializedView()
 ```ts
@@ -8,19 +8,46 @@
 A handle to an operation that may still be running.
-## Constructors
+The operation may already be complete when the handle is created.
-### new Job()
+The detail getters read what the handle last observed. Submitting an
 operation returns only a job id, so populating them eagerly would cost an
 extra round trip on every call:
 - [Job.refresh](Job.md#refresh) and [Job.status](Job.md#status) fetch the whole record.
 - [Job.wait](Job.md#wait) records the terminal state it establishes, but not the
  rest of the record.
 - Everything is null until one of those runs.
 ## Accessors
 ### creationMs
 ```ts
-new Job(): Job
+get creationMs(): null | number
 ```
 When the job was created, in milliseconds since the epoch.
 #### Returns
-[`Job`](Job.md)
+`null` \| `number`
-## Accessors
+***
 ### failure
 ```ts
 get failure(): null | JobFailureInfo
 ```
 Why the job failed, when it failed and the server reports a reason.
 #### Returns
 `null` \| [`JobFailureInfo`](../interfaces/JobFailureInfo.md)
 ***
 ### id
@@ -28,8 +55,69 @@ new Job(): Job
 get id(): null | string
 ```
-Identifies the operation on the server that is running it. Operations
+Identifies the operation on the server that is running it.
-that run in this process have no server id. The value is opaque.
+
 Operations that run in this process have no server id. The value is
 opaque: parsing it or storing it to resume the job later is not supported.
 #### Returns
 `null` \| `string`
 ***
 ### jobType
 ```ts
 get jobType(): null | string
 ```
 The job's type, as the server names it. Null for an in-process job, which
 has no server-side record.
 #### Returns
 `null` \| `string`
 ***
 ### result
 ```ts
 get result(): any
 ```
 The job-type-specific terminal result. Null until the job succeeds, so a
 job that never terminates reports its progress through [Job.events](Job.md#events)
 instead.
 #### Returns
 `any`
 ***
 ### spec
 ```ts
 get spec(): any
 ```
 The job-type-specific specification it was submitted with.
 #### Returns
 `any`
 ***
 ### state
 ```ts
 get state(): null | string
 ```
 The last observed lifecycle state, without contacting the backend.
 #### Returns
@@ -51,18 +139,61 @@ Request cancellation. Cancelling a finished operation is a no-op.
 ***
 ### events()
 ```ts
 events(options?): Promise<Table<any>>
 ```
 This job's recorded lifecycle events.
 Where the getters above report a terminal result only once the job reaches
 one, events are written as the job runs and outlive the workers that
 produced them. A distributed job records a `claim`/`claim_complete` pair
 per unit of work, each carrying `rows_processed`, so a job that never
 finishes still accounts for what it did.
 The server caps results at 1000 rows by default and 10,000 at most, and
 truncates without saying so, so pass `limit` for a job that emits an event
 per fragment. `filter` is a SQL-like expression over the `state`,
 `updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
 #### Parameters
 * **options?**: [`JobEventsOptions`](../interfaces/JobEventsOptions.md)
 #### Returns
 `Promise`&lt;`Table`&lt;`any`&gt;&gt;
 ***
 ### refresh()
 ```ts
 refresh(): Promise<void>
 ```
 Ask the backend for this job's current state, and for a server-side job
 its full record, then cache it for the getters above.
 #### Returns
 `Promise`&lt;`void`&gt;
 ***
 ### status()
 ```ts
 status(): Promise<string>
 ```
-The operation's current lifecycle state: "running", "finished",
+The operation's current lifecycle state: "running", "finished", "failed",
-"failed", or "cancelled".
+or "cancelled".
-A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject
+A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject on a
-on a terminal failure state. States a newer server reports that this
+terminal failure state. Also refreshes the getters above.
 client version does not know pass through as-is.
 #### Returns
@@ -70,6 +201,22 @@ client version does not know pass through as-is.
 ***
 ### toString()
 ```ts
 toString(): string
 ```
 Every field the handle currently knows, one per line, with the JSON
 payloads indented -- a refresh job's spec and result are the point of
 printing it.
 #### Returns
 `string`
 ***
 ### wait()
 ```ts
@@ -676,9 +676,17 @@ List all the versions of the table
 abstract mergeInsert(on): MergeInsertBuilder
 ```
 Create a [MergeInsertBuilder](MergeInsertBuilder.md), which combines new data with the
 existing table in a single transaction — inserting, updating and deleting
 rows depending on how they match.
 #### Parameters
 * **on**: `string` \| `string`[]
    The column, or columns, to match source rows against target
    rows on. Typically a key or id column. Several columns match on the
    composite key: a source row updates a target row only when it agrees on
    every one of them.
 #### Returns
@@ -942,7 +950,7 @@ Get the schema of the table.
 abstract search(
   query,
   queryType?,
-   ftsColumns?): Query | VectorQuery
+   ftsColumns?): Query | VectorQuery | AutoQuery
 ```
 Create a search query to find the nearest neighbors
@@ -964,7 +972,7 @@ of the given query
 #### Returns
-[`Query`](Query.md) \| [`VectorQuery`](VectorQuery.md)
+[`Query`](Query.md) \| [`VectorQuery`](VectorQuery.md) \| [`AutoQuery`](AutoQuery.md)
 ***
@@ -1292,6 +1300,18 @@ abstract updateFieldMetadata(updates): Promise<UpdateFieldMetadataResult>
 Update per-field (column) metadata.
 The following keys are treated specially, by convention, and should be
 used when appropriate:
 - `lancedb:description`: for a human-readable description of a field.
 - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
  names the tag category; e.g. `lancedb:tag:model: "clip"`.
 - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
  `feature_v2` might be in the same logical column.
 - `lancedb:status`: for status options (`production`, `candidate`,
  `deprecated`, `archived`) to designate the current life cycle state of
  this column.
 #### Parameters
 * **updates**: [`FieldMetadataUpdate`](../interfaces/FieldMetadataUpdate.md)[]
@@ -18,6 +18,7 @@
 ## Classes
 - [AutoQuery](classes/AutoQuery.md)
 - [BooleanQuery](classes/BooleanQuery.md)
 - [BoostQuery](classes/BoostQuery.md)
 - [BranchContents](classes/BranchContents.md)
@@ -95,7 +96,7 @@
 - [IvfFlatOptions](interfaces/IvfFlatOptions.md)
 - [IvfPqOptions](interfaces/IvfPqOptions.md)
 - [IvfRqOptions](interfaces/IvfRqOptions.md)
- [JobDescription](interfaces/JobDescription.md)
+- [JobEventsOptions](interfaces/JobEventsOptions.md)
 - [JobFailureInfo](interfaces/JobFailureInfo.md)
 - [JobInfo](interfaces/JobInfo.md)
 - [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
@@ -17,7 +17,8 @@ metadata: Record<string, null | string>;
 ```
 Metadata key/value pairs. Merged into the field's existing metadata by
-default; a value of `null` deletes that key.
+default; a value of `null` deletes that key. See
 [Table.updateFieldMetadata](../classes/Table.md#updatefieldmetadata) for the conventional `lancedb:*` keys.
 ***
@@ -1,66 +0,0 @@
 [**@lancedb/lancedb**](../README.md) • **Docs**
 ***
 [@lancedb/lancedb](../globals.md) / JobDescription
 # Interface: JobDescription
 A described job from `Connection.getJob`.
 ## Properties
 ### creationMs
 ```ts
 creationMs: number;
 ```
 When the job was created, in milliseconds since the epoch.
 ***
 ### failure?
 ```ts
 optional failure: JobFailureInfo;
 ```
 Why the job failed, when the job is failed and the server reports a
 reason.
 ***
 ### jobId
 ```ts
 jobId: string;
 ```
 ***
 ### jobType
 ```ts
 jobType: string;
 ```
 ***
 ### specJson?
 ```ts
 optional specJson: string;
 ```
 The job-type-specific specification as a JSON string, when present.
 ***
 ### state
 ```ts
 state: string;
 ```
 Lifecycle state: "running", "finished", "failed", or "cancelled".
@@ -0,0 +1,29 @@
 [**@lancedb/lancedb**](../README.md) • **Docs**
 ***
 [@lancedb/lancedb](../globals.md) / JobEventsOptions
 # Interface: JobEventsOptions
 Which of a job's events [Job.events](../classes/Job.md#events) returns.
 ## Properties
 ### filter?
 ```ts
 optional filter: string;
 ```
 SQL-like filter over the event columns.
 ***
 ### limit?
 ```ts
 optional limit: number;
 ```
 Maximum event rows to return, up to the server maximum of 10,000.
@@ -26,7 +26,7 @@ When the job was created, in milliseconds since the epoch.
 jobId: string;
 ```
-The job id -- what `Connection.getJob` and `Connection.cancelJob`
+The job id -- what `Connection.openJob` and `Connection.cancelJob`
 accept.
 ***
@@ -50,6 +50,16 @@ projections: [string, string][];
 ***
 ### sourceNamespace
 ```ts
 sourceNamespace: string[];
 ```
 Namespace holding the source table; empty is the root namespace.
 ***
 ### sourceTable
 ```ts
@@ -10,16 +10,12 @@
 function getRegistry(): EmbeddingFunctionRegistry
 ```
-Utility function to get the global instance of the registry
+Get the global embedding function registry.
 LanceDB built-in providers are initialized when this public API is first
 used, so importing the root package does not change automatic search
 selection for tables without embedding metadata.
 ## Returns
 [`EmbeddingFunctionRegistry`](../classes/EmbeddingFunctionRegistry.md)
 `EmbeddingFunctionRegistry` The global instance of the registry
 ## Example
 ```ts
 const registry = getRegistry();
 const openai = registry.get("openai").create();
@@ -28,6 +28,59 @@ is also an [asynchronous API client](#connections-asynchronous).
 ::: lancedb.Session
 ## Remote SQL
 Submit SQL against a remote LanceDB database through the connection.
 The connected database and `default_namespace_path=["public"]` are used for
 unqualified tables. Fully qualified references can still query other databases
 and namespaces available to the same deployment. `execute_query` returns a
 reader as soon as its initial result stream is available. `execute_query_async`
 returns a query handle immediately; use it to inspect progress, open a reader,
 or cancel the query. The SQL client is initialized by the first query and
 retained for the lifetime of the remote connection. Query ids are random,
 connection-scoped references rather than encoded SQL or durable resume tokens:
 ```python
 import lancedb
 db = lancedb.connect(
    "db://analytics",
    api_key="ldb_...",
    host_override="https://api.example.com",
    sql_host_override="grpc+tls://sql.example.com:10026",
 )
 reader = db.execute_query(
    """
    SELECT events.id, accounts.name
    FROM analytics.public.events AS events
    JOIN users.public.accounts AS accounts ON events.user_id = accounts.id
    """,
    default_namespace_path=["public"],
 )
 for batch in reader:
    print(batch.num_rows)
 query = db.execute_query_async("SELECT * FROM events")
 print(query.id)
 print(query.describe().status)
 for batch in query.reader():
    print(batch.num_rows)
 # The async connection exposes the same lifecycle without blocking:
 # async_db = await lancedb.connect_async(
 #     "db://analytics",
 #     api_key="ldb_...",
 #     host_override="https://api.example.com",
 #     sql_host_override="grpc+tls://sql.example.com:10026",
 # )
 # reader = await async_db.execute_query("SELECT * FROM events")
 # query = await async_db.execute_query_async("SELECT * FROM events")
 # description = await async_db.describe_query(query.id)
 # async for batch in await query.reader():
 #     print(batch.num_rows)
 # await query.cancel()
 ```
 ## Namespaces (Synchronous)
 A namespace-backed connection resolves tables through a
@@ -72,6 +125,10 @@ listing a storage directory.
 ::: lancedb.functions.UdfDefinition
 ::: lancedb.secrets.EnvVarSecret
 ::: lancedb.secrets.SecretInfo
 ::: lancedb.functions.FunctionRegistrationRequest
 ::: lancedb.functions.FunctionArtifactRequest
@@ -94,6 +151,8 @@ listing a storage directory.
 ::: lancedb.functions.OutputMapping
 ::: lancedb.functions.AssignmentMapping
 ::: lancedb.functions.FunctionBinding
 ::: lancedb.functions.RefreshColumnResult
@@ -102,6 +161,18 @@ listing a storage directory.
 ::: lancedb.job.AsyncJob
 ::: lancedb.job.JobInfo
 ::: lancedb.job.JobDescription
 ::: lancedb.job.JobFailureInfo
 ::: lancedb.sql.Query
 ::: lancedb.sql.AsyncQuery
 ::: lancedb.sql.QueryDescription
 ## Materialized Views (Synchronous)
 ::: lancedb.materialized_view.MaterializedView
@@ -159,6 +230,8 @@ and combined with [BooleanQuery][lancedb.query.BooleanQuery].
 ::: lancedb.query.FullTextOperator
 ::: lancedb.query.DocumentGranularity
 ::: lancedb.query.Occur
 ## Embeddings
@@ -221,9 +294,13 @@ tokens = list(
 Blob columns store large binary values out of line so they can be read lazily
 instead of being materialized with the rest of the row.
-::: lancedb.blob
+`lancedb.BlobType` is `lance.blob.BlobType` when pylance is installed. Without
 pylance, LanceDB uses a matching `lance.blob.v2` extension type so blob columns
 still work. Queries return descriptors. Call
 [`fetch_blob_files`][lancedb.table.Table.fetch_blob_files] for lazy reads or
 [`fetch_blobs`][lancedb.table.Table.fetch_blobs] for eager bytes.
-::: lancedb.BlobType
+::: lancedb.blob
 ::: lancedb._blob.BlobFile
    options:
@@ -243,6 +320,12 @@ instead of being materialized with the rest of the row.
 ::: lancedb.exceptions.MissingColumnError
 ::: lancedb.exceptions.JobNotFoundError
 ::: lancedb.exceptions.JobFailedError
 ::: lancedb.exceptions.JobCancelledError
 ## Integrations
 ## Pydantic
@@ -1,17 +0,0 @@
 {
  "include": [
    "src/*.ts",
  ],
  "compilerOptions": {
    "target": "es2022",
    "module": "nodenext",
    "declaration": true,
    "outDir": "./dist",
    "strict": true,
    "allowJs": true,
    "resolveJsonModule": true,
  },
  "exclude": [
    "./dist/*",
  ]
 }
@@ -8,7 +8,7 @@
    <parent>
      <groupId>com.lancedb</groupId>
      <artifactId>lancedb-parent</artifactId>
-      <version>0.38.0-beta.9</version>
+      <version>0.39.0-beta.4</version>
      <relativePath>../pom.xml</relativePath>
    </parent>
@@ -6,7 +6,7 @@
    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-parent</artifactId>
-    <version>0.38.0-beta.9</version>
+    <version>0.39.0-beta.4</version>
    <packaging>pom</packaging>
    <name>${project.artifactId}</name>
    <description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>11.0.0-beta.22</lance-core.version>
+        <lance-core.version>12.0.0-beta.14</lance-core.version>
        <spotless.skip>false</spotless.skip>
        <spotless.version>2.30.0</spotless.version>
        <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.38.0-beta.9"
+version = "0.39.0-beta.4"
 publish = false
 license.workspace = true
 description.workspace = true
@@ -1,11 +1,16 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 import * as fs from "node:fs";
 import * as vm from "node:vm";
 import * as arrow15 from "apache-arrow-15";
 import * as arrow16 from "apache-arrow-16";
 import * as arrow17 from "apache-arrow-17";
 import * as arrow18 from "apache-arrow-18";
 import {
  Field as CurrentField,
  LargeBinary as CurrentLargeBinary,
  Schema as CurrentSchema,
  Vector as CurrentVector,
  convertToTable,
  tableFromIPC as currentTableFromIPC,
@@ -36,6 +41,59 @@ function sampleRecords(): Array<Record<string, any>> {
    },
  ];
 }
 it("serializes an Arrow Table created in another JavaScript realm", async () => {
  const context = vm.createContext({
    TextDecoder,
    TextEncoder,
    console,
    setTimeout,
    clearTimeout,
  });
  vm.runInContext(
    fs.readFileSync(
      require.resolve("apache-arrow-15/Arrow.es2015.min"),
      "utf8",
    ),
    context,
  );
  const foreignTable: unknown = vm.runInContext(
    "Arrow.tableFromArrays({ id: new Int32Array([1, 2, 3]), text: ['foo', 'bar', 'baz'] })",
    context,
  );
  const foreignMetadata = (
    foreignTable as { schema: { metadata: Map<string, string> } }
  ).schema.metadata;
  expect(foreignMetadata).not.toBeInstanceOf(Map);
  const buf = await fromDataToBuffer(
    foreignTable as Parameters<typeof fromDataToBuffer>[0],
  );
  const actual = currentTableFromIPC(buf);
  expect(actual.numRows).toBe(3);
  expect(actual.getChild("id")?.toJSON()).toEqual([1, 2, 3]);
  expect(actual.getChild("text")?.toJSON()).toEqual(["foo", "bar", "baz"]);
 });
 it("preserves field metadata from a provided schema", async function () {
  const jsonMetadata = new Map([["ARROW:extension:name", "lance.json"]]);
  const schema = new CurrentSchema([
    new CurrentField("meta", new CurrentLargeBinary(), true, jsonMetadata),
  ]);
  const table = makeArrowTable(
    [{ meta: Buffer.from(JSON.stringify({ source: "test" })) }],
    { schema },
  );
  expect(table.schema.fields[0].metadata).toEqual(jsonMetadata);
  const roundTripped = currentTableFromIPC(await fromTableToBuffer(table));
  expect(roundTripped.schema.fields[0].metadata).toEqual(jsonMetadata);
 });
 describe.each([arrow15, arrow16, arrow17, arrow18])(
  "Arrow",
  (
@@ -187,6 +187,58 @@ describe("embedding functions", () => {
    const vector0 = JSON.parse(JSON.stringify(arr[0].vector));
    expect(vector0).toEqual([1, 2, 3]);
  });
  it("should append multiple Python embeddings with the same alias", async () => {
    @register("python-mock")
    // biome-ignore lint/correctness/noUnusedVariables: the decorator registers this class
    class MockEmbeddingFunction extends EmbeddingFunction<string> {
      ndims() {
        return 3;
      }
      embeddingDataType(): Float {
        return new Float32();
      }
      async computeQueryEmbeddings(_data: string) {
        return [1, 2, 3];
      }
      async computeSourceEmbeddings(data: string[]) {
        return data.map((value) =>
          value === "hello world" ? [1, 2, 3] : [4, 5, 6],
        );
      }
    }
    const metadata = new Map([
      [
        "embedding_functions",
        '[{"source_column":"text1","vector_column":"vector1","name":"python-mock","model":{}},{"source_column":"text2","vector_column":"vector2","name":"python-mock","model":{}}]',
      ],
    ]);
    const schema = new Schema(
      [
        new Field("text1", new Utf8(), true),
        new Field("text2", new Utf8(), true),
        new Field(
          "vector1",
          new FixedSizeList(3, new Field("item", new Float32(), true)),
          true,
        ),
        new Field(
          "vector2",
          new FixedSizeList(3, new Field("item", new Float32(), true)),
          true,
        ),
      ],
      metadata,
    );
    const db = await connect(tmpDir.name);
    const table = await db.createEmptyTable("test", schema);
    await table.add([{ text1: "hello world", text2: "goodbye world" }]);
    const rows = await table.query().toArray();
    expect(JSON.parse(JSON.stringify(rows[0].vector1))).toEqual([1, 2, 3]);
    expect(JSON.parse(JSON.stringify(rows[0].vector2))).toEqual([4, 5, 6]);
  });
  it("should append generated vectors to a non-nullable schema", async () => {
    @register("non_nullable_schema_test")
@@ -0,0 +1,95 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 import { execFileSync } from "node:child_process";
 import { resolve } from "node:path";
 import type { OpenAIEmbeddingFunction } from "../lancedb/embedding/openai";
 import type { EmbeddingFunctionRegistry } from "../lancedb/embedding/registry";
 type EmbeddingModule = typeof import("../lancedb/embedding");
 type OpenAIModule = typeof import("../lancedb/embedding/openai");
 type RegistryModule = typeof import("../lancedb/embedding/registry");
 describe("embedding function registry", () => {
  const registries: EmbeddingFunctionRegistry[] = [];
  afterEach(() => {
    for (const registry of registries) {
      registry.reset();
    }
    registries.length = 0;
  });
  it("defers built-in providers until the public registry API is used", () => {
    jest.isolateModules(() => {
      const embedding = require("../lancedb/embedding") as EmbeddingModule;
      const { getRegistry: getInternalRegistry } =
        require("../lancedb/embedding/registry") as RegistryModule;
      const registry = getInternalRegistry();
      registries.push(registry);
      expect(registry.length()).toBe(0);
      expect(embedding.getRegistry()).toBe(registry);
      expect(registry.get("openai")).toBeDefined();
      expect(registry.get("huggingface")).toBeDefined();
    });
  });
  it("preserves automatic FTS search in a fresh process", () => {
    execFileSync(
      process.execPath,
      [resolve(__dirname, "fixtures", "auto_fts_search.cjs")],
      { stdio: "pipe" },
    );
  });
  it("shares registrations across duplicated provider module graphs", () => {
    let registeringRegistry: EmbeddingFunctionRegistry | undefined;
    let latestOpenAIConstructor: typeof OpenAIEmbeddingFunction | undefined;
    jest.isolateModules(() => {
      require("../lancedb/embedding/openai");
      const { getRegistry } =
        require("../lancedb/embedding/registry") as RegistryModule;
      registeringRegistry = getRegistry();
      registries.push(registeringRegistry);
      expect(registeringRegistry.get("openai")).toBeDefined();
    });
    expect(() => {
      jest.isolateModules(() => {
        const { OpenAIEmbeddingFunction } =
          require("../lancedb/embedding/openai") as OpenAIModule;
        latestOpenAIConstructor = OpenAIEmbeddingFunction;
        const { getRegistry } =
          require("../lancedb/embedding/registry") as RegistryModule;
        registries.push(getRegistry());
      });
    }).not.toThrow();
    const previousApiKey = process.env.OPENAI_API_KEY;
    process.env.OPENAI_API_KEY = "test";
    try {
      const latestOpenAI = registeringRegistry!
        .get<OpenAIEmbeddingFunction>("openai")!
        .create();
      expect(latestOpenAI).toBeInstanceOf(latestOpenAIConstructor!);
    } finally {
      if (previousApiKey === undefined) {
        delete process.env.OPENAI_API_KEY;
      } else {
        process.env.OPENAI_API_KEY = previousApiKey;
      }
    }
    jest.isolateModules(() => {
      const { getRegistry } =
        require("../lancedb/embedding") as EmbeddingModule;
      const publicRegistry = getRegistry();
      registries.push(publicRegistry);
      expect(publicRegistry).toBe(registeringRegistry);
      expect(publicRegistry.get("openai")).toBeDefined();
    });
  });
 });
@@ -0,0 +1,33 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 const assert = require("node:assert/strict");
 const tmp = require("tmp");
 const { connect, embedding, Index } = require("../../dist");
 const { getRegistry } = require("../../dist/embedding/registry");
 async function main() {
  assert.equal(typeof embedding.getRegistry, "function");
  assert.equal(getRegistry().length(), 0);
  assert.equal(embedding.getRegistry(), getRegistry());
  assert.equal(getRegistry().length(), 2);
  const dir = tmp.dirSync({ unsafeCleanup: true });
  let db;
  try {
    db = await connect(dir.name);
    const table = await db.createTable("docs", [{ text: "hello world" }]);
    await table.createIndex("text", { config: Index.fts() });
    const rows = await table.search("hello").toArray();
    assert.equal(rows[0].text, "hello world");
  } finally {
    db?.close();
    dir.removeCallback();
  }
 }
 main().catch((error) => {
  console.error(error);
  process.exitCode = 1;
 });
@@ -48,6 +48,28 @@ describe("materialized views", () => {
    expect(definitionFromMetadata(safe, "v").limit).toBe(42);
  });
  it("reads the namespaced select kind and refuses unknown kinds", () => {
    // "namespaced_select" is the namespaced form of "select": same shape, a
    // separate kind so readers that predate it refuse instead of resolving
    // the source at the root.
    const namespaced = new Map([
      [
        DEFINITION_META_KEY,
        '{"kind":"namespaced_select","source_table":"people","source_namespace":["ns"]}',
      ],
    ]);
    const definition = definitionFromMetadata(namespaced, "v");
    expect(definition.sourceTable).toBe("people");
    expect(definition.sourceNamespace).toEqual(["ns"]);
    const unknown = new Map([
      [DEFINITION_META_KEY, '{"kind":"select_v3","source_table":"people"}'],
    ]);
    expect(() => definitionFromMetadata(unknown, "v")).toThrow(
      /cannot refresh/,
    );
  });
  it("creates, refreshes and queries a view", async () => {
    const view = await db.createMaterializedView("adults", "people", {
      select: ["name", ["shout", "upper(name)"]],
@@ -5,8 +5,8 @@ import packageJson = require("../package.json");
 describe("package metadata", () => {
  it("requires Node.js type declarations compatible with the runtime", () => {
-    expect(packageJson.engines.node).toBe(">= 18");
+    expect(packageJson.engines.node).toBe(">= 22");
-    expect(packageJson.peerDependencies["@types/node"]).toBe(">=18");
+    expect(packageJson.peerDependencies["@types/node"]).toBe(">=22");
    expect(packageJson.peerDependenciesMeta["@types/node"]).toEqual({
      optional: true,
    });
@@ -3,6 +3,7 @@
 import * as http from "http";
 import { RequestListener } from "http";
 import packageJson = require("../package.json");
 import {
  ClientConfig,
  Connection,
@@ -70,7 +71,13 @@ async function withMockDatabase(
  try {
    await callback(db);
  } finally {
-    server.close();
+    // `close()` alone leaves the port bound until keep-alive sockets drain, so
    // a single failing test would cascade into EADDRINUSE for every test after
    // it. Destroy the connections and wait for the port to actually be free.
    await new Promise<void>((resolve) => {
      server.closeAllConnections();
      server.close(() => resolve());
    });
  }
 }
@@ -131,7 +138,7 @@ describe("remote connection", () => {
      (req, res) => {
        expect(req.headers["x-api-key"]).toEqual("fake");
        expect(req.headers["user-agent"]).toEqual(
-          `LanceDB-Node-Client/${process.env.npm_package_version}`,
+          `LanceDB-Node-Client/${packageJson.version}`,
        );
        const body = JSON.stringify({ tables: [] });
@@ -932,6 +939,7 @@ describe("remote connection jobs surface", () => {
    const { tableFromArrays, tableToIPC } = await import("apache-arrow");
    const eventsTable = tableFromArrays({ state: ["created", "succeeded"] });
    const eventsBody = Buffer.from(tableToIPC(eventsTable, "stream"));
    const queryEventsPayloads: Record<string, unknown>[] = [];
    await withMockDatabase(
      (req, res) => {
@@ -960,6 +968,16 @@ describe("remote connection jobs surface", () => {
                );
            }
          } else if (req.url === "/v1/jobs/describe") {
            if (payload["job_id"] === "job-2") {
              res
                .writeHead(200, { "Content-Type": "application/json" })
                .end(
                  '{"job_id": "job-2", "job_type": "refresh_column", ' +
                    '"job_state": "DONE", "creation_ms": 2000, ' +
                    '"result": {"rows_assigned": 1000000}}',
                );
              return;
            }
            if (payload["job_id"] !== "job-1") {
              res.writeHead(404).end("no such job");
              return;
@@ -981,6 +999,7 @@ describe("remote connection jobs surface", () => {
              .writeHead(200, { "Content-Type": "application/json" })
              .end('{"job_id": "job-1"}');
          } else if (req.url === "/v1/jobs/query_events") {
            queryEventsPayloads.push(payload);
            res
              .writeHead(200, {
                "Content-Type": "application/vnd.apache.arrow.stream",
@@ -997,22 +1016,65 @@ describe("remote connection jobs surface", () => {
        expect(jobs[0].state).toEqual("running");
        expect(jobs[1].state).toEqual("finished");
        const description = await db.getJob("job-1");
        expect(description?.state).toEqual("failed");
        expect(JSON.parse(description?.specJson ?? "")).toEqual({
          column: "vec",
        });
        expect(description?.failure?.message).toEqual("worker died");
        expect(await db.getJob("missing")).toBeNull();
        expect(await db.cancelJob("job-1")).toBe(true);
        expect(await db.cancelJob("missing")).toBe(false);
-        const history = await db.jobHistory("job-1");
+        // Opening a job hands back a populated handle; a missing one rejects.
-        expect(history.numRows).toEqual(2);
+        await expect(db.openJob("missing")).rejects.toThrow("not found");
        const finished = await db.openJob("job-2");
        expect(finished.state).toEqual("finished");
        expect(finished.result).toEqual({
          // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
          rows_assigned: 1000000,
        });
-        const job = db.job("job-1");
+        const job = await db.openJob("job-1");
        expect(job.id).toEqual("job-1");
        // openJob already populated the handle; refresh() re-reads it.
        expect(job.state).toEqual("failed");
        await job.refresh();
        expect(job.state).toEqual("failed");
        expect(job.jobType).toEqual("create_index");
        expect(job.creationMs).toEqual(1000);
        expect(job.spec).toEqual({ column: "vec" });
        expect(job.result).toBeNull();
        expect(job.failure?.message).toEqual("worker died");
        // The handle reaches its own events, supplying its job id.
        const jobEvents = await job.events({
          limit: 500,
          filter: "state = 'claim_complete'",
        });
        expect(jobEvents.numRows).toEqual(2);
        expect(queryEventsPayloads.pop()).toEqual({
          // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
          job_id: "job-1",
          limit: 500,
          filter: "state = 'claim_complete'",
        });
        // Printing lays every known field out on its own line, with the JSON
        // payloads indented rather than crammed onto one line.
        expect(`${job}`).toEqual(
          [
            "Job(",
            '    id="job-1",',
            '    state="failed",',
            '    jobType="create_index",',
            "    creationMs=1000,",
            "    spec={",
            '        "column": "vec"',
            "    },",
            "    failure={",
            '        "phase": "execute",',
            '        "message": "worker died",',
            '        "retryable": true',
            "    },",
            ")",
          ].join("\n"),
        );
        expect(await job.status()).toEqual("failed");
        await expect(job.wait()).rejects.toThrow("worker died");
      },
@@ -11,10 +11,13 @@ import * as arrow17 from "apache-arrow-17";
 import * as arrow18 from "apache-arrow-18";
 import {
  AutoQuery,
  Connection,
  MatchQuery,
  PhraseQuery,
  Query,
  Table,
  VectorQuery,
  connect,
  tokenize,
 } from "../lancedb";
@@ -682,13 +685,64 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
  },
 );
 // https://github.com/lancedb/lancedb/issues/1963
 it("should query documents with LangChain PDF metadata", async () => {
  const tmpDir = tmp.dirSync({ unsafeCleanup: true });
  try {
    const db = await connect(tmpDir.name);
    const documents = [
      {
        text: "first page",
        vector: [1, 0],
        source: "first.pdf",
        loc: { pageNumber: 1, lines: { from: 1, to: 12 } },
        pdf: {
          version: "1.10.100",
          info: {
            format: "PDF 1.7",
            producer: "pdf.js",
            creator: "Writer",
          },
          totalPages: 2,
        },
      },
      {
        text: "second page",
        vector: [0, 1],
        source: "second.pdf",
        loc: { pageNumber: 2, lines: { from: 13, to: 24 } },
        pdf: {
          version: "1.10.100",
          info: {
            format: "PDF 1.7",
            producer: "pdf.js",
            creator: "Writer",
          },
          totalPages: 2,
        },
      },
    ];
    const documentsTable = await db.createTable("documents", documents);
    const results = await documentsTable.query().toArray();
    expect(results).toHaveLength(2);
    expect(results[0].source).toBe("first.pdf");
    expect(results[0].pdf.info.producer).toBe("pdf.js");
    expect(results[1].loc.pageNumber).toBe(2);
  } finally {
    tmpDir.removeCallback();
  }
 });
 describe("merge insert", () => {
  let tmpDir: tmp.DirResult;
  let conn: Connection;
  let table: Table;
  beforeEach(async () => {
    tmpDir = tmp.dirSync({ unsafeCleanup: true });
-    const conn = await connect(tmpDir.name);
+    conn = await connect(tmpDir.name);
    table = await conn.createTable("some_table", [
      { a: 1, b: "a" },
@@ -726,6 +780,38 @@ describe("merge insert", () => {
    expect(result.map((row) => ({ ...row }))).toEqual(expected);
  });
  test("upsert on a composite key", async () => {
    const composite = await conn.createTable("composite", [
      { shard: "a", id: 1, val: "x" },
      { shard: "a", id: 2, val: "y" },
      { shard: "b", id: 1, val: "z" },
    ]);
    // ("a", 1) matches an existing row and updates it. ("b", 2) agrees with an
    // existing row on each key column separately but on neither pair, so it is
    // an insert.
    const mergeInsertRes = await composite
      .mergeInsert(["shard", "id"])
      .whenMatchedUpdateAll()
      .whenNotMatchedInsertAll()
      .execute([
        { shard: "a", id: 1, val: "X" },
        { shard: "b", id: 2, val: "W" },
      ]);
    expect(mergeInsertRes.numUpdatedRows).toBe(1);
    expect(mergeInsertRes.numInsertedRows).toBe(1);
    const result = (await composite.toArrow())
      .toArray()
      .sort((a, b) => a.shard.localeCompare(b.shard) || a.id - b.id);
    expect(result.map((row) => ({ ...row }))).toEqual([
      { shard: "a", id: 1, val: "X" },
      { shard: "a", id: 2, val: "y" },
      { shard: "b", id: 1, val: "z" },
      { shard: "b", id: 2, val: "W" },
    ]);
  });
  test("conditional update", async () => {
    const newData = [
      { a: 2, b: "x" },
@@ -1777,6 +1863,194 @@ describe("Read consistency interval", () => {
  });
 });
 describe("automatic search schema consistency", () => {
  let tmpDir: tmp.DirResult;
  class SchemaRefreshEmbedding extends EmbeddingFunction<string> {
    ndims() {
      return 2;
    }
    embeddingDataType() {
      return new Float32();
    }
    async computeSourceEmbeddings(data: string[]) {
      return data.map((value) => [value.length, 1]);
    }
    async computeQueryEmbeddings(value: string) {
      return [value.length, 1];
    }
  }
  function embeddingSchema() {
    const func = new SchemaRefreshEmbedding();
    return LanceSchema({
      text: func.sourceField(new Utf8()),
      vector: func.vectorField(),
    });
  }
  beforeEach(() => {
    getRegistry().reset();
    register("schema-refresh")(SchemaRefreshEmbedding);
    tmpDir = tmp.dirSync({ unsafeCleanup: true });
  });
  afterEach(() => {
    getRegistry().reset();
    tmpDir.removeCallback();
  });
  it("uses the schema refreshed from another connection", async () => {
    const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
    const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
    try {
      const stale = await first.createTable("docs", [{ text: "before" }], {
        schema: embeddingSchema(),
      });
      const replacement = await second.createTable(
        "docs",
        [{ text: "after hello" }],
        { mode: "overwrite" },
      );
      await replacement.createIndex("text", { config: Index.fts() });
      const search = stale.search("hello");
      expect(search).toBeInstanceOf(AutoQuery);
      expect(search).not.toBeInstanceOf(Query);
      expect(search).not.toBeInstanceOf(VectorQuery);
      expect("nprobes" in search).toBe(false);
      const rows = await search.toArray();
      expect(rows[0].text).toBe("after hello");
      expect((await stale.schema()).metadata.has("embedding_functions")).toBe(
        false,
      );
    } finally {
      first.close();
      second.close();
    }
  });
  it("tracks embedding metadata across checkout and restore", async () => {
    const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
    const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
    try {
      await first.createTable("docs", [{ text: "before" }], {
        schema: embeddingSchema(),
      });
      const table = await second.createTable(
        "docs",
        [{ text: "after hello" }],
        { mode: "overwrite" },
      );
      await table.createIndex("text", { config: Index.fts() });
      await table.checkout(1);
      expect((await table.search("before").toArray())[0].text).toBe("before");
      await table.checkoutLatest();
      expect((await table.search("hello").toArray())[0].text).toBe(
        "after hello",
      );
      await table.checkout(1);
      await table.restore();
      expect((await table.search("before").toArray())[0].text).toBe("before");
    } finally {
      first.close();
      second.close();
    }
  });
  it("pins automatic search while computing an embedding", async () => {
    let markStarted!: () => void;
    let releaseEmbedding!: () => void;
    const started = new Promise<void>((resolve) => {
      markStarted = resolve;
    });
    const released = new Promise<void>((resolve) => {
      releaseEmbedding = resolve;
    });
    class BlockingEmbedding extends SchemaRefreshEmbedding {
      async computeQueryEmbeddings(value: string) {
        markStarted();
        await released;
        return [value.length, 1];
      }
    }
    register("schema-refresh-blocking")(BlockingEmbedding);
    const func = new BlockingEmbedding();
    const schema = LanceSchema({
      text: func.sourceField(new Utf8()),
      vector: func.vectorField(),
    });
    const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
    const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
    try {
      const table = await first.createTable(
        "docs",
        [{ text: "hello before" }],
        { schema },
      );
      const pending = table.search("hello").toArray();
      await started;
      const replacement = await second.createTable(
        "docs",
        [{ text: "hello after" }],
        { mode: "overwrite" },
      );
      await replacement.createIndex("text", { config: Index.fts() });
      releaseEmbedding();
      expect((await pending)[0].text).toBe("hello before");
    } finally {
      releaseEmbedding();
      first.close();
      second.close();
    }
  });
  it("refreshes a reused automatic search for every execution", async () => {
    const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
    const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
    try {
      const table = await first.createTable("docs", [
        { text: "hello before", marker: "before" },
      ]);
      await table.createIndex("text", { config: Index.fts() });
      const search = table.search("hello").select(["text"]);
      const before = (await search.toArray())[0];
      expect(before.text).toBe("hello before");
      expect(before.marker).toBeUndefined();
      const replacement = await second.createTable(
        "docs",
        [{ text: "hello after", marker: "after" }],
        { mode: "overwrite" },
      );
      await replacement.createIndex("text", { config: Index.fts() });
      const after = (await search.toArray())[0];
      expect(after.text).toBe("hello after");
      expect(after.marker).toBeUndefined();
    } finally {
      first.close();
      second.close();
    }
  });
 });
 describe("schema evolution", function () {
  let tmpDir: tmp.DirResult;
  beforeEach(() => {
@@ -2344,7 +2618,24 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
      );
    });
-    test("full text search if no embedding function provided", async () => {
+    test("full text search if only an unrelated embedding function is registered", async () => {
      register("unused")(
        class extends EmbeddingFunction<string> {
          ndims() {
            return 3;
          }
          embeddingDataType() {
            return new Float32();
          }
          async computeQueryEmbeddings(_data: string) {
            return [1, 2, 3];
          }
          async computeSourceEmbeddings(data: string[]) {
            return data.map(() => [1, 2, 3]);
          }
        },
      );
      const db = await connect(tmpDir.name);
      const data = [
        { text: "hello world", vector: [0.1, 0.2, 0.3] },
@@ -2366,6 +2657,306 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
      expect(results2[0].text).toBe(data[1].text);
    });
    test("auto search stays consistent with the active revision", async () => {
      let initCalls = 0;
      let queryCalls = 0;
      let markStarted!: () => void;
      const started = new Promise<void>((resolve) => {
        markStarted = resolve;
      });
      let releaseEmbedding!: () => void;
      const embeddingReleased = new Promise<void>((resolve) => {
        releaseEmbedding = resolve;
      });
      @register("refresh-test")
      class TestEmbedding extends EmbeddingFunction<string> {
        async init() {
          initCalls += 1;
        }
        ndims() {
          return 1;
        }
        embeddingDataType() {
          return new arrow.Float32();
        }
        async computeQueryEmbeddings(value: string) {
          queryCalls += 1;
          if (value === "blocked") {
            markStarted();
            await embeddingReleased;
          }
          return value === "greetings" ? [0.1] : [0.2];
        }
        async computeSourceEmbeddings(values: string[]) {
          return values.map((value) =>
            value === "hello world" ? [0.1] : [0.2],
          );
        }
      }
      const writer = await connect(tmpDir.name);
      await writer.createTable("test", [{ text: "plain", vector: [0.0] }]);
      const reader = await connect(tmpDir.name, {
        readConsistencyInterval: 0,
      });
      const tracked = await reader.openTable("test");
      type SnapshotCountingNative = {
        querySnapshot: () => Promise<unknown>;
      };
      const native = (tracked as unknown as { inner: SnapshotCountingNative })
        .inner;
      const querySnapshot = native.querySnapshot.bind(native);
      let snapshotCalls = 0;
      native.querySnapshot = async () => {
        snapshotCalls += 1;
        return await querySnapshot();
      };
      const autoQuery = tracked.search("greetings").select(["text"]).limit(1);
      const func = new TestEmbedding();
      const schema = LanceSchema({
        text: func.sourceField(new arrow.Utf8()),
        vector: func.vectorField(),
      });
      const data = [{ text: "hello world" }, { text: "goodbye world" }];
      await writer.createTable("test", data, { mode: "overwrite", schema });
      const baselineInitCalls = initCalls;
      expect(
        (await tracked.schema()).metadata.get("embedding_functions"),
      ).toBeDefined();
      const results = await autoQuery.toArray();
      expect(results[0].text).toBe(data[0].text);
      expect(initCalls).toBe(baselineInitCalls + 1);
      expect(queryCalls).toBe(1);
      expect(snapshotCalls).toBe(1);
      const repeatedResults = await autoQuery.toArray();
      expect(repeatedResults[0].text).toBe(data[0].text);
      expect(initCalls).toBe(baselineInitCalls + 1);
      expect(queryCalls).toBe(1);
      expect(snapshotCalls).toBe(2);
      const pending = tracked
        .search("blocked")
        .select(["text"])
        .limit(1)
        .toArray();
      await started;
      const ftsData = [
        { text: "greetings from full text", vector: [0.0] },
        { text: "blocked from full text", vector: [0.0] },
      ];
      const ftsTable = await writer.createTable("test", ftsData, {
        mode: "overwrite",
      });
      await ftsTable.createIndex("text", { config: Index.fts() });
      releaseEmbedding();
      const pendingResults = await pending;
      expect(pendingResults[0].text).toBe(data[1].text);
      expect(
        (await tracked.schema()).metadata.get("embedding_functions"),
      ).toBeUndefined();
      const ftsResults = await autoQuery.toArray();
      expect(ftsResults[0].text).toBe(ftsData[0].text);
    });
    test("auto search keeps newer preparation during a revision race", async () => {
      let aCalls = 0;
      let bCalls = 0;
      let markAStarted!: () => void;
      const aStarted = new Promise<void>((resolve) => {
        markAStarted = resolve;
      });
      let releaseA!: () => void;
      const aReleased = new Promise<void>((resolve) => {
        releaseA = resolve;
      });
      let markBStarted!: () => void;
      const bStarted = new Promise<void>((resolve) => {
        markBStarted = resolve;
      });
      let releaseB!: () => void;
      const bReleased = new Promise<void>((resolve) => {
        releaseB = resolve;
      });
      @register("race-a")
      class EmbeddingA extends EmbeddingFunction<string> {
        ndims() {
          return 1;
        }
        embeddingDataType() {
          return new arrow.Float32();
        }
        async computeQueryEmbeddings() {
          aCalls += 1;
          markAStarted();
          await aReleased;
          return [0.1];
        }
        async computeSourceEmbeddings(values: string[]) {
          return values.map(() => [0.1]);
        }
      }
      @register("race-b")
      class EmbeddingB extends EmbeddingFunction<string> {
        ndims() {
          return 1;
        }
        embeddingDataType() {
          return new arrow.Float32();
        }
        async computeQueryEmbeddings() {
          bCalls += 1;
          markBStarted();
          await bReleased;
          return [0.2];
        }
        async computeSourceEmbeddings(values: string[]) {
          return values.map(() => [0.2]);
        }
      }
      const writer = await connect(tmpDir.name);
      const embeddingA = new EmbeddingA();
      const schemaA = LanceSchema({
        text: embeddingA.sourceField(new arrow.Utf8()),
        vector: embeddingA.vectorField(),
      });
      await writer.createTable("race", [{ text: "revision a" }], {
        schema: schemaA,
      });
      const reader = await connect(tmpDir.name, {
        readConsistencyInterval: 0,
      });
      const tracked = await reader.openTable("race");
      const query = tracked.search("query");
      const first = query.toArray();
      await aStarted;
      const embeddingB = new EmbeddingB();
      const schemaB = LanceSchema({
        text: embeddingB.sourceField(new arrow.Utf8()),
        vector: embeddingB.vectorField(),
      });
      await writer.createTable("race", [{ text: "revision b" }], {
        mode: "overwrite",
        schema: schemaB,
      });
      const second = query.toArray();
      await bStarted;
      releaseA();
      releaseB();
      await Promise.all([first, second]);
      expect(aCalls).toBe(1);
      expect(bCalls).toBe(1);
    });
    test("stale FTS routing keeps newer vector preparation", async () => {
      let vectorCalls = 0;
      let markVectorStarted!: () => void;
      const vectorStarted = new Promise<void>((resolve) => {
        markVectorStarted = resolve;
      });
      let releaseVector!: () => void;
      const vectorReleased = new Promise<void>((resolve) => {
        releaseVector = resolve;
      });
      @register("stale-fts-race")
      class RaceEmbedding extends EmbeddingFunction<string> {
        ndims() {
          return 1;
        }
        embeddingDataType() {
          return new arrow.Float32();
        }
        async computeQueryEmbeddings() {
          vectorCalls += 1;
          markVectorStarted();
          await vectorReleased;
          return [0.1];
        }
        async computeSourceEmbeddings(values: string[]) {
          return values.map(() => [0.1]);
        }
      }
      const writer = await connect(tmpDir.name);
      const ftsTable = await writer.createTable("stale_fts", [
        { text: "hello", vector: [0.0] },
      ]);
      await ftsTable.createIndex("text", { config: Index.fts() });
      const reader = await connect(tmpDir.name, {
        readConsistencyInterval: 0,
      });
      const tracked = await reader.openTable("stale_fts");
      type Snapshot = {
        schema: () => Promise<Buffer>;
      };
      type NativeWithSnapshot = {
        querySnapshot: () => Promise<Snapshot>;
      };
      const native = (tracked as unknown as { inner: NativeWithSnapshot })
        .inner;
      const querySnapshot = native.querySnapshot.bind(native);
      let snapshotCalls = 0;
      let markStaleSchemaStarted!: () => void;
      const staleSchemaStarted = new Promise<void>((resolve) => {
        markStaleSchemaStarted = resolve;
      });
      let releaseStaleSchema!: () => void;
      const staleSchemaReleased = new Promise<void>((resolve) => {
        releaseStaleSchema = resolve;
      });
      native.querySnapshot = async () => {
        const snapshot = await querySnapshot();
        snapshotCalls += 1;
        if (snapshotCalls === 1) {
          const schema = snapshot.schema.bind(snapshot);
          snapshot.schema = async () => {
            markStaleSchemaStarted();
            await staleSchemaReleased;
            return await schema();
          };
        }
        return snapshot;
      };
      const query = tracked.search("hello");
      const staleFtsExecution = query.toArray();
      await staleSchemaStarted;
      const embedding = new RaceEmbedding();
      const vectorSchema = LanceSchema({
        text: embedding.sourceField(new arrow.Utf8()),
        vector: embedding.vectorField(),
      });
      await writer.createTable("stale_fts", [{ text: "hello" }], {
        mode: "overwrite",
        schema: vectorSchema,
      });
      const vectorExecution = query.toArray();
      await vectorStarted;
      releaseStaleSchema();
      await staleFtsExecution;
      releaseVector();
      await vectorExecution;
      await query.toArray();
      expect(vectorCalls).toBe(1);
    });
    test("tokenizes FTS queries by column or index name", async () => {
      const db = await connect(tmpDir.name);
      const data = [
@@ -2916,6 +3507,30 @@ describe("column name options", () => {
    expect(results[1].query_index).toBe(1);
  });
  test("observes promised additional vectors while the query is pending", async () => {
    const initialVector = new Promise<number[]>(() => undefined);
    const query = table.query().nearestTo(initialVector);
    const unhandled: unknown[] = [];
    const onUnhandled = (reason: unknown) => unhandled.push(reason);
    process.on("unhandledRejection", onUnhandled);
    try {
      query.addQueryVector(Promise.reject(new Error("extra vector failed")));
      await new Promise<void>((resolve) => setImmediate(resolve));
      expect(unhandled).toEqual([]);
      const rejectedQuery = table
        .query()
        .nearestTo([0.1, 0.2])
        .addQueryVector(Promise.reject(new Error("consumed vector failed")));
      await expect(rejectedQuery.toArray()).rejects.toThrow(
        "consumed vector failed",
      );
    } finally {
      process.off("unhandledRejection", onUnhandled);
    }
  });
  test("index and search multivectors", async () => {
    const db = await connect(tmpDir.name);
    const data = [];
@@ -2979,6 +3594,27 @@ describe("when creating an empty table", () => {
    expect((actualSchema.fields[1].type as Float64).precision).toBe(2);
  });
  it("can add and query JSON data", async () => {
    const schema = new Schema([
      new Field("id", new Int32(), true),
      new Field(
        "meta",
        new Utf8(),
        true,
        new Map([["ARROW:extension:name", "arrow.json"]]),
      ),
    ]);
    const table = await con.createEmptyTable("json", schema);
    const meta = JSON.stringify({ x: 1 });
    await table.add([{ id: 1, meta }]);
    const rows = await table.query().toArray();
    expect(rows).toHaveLength(1);
    expect(rows[0].id).toBe(1);
    expect(rows[0].meta).toBe(meta);
  });
  it("can create an empty table from schema that specifies field types by name", async () => {
    const schemaLike = {
      fields: [
@@ -170,7 +170,7 @@ test("basic table examples", async () => {
    // --8<-- [end:create_index]
    // --8<-- [start:delete_rows]
-    await tbl.delete('item = "fizz"');
+    await tbl.delete("item = 'fizz'");
    // --8<-- [end:delete_rows]
    // --8<-- [start:drop_table]
@@ -8,7 +8,8 @@
    "//1": "--experimental-vm-modules is needed to run jest with sentence-transformers",
    "//2": "--testEnvironment is needed to run jest with sentence-transformers",
    "//3": "See: https://github.com/huggingface/transformers.js/issues/57",
-    "test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
+    "//4": "jest is invoked by its JS entry, not node_modules/.bin/jest: under pnpm that path is a shell shim, which `node` cannot execute",
    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testEnvironment jest-environment-node-single-context --verbose",
    "lint": "biome check *.ts && biome format *.ts",
    "lint-ci": "biome ci .",
    "lint-fix": "biome check --write *.ts && pnpm format",
@@ -72,8 +72,7 @@ export type FieldLike =
    };
 export type DataLike =
-  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  | import("apache-arrow").Data
  | import("apache-arrow").Data<Struct<any>>
  | {
      // biome-ignore lint/suspicious/noExplicitAny: <explanation>
      type: any;
@@ -82,6 +81,7 @@ export type DataLike =
      stride: number;
      nullable: boolean;
      children: DataLike[];
      dictionary?: { data: readonly DataLike[] };
      get nullCount(): number;
      // biome-ignore lint/suspicious/noExplicitAny: <explanation>
      values: Buffers<any>[BufferType.DATA];
@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 import { tableFromIPC } from "apache-arrow";
 import {
  Data,
  SchemaLike,
@@ -16,6 +15,7 @@ import {
  makeEmptyTable,
 } from "./arrow";
 import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
 import { Job } from "./job";
 import {
  MaterializedView,
  MaterializedViewSelect,
@@ -27,8 +27,6 @@ import type {
  CreateNamespaceResponse,
  DescribeNamespaceResponse,
  DropNamespaceResponse,
  Job,
  JobDescription,
  JobInfo,
  ListNamespacesResponse,
  ListTablesResponse,
@@ -557,24 +555,19 @@ export abstract class Connection {
  ): Promise<void>;
  /**
-   * A {@link Job} handle for a server-side job by id.
+   * Open a server-side job by id, returning a handle with its record already
   * populated. Rejects when the server has no such job, the way
   * {@link Connection.openTable} does for a missing table.
   *
-   * The handle is constructed without a server round trip; an unknown id
+   * The returned {@link Job} answers for its own state, specification,
-   * surfaces when the handle is used. Dropping the handle has no effect on
+   * result, failure and event history, so there is no separate
-   * the job itself.
+   * connection-level call for any of them.
   */
-  abstract job(jobId: string): Job;
+  abstract openJob(jobId: string): Promise<Job>;
  /** List server-side jobs across the database's tables. */
  abstract listJobs(): Promise<JobInfo[]>;
  /**
   * Describe a single server-side job by id.
   *
   * Resolves to `null` when the server has no such job.
   */
  abstract getJob(jobId: string): Promise<JobDescription | null>;
  /**
   * Request cancellation of a server-side job by id.
   *
@@ -582,13 +575,6 @@ export abstract class Connection {
   * such job exists. Cancelling an already-terminal job is a no-op success.
   */
  abstract cancelJob(jobId: string): Promise<boolean>;
  /**
   * The lifecycle event history of a server-side job, as an Arrow table.
   *
   * Lists history across all jobs when `jobId` is omitted.
   */
  abstract jobHistory(jobId?: string): Promise<ArrowTable>;
 }
 /** @hideconstructor */
@@ -869,7 +855,7 @@ export class LocalConnection extends Connection {
  }
  async dropTableAsync(name: string, namespacePath?: string[]): Promise<Job> {
-    return this.inner.dropTableAsync(name, namespacePath ?? []);
+    return new Job(await this.inner.dropTableAsync(name, namespacePath ?? []));
  }
  async dropAllTables(namespacePath?: string[]): Promise<void> {
@@ -928,29 +914,17 @@ export class LocalConnection extends Connection {
    );
  }
-  job(jobId: string): Job {
+  async openJob(jobId: string): Promise<Job> {
-    return this.inner.job(jobId);
+    return new Job(await this.inner.openJob(jobId));
  }
  async listJobs(): Promise<JobInfo[]> {
    return this.inner.listJobs();
  }
  async getJob(jobId: string): Promise<JobDescription | null> {
    return this.inner.getJob(jobId);
  }
  async cancelJob(jobId: string): Promise<boolean> {
    return this.inner.cancelJob(jobId);
  }
  async jobHistory(jobId?: string): Promise<ArrowTable> {
    const buf = await this.inner.jobHistory(jobId);
    if (buf.length === 0) {
      return new ArrowTable();
    }
    return tableFromIPC(buf);
  }
 }
 /**
@@ -4,7 +4,15 @@
 import { Field, Schema } from "../arrow";
 import { sanitizeType } from "../sanitize";
 import { EmbeddingFunction } from "./embedding_function";
-import { EmbeddingFunctionConfig, getRegistry } from "./registry";
+import {
  EmbeddingFunctionConfig,
  EmbeddingFunctionRegistry,
  getRegistry as getGlobalRegistry,
  registerBuiltIn,
 } from "./registry";
 type OpenAIModule = typeof import("./openai");
 type TransformersModule = typeof import("./transformers");
 export {
  FieldOptions,
@@ -14,7 +22,39 @@ export {
  EmbeddingFunctionConstructor,
 } from "./embedding_function";
-export * from "./registry";
+export {
  EmbeddingFunctionRegistry,
  parseEmbeddingMetadata,
  register,
 } from "./registry";
 export type {
  CreateReturnType,
  EmbeddingFunctionConfig,
  EmbeddingFunctionCreate,
  EmbeddingMetadataEntry,
  ResolvedEmbeddingFunctionConfig,
 } from "./registry";
 function initializeBuiltInProviders() {
  const { OpenAIEmbeddingFunction } = require("./openai") as OpenAIModule;
  const { TransformersEmbeddingFunction } =
    require("./transformers") as TransformersModule;
  registerBuiltIn("openai", OpenAIEmbeddingFunction);
  registerBuiltIn("huggingface", TransformersEmbeddingFunction);
 }
 /**
 * Get the global embedding function registry.
 *
 * LanceDB built-in providers are initialized when this public API is first
 * used, so importing the root package does not change automatic search
 * selection for tables without embedding metadata.
 */
 export function getRegistry(): EmbeddingFunctionRegistry {
  initializeBuiltInProviders();
  return getGlobalRegistry();
 }
 /**
 * Create a schema with embedding functions.
@@ -5,14 +5,13 @@ import type OpenAI from "openai";
 import type { EmbeddingCreateParams } from "openai/resources/index";
 import { Float, Float32 } from "../arrow";
 import { EmbeddingFunction } from "./embedding_function";
-import { register } from "./registry";
+import { registerBuiltIn } from "./registry";
 export type OpenAIOptions = {
  apiKey: string;
  model: EmbeddingCreateParams["model"];
 };
@register("openai")
 export class OpenAIEmbeddingFunction extends EmbeddingFunction<
  string,
  Partial<OpenAIOptions>
@@ -100,3 +99,5 @@ export class OpenAIEmbeddingFunction extends EmbeddingFunction<
    return response.data[0].embedding;
  }
 }
 registerBuiltIn("openai", OpenAIEmbeddingFunction);
@@ -7,6 +7,10 @@ import {
 } from "./embedding_function";
 import "reflect-metadata";
 const builtInFunctionsKey = Symbol.for(
  "@lancedb/lancedb::embedding-built-in-functions::v1",
 );
 export type CreateReturnType<T> = T extends { init: () => Promise<void> }
  ? Promise<T>
  : T;
@@ -59,6 +63,15 @@ export class EmbeddingFunctionRegistry {
    };
  }
  /** @ignore */
  setBuiltIn<
    T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
  >(name: string, ctor: T): T {
    this.#functions.set(name, ctor);
    Reflect.defineMetadata("lancedb::embedding::name", name, ctor);
    return ctor;
  }
  get<T extends EmbeddingFunction<unknown>>(
    name: string,
  ): EmbeddingFunctionCreate<T> | undefined;
@@ -96,6 +109,7 @@ export class EmbeddingFunctionRegistry {
   */
  reset(this: EmbeddingFunctionRegistry) {
    this.#functions.clear();
    getBuiltInFunctions(this).clear();
  }
  /**
@@ -183,12 +197,56 @@ export class EmbeddingFunctionRegistry {
  }
 }
-const _REGISTRY = new EmbeddingFunctionRegistry();
+function getBuiltInFunctions(registry: EmbeddingFunctionRegistry): Set<string> {
  const registryWithBuiltIns = registry as EmbeddingFunctionRegistry & {
    [key: symbol]: Set<string> | undefined;
  };
  let builtInFunctions = registryWithBuiltIns[builtInFunctionsKey];
  if (builtInFunctions === undefined) {
    builtInFunctions = new Set<string>();
    registryWithBuiltIns[builtInFunctionsKey] = builtInFunctions;
  }
  return builtInFunctions;
 }
 // Server bundlers can load the side-effect embedding entry points and the public
 // embedding API from separate module graphs. Keep their registry shared.
 const registryKey = Symbol.for(
  "@lancedb/lancedb::embedding-function-registry::v1",
 );
 const registryGlobal = globalThis as typeof globalThis & {
  [key: symbol]: EmbeddingFunctionRegistry | undefined;
 };
 function getGlobalRegistry(): EmbeddingFunctionRegistry {
  const existingRegistry = registryGlobal[registryKey];
  if (existingRegistry !== undefined) {
    return existingRegistry;
  }
  const registry = new EmbeddingFunctionRegistry();
  registryGlobal[registryKey] = registry;
  return registry;
 }
 const _REGISTRY = getGlobalRegistry();
 export function register(name?: string) {
  return _REGISTRY.register(name);
 }
 /** @ignore */
 export function registerBuiltIn<
  T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
 >(name: string, ctor: T): T {
  const builtInFunctions = getBuiltInFunctions(_REGISTRY);
  if (builtInFunctions.has(name)) {
    return _REGISTRY.setBuiltIn(name, ctor);
  }
  _REGISTRY.register(name)(ctor);
  builtInFunctions.add(name);
  return ctor;
 }
 /**
 * Utility function to get the global instance of the registry
 * @returns `EmbeddingFunctionRegistry` The global instance of the registry
@@ -3,7 +3,7 @@
 import { Float, Float32 } from "../arrow";
 import { EmbeddingFunction } from "./embedding_function";
-import { register } from "./registry";
+import { registerBuiltIn } from "./registry";
 export type XenovaTransformerOptions = {
  /** The wasm compatible model to use */
@@ -31,7 +31,6 @@ export type XenovaTransformerOptions = {
  };
 };
@register("huggingface")
 export class TransformersEmbeddingFunction extends EmbeddingFunction<
  string,
  Partial<XenovaTransformerOptions>
@@ -158,6 +157,8 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
  }
 }
 registerBuiltIn("huggingface", TransformersEmbeddingFunction);
 const tensorDiv = (
  src: import("@huggingface/transformers").Tensor,
  divBy: number,
@@ -94,15 +94,12 @@ export {
  RenameTableOptions,
 } from "./connection";
-export {
+export { JobFailureInfo, JobInfo, Session } from "./native.js";
-  Job,
+
-  JobDescription,
+export { Job, JobEventsOptions } from "./job";
  JobFailureInfo,
  JobInfo,
  Session,
 } from "./native.js";
 export {
  AutoQuery,
  ExecutableQuery,
  Query,
  QueryBase,
@@ -0,0 +1,188 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 import { Table as ArrowTable, tableFromIPC } from "apache-arrow";
 import { JobFailureInfo, Job as NativeJob } from "./native";
 /** Which of a job's events {@link Job.events} returns. */
 export interface JobEventsOptions {
  /** Maximum event rows to return, up to the server maximum of 10,000. */
  limit?: number;
  /** SQL-like filter over the event columns. */
  filter?: string;
 }
 /**
 * A handle to an operation that may still be running.
 *
 * The operation may already be complete when the handle is created.
 *
 * The detail getters read what the handle last observed. Submitting an
 * operation returns only a job id, so populating them eagerly would cost an
 * extra round trip on every call:
 *
 * - {@link Job.refresh} and {@link Job.status} fetch the whole record.
 * - {@link Job.wait} records the terminal state it establishes, but not the
 *   rest of the record.
 * - Everything is null until one of those runs.
 *
 * @hideconstructor
 */
 export class Job {
  private readonly inner: NativeJob;
  constructor(inner: NativeJob) {
    this.inner = inner;
  }
  /**
   * Identifies the operation on the server that is running it.
   *
   * Operations that run in this process have no server id. The value is
   * opaque: parsing it or storing it to resume the job later is not supported.
   */
  get id(): string | null {
    return this.inner.id ?? null;
  }
  /** The last observed lifecycle state, without contacting the backend. */
  get state(): string | null {
    return this.inner.state ?? null;
  }
  /**
   * The job's type, as the server names it. Null for an in-process job, which
   * has no server-side record.
   */
  get jobType(): string | null {
    return this.inner.jobType ?? null;
  }
  /** When the job was created, in milliseconds since the epoch. */
  get creationMs(): number | null {
    return this.inner.creationMs ?? null;
  }
  /** The job-type-specific specification it was submitted with. */
  // biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
  get spec(): any | null {
    return parseJson(this.inner.specJson);
  }
  /**
   * The job-type-specific terminal result. Null until the job succeeds, so a
   * job that never terminates reports its progress through {@link Job.events}
   * instead.
   */
  // biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
  get result(): any | null {
    return parseJson(this.inner.resultJson);
  }
  /** Why the job failed, when it failed and the server reports a reason. */
  get failure(): JobFailureInfo | null {
    return this.inner.failure ?? null;
  }
  /**
   * The operation's current lifecycle state: "running", "finished", "failed",
   * or "cancelled".
   *
   * A point snapshot; unlike {@link Job.wait} it does not block or reject on a
   * terminal failure state. Also refreshes the getters above.
   */
  async status(): Promise<string> {
    return this.inner.status();
  }
  /** Wait until the operation reaches a terminal state. */
  async wait(): Promise<void> {
    return this.inner.wait();
  }
  /** Request cancellation. Cancelling a finished operation is a no-op. */
  async cancel(): Promise<void> {
    return this.inner.cancel();
  }
  /**
   * Ask the backend for this job's current state, and for a server-side job
   * its full record, then cache it for the getters above.
   */
  async refresh(): Promise<void> {
    return this.inner.refresh();
  }
  /**
   * This job's recorded lifecycle events.
   *
   * Where the getters above report a terminal result only once the job reaches
   * one, events are written as the job runs and outlive the workers that
   * produced them. A distributed job records a `claim`/`claim_complete` pair
   * per unit of work, each carrying `rows_processed`, so a job that never
   * finishes still accounts for what it did.
   *
   * The server caps results at 1000 rows by default and 10,000 at most, and
   * truncates without saying so, so pass `limit` for a job that emits an event
   * per fragment. `filter` is a SQL-like expression over the `state`,
   * `updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
   */
  async events(options?: JobEventsOptions): Promise<ArrowTable> {
    const buf = await this.inner.events(options?.limit, options?.filter);
    if (buf.length === 0) {
      return new ArrowTable();
    }
    return tableFromIPC(buf);
  }
  /**
   * Every field the handle currently knows, one per line, with the JSON
   * payloads indented -- a refresh job's spec and result are the point of
   * printing it.
   */
  toString(): string {
    if (this.state === null) {
      const known = this.id === null ? "" : `id=${JSON.stringify(this.id)}, `;
      return `Job(${known}not refreshed)`;
    }
    const fields: string[] = [];
    if (this.id !== null) {
      fields.push(`id=${JSON.stringify(this.id)}`);
    }
    fields.push(`state=${JSON.stringify(this.state)}`);
    if (this.jobType !== null) {
      fields.push(`jobType=${JSON.stringify(this.jobType)}`);
    }
    if (this.creationMs !== null) {
      fields.push(`creationMs=${this.creationMs}`);
    }
    for (const [name, value] of [
      ["spec", this.spec],
      ["result", this.result],
    ] as const) {
      if (value !== null) {
        fields.push(`${name}=${indentJson(value)}`);
      }
    }
    if (this.failure !== null) {
      fields.push(`failure=${indentJson(this.failure)}`);
    }
    return `Job(${fields.map((field) => `\n${REPR_INDENT}${field},`).join("")}\n)`;
  }
  [Symbol.for("nodejs.util.inspect.custom")](): string {
    return this.toString();
  }
 }
 const REPR_INDENT = "    ";
 // biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
 function indentJson(value: any): string {
  return JSON.stringify(value, null, 4).replace(/\n/g, `\n${REPR_INDENT}`);
 }
 // biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
 function parseJson(raw: string | null | undefined): any | null {
  return raw === null || raw === undefined ? null : JSON.parse(raw);
 }
@@ -19,6 +19,8 @@ export interface MaterializedViewDefinition {
  limit?: number;
  /** Source columns the projections and filter read. */
  inputs: string[];
  /** Namespace holding the source table; empty is the root namespace. */
  sourceNamespace: string[];
 }
 /**
@@ -78,7 +80,8 @@ export function definitionFromMetadata(
  }
  // biome-ignore lint/suspicious/noExplicitAny: raw JSON
  const value: any = JSON.parse(raw);
-  if (value.kind !== "select") {
+  // "namespaced_select" keeps older readers from resolving the source at root.
  if (value.kind !== "select" && value.kind !== "namespaced_select") {
    throw new Error(
      `materialized view '${name}' is defined by '${value.kind}', which this ` +
        "version of lancedb cannot refresh",
@@ -103,6 +106,7 @@ export function definitionFromMetadata(
    filter: value.filter ?? undefined,
    limit,
    inputs: value.inputs ?? [],
    sourceNamespace: value.source_namespace ?? [],
  };
 }
@@ -100,6 +100,29 @@ export interface FullTextSearchOptions {
  columns?: string | string[];
 }
 function nearestToNative(
  inner: NativeQuery,
  vector: Awaited<IntoVector>,
 ): NativeVectorQuery {
  const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector);
  if (raw) {
    return inner.nearestToRaw(raw.data, raw.dtype);
  }
  return inner.nearestTo(Float32Array.from(vector as number[]));
 }
 function addQueryVectorToNative(
  inner: NativeVectorQuery,
  vector: Awaited<IntoVector>,
 ) {
  const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector);
  if (raw) {
    inner.addQueryVectorRaw(raw.data, raw.dtype);
  } else {
    inner.addQueryVector(Float32Array.from(vector as number[]));
  }
 }
 /** Common methods supported by all query types
 *
 * @see {@link Query}
@@ -111,13 +134,15 @@ export class QueryBase<
  NativeQueryType extends NativeQuery | NativeVectorQuery | NativeTakeQuery,
 > implements AsyncIterable<RecordBatch>
 {
  protected inner!: NativeQueryType | Promise<NativeQueryType>;
  /**
   * @hidden
   */
-  protected constructor(
+  protected constructor(inner?: NativeQueryType | Promise<NativeQueryType>) {
-    protected inner: NativeQueryType | Promise<NativeQueryType>,
+    if (inner !== undefined) {
-  ) {
+      this.inner = inner;
-    // intentionally empty
+    }
  }
  // call a function on the inner (either a promise or the actual object)
@@ -135,6 +160,15 @@ export class QueryBase<
    }
  }
  /**
   * Return the native query used by the next terminal operation.
   *
   * @hidden
   */
  protected async getInner(): Promise<NativeQueryType> {
    return this.inner;
  }
  /**
   * Return only the specified columns.
   *
@@ -207,16 +241,11 @@ export class QueryBase<
  /**
   * @hidden
   */
-  protected nativeExecute(
+  protected async nativeExecute(
    options?: Partial<QueryExecutionOptions>,
  ): Promise<NativeBatchIterator> {
-    if (this.inner instanceof Promise) {
+    const inner = await this.getInner();
-      return this.inner.then((inner) =>
+    return inner.execute(options?.maxBatchLength, options?.timeoutMs);
        inner.execute(options?.maxBatchLength, options?.timeoutMs),
      );
    } else {
      return this.inner.execute(options?.maxBatchLength, options?.timeoutMs);
    }
  }
  /**
@@ -245,12 +274,7 @@ export class QueryBase<
  /** Collect the results as an Arrow @see {@link ArrowTable}. */
  async toArrow(options?: Partial<QueryExecutionOptions>): Promise<ArrowTable> {
    const batches = [];
-    let inner;
+    const inner = await this.getInner();
    if (this.inner instanceof Promise) {
      inner = await this.inner;
    } else {
      inner = this.inner;
    }
    for await (const batch of new RecordBatchIterable(inner, options)) {
      batches.push(batch);
    }
@@ -279,11 +303,8 @@ export class QueryBase<
   * @returns A Promise that resolves to a string containing the query execution plan explanation.
   */
  async explainPlan(verbose = false): Promise<string> {
-    if (this.inner instanceof Promise) {
+    const inner = await this.getInner();
-      return this.inner.then((inner) => inner.explainPlan(verbose));
+    return inner.explainPlan(verbose);
    } else {
      return this.inner.explainPlan(verbose);
    }
  }
  /**
@@ -321,13 +342,8 @@ export class QueryBase<
    distributedMetrics?: AnalyzePlanDistributedMetrics,
  ): Promise<string> {
    const distributedMetricsMode = distributedMetrics ?? "aggregate";
-    if (this.inner instanceof Promise) {
+    const inner = await this.getInner();
-      return this.inner.then((inner) =>
+    return inner.analyzePlan(distributedMetricsMode);
        inner.analyzePlan(distributedMetricsMode),
      );
    } else {
      return this.inner.analyzePlan(distributedMetricsMode);
    }
  }
  /**
@@ -339,12 +355,8 @@ export class QueryBase<
   * @returns An Arrow Schema describing the output columns.
   */
  async outputSchema(): Promise<import("./arrow").Schema> {
-    let schemaBuffer: Buffer;
+    const inner = await this.getInner();
-    if (this.inner instanceof Promise) {
+    const schemaBuffer = await inner.outputSchema();
      schemaBuffer = await this.inner.then((inner) => inner.outputSchema());
    } else {
      schemaBuffer = await this.inner.outputSchema();
    }
    const schema = tableFromIPC(schemaBuffer).schema;
    return schema;
  }
@@ -356,7 +368,7 @@ export class StandardQueryBase<
  extends QueryBase<NativeQueryType>
  implements ExecutableQuery
 {
-  constructor(inner: NativeQueryType | Promise<NativeQueryType>) {
+  constructor(inner?: NativeQueryType | Promise<NativeQueryType>) {
    super(inner);
  }
@@ -510,6 +522,13 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
    super(inner);
  }
  /**
   * @hidden
   */
  protected doVectorCall(fn: (inner: NativeVectorQuery) => void) {
    super.doCall(fn);
  }
  /**
   * Set the number of partitions to search (probe)
   *
@@ -537,7 +556,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
   * the minimum and maximum to the same value.
   */
  nprobes(nprobes: number): VectorQuery {
-    super.doCall((inner) => inner.nprobes(nprobes));
+    this.doVectorCall((inner) => inner.nprobes(nprobes));
    return this;
  }
@@ -551,7 +570,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
   * but will also increase latency.
   */
  minimumNprobes(minimumNprobes: number): VectorQuery {
-    super.doCall((inner) => inner.minimumNprobes(minimumNprobes));
+    this.doVectorCall((inner) => inner.minimumNprobes(minimumNprobes));
    return this;
  }
@@ -565,7 +584,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
   * potential false negatives.
   */
  maximumNprobes(maximumNprobes: number): VectorQuery {
-    super.doCall((inner) => inner.maximumNprobes(maximumNprobes));
+    this.doVectorCall((inner) => inner.maximumNprobes(maximumNprobes));
    return this;
  }
@@ -578,7 +597,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
   * `undefined` means no lower or upper bound.
   */
  distanceRange(lowerBound?: number, upperBound?: number): VectorQuery {
-    super.doCall((inner) => inner.distanceRange(lowerBound, upperBound));
+    this.doVectorCall((inner) => inner.distanceRange(lowerBound, upperBound));
    return this;
  }
@@ -592,7 +611,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
   * also increase the latency of your query. The default value is 1.5*limit.
   */
  ef(ef: number): VectorQuery {
-    super.doCall((inner) => inner.ef(ef));
+    this.doVectorCall((inner) => inner.ef(ef));
    return this;
  }
@@ -606,7 +625,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
   * whose data type is a fixed-size-list of floats.
   */
  column(column: string): VectorQuery {
-    super.doCall((inner) => inner.column(column));
+    this.doVectorCall((inner) => inner.column(column));
    return this;
  }
@@ -627,7 +646,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
  distanceType(
    distanceType: Required<IvfPqOptions>["distanceType"],
  ): VectorQuery {
-    super.doCall((inner) => inner.distanceType(distanceType));
+    this.doVectorCall((inner) => inner.distanceType(distanceType));
    return this;
  }
@@ -661,7 +680,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
   * distance between the query vector and the actual uncompressed vector.
   */
  refineFactor(refineFactor: number): VectorQuery {
-    super.doCall((inner) => inner.refineFactor(refineFactor));
+    this.doVectorCall((inner) => inner.refineFactor(refineFactor));
    return this;
  }
@@ -686,7 +705,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
   * factor can often help restore some of the results lost by post filtering.
   */
  postfilter(): VectorQuery {
-    super.doCall((inner) => inner.postfilter());
+    this.doVectorCall((inner) => inner.postfilter());
    return this;
  }
@@ -700,7 +719,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
   * calculate your recall to select an appropriate value for nprobes.
   */
  bypassVectorIndex(): VectorQuery {
-    super.doCall((inner) => inner.bypassVectorIndex());
+    this.doVectorCall((inner) => inner.bypassVectorIndex());
    return this;
  }
@@ -708,43 +727,39 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
   * Add a query vector to the search
   *
   * This method can be called multiple times to add multiple query vectors
-   * to the search. If multiple query vectors are added, then they will be searched
+   * to the search. A column called `query_index` will be added to indicate the index
-   * in parallel, and the results will be concatenated. A column called `query_index`
+   * of the query vector that produced the result. Flat searches share one table scan
-   * will be added to indicate the index of the query vector that produced the result.
+   * across the query vectors, avoiding the scan and memory amplification of running
-   *
+   * multiple queries concurrently. Indexed searches may still perform per-vector
-   * Performance wise, this is equivalent to running multiple queries concurrently.
+   * index work.
   */
  addQueryVector(vector: IntoVector): VectorQuery {
    if (vector instanceof Promise) {
      // Observe the promise as soon as it is accepted. The existing native
      // query may still be pending, and delaying observation until it resolves
      // can otherwise surface a fast rejection as unhandled.
      const settledVector = vector.then(
        (value) => ({ status: "fulfilled" as const, value }),
        (reason) => ({ status: "rejected" as const, reason }),
      );
      const res = (async () => {
-        try {
+        const inner = await this.getInner();
-          const v = await vector;
+        const outcome = await settledVector;
-          // biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
+        if (outcome.status === "rejected") {
-          const value: any = this.addQueryVector(v);
+          throw outcome.reason;
          const inner = value.inner as
            | NativeVectorQuery
            | Promise<NativeVectorQuery>;
          return inner;
        } catch (e) {
          return Promise.reject(e);
        }
        addQueryVectorToNative(inner, outcome.value);
        return inner;
      })();
      return new VectorQuery(res);
    } else {
-      super.doCall((inner) => {
+      this.doVectorCall((inner) => addQueryVectorToNative(inner, vector));
        const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector);
        if (raw) {
          inner.addQueryVectorRaw(raw.data, raw.dtype);
        } else {
          inner.addQueryVector(Float32Array.from(vector as number[]));
        }
      });
      return this;
    }
  }
  rerank(reranker: Reranker): VectorQuery {
-    super.doCall((inner) =>
+    this.doVectorCall((inner) =>
      inner.rerank(async (args) => {
        const vecResults = await fromBufferToRecordBatch(args.vecResults);
        const ftsResults = await fromBufferToRecordBatch(args.ftsResults);
@@ -763,6 +778,71 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
  }
 }
 /**
 * Create a string query whose vector/FTS routing is resolved against the active
 * table schema when the query executes.
 *
 * @hidden
 */
 export function createAutoQuery(
  table: NativeTable,
  query: string,
  columns: string[] | null,
  getVector: (metadata: string) => Promise<Awaited<IntoVector>>,
 ): AutoQuery {
  type RouteSnapshot = {
    table: NativeTable;
    embeddingMetadata: string | undefined;
  };
  type CachedPreparation = {
    metadata: string;
    vector: Promise<Awaited<IntoVector>>;
  };
  let cachedPreparation: CachedPreparation | undefined;
  const snapshotRoute = async (): Promise<RouteSnapshot> => {
    const snapshot = await table.querySnapshot();
    const schema = tableFromIPC(await snapshot.schema()).schema;
    return {
      table: snapshot,
      embeddingMetadata: schema.metadata.get("embedding_functions"),
    };
  };
  const createInner = async (): Promise<NativeQuery | NativeVectorQuery> => {
    const route = await snapshotRoute();
    if (route.embeddingMetadata === undefined) {
      const inner = route.table.query();
      inner.fullTextSearch({ query, columns });
      return inner;
    }
    const metadata = route.embeddingMetadata;
    if (cachedPreparation?.metadata !== metadata) {
      cachedPreparation = {
        metadata,
        vector: Promise.resolve().then(() => getVector(metadata)),
      };
    }
    const preparation = cachedPreparation;
    let vector: Awaited<IntoVector>;
    try {
      vector = await preparation.vector;
    } catch (error) {
      if (cachedPreparation === preparation) {
        cachedPreparation = undefined;
      }
      throw error;
    }
    return nearestToNative(route.table.query(), vector);
  };
  return new AutoQuery(createInner);
 }
 /**
 * A query that returns a subset of the rows in the table.
 *
@@ -788,6 +868,51 @@ export class TakeQuery extends QueryBase<NativeTakeQuery> {
  }
 }
 /**
 * A builder for automatic string searches.
 *
 * Automatic search determines whether to use full-text or vector search from
 * the table revision selected for each execution. This builder exposes the
 * common operations supported by both query families.
 *
 * @hideconstructor
 */
 export class AutoQuery extends StandardQueryBase<
  NativeQuery | NativeVectorQuery
 > {
  private readonly calls: Array<
    (inner: NativeQuery | NativeVectorQuery) => void
  > = [];
  /** @hidden */
  constructor(
    private readonly createInner: () => Promise<
      NativeQuery | NativeVectorQuery
    >,
  ) {
    super();
  }
  /** @hidden */
  protected override doCall(
    fn: (inner: NativeQuery | NativeVectorQuery) => void,
  ) {
    this.calls.push(fn);
  }
  /** @hidden */
  protected override async getInner(): Promise<
    NativeQuery | NativeVectorQuery
  > {
    const calls = [...this.calls];
    const inner = await this.createInner();
    for (const call of calls) {
      call(inner);
    }
    return inner;
  }
 }
 /** A builder for LanceDB queries.
 *
 * @see {@link Table#query}, {@link Table#search}
@@ -840,45 +965,19 @@ export class Query extends StandardQueryBase<NativeQuery> {
   * a default `limit` of 10 will be used.  @see {@link Query#limit}
   */
  nearestTo(vector: IntoVector): VectorQuery {
-    const callNearestTo = (
+    const inner = this.inner;
-      inner: NativeQuery,
+    if (inner instanceof Promise) {
-      resolved: Float32Array | Float64Array | Uint8Array | number[],
+      const nativeQuery = inner.then(async (resolvedInner) =>
-    ): NativeVectorQuery => {
+        nearestToNative(resolvedInner, await vector),
-      const raw = Array.isArray(resolved)
+      );
        ? null
        : extractVectorBuffer(resolved);
      if (raw) {
        return inner.nearestToRaw(raw.data, raw.dtype);
      }
      return inner.nearestTo(Float32Array.from(resolved as number[]));
    };
    if (this.inner instanceof Promise) {
      const nativeQuery = this.inner.then(async (inner) => {
        const resolved = vector instanceof Promise ? await vector : vector;
        return callNearestTo(inner, resolved);
      });
      return new VectorQuery(nativeQuery);
    }
    if (vector instanceof Promise) {
-      const res = (async () => {
+      return new VectorQuery(
-        try {
+        vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)),
-          const v = await vector;
+      );
          // biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
          const value: any = this.nearestTo(v);
          const inner = value.inner as
            | NativeVectorQuery
            | Promise<NativeVectorQuery>;
          return inner;
        } catch (e) {
          return Promise.reject(e);
        }
      })();
      return new VectorQuery(res);
    } else {
      const vectorQuery = callNearestTo(this.inner, vector);
      return new VectorQuery(vectorQuery);
    }
    return new VectorQuery(nearestToNative(inner, vector));
  }
  nearestToText(query: string | FullTextQuery, columns?: string[]): Query {
@@ -94,17 +94,24 @@ export function sanitizeMetadata(
  if (metadataLike === undefined || metadataLike === null) {
    return undefined;
  }
-  if (!(metadataLike instanceof Map)) {
+
  let entries: IterableIterator<[unknown, unknown]>;
  try {
    entries = Map.prototype.entries.call(metadataLike);
  } catch {
    throw Error("Expected metadata, if present, to be a Map<string, string>");
  }
-  for (const item of metadataLike) {
+
-    if (typeof item[0] !== "string" || typeof item[1] !== "string") {
+  const metadata = new Map<string, string>();
  for (const [key, value] of entries) {
    if (typeof key !== "string" || typeof value !== "string") {
      throw Error(
        "Expected metadata, if present, to be a Map<string, string> but it had non-string keys or values",
      );
    }
    metadata.set(key, value);
  }
-  return metadataLike as Map<string, string>;
+  return metadata;
 }
 export function sanitizeInt(typeLike: object) {
@@ -406,10 +406,11 @@ function matchingFields(fields: Field[], tree: FieldTree): Field[] {
          field.name,
          new Struct(matchingFields(struct.children, value)),
          field.nullable,
          field.metadata,
        ),
      );
    } else {
-      matches.push(new Field(field.name, value as DataType, field.nullable));
+      matches.push(field);
    }
  }
  return matches;
@@ -19,6 +19,7 @@ import {
 import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
 import { IndexOptions } from "./indices";
 import { Job } from "./job";
 import { MergeInsertBuilder } from "./merge";
 import {
  AddColumnsResult,
@@ -30,7 +31,6 @@ import {
  DropColumnsResult,
  IndexConfig,
  IndexStatistics,
  Job,
  LsmStats,
  Branches as NativeBranches,
  OptimizeStats,
@@ -43,10 +43,12 @@ import {
  Table as _NativeTable,
 } from "./native";
 import {
  AutoQuery,
  FullTextQuery,
  Query,
  TakeQuery,
  VectorQuery,
  createAutoQuery,
  instanceOfFullTextQuery,
 } from "./query";
 import { sanitizeType } from "./sanitize";
@@ -523,7 +525,7 @@ export abstract class Table {
    query: string | IntoVector | MultiVector | FullTextQuery,
    queryType?: string,
    ftsColumns?: string | string[],
-  ): VectorQuery | Query;
+  ): VectorQuery | Query | AutoQuery;
  /**
   * Search the table with a given query vector.
   *
@@ -628,6 +630,18 @@ export abstract class Table {
  /**
   * Update per-field (column) metadata.
   *
   * The following keys are treated specially, by convention, and should be
   * used when appropriate:
   *
   * - `lancedb:description`: for a human-readable description of a field.
   * - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
   *   names the tag category; e.g. `lancedb:tag:model: "clip"`.
   * - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
   *   `feature_v2` might be in the same logical column.
   * - `lancedb:status`: for status options (`production`, `candidate`,
   *   `deprecated`, `archived`) to designate the current life cycle state of
   *   this column.
   * @param {FieldMetadataUpdate[]} updates One or more per-field updates. Each
   * update's metadata is merged into the field's existing metadata by default;
   * a value of `null` deletes that key, and `replace: true` swaps the whole map.
@@ -905,6 +919,16 @@ export abstract class Table {
  /** Return the table as an arrow table */
  abstract toArrow(): Promise<ArrowTable>;
  /**
   * Create a {@link MergeInsertBuilder}, which combines new data with the
   * existing table in a single transaction — inserting, updating and deleting
   * rows depending on how they match.
   *
   * @param on - The column, or columns, to match source rows against target
   * rows on. Typically a key or id column. Several columns match on the
   * composite key: a source row updates a target row only when it agrees on
   * every one of them.
   */
  abstract mergeInsert(on: string | string[]): MergeInsertBuilder;
  /** List all the stats of a specified index
@@ -975,10 +999,11 @@ export class LocalTable extends Table {
    return this.inner.display();
  }
-  private async getEmbeddingFunctions(): Promise<
+  private async getEmbeddingFunctions(
-    Map<string, EmbeddingFunctionConfig>
+    inner: _NativeTable = this.inner,
-  > {
+  ): Promise<Map<string, EmbeddingFunctionConfig>> {
-    const schema = await this.schema();
+    const schemaBuf = await inner.schema();
    const schema = tableFromIPC(schemaBuf).schema;
    const registry = getRegistry();
    return registry.parseFunctions(schema.metadata);
  }
@@ -1099,13 +1124,15 @@ export class LocalTable extends Table {
  ): Promise<Job> {
    // biome-ignore lint/suspicious/noExplicitAny: skip
    const nativeIndex = (options?.config as any)?.inner;
-    return await this.inner.createIndexAsync(
+    return new Job(
-      nativeIndex,
+      await this.inner.createIndexAsync(
-      column,
+        nativeIndex,
-      options?.replace,
+        column,
-      options?.waitTimeoutSeconds,
+        options?.replace,
-      options?.name,
+        options?.waitTimeoutSeconds,
-      options?.train,
+        options?.name,
        options?.train,
      ),
    );
  }
@@ -1160,7 +1187,7 @@ export class LocalTable extends Table {
    query: string | IntoVector | MultiVector | FullTextQuery,
    queryType: string = "auto",
    ftsColumns?: string | string[],
-  ): VectorQuery | Query {
+  ): VectorQuery | Query | AutoQuery {
    if (typeof query !== "string" && !instanceOfFullTextQuery(query)) {
      if (queryType === "fts") {
        throw new Error("Cannot perform full text search on a vector query");
@@ -1175,14 +1202,28 @@ export class LocalTable extends Table {
      });
    }
-    // The query type is auto or vector
+    if (queryType === "auto") {
-    // fall back to full text search if no embedding functions are defined and the query is a string
+      if (instanceOfFullTextQuery(query)) {
-    if (
+        return this.query().fullTextSearch(query, {
-      queryType === "auto" &&
+          columns: ftsColumns,
-      (getRegistry().length() === 0 || instanceOfFullTextQuery(query))
+        });
-    ) {
+      }
-      return this.query().fullTextSearch(query, {
+
-        columns: ftsColumns,
+      const columns =
        typeof ftsColumns === "string" ? [ftsColumns] : (ftsColumns ?? null);
      return createAutoQuery(this.inner, query, columns, async (metadata) => {
        const functions = await getRegistry().parseFunctions(
          new Map([["embedding_functions", metadata]]),
        );
        // TODO: Support multiple embedding functions
        const embeddingFunc: EmbeddingFunctionConfig | undefined = functions
          .values()
          .next().value;
        // The route only calls this callback when embedding metadata exists.
        // parseFunctions either yields a provider or reports malformed metadata.
        if (!embeddingFunc)
          throw new Error("Invalid embedding function metadata");
        return await embeddingFunc.function.computeQueryEmbeddings(query);
      });
    }
@@ -1274,7 +1315,7 @@ export class LocalTable extends Table {
  }
  async refreshColumnAsync(column: string): Promise<Job> {
-    return await this.inner.refreshColumnAsync(column);
+    return new Job(await this.inner.refreshColumnAsync(column));
  }
  async refreshMaterializedView(
@@ -1538,7 +1579,8 @@ export interface FieldMetadataUpdate {
  path: string;
  /**
   * Metadata key/value pairs. Merged into the field's existing metadata by
-   * default; a value of `null` deletes that key.
+   * default; a value of `null` deletes that key. See
   * {@link Table.updateFieldMetadata} for the conventional `lancedb:*` keys.
   */
  metadata: Record<string, string | null>;
  /** If true, replace the field's entire metadata map instead of merging. */
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.38.0-beta.9",
+	"version": "0.39.0-beta.4",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.38.0-beta.9",
+	"version": "0.39.0-beta.4",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.38.0-beta.9",
+	"version": "0.39.0-beta.4",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.38.0-beta.9",
+	"version": "0.39.0-beta.4",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.38.0-beta.9",
+	"version": "0.39.0-beta.4",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",
@@ -1,6 +1,6 @@
 {
  "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.38.0-beta.9",
+  "version": "0.39.0-beta.4",
  "os": [
    "win32"
  ],
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.38.0-beta.9",
+	"version": "0.39.0-beta.4",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",
@@ -11,7 +11,7 @@
    "ann"
  ],
  "private": false,
-  "version": "0.38.0-beta.9",
+  "version": "0.39.0-beta.4",
  "main": "dist/index.js",
  "exports": {
    ".": "./dist/index.js",
@@ -44,7 +44,7 @@
    "@biomejs/biome": "^1.7.3",
    "@jest/globals": "^29.7.0",
    "@napi-rs/cli": "3.7.0",
-    "@opentelemetry/sdk-metrics": "^1.30.0",
+    "@opentelemetry/sdk-metrics": "^2.10.0",
    "@types/axios": "^0.14.0",
    "@types/jest": "^29.1.2",
    "@types/node": "22.7.4",
@@ -56,7 +56,7 @@
    "eslint": "^8.57.0",
    "jest": "^29.7.0",
    "shx": "^0.3.4",
-    "tmp": "^0.2.3",
+    "tmp": "^0.2.7",
    "ts-jest": "^29.1.2",
    "typedoc": "0.26.4",
    "typedoc-plugin-markdown": "4.2.1",
@@ -67,7 +67,7 @@
    "timeout": "3m"
  },
  "engines": {
-    "node": ">= 18"
+    "node": ">= 22"
  },
  "packageManager": "pnpm@11.1.1",
  "cpu": ["x64", "arm64"],
@@ -101,7 +101,7 @@
    "openai": "4.29.2"
  },
  "peerDependencies": {
-    "@types/node": ">=18",
+    "@types/node": ">=22",
    "apache-arrow": ">=15.0.0 <=18.1.0"
  },
  "peerDependenciesMeta": {
@@ -16,3 +16,41 @@ allowBuilds:
  onnxruntime-node: true
  protobufjs: true
  sharp: true
 minimumReleaseAgeExclude:
  - protobufjs@7.5.8
  - tmp@0.2.6
  - form-data@4.0.6
  - tar@7.5.16
  - markdown-it@14.1.2
  - linkify-it@5.0.1
  - js-yaml@3.15.0
  - js-yaml@4.1.2
  - protobufjs@7.6.1
  - protobufjs@7.6.3
  - '@babel/core@7.29.1'
  - axios@1.18.0
  - brace-expansion@2.1.2
  - brace-expansion@1.1.16
  - js-yaml@4.3.0
  - tar@7.5.18
  - tar@7.5.19
  - tar@7.5.17
  - protobufjs@7.6.5
  - linkify-it@5.0.2
  - sharp@0.35.0
  - brace-expansion@1.1.17
  - brace-expansion@2.1.3
  - brace-expansion@2.1.4
  - brace-expansion@1.1.18
  - js-yaml@3.15.1
  - js-yaml@4.3.1
  - tar@7.5.21
  - '@opentelemetry/core@2.8.0'
 # @huggingface/transformers pins sharp ^0.33.5 and no released version has moved
 # past ^0.34.5, all of which inherit the libvips CVEs in GHSA-f88m-g3jw-g9cj.
 # Force the patched line. sharp is only reached by transformers' image pipeline,
 # which LanceDB's text embedding function never uses.
 overrides:
  sharp: ^0.35.4
@@ -442,13 +442,15 @@ impl Connection {
        self.get_inner()?.drop_all_tables(&ns).await.default_error()
    }
-    /// A `Job` handle for a server-side job by id.
+    /// Open a server-side job by id, returning a handle with its record
    /// already populated. Rejects when the server has no such job.
    ///
-    /// The handle is constructed without a server round trip; an unknown id
+    /// The returned handle answers for its own state, specification, result,
-    /// surfaces when the handle is used.
+    /// failure and event history, so there is no separate connection-level
-    #[napi]
+    /// call for any of them.
-    pub fn job(&self, job_id: String) -> napi::Result<crate::job::Job> {
+    #[napi(catch_unwind)]
-        let job = self.get_inner()?.job(job_id).default_error()?;
+    pub async fn open_job(&self, job_id: String) -> napi::Result<crate::job::Job> {
        let job = self.get_inner()?.open_job(&job_id).await.default_error()?;
        Ok(crate::job::Job::new(job))
    }
@@ -459,17 +461,6 @@ impl Connection {
        Ok(jobs.into_iter().map(Into::into).collect())
    }
    /// Describe a single server-side job by id. `null` when the server has
    /// no such job.
    #[napi(catch_unwind)]
    pub async fn get_job(
        &self,
        job_id: String,
    ) -> napi::Result<Option<crate::job::JobDescription>> {
        let description = self.get_inner()?.get_job(&job_id).await.default_error()?;
        Ok(description.map(Into::into))
    }
    /// Request cancellation of a server-side job by id. Returns true if the
    /// server accepted the cancellation, false if no such job exists.
    #[napi(catch_unwind)]
@@ -477,34 +468,6 @@ impl Connection {
        self.get_inner()?.cancel_job(&job_id).await.default_error()
    }
    /// The lifecycle event history of a server-side job (all jobs when
    /// `job_id` is null), as an Arrow IPC stream buffer. Empty when there is
    /// no history.
    #[napi(catch_unwind)]
    pub async fn job_history(&self, job_id: Option<String>) -> napi::Result<Buffer> {
        let batches = self
            .get_inner()?
            .job_history(job_id.as_deref())
            .await
            .default_error()?;
        let Some(first) = batches.first() else {
            return Ok(Buffer::from(Vec::<u8>::new()));
        };
        let mut out = Vec::new();
        let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut out, &first.schema())
            .map_err(|e| napi::Error::from_reason(e.to_string()))?;
        for batch in &batches {
            writer
                .write(batch)
                .map_err(|e| napi::Error::from_reason(e.to_string()))?;
        }
        writer
            .finish()
            .map_err(|e| napi::Error::from_reason(e.to_string()))?;
        drop(writer);
        Ok(Buffer::from(out))
    }
    #[napi(catch_unwind)]
    /// Describe a namespace and return its properties.
    pub async fn describe_namespace(
@@ -3,6 +3,9 @@
 use std::sync::Arc;
 use arrow_array::RecordBatch;
 use lancedb::job::JobEventsRequest;
 use napi::bindgen_prelude::Buffer;
 use napi_derive::napi;
 use crate::error::NapiErrorExt;
@@ -55,12 +58,98 @@ impl Job {
    pub async fn cancel(&self) -> napi::Result<()> {
        self.inner.cancel().await.default_error()
    }
    /// Ask the backend for this job's current state, and for a server-side job
    /// its full record, then cache it for the getters below.
    ///
    /// They are all null until this runs, because submitting an operation
    /// returns only a job id. {@link Job.status} fetches the whole record too;
    /// {@link Job.wait} records only the terminal state it establishes.
    #[napi(catch_unwind)]
    pub async fn refresh(&self) -> napi::Result<()> {
        self.inner.refresh().await.default_error()
    }
    /// The last observed lifecycle state, without contacting the backend.
    #[napi(getter)]
    pub fn state(&self) -> Option<String> {
        self.inner.state()
    }
    /// The job's type, as the server names it. Null for an in-process job,
    /// which has no server-side record.
    #[napi(getter)]
    pub fn job_type(&self) -> Option<String> {
        self.inner.job_type()
    }
    /// When the job was created, in milliseconds since the epoch.
    #[napi(getter)]
    pub fn creation_ms(&self) -> Option<i64> {
        self.inner.creation_ms()
    }
    /// The job-type-specific specification as a JSON string, when present.
    #[napi(getter)]
    pub fn spec_json(&self) -> Option<String> {
        self.inner.spec().map(|spec| spec.to_string())
    }
    /// The job-type-specific terminal result as a JSON string. Null until the
    /// job succeeds, so a job that never terminates reports its progress
    /// through {@link Job.events} instead.
    #[napi(getter)]
    pub fn result_json(&self) -> Option<String> {
        self.inner.result().map(|result| result.to_string())
    }
    /// Why the job failed, when it failed and the server reports a reason.
    #[napi(getter)]
    pub fn failure(&self) -> Option<JobFailureInfo> {
        self.inner.failure().map(|failure| JobFailureInfo {
            phase: failure.phase,
            message: failure.message,
            retryable: failure.retryable,
        })
    }
    /// This job's recorded lifecycle events, as an Arrow IPC stream buffer.
    /// The TypeScript wrapper turns it into an Arrow table.
    #[napi(catch_unwind)]
    pub async fn events(&self, limit: Option<u32>, filter: Option<String>) -> napi::Result<Buffer> {
        let batches = self
            .inner
            .events(JobEventsRequest { limit, filter })
            .await
            .default_error()?;
        batches_to_ipc_buffer(&batches)
    }
 }
 /// Serialise Arrow batches as a single IPC stream for the TypeScript layer.
 fn batches_to_ipc_buffer(batches: &[RecordBatch]) -> napi::Result<Buffer> {
    let Some(first) = batches.first() else {
        return Ok(Buffer::from(Vec::<u8>::new()));
    };
    let mut out = Vec::new();
    let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut out, &first.schema())
        .map_err(|e| napi::Error::from_reason(e.to_string()))?;
    for batch in batches {
        writer
            .write(batch)
            .map_err(|e| napi::Error::from_reason(e.to_string()))?;
    }
    writer
        .finish()
        .map_err(|e| napi::Error::from_reason(e.to_string()))?;
    drop(writer);
    Ok(Buffer::from(out))
 }
 /// A row from `Connection.listJobs`: one server-side job.
 #[napi(object)]
 pub struct JobInfo {
-    /// The job id -- what `Connection.getJob` and `Connection.cancelJob`
+    /// The job id -- what `Connection.openJob` and `Connection.cancelJob`
    /// accept.
    pub job_id: String,
    /// The table the job runs against, without URI or namespace.
@@ -91,36 +180,3 @@ pub struct JobFailureInfo {
    pub message: Option<String>,
    pub retryable: Option<bool>,
 }
 /// A described job from `Connection.getJob`.
 #[napi(object)]
 pub struct JobDescription {
    pub job_id: String,
    pub job_type: String,
    /// Lifecycle state: "running", "finished", "failed", or "cancelled".
    pub state: String,
    /// When the job was created, in milliseconds since the epoch.
    pub creation_ms: i64,
    /// The job-type-specific specification as a JSON string, when present.
    pub spec_json: Option<String>,
    /// Why the job failed, when the job is failed and the server reports a
    /// reason.
    pub failure: Option<JobFailureInfo>,
 }
 impl From<lancedb::database::JobDescription> for JobDescription {
    fn from(description: lancedb::database::JobDescription) -> Self {
        Self {
            job_id: description.job_id,
            job_type: description.job_type,
            state: description.state,
            creation_ms: description.creation_ms,
            spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
            failure: description.failure.map(|failure| JobFailureInfo {
                phase: failure.phase,
                message: failure.message,
                retryable: failure.retryable,
            }),
        }
    }
 }
@@ -664,7 +664,11 @@ impl JsFullTextQuery {
 }
 fn parse_fts_query(query: Object) -> napi::Result<FullTextSearchQuery> {
-    if let Ok(Some(query)) = query.get::<&JsFullTextQuery>("query") {
+    // `&JsFullTextQuery` recovers a native class reference through napi's borrow-tracked
    // path, which is only usable from generated `#[napi]` argument conversion. This is a
    // manual lookup on a nested `Object` property instead, so use `ClassInstance`, which
    // unwraps the class without requiring a borrow scope.
    if let Ok(Some(query)) = query.get::<ClassInstance<JsFullTextQuery>>("query") {
        Ok(FullTextSearchQuery::new_query(query.inner.clone()))
    } else if let Ok(Some(query_text)) = query.get::<String>("query") {
        let mut query_text = query_text;
@@ -278,6 +278,13 @@ impl Table {
        Ok(Query::new(self.inner_ref()?.query()))
    }
    /// Return a read-only table handle pinned to the current query revision.
    #[napi(catch_unwind)]
    pub async fn query_snapshot(&self) -> napi::Result<Self> {
        let snapshot = self.inner_ref()?.query_snapshot().await.default_error()?;
        Ok(Self::new(snapshot))
    }
    #[napi(catch_unwind)]
    pub fn take_offsets(&self, offsets: Vec<i64>) -> napi::Result<TakeQuery> {
        Ok(TakeQuery::new(
@@ -554,6 +561,12 @@ impl Table {
            .default_error()
    }
    #[napi(catch_unwind)]
    pub async fn checkout_current(&self) -> napi::Result<Self> {
        let table = self.inner_ref()?.checkout_current().await.default_error()?;
        Ok(Self::new(table))
    }
    #[napi(catch_unwind)]
    pub async fn checkout(&self, version: i64) -> napi::Result<()> {
        self.inner_ref()?
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.38.0-beta.9"
+version = "0.39.0-beta.4"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"
@@ -28,7 +28,7 @@ env_logger.workspace = true
 log.workspace = true
 # Maturin enables extension-module mode for Python builds. Keeping it out of
 # Cargo features lets Rust unit tests link against libpython.
-pyo3 = { version = "0.28", features = ["abi3-py310", "chrono"] }
+pyo3 = { version = "0.28", features = ["abi3-py310", "chrono", "uuid"] }
 chrono.workspace = true
 pyo3-async-runtimes = { version = "0.28", features = [
    "attributes",
@@ -40,6 +40,7 @@ serde.workspace = true
 serde_json.workspace = true
 snafu.workspace = true
 tokio.workspace = true
 uuid.workspace = true
 libc = "0.2"
 [build-dependencies]
@@ -101,9 +101,12 @@ azure = ["adlfs>=2024.2.0"]
 [tool.maturin]
 python-source = "python"
 module-name = "lancedb._lancedb"
 # uv installs the project as an editable package before `uv run`, so keep that
 # bootstrap build consistent with `maturin develop`.
 editable-profile = "dev"
 [build-system]
-requires = ["maturin>=1.9.4"]
+requires = ["maturin>=1.10"]
 build-backend = "maturin"
 [tool.ruff.lint]
@@ -136,6 +139,7 @@ include = [
    "python/lancedb/exceptions.py",
    "python/lancedb/background_loop.py",
    "python/lancedb/schema.py",
    "python/lancedb/sql.py",
    "python/lancedb/remote/__init__.py",
    "python/lancedb/remote/errors.py",
    "python/lancedb/embeddings/__init__.py",
@@ -6,7 +6,7 @@ import importlib.metadata
 import os
 from concurrent.futures import ThreadPoolExecutor
 from datetime import timedelta
-from typing import Dict, Optional, Union, Any, List, Iterable
+from typing import Dict, Optional, Union, Any, List, Iterable, TYPE_CHECKING
 __version__ = importlib.metadata.version("lancedb")
@@ -20,9 +20,13 @@ from .db import AsyncConnection, DBConnection, LanceDBConnection
 from .remote import ClientConfig
 from .remote.db import RemoteDBConnection
 from .expr import Expr, col, lit, func
-from .schema import blob, vector, BlobType
+from .schema import blob, vector
 from .job import AsyncJob, Job
 from .sql import AsyncQuery as AsyncSqlQuery
 from .sql import Query as SqlQuery
 from .sql import QueryDescription
 from .functions import (
    AssignmentMapping as AssignmentMapping,
    FunctionArtifactRequest as FunctionArtifactRequest,
    FunctionApplication as FunctionApplication,
    FunctionBinding as FunctionBinding,
@@ -33,6 +37,8 @@ from .functions import (
    UdfDefinition as UdfDefinition,
    udf as udf,
 )
 from .secrets import EnvVarSecret as EnvVarSecret
 from .secrets import SecretInfo as SecretInfo
 from .materialized_view import (
    AsyncMaterializedView,
    MaterializedView,
@@ -49,6 +55,19 @@ from .namespace import (
 )
 if TYPE_CHECKING:
    from lance.blob import BlobType as BlobType
 def __getattr__(name: str):
    if name == "BlobType":
        from .schema import BlobType
        globals()["BlobType"] = BlobType
        return BlobType
    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
 def _check_s3_bucket_with_dots(
    uri: str, storage_options: Optional[Dict[str, str]]
 ) -> None:
@@ -88,6 +107,7 @@ def connect(
    api_key: Optional[str] = None,
    region: str = "us-east-1",
    host_override: Optional[str] = None,
    sql_host_override: Optional[str] = None,
    read_consistency_interval: Optional[timedelta] = None,
    request_thread_pool: Optional[Union[int, ThreadPoolExecutor]] = None,
    client_config: Union[ClientConfig, Dict[str, Any], None] = None,
@@ -116,6 +136,9 @@ def connect(
        The region to use for LanceDB Cloud.
    host_override: str, optional
        The override url for LanceDB Cloud.
    sql_host_override: str, optional
        The remote SQL service endpoint override. The client connects lazily when SQL
        is first executed and retains that connection.
    read_consistency_interval: timedelta, default None
        The interval at which to check for updates to the table from other
        processes. If None, then consistency is not checked. For performance
@@ -179,6 +202,18 @@ def connect(
    ...     },
    ... )
    For Azure Blob Storage, credentials can be passed directly without setting
    environment variables:
    >>> azure_storage_options = {
    ...     "account_name": "some-account",
    ...     "account_key": "some-key",
    ... }
    >>> db = lancedb.connect(  # doctest: +SKIP
    ...     "az://my-container/my-database",
    ...     storage_options=azure_storage_options,
    ... )
    For tests and temporary data, use an in-memory database:
    >>> db = lancedb.connect("memory://")
@@ -245,6 +280,7 @@ def connect(
            api_key,
            region,
            host_override,
            sql_host_override=sql_host_override,
            # TODO: remove this (deprecation warning downstream)
            request_thread_pool=request_thread_pool,
            client_config=client_config,
@@ -387,6 +423,7 @@ def deserialize_conn(
            parsed["api_key"],
            parsed.get("region", "us-east-1"),
            host_override=parsed.get("host_override"),
            sql_host_override=parsed.get("sql_host_override"),
            client_config=parsed.get("client_config"),
            storage_options=storage_options,
        )
@@ -400,6 +437,7 @@ async def connect_async(
    api_key: Optional[str] = None,
    region: str = "us-east-1",
    host_override: Optional[str] = None,
    sql_host_override: Optional[str] = None,
    read_consistency_interval: Optional[timedelta] = None,
    client_config: Optional[Union[ClientConfig, Dict[str, Any]]] = None,
    storage_options: Optional[Dict[str, str]] = None,
@@ -422,6 +460,9 @@ async def connect_async(
        The region to use for LanceDB Cloud.
    host_override: str, optional
        The override url for LanceDB Cloud.
    sql_host_override: str, optional
        The remote SQL service endpoint override. The client connects lazily when SQL
        is first executed and retains that connection.
    read_consistency_interval: timedelta, default None
        The interval at which to check for updates to the table from other
        processes. If None, then consistency is not checked. For performance
@@ -465,6 +506,10 @@ async def connect_async(
    --------
    >>> import lancedb
    >>> azure_storage_options = {
    ...     "account_name": "some-account",
    ...     "account_key": "some-key",
    ... }
    >>> async def doctest_example():
    ...     # For a local directory, provide a path to the database
    ...     db = await lancedb.connect_async("~/.lancedb")
@@ -472,6 +517,11 @@ async def connect_async(
    ...     db = await lancedb.connect_async("s3://my-bucket/lancedb",
    ...                                      storage_options={
    ...                                          "aws_access_key_id": "***"})
    ...     # Azure credentials can also be passed directly
    ...     db = await lancedb.connect_async(
    ...         "az://my-container/my-database",
    ...         storage_options=azure_storage_options,
    ...     )
    ...     # For tests and temporary data, use an in-memory database
    ...     db = await lancedb.connect_async("memory://")
    ...     # Connect to LanceDB cloud
@@ -500,6 +550,7 @@ async def connect_async(
            api_key,
            region,
            host_override,
            sql_host_override,
            read_consistency_interval_secs,
            client_config,
            storage_options,
@@ -522,6 +573,7 @@ __all__ = [
    "connect_namespace_async",
    "AsyncConnection",
    "AsyncJob",
    "AsyncSqlQuery",
    "AsyncLanceNamespaceDBConnection",
    "AsyncTable",
    "FtsToken",
@@ -536,6 +588,8 @@ __all__ = [
    "vector",
    "DBConnection",
    "Job",
    "QueryDescription",
    "SqlQuery",
    "LanceDBConnection",
    "LanceNamespaceDBConnection",
    "LsmWriteSpec",
@@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Optional, Union
 import pyarrow as pa
 from .expr import Expr
-from .schema import blob_v2_column_paths
+from .schema import row_addressable_blob_v2_paths
 from .types import BlobMode, QueryProjection, QueryProjectionSpec
 if TYPE_CHECKING:
@@ -119,7 +119,7 @@ def blob_v2_projection_sources(
    schema: pa.Schema,
    projection: QueryProjection,
 ) -> dict[str, str]:
-    blob_columns = blob_v2_column_paths(schema)
+    blob_columns = row_addressable_blob_v2_paths(schema)
    if not blob_columns:
        return {}
    columns = set(blob_columns)
@@ -140,7 +140,9 @@ def v2_projection_needs_row_id(
 ) -> bool:
    if with_row_id:
        return False
-    return projection_includes_blob_column(projection, blob_v2_column_paths(schema))
+    return projection_includes_blob_column(
        projection, row_addressable_blob_v2_paths(schema)
    )
 def blob_auto_row_id_for_scan(
@@ -270,7 +272,8 @@ def _iter_projection_pairs(
            if isinstance(expr, str):
                yield name, expr
            elif isinstance(expr, Expr):
-                yield name, expr.to_sql()
+                source = expr._column_name()
                yield name, source if source is not None else expr.to_sql()
        return
    for column in projection:
        if isinstance(column, str):
@@ -280,7 +283,8 @@ def _iter_projection_pairs(
            if isinstance(expr, str):
                yield name, expr
            elif isinstance(expr, Expr):
-                yield name, expr.to_sql()
+                source = expr._column_name()
                yield name, source if source is not None else expr.to_sql()
 def _set_blob_column(tbl: pa.Table, output_name: str, blobs: pa.Array) -> pa.Table:
@@ -1,6 +1,7 @@
 from datetime import date, datetime, timedelta
 from decimal import Decimal
 from typing import Dict, List, Optional, Tuple, Any, TypedDict, Union, Literal
 from uuid import UUID
 import pyarrow as pa
@@ -87,6 +88,7 @@ class PyExpr:
    def contains(self, substr: "PyExpr") -> "PyExpr": ...
    def isin(self, values: List["PyExpr"]) -> "PyExpr": ...
    def cast(self, data_type: pa.DataType) -> "PyExpr": ...
    def column_name(self) -> Optional[str]: ...
    def to_sql(self) -> str: ...
 def expr_col(name: str) -> PyExpr: ...
@@ -146,15 +148,25 @@ class Connection(object):
        start_after: Optional[str],
        limit: Optional[int],
    ) -> list[str]: ...  # Deprecated: Use list_tables instead
-    def job(self, job_id: str) -> Job: ...
+    async def open_job(self, job_id: str) -> Job: ...
    async def create_function_async(self, request_json: str) -> Job: ...
    async def get_function(self, name: str, version: str) -> str: ...
    async def list_functions(self) -> List[str]: ...
    async def drop_function(self, name: str, version: str) -> bool: ...
    async def create_secret(self, name: str, value: str) -> None: ...
    async def alter_secret(self, name: str, value: str) -> None: ...
    async def list_secrets(self) -> List[str]: ...
    async def drop_secret(self, name: str) -> None: ...
    async def describe_secret(self, name: str) -> Dict[str, str]: ...
    async def list_jobs(self) -> List[JobInfo]: ...
    async def get_job(self, job_id: str) -> Optional[JobDescription]: ...
    async def cancel_job(self, job_id: str) -> bool: ...
-    async def job_history(
+    async def execute_query_async(
-        self, job_id: Optional[str] = None
+        self,
-    ) -> List[pa.RecordBatch]: ...
+        query: str,
        *,
        default_namespace_path: Optional[List[str]] = None,
    ) -> SqlQuery: ...
    async def describe_query(self, query_id: UUID) -> QueryDescription: ...
    async def create_table(
        self,
        name: str,
@@ -233,9 +245,20 @@ class BlobFile:
 class Job:
    @property
    def id(self) -> Optional[str]: ...
    @property
    def _state(self) -> Optional[str]: ...
    @property
    def _description(self) -> Optional[JobDescription]: ...
    async def status(self) -> str: ...
    async def wait(self) -> Optional[str]: ...
    async def cancel(self) -> None: ...
    async def refresh(self) -> None: ...
    async def events(
        self,
        *,
        limit: Optional[int] = None,
        filter: Optional[str] = None,
    ) -> pa.Table: ...
 class JobInfo:
    @property
@@ -267,10 +290,33 @@ class JobDescription:
    @property
    def creation_ms(self) -> int: ...
    @property
-    def spec_json(self) -> Optional[str]: ...
+    def _spec_json(self) -> Optional[str]: ...
    @property
    def _result_json(self) -> Optional[str]: ...
    @property
    def spec(self) -> Optional[Any]: ...
    @property
    def result(self) -> Optional[Any]: ...
    @property
    def failure(self) -> Optional[JobFailureInfo]: ...
 class SqlQuery:
    @property
    def id(self) -> UUID: ...
    async def describe(self) -> QueryDescription: ...
    async def reader(self) -> RecordBatchStream: ...
    async def cancel(self) -> None: ...
 class QueryDescription:
    @property
    def id(self) -> UUID: ...
    @property
    def status(self) -> str: ...
    @property
    def progress(self) -> Optional[float]: ...
    @property
    def expires_at(self) -> Optional[datetime]: ...
 class Table:
    def name(self) -> str: ...
    def __repr__(self) -> str: ...
@@ -283,6 +329,7 @@ class Table:
        mode: Literal["append", "overwrite"],
        progress: Optional[Any] = None,
        write_parallelism: Optional[int] = None,
        allow_external_blob_outside_bases: bool = False,
    ) -> AddResult: ...
    async def update(
        self, updates: Dict[str, str], where: Optional[str]
@@ -448,6 +495,7 @@ async def connect(
    api_key: Optional[str],
    region: Optional[str],
    host_override: Optional[str],
    sql_host_override: Optional[str],
    read_consistency_interval: Optional[float],
    client_config: Optional[Union[ClientConfig, Dict[str, Any]]],
    storage_options: Optional[Dict[str, str]],
@@ -604,9 +652,11 @@ class FullTextQuery:
 class PyQueryRequest:
    limit: Optional[int]
    offset: Optional[int]
    take_offsets: Optional[List[int]]
    filter: Optional[Union[str, bytes]]
    full_text_search: Optional[FullTextQuery]
    select: Optional[Union[str, List[str]]]
    select_source_columns: Optional[Dict[str, str]]
    fast_search: Optional[bool]
    with_row_id: Optional[bool]
    use_lsm: Optional[bool]
@@ -17,8 +17,10 @@ from typing import (
    List,
    Literal,
    Optional,
    Sequence,
    Union,
 )
 from uuid import UUID
 if sys.version_info >= (3, 12):
    from typing import override
@@ -47,12 +49,16 @@ from . import __version__
 from ._lancedb import connect as lancedb_connect  # type: ignore
 from .functions import FunctionVersion, UdfDefinition
 from .job import AsyncJob, Job, _typed_job
 from .sql import AsyncQuery as AsyncSqlQuery
 from .sql import Query as SqlQuery
 from .sql import QueryDescription
 from .materialized_view import (
    AsyncMaterializedView,
    MaterializedView,
    SelectArg,
    normalize_select,
 )
 from .secrets import EnvVarSecret, SecretInfo, validate_secret_name
 from .table import (
    AsyncTable,
    LanceTable,
@@ -68,10 +74,11 @@ import deprecation
 if TYPE_CHECKING:
    import pyarrow as pa
    from .arrow import AsyncRecordBatchReader
    from .pydantic import LanceModel
    from ._lancedb import Connection as LanceDbConnection
-    from ._lancedb import JobDescription, JobInfo
+    from ._lancedb import JobInfo
    from .common import DATA, URI
    from .embeddings import EmbeddingFunctionConfig
    from ._lancedb import Session
@@ -687,15 +694,47 @@ class DBConnection(EnforceOverrides):
        """
        raise NotImplementedError("serialize is not supported for this connection type")
-    def create_function(self, definition: UdfDefinition) -> FunctionVersion:
+    def create_function(
        self,
        definition: UdfDefinition,
        *,
        secrets: Optional[Sequence[EnvVarSecret]] = None,
    ) -> FunctionVersion:
        """Register a scalar Python UDF and wait for its immutable version.
        This is the blocking counterpart of :meth:`create_function_async`.
        Local connections raise ``NotImplementedError``.
        """
        return self.create_function_async(definition).wait()
-    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
+        Parameters
        ----------
        definition : UdfDefinition
            A callable decorated with [udf][lancedb.udf].
        secrets : sequence of EnvVarSecret, optional
            One [EnvVarSecret][lancedb.secrets.EnvVarSecret] per credential the
            Function needs, each naming a Secret and the environment variable
            its value arrives in. The Function's source is unchanged by this;
            it reads the variable the way it already did.
        Examples
        --------
        ```python
        db.create_secret("openai-prod", os.environ["OPENAI_API_KEY"])
        db.create_function(
            analyze_caption,
            secrets=[
                EnvVarSecret(secret="openai-prod", env_variable="OPENAI_API_KEY")
            ],
        )
        ```
        """
        return self.create_function_async(definition, secrets=secrets).wait()
    def create_function_async(
        self,
        definition: UdfDefinition,
        *,
        secrets: Optional[Sequence[EnvVarSecret]] = None,
    ) -> Job[FunctionVersion]:
        """Register a scalar Python UDF through the remote Function catalog.
        Submission returns a typed job. The immutable Function version becomes
@@ -712,26 +751,107 @@ class DBConnection(EnforceOverrides):
            "Function catalog operations are not supported for this connection type"
        )
-    def job(self, job_id: str) -> Job:
+    def list_functions(self) -> List[FunctionVersion]:
-        """A [Job][lancedb.job.Job] handle for a server-side job by id.
+        """List every published immutable Function version.
-        The handle is constructed without a server round trip; an unknown id
+        Results are ordered by Function name then version. Local connections
-        surfaces when the handle is used. Dropping the handle has no effect
+        raise ``NotImplementedError``.
-        on the job itself.
+
        Examples
        --------
        List the identities available to use in Function-backed columns:
        ```python
        [(function.name, function.version) for function in db.list_functions()]
        ```
        """
-        raise NotImplementedError("job is not supported for this connection type")
+        raise NotImplementedError(
            "Function catalog operations are not supported for this connection type"
        )
    def drop_function(self, name: str, *, version: str) -> bool:
        """Drop one exact immutable Function version from the remote catalog.
        Returns True when the version changed to Dropped and False for an
        idempotent replay. Local connections raise NotImplementedError.
        """
        raise NotImplementedError(
            "Function catalog operations are not supported for this connection type"
        )
    def create_secret(self, name: str, value: str) -> None:
        """Create a named Secret in this database.
        Fails if the name is taken, so a create never silently becomes a
        rotation. Nothing reads the value back: it is bound to a Function by
        name and resolved by the service when that Function runs. Local
        connections raise ``NotImplementedError``.
        """
        raise NotImplementedError(
            "Secret operations are not supported for this connection type"
        )
    def alter_secret(self, name: str, value: str) -> None:
        """Replace the credential behind an existing Secret.
        Fails if it does not exist. Every Function bound to the Secret uses the
        new value from its next job, and no new Function version is created --
        which is how a rotation reaches columns pinned to a version registered
        before it. Local connections raise ``NotImplementedError``.
        """
        raise NotImplementedError(
            "Secret operations are not supported for this connection type"
        )
    def list_secrets(self) -> List[str]:
        """The names of every Secret in this database.
        Names only. No method returns a stored credential, by construction
        rather than by policy. Local connections raise ``NotImplementedError``.
        """
        raise NotImplementedError(
            "Secret operations are not supported for this connection type"
        )
    def drop_secret(self, name: str) -> None:
        """Drop a Secret.
        Functions bound to it fail at their next job, naming the Secret; that
        is the revocation path. The name becomes free to reuse, and a new
        Secret under it is picked up by everything still bound to that name.
        Local connections raise ``NotImplementedError``.
        """
        raise NotImplementedError(
            "Secret operations are not supported for this connection type"
        )
    def describe_secret(self, name: str) -> SecretInfo:
        """What this database records about a Secret: name and timestamps.
        Never the value -- there is no code path that could return one. Local
        connections raise ``NotImplementedError``.
        """
        raise NotImplementedError(
            "Secret operations are not supported for this connection type"
        )
    def open_job(self, job_id: str) -> Job:
        """Open a server-side job by id, returning a handle with its record
        already populated.
        The returned [Job][lancedb.job.Job] answers for its own state,
        specification, result, failure and event history, so there is no
        separate connection-level call for any of them.
        Raises `JobNotFoundError` when the server has no such job, the way
        `open_table` does for a missing table.
        """
        raise NotImplementedError("open_job is not supported for this connection type")
    def list_jobs(self) -> List[JobInfo]:
        """List server-side jobs across the database's tables."""
        raise NotImplementedError("list_jobs is not supported for this connection type")
    def get_job(self, job_id: str) -> Optional[JobDescription]:
        """Describe a single server-side job by id.
        Returns None when the server has no such job.
        """
        raise NotImplementedError("get_job is not supported for this connection type")
    def cancel_job(self, job_id: str) -> bool:
        """Request cancellation of a server-side job by id.
@@ -743,14 +863,38 @@ class DBConnection(EnforceOverrides):
            "cancel_job is not supported for this connection type"
        )
-    def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
+    def execute_query(
-        """The lifecycle event history of a server-side job, as Arrow batches.
+        self,
        query: str,
        *,
        default_namespace_path: Optional[List[str]] = None,
    ) -> pa.RecordBatchReader:
        """Execute SQL and return a blocking Arrow reader.
-        Lists history across all jobs when `job_id` is None.
+        This submits through :meth:`execute_query_async` and waits until the
        initial result stream is readable. It does not wait for the full query
        to finish.
        """
-        raise NotImplementedError(
+        return self.execute_query_async(
-            "job_history is not supported for this connection type"
+            query,
-        )
+            default_namespace_path=default_namespace_path,
        ).reader()
    def execute_query_async(
        self,
        query: str,
        *,
        default_namespace_path: Optional[List[str]] = None,
    ) -> SqlQuery:
        """Start executing SQL and return its query handle.
        Local connections do not support SQL.
        """
        raise NotImplementedError("SQL is not supported for this connection type")
    def describe_query(self, query_id: UUID) -> QueryDescription:
        """Describe a submitted SQL query by its connection-scoped id."""
        raise NotImplementedError("SQL is not supported for this connection type")
 class LanceDBConnection(DBConnection):
@@ -847,6 +991,7 @@ class LanceDBConnection(DBConnection):
                None,
                None,
                None,
                None,
                read_consistency_interval_secs,
                None,
                storage_options,
@@ -1395,37 +1540,59 @@ class LanceDBConnection(DBConnection):
        )
    @override
-    def job(self, job_id: str) -> Job:
+    def open_job(self, job_id: str) -> Job:
-        """A [Job][lancedb.job.Job] handle for a server-side job by id.
+        """Open a server-side job by id. See
-
+        [DBConnection.open_job][lancedb.db.DBConnection.open_job].
        The handle is constructed without a server round trip; an unknown id
        surfaces when the handle is used. Dropping the handle has no effect
        on the job itself.
        """
-        return Job(self._conn.job(job_id))
+        return Job(LOOP.run(self._conn.open_job(job_id)))
    @override
-    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
+    def create_function_async(
-        job = LOOP.run(self._conn.create_function_async(definition))
+        self,
        definition: UdfDefinition,
        *,
        secrets: Optional[Sequence[EnvVarSecret]] = None,
    ) -> Job[FunctionVersion]:
        job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets))
        return Job(job)
    @override
    def get_function(self, name: str, *, version: str) -> FunctionVersion:
        return LOOP.run(self._conn.get_function(name, version=version))
    @override
    def list_functions(self) -> List[FunctionVersion]:
        return LOOP.run(self._conn.list_functions())
    @override
    def drop_function(self, name: str, *, version: str) -> bool:
        return LOOP.run(self._conn.drop_function(name, version=version))
    @override
    def create_secret(self, name: str, value: str) -> None:
        LOOP.run(self._conn.create_secret(name, value))
    @override
    def alter_secret(self, name: str, value: str) -> None:
        LOOP.run(self._conn.alter_secret(name, value))
    @override
    def list_secrets(self) -> List[str]:
        return LOOP.run(self._conn.list_secrets())
    @override
    def drop_secret(self, name: str) -> None:
        LOOP.run(self._conn.drop_secret(name))
    @override
    def describe_secret(self, name: str) -> SecretInfo:
        return LOOP.run(self._conn.describe_secret(name))
    @override
    def list_jobs(self) -> List[JobInfo]:
        """List server-side jobs across the database's tables."""
        return LOOP.run(self._conn.list_jobs())
    @override
    def get_job(self, job_id: str) -> Optional[JobDescription]:
        """Describe a single server-side job by id.
        Returns None when the server has no such job.
        """
        return LOOP.run(self._conn.get_job(job_id))
    @override
    def cancel_job(self, job_id: str) -> bool:
        """Request cancellation of a server-side job by id.
@@ -1436,14 +1603,6 @@ class LanceDBConnection(DBConnection):
        """
        return LOOP.run(self._conn.cancel_job(job_id))
    @override
    def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
        """The lifecycle event history of a server-side job, as Arrow batches.
        Lists history across all jobs when `job_id` is None.
        """
        return LOOP.run(self._conn.job_history(job_id))
    @override
    def namespace_client(self) -> LanceNamespace:
        """Get the equivalent namespace client for this connection.
@@ -2214,46 +2373,85 @@ class AsyncConnection(object):
            namespace_path = []
        await self._inner.drop_all_tables(namespace_path=namespace_path)
-    def job(self, job_id: str) -> AsyncJob:
+    async def open_job(self, job_id: str) -> AsyncJob:
-        """An [AsyncJob][lancedb.job.AsyncJob] handle for a server-side job
+        """Open a server-side job by id. See
-        by id.
+        [DBConnection.open_job][lancedb.db.DBConnection.open_job].
        The handle is constructed without a server round trip; an unknown id
        surfaces when the handle is used. Dropping the handle has no effect
        on the job itself.
        """
-        return AsyncJob(self._inner.job(job_id))
+        return AsyncJob(await self._inner.open_job(job_id))
    async def create_function_async(
-        self, definition: UdfDefinition
+        self,
        definition: UdfDefinition,
        *,
        secrets: Optional[Sequence[EnvVarSecret]] = None,
    ) -> AsyncJob[FunctionVersion]:
        """Register a scalar Python UDF through the remote Function catalog.
        The returned typed job resolves to the immutable Function version.
-        Local connections raise ``NotImplementedError``.
+        ``secrets`` is a sequence of
        [EnvVarSecret][lancedb.secrets.EnvVarSecret], each naming a Secret and
        the environment variable its value arrives in. Local connections raise
        ``NotImplementedError``.
        """
        if not isinstance(definition, UdfDefinition):
            raise TypeError("create_function_async requires a @udf definition")
-        inner = await self._inner.create_function_async(
+        request = definition.bind_secrets(secrets)
-            definition.registration_request.to_canonical_json()
+        inner = await self._inner.create_function_async(request.to_canonical_json())
        )
        return _typed_job(inner, FunctionVersion.from_json)
    async def get_function(self, name: str, *, version: str) -> FunctionVersion:
        """Open one exact immutable Function version from the remote catalog."""
        return FunctionVersion.from_json(await self._inner.get_function(name, version))
    async def list_functions(self) -> List[FunctionVersion]:
        """List every published immutable Function version.
        Results are ordered by Function name then version. Local connections
        raise ``NotImplementedError``.
        """
        return [
            FunctionVersion.from_json(value)
            for value in await self._inner.list_functions()
        ]
    async def drop_function(self, name: str, *, version: str) -> bool:
        """Drop one exact immutable Function version from the remote catalog."""
        return await self._inner.drop_function(name, version)
    async def create_secret(self, name: str, value: str) -> None:
        """Create a named Secret in this database.
        Fails if the name is taken, so a create never silently becomes a
        rotation. Nothing reads the value back.
        """
        await self._inner.create_secret(validate_secret_name(name), value)
    async def alter_secret(self, name: str, value: str) -> None:
        """Replace the credential behind an existing Secret.
        Fails if it does not exist. Bound Functions use the new value from
        their next job, with no new Function version.
        """
        await self._inner.alter_secret(validate_secret_name(name), value)
    async def list_secrets(self) -> List[str]:
        """The names of every Secret in this database. Names only."""
        return await self._inner.list_secrets()
    async def drop_secret(self, name: str) -> None:
        """Drop a Secret. Bound Functions fail at their next job."""
        await self._inner.drop_secret(validate_secret_name(name))
    async def describe_secret(self, name: str) -> SecretInfo:
        """What this database records about a Secret. Never the value."""
        return SecretInfo.from_json(
            await self._inner.describe_secret(validate_secret_name(name))
        )
    async def list_jobs(self) -> List[JobInfo]:
        """List server-side jobs across the database's tables."""
        return await self._inner.list_jobs()
    async def get_job(self, job_id: str) -> Optional[JobDescription]:
        """Describe a single server-side job by id.
        Returns None when the server has no such job.
        """
        return await self._inner.get_job(job_id)
    async def cancel_job(self, job_id: str) -> bool:
        """Request cancellation of a server-side job by id.
@@ -2263,12 +2461,46 @@ class AsyncConnection(object):
        """
        return await self._inner.cancel_job(job_id)
-    async def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
+    async def execute_query(
-        """The lifecycle event history of a server-side job, as Arrow batches.
+        self,
        query: str,
        *,
        default_namespace_path: Optional[List[str]] = None,
    ) -> AsyncRecordBatchReader:
        """Execute SQL and return an asynchronous Arrow reader.
-        Lists history across all jobs when `job_id` is None.
+        This submits through :meth:`execute_query_async` and waits until the
        initial result stream is readable. It does not wait for the full query
        to finish.
        """
-        return await self._inner.job_history(job_id)
+        submitted = await self.execute_query_async(
            query,
            default_namespace_path=default_namespace_path,
        )
        return await submitted.reader()
    async def execute_query_async(
        self,
        query: str,
        *,
        default_namespace_path: Optional[List[str]] = None,
    ) -> AsyncSqlQuery:
        """Start executing SQL and return its query handle.
        The database from ``connect_async`` is used for unqualified database
        references. The namespace defaults to ``["public"]``. Local
        connections raise ``NotImplementedError``.
        """
        return AsyncSqlQuery(
            await self._inner.execute_query_async(
                query,
                default_namespace_path=default_namespace_path,
            )
        )
    async def describe_query(self, query_id: UUID) -> QueryDescription:
        """Describe a submitted SQL query by its connection-scoped id."""
        return await self._inner.describe_query(query_id)
    async def namespace_client(self) -> LanceNamespace:
        """Get the equivalent namespace client for this connection.
@@ -35,3 +35,9 @@ class JobCancelledError(RuntimeError):
    """Exception raised when an asynchronous job was cancelled."""
    pass
 class JobNotFoundError(ValueError):
    """Exception raised when opening a job the server does not have."""
    pass
@@ -249,6 +249,10 @@ class Expr:
    # ── utilities ────────────────────────────────────────────────────────────
    def _column_name(self) -> str | None:
        """Return the source name when this is a bare column expression."""
        return self._inner.column_name()
    def to_sql(self) -> str:
        """Render the expression as a SQL string (useful for debugging)."""
        return self._inner.to_sql()
@@ -312,7 +316,7 @@ def func(name: str, *args: ExprLike) -> Expr:
    --------
    >>> from lancedb.expr import col, func
    >>> func("lower", col("name"))
-    Expr(lower(name))
+    Expr(lower(`name`))
    """
    inner_args = [_coerce(a)._inner for a in args]
    return Expr(expr_func(name, inner_args))
@@ -4,7 +4,7 @@
 """Canonical Function values exchanged with LanceDB Enterprise services.
 These immutable models contain client/wire state only. Catalog persistence,
-environment bake, and execution are owned by Sophon.
+environment bake, secret resolution, and execution are owned by Sophon.
 ``RefreshColumnResult`` is also the backend-neutral result of a local
 expression-backed refresh job.
 """
@@ -25,7 +25,7 @@ import re
 import sys
 import textwrap
 import types
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from datetime import date, datetime
 from typing import (
    Annotated,
@@ -49,11 +49,26 @@ from pydantic import (
    model_validator,
 )
 from .schema import is_blob_v2_field as _is_blob_v2_field
 from .secrets import EnvVarSecret
 _Int32 = conint(strict=True, ge=-(2**31), le=2**31 - 1)
 _UInt32 = conint(strict=True, ge=0, le=2**32 - 1)
 _UInt64 = conint(strict=True, ge=0, le=2**64 - 1)
 def _validate_gpu_wire_marker(value: Any) -> bool:
    if value is not True:
        raise ValueError("runtime.gpu must be true")
    return True
 def _normalize_gpu_marker(value: bool) -> Optional[bool]:
    if not isinstance(value, bool):
        raise ValueError("gpu must be a boolean")
    return True if value else None
 class _FrozenDict(dict):
    def _immutable(self, *args, **kwargs):
        raise TypeError("remote canonical values are immutable")
@@ -222,6 +237,7 @@ class PythonEnvironmentSpec(_RemoteValue):
    kind: str
    packages: tuple[str, ...] = ()
    channels: tuple[str, ...] = ()
    path: Optional[str] = None
    modules: tuple[str, ...] = ()
    image: Optional[str] = None
@@ -238,6 +254,23 @@ class PythonRuntimeSpec(_RemoteValue):
    python_version: Optional[str] = None
    environment: Optional[PythonEnvironmentSpec] = None
    env: Optional[Mapping[str, str]] = None
    gpu: Optional[bool] = None
    @model_validator(mode="before")
    @classmethod
    def _discard_unknown_runtime_payload(cls, value):
        if isinstance(value, Mapping):
            kind = value.get("kind")
            if isinstance(kind, str) and kind not in {"python", "python_v2"}:
                return {"kind": kind}
        return value
    @field_validator("gpu", mode="before")
    @classmethod
    def _validate_gpu_marker(cls, value):
        if value is None:
            return None
        return _validate_gpu_wire_marker(value)
    @model_validator(mode="after")
    def _validate_runtime_kind(self):
@@ -246,18 +279,28 @@ class PythonRuntimeSpec(_RemoteValue):
                raise ValueError("python runtime requires python_version")
            if self.environment is None:
                raise ValueError("python runtime requires environment")
            if self.gpu is not None:
                raise ValueError("python runtime with gpu requires kind='python_v2'")
        elif self.kind == "python_v2":
            if self.python_version is None:
                raise ValueError("python_v2 runtime requires python_version")
            if self.environment is None:
                raise ValueError("python_v2 runtime requires environment")
            if self.gpu is None:
                raise ValueError("python_v2 runtime requires gpu")
        else:
            object.__setattr__(self, "python_version", None)
            object.__setattr__(self, "environment", None)
            object.__setattr__(self, "env", None)
            object.__setattr__(self, "gpu", None)
        return self
 class FunctionVersion(_RemoteValue):
    """An exact immutable Function version returned by Enterprise.
-    Scheduling resources, priority, concurrency, and retry policy belong to
+    The GPU execution requirement is part of this identity. CPU and memory sizing,
-    the submitting Job and are not part of this identity.
+    priority, concurrency, and retry policy belong to the execution platform.
    """
    name: str
@@ -267,6 +310,7 @@ class FunctionVersion(_RemoteValue):
    runtime: PythonRuntimeSpec
    runtime_digest: str
    environment_digest: str
    secret_env_bindings: Mapping[str, str] = {}
    created_at: str
    def __call__(self, **inputs: Any) -> FunctionApplication:
@@ -328,12 +372,18 @@ class FunctionVersion(_RemoteValue):
 class FunctionRegistrationRequest(_RemoteValue):
-    """Stable remote registration envelope produced by :func:`udf`."""
+    """Stable remote registration envelope produced by :func:`udf`.
    Credential values deliberately have no field here. The only secret-shaped
    thing a client sends is ``secret_env_bindings``: the name of a Secret the
    database already holds, which the remote service resolves at execution.
    """
    name: str
    artifact: FunctionArtifactRequest
    signature: FunctionSignature
    runtime: PythonRuntimeSpec
    secret_env_bindings: Mapping[str, str] = {}
 class FunctionVersionRef(_OpenRemoteValue):
@@ -428,11 +478,7 @@ class InputBinding(_RemoteValue):
 class OutputMapping(_RemoteValue):
-    """One stable result-field mapping.
+    """One stable result-field mapping."""
    Assignment state is outside the Slice 1 client contract. During the NULL
    transition Lance exposes no public cell-flag identifier to persist here.
    """
    result_field: str
    output_name: str
@@ -442,6 +488,13 @@ class OutputMapping(_RemoteValue):
    nullable: bool
 class AssignmentMapping(_RemoteValue):
    """Internal physical column preserving flattened struct validity."""
    output_name: str
    output_field_id: _Int32
 class FunctionBinding(_RemoteValue):
    """Immutable Function binding persisted by the Enterprise table service."""
@@ -449,6 +502,7 @@ class FunctionBinding(_RemoteValue):
    function: FunctionVersionRef
    inputs: tuple[InputBinding, ...]
    outputs: tuple[OutputMapping, ...]
    assignment: Optional[AssignmentMapping] = None
    input_schema: Optional[Mapping[str, Any]] = None
    output_schema: Optional[Mapping[str, Any]] = None
@@ -478,6 +532,15 @@ class RefreshColumnResult(_RemoteValue):
 _FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$")
 _DECLARED_SECRET = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
 _FUNCTION_BLOB_V2_TYPE = "blob_v2"
 _ARROW_EXTENSION_NAME_KEY = "ARROW:extension:name"
 _BLOB_V2_EXTENSION_NAME = "lance.blob.v2"
 _NESTED_BLOB_COLLECTION_ERROR = (
    "unsupported Arrow type for Function signature: Blob v2 fields nested under "
    "collection types are not supported"
 )
 _GRAMMAR_PRIMITIVES = (
@@ -494,6 +557,7 @@ _GRAMMAR_PRIMITIVES = (
    (pa.float32(), "float32"),
    (pa.float64(), "float64"),
    (pa.string(), "utf8"),
    (pa.large_string(), "large_utf8"),
    (pa.binary(), "binary"),
    (pa.date32(), "date32"),
    (pa.date64(), "date64"),
@@ -501,31 +565,258 @@ _GRAMMAR_PRIMITIVES = (
 def _canonical_arrow_type(data_type: pa.DataType) -> str:
-    """The server's V1 Function type grammar. Anything outside it is rejected
+    """The compact Function grammar, or canonical exact JSON for nested types."""
-    here rather than at registration."""
+    grammar = _grammar_arrow_type(data_type)
    if grammar is not None:
        return grammar
    exact = _exact_arrow_type(data_type)
    return json.dumps(exact, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
 def _grammar_arrow_type(data_type: pa.DataType) -> Optional[str]:
    for candidate, name in _GRAMMAR_PRIMITIVES:
        if data_type == candidate:
            return name
    if pa.types.is_list(data_type) or pa.types.is_large_list(data_type):
        item = _grammar_list_item(data_type)
        if item is None:
            return None
        prefix = "list" if pa.types.is_list(data_type) else "large_list"
-        return f"{prefix}<{_canonical_list_item(data_type)}>"
+        return f"{prefix}<{item}>"
    if pa.types.is_fixed_size_list(data_type) and data_type.list_size > 0:
-        return (
+        item = _grammar_list_item(data_type)
-            f"fixed_size_list<{_canonical_list_item(data_type)}, {data_type.list_size}>"
+        if item is not None:
-        )
+            return f"fixed_size_list<{item}, {data_type.list_size}>"
-    raise TypeError(f"unsupported Arrow type for Function signature: {data_type}")
+    return None
-def _canonical_list_item(data_type: pa.DataType) -> str:
+def _grammar_list_item(data_type: pa.DataType) -> Optional[str]:
    """The grammar names only the item type; it always means a non-nullable
-    child called `item`, so any other child metadata cannot be represented."""
+    child called `item`, so other child properties require exact JSON."""
    child = data_type.value_field
    if child.name != "item" or child.nullable or child.metadata:
        return None
    return _grammar_arrow_type(child.type)
 def _validate_exact_arrow_field(field: pa.Field) -> None:
    if not field.name:
        raise TypeError(
-            "unsupported Arrow type for Function signature: list items must be a "
+            "unsupported Arrow type for Function signature: field names "
-            f"non-nullable field named 'item', got {child}"
+            "must not be empty"
        )
-    return _canonical_arrow_type(child.type)
+    if _is_blob_v2_field(field):
        if not _has_supported_blob_v2_layout(field):
            raise TypeError(
                "unsupported Arrow type for Function signature: lance.blob.v2 "
                f"requires a supported Blob storage layout, got {field}"
            )
        metadata = {
            (key.decode() if isinstance(key, bytes) else key): (
                value.decode() if isinstance(value, bytes) else value
            )
            for key, value in (field.metadata or {}).items()
        }
        if metadata and metadata != {
            _ARROW_EXTENSION_NAME_KEY: _BLOB_V2_EXTENSION_NAME
        }:
            raise TypeError(
                "unsupported Arrow type for Function signature: lance.blob.v2 "
                "field metadata must contain only its canonical extension marker"
            )
    elif field.metadata:
        raise TypeError(
            "unsupported Arrow type for Function signature: field metadata "
            f"is not supported, got {field}"
        )
 def _has_supported_blob_v2_layout(field: pa.Field) -> bool:
    data_type = field.type
    if isinstance(data_type, pa.ExtensionType):
        data_type = data_type.storage_type
    if not pa.types.is_struct(data_type):
        return False
    fields = tuple(data_type)
    def matches(spec, compare_nullable) -> bool:
        return len(fields) == len(spec) and all(
            actual.name == name
            and actual.type == expected_type
            and (not check_nullable or actual.nullable == nullable)
            for actual, (name, expected_type, nullable), check_nullable in zip(
                fields, spec, compare_nullable
            )
        )
    logical_minimal = (
        ("data", pa.large_binary(), True),
        ("uri", pa.utf8(), True),
    )
    logical_full = logical_minimal + (
        ("position", pa.uint64(), True),
        ("size", pa.uint64(), True),
    )
    prepared = (
        ("kind", pa.uint8(), True),
        ("data", pa.large_binary(), True),
        ("uri", pa.utf8(), True),
        ("blob_id", pa.uint32(), True),
        ("blob_size", pa.uint64(), True),
        ("position", pa.uint64(), True),
    )
    descriptor = (
        ("kind", pa.uint8(), False),
        ("position", pa.uint64(), False),
        ("size", pa.uint64(), False),
        ("blob_id", pa.uint32(), False),
        ("blob_uri", pa.utf8(), False),
    )
    return (
        matches(logical_minimal, (True, True))
        or matches(logical_full, (True, True, False, False))
        or matches(prepared, (True,) * len(prepared))
        or matches(descriptor, (False,) * len(descriptor))
    )
 def _canonical_arrow_field(field: pa.Field) -> str:
    _validate_exact_arrow_field(field)
    if _is_blob_v2_field(field):
        return _FUNCTION_BLOB_V2_TYPE
    return _canonical_arrow_type(field.type)
 def _blob_storage_type(field: pa.Field) -> pa.DataType:
    data_type = field.type
    if isinstance(data_type, pa.ExtensionType):
        return data_type.storage_type
    return data_type
 def _exact_blob_storage_type(field: pa.Field) -> dict[str, Any]:
    storage = _blob_storage_type(field)
    if not pa.types.is_struct(storage):
        raise TypeError(
            "unsupported Arrow type for Function signature: lance.blob.v2 "
            "requires struct storage"
        )
    return {
        "type": "struct",
        "fields": [
            {
                "name": child.name,
                "nullable": child.nullable,
                "type": (
                    {"type": "large_binary"}
                    if pa.types.is_large_binary(child.type)
                    else _exact_arrow_type(child.type)
                ),
            }
            for child in storage
        ],
    }
 def _data_type_has_blob_v2(data_type: pa.DataType) -> bool:
    if pa.types.is_struct(data_type):
        return any(
            _is_blob_v2_field(field) or _data_type_has_blob_v2(field.type)
            for field in data_type
        )
    if (
        pa.types.is_list(data_type)
        or pa.types.is_large_list(data_type)
        or pa.types.is_fixed_size_list(data_type)
    ):
        field = data_type.value_field
        return _is_blob_v2_field(field) or _data_type_has_blob_v2(field.type)
    if pa.types.is_map(data_type):
        return any(
            _is_blob_v2_field(field) or _data_type_has_blob_v2(field.type)
            for field in (data_type.key_field, data_type.item_field)
        )
    return False
 def _exact_arrow_field(
    field: pa.Field, *, inside_collection: bool = False
 ) -> dict[str, Any]:
    _validate_exact_arrow_field(field)
    if _is_blob_v2_field(field):
        if inside_collection:
            raise TypeError(_NESTED_BLOB_COLLECTION_ERROR)
        return {
            "name": field.name,
            "nullable": field.nullable,
            "type": _exact_blob_storage_type(field),
            "metadata": {
                _ARROW_EXTENSION_NAME_KEY: _BLOB_V2_EXTENSION_NAME,
            },
        }
    value = {
        "name": field.name,
        "nullable": field.nullable,
        "type": _exact_arrow_type(field.type, inside_collection=inside_collection),
    }
    return value
 def _exact_arrow_type(
    data_type: pa.DataType, *, inside_collection: bool = False
 ) -> dict[str, Any]:
    for candidate, name in _GRAMMAR_PRIMITIVES:
        if data_type == candidate:
            return {"type": name}
    if pa.types.is_struct(data_type):
        fields = list(data_type)
        names = [field.name for field in fields]
        if not fields or len(set(names)) != len(names):
            raise TypeError(
                "unsupported Arrow type for Function signature: structs must have "
                "non-empty, uniquely named fields"
            )
        return {
            "type": "struct",
            "fields": [
                _exact_arrow_field(field, inside_collection=inside_collection)
                for field in fields
            ],
        }
    if (
        pa.types.is_list(data_type)
        or pa.types.is_large_list(data_type)
        or pa.types.is_fixed_size_list(data_type)
    ):
        if pa.types.is_fixed_size_list(data_type):
            if data_type.value_field.name != "item":
                raise TypeError(
                    "unsupported Arrow type for Function signature: fixed-size list "
                    "items must be named 'item'"
                )
            if data_type.list_size <= 0:
                raise TypeError(
                    f"unsupported Arrow type for Function signature: {data_type}"
                )
        value: dict[str, Any] = {
            "type": (
                "list"
                if pa.types.is_list(data_type)
                else "large_list"
                if pa.types.is_large_list(data_type)
                else "fixed_size_list"
            ),
            "fields": [
                _exact_arrow_field(data_type.value_field, inside_collection=True)
            ],
        }
        if pa.types.is_fixed_size_list(data_type):
            value["length"] = data_type.list_size
        return value
    if pa.types.is_map(data_type) and _data_type_has_blob_v2(data_type):
        raise TypeError(_NESTED_BLOB_COLLECTION_ERROR)
    raise TypeError(f"unsupported Arrow type for Function signature: {data_type}")
 def _list_of(item: pa.DataType) -> pa.DataType:
@@ -599,8 +890,15 @@ def _callable_parameters(function: Callable[..., Any]) -> tuple[inspect.Paramete
 def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutput:
    if isinstance(output, pa.Schema):
        if output.metadata:
            raise TypeError("Function output schema metadata is not supported")
        fields = tuple(output)
-    elif isinstance(output, pa.Field) and pa.types.is_struct(output.type):
+    elif (
        isinstance(output, pa.Field)
        and not _is_blob_v2_field(output)
        and pa.types.is_struct(output.type)
    ):
        _validate_exact_arrow_field(output)
        if output.nullable:
            raise ValueError("Function output must be non-nullable")
        fields = tuple(output.type)
@@ -616,18 +914,19 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
            raise TypeError(
                "output_schema must be a PyArrow DataType, Field, or Schema"
            )
        _validate_exact_arrow_field(field)
        if field.nullable:
            raise ValueError("Function output must be non-nullable")
        return FunctionOutput(
            kind="scalar",
-            arrow_type=_canonical_arrow_type(field.type),
+            arrow_type=_canonical_arrow_field(field),
            nullable=False,
        )
    if not fields:
        raise ValueError("named-struct Function output must contain at least one field")
-    if any(field.nullable for field in fields):
+    for field in fields:
-        raise ValueError("Function output fields must be non-nullable")
+        _validate_exact_arrow_field(field)
    names = [field.name for field in fields]
    if len(set(names)) != len(names):
        raise ValueError("Function output field names must be unique")
@@ -636,8 +935,8 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
        fields=tuple(
            FunctionResultField(
                name=field.name,
-                arrow_type=_canonical_arrow_type(field.type),
+                arrow_type=_canonical_arrow_field(field),
-                nullable=False,
+                nullable=field.nullable,
            )
            for field in fields
        ),
@@ -656,6 +955,10 @@ def _infer_signature(
    if input_schema is not None:
        if not isinstance(input_schema, pa.Schema):
            raise TypeError("input_schema must be a PyArrow Schema")
        if input_schema.metadata:
            raise TypeError("Function input schema metadata is not supported")
        for field in input_schema:
            _validate_exact_arrow_field(field)
        expected = tuple(parameter.name for parameter in parameters)
        actual = tuple(input_schema.names)
        if actual != expected:
@@ -666,7 +969,7 @@ def _infer_signature(
        inputs = tuple(
            FunctionParameter(
                name=field.name,
-                arrow_type=_canonical_arrow_type(field.type),
+                arrow_type=_canonical_arrow_field(field),
                nullable=field.nullable,
            )
            for field in input_schema
@@ -689,7 +992,9 @@ def _infer_signature(
        inputs.append(
            FunctionParameter(
                name=parameter.name,
-                arrow_type=_canonical_arrow_type(data_type),
+                arrow_type=_canonical_arrow_field(
                    pa.field(parameter.name, data_type, nullable=nullable)
                ),
                nullable=nullable,
            )
        )
@@ -909,13 +1214,26 @@ class UdfDefinition:
        pip: tuple[str, ...],
        env: Mapping[str, str],
        python_version: Optional[str],
        gpu: bool = False,
        conda: tuple[str, ...] = (),
        conda_channels: tuple[str, ...] = (),
    ):
        function_name = name or function.__name__
        if not _FUNCTION_NAME.fullmatch(function_name):
            raise ValueError(f"invalid Function name: {function_name!r}")
-        packages = tuple(sorted(set(pip)))
+        if pip and conda:
            raise ValueError("a Function environment is pip or conda, not both")
        if conda_channels and not conda:
            raise ValueError("conda_channels requires conda packages")
        packages = tuple(sorted(set(conda if conda else pip)))
        if any(not package or package != package.strip() for package in packages):
-            raise ValueError("pip requirements must be non-empty and trimmed")
+            raise ValueError("package requirements must be non-empty and trimmed")
        if conda:
            environment_spec = PythonEnvironmentSpec(
                kind="conda", packages=packages, channels=tuple(conda_channels)
            )
        else:
            environment_spec = PythonEnvironmentSpec(kind="pip", packages=packages)
        environment = dict(env)
        if any(
            not isinstance(key, str) or not isinstance(value, str)
@@ -925,12 +1243,14 @@ class UdfDefinition:
        signature = _infer_signature(function, input_schema, output_schema)
        source = _package_source(function)
        digest = f"sha256:{hashlib.sha256(source).hexdigest()}"
        gpu_marker = _normalize_gpu_marker(gpu)
        runtime = PythonRuntimeSpec(
-            kind="python",
+            kind="python_v2" if gpu_marker is not None else "python",
            python_version=python_version
            or f"{sys.version_info.major}.{sys.version_info.minor}",
-            environment=PythonEnvironmentSpec(kind="pip", packages=packages),
+            environment=environment_spec,
            env=environment,
            gpu=gpu_marker,
        )
        self._function = function
        self._request = FunctionRegistrationRequest(
@@ -955,9 +1275,56 @@ class UdfDefinition:
    @property
    def registration_request(self) -> FunctionRegistrationRequest:
-        """The immutable request sent by ``create_function_async``."""
+        """The immutable request sent by ``create_function_async``.
        Carries no secret bindings. Binding is a registration-time decision,
        so a Function bound to Secrets is registered through :meth:`bind_secrets`,
        which is what ``create_function`` calls.
        """
        return self._request
    def bind_secrets(
        self, secrets: Optional[Sequence[EnvVarSecret]]
    ) -> FunctionRegistrationRequest:
        """The registration request for this definition bound to ``secrets``.
        Binding does not change the Function's source: each
        [EnvVarSecret][lancedb.secrets.EnvVarSecret] names a Secret and the
        environment variable its value should arrive in, and the Function reads
        that variable the way it already did. Whether the named Secrets exist is
        the server's answer, not this one.
        """
        bindings = () if secrets is None else tuple(secrets)
        wrong_type = [
            binding for binding in bindings if not isinstance(binding, EnvVarSecret)
        ]
        if wrong_type:
            kinds = sorted({type(binding).__name__ for binding in wrong_type})
            raise TypeError(
                f"Function secrets must be EnvVarSecret values, not {kinds!r}; a "
                "credential value is never sent to this API"
            )
        variables = [binding.env_variable for binding in bindings]
        duplicates = sorted({name for name in variables if variables.count(name) > 1})
        if duplicates:
            raise ValueError(
                "a Function binds each environment variable once; duplicated: "
                f"{duplicates!r}"
            )
        # `env` is ordinary configuration carried in the definition, so a name in
        # both would have a value visible in the Function's record and a value
        # that is not. Refuse rather than pick.
        environment = self._request.runtime.env or {}
        overlap = sorted(set(environment) & set(variables))
        if overlap:
            raise ValueError(
                f"Function env and secret bindings must be disjoint: {overlap!r}"
            )
        if not bindings:
            return self._request
        resolved = {binding.env_variable: binding.secret for binding in bindings}
        return self._request._copy(update={"secret_env_bindings": resolved})
    def __call__(self, *args, **kwargs):
        return self._function(*args, **kwargs)
@@ -976,6 +1343,9 @@ def udf(
    pip: tuple[str, ...] | list[str] = (),
    env: Optional[Mapping[str, str]] = None,
    python_version: Optional[str] = None,
    gpu: bool = False,
    conda: tuple[str, ...] | list[str] = (),
    conda_channels: tuple[str, ...] | list[str] = (),
 ) -> Callable[[Callable[..., Any]], UdfDefinition]: ...
@@ -988,13 +1358,17 @@ def udf(
    pip: tuple[str, ...] | list[str] = (),
    env: Optional[Mapping[str, str]] = None,
    python_version: Optional[str] = None,
    gpu: bool = False,
    conda: tuple[str, ...] | list[str] = (),
    conda_channels: tuple[str, ...] | list[str] = (),
 ):
    """Prepare a scalar Python callable for remote Function registration.
    Input and output signatures are inferred from supported annotations. For
    Arrow types annotations cannot express precisely, pass ``input_schema``
-    and ``output_schema`` together. Nullable outputs are rejected because V1
+    and ``output_schema`` together. Scalar outputs must be non-nullable. Every
-    uses physical NULL to represent unassigned computed-column rows.
+    named-struct field may be nullable; Enterprise preserves the struct's
    validity when the result is expanded into sibling columns.
    Parameters
    ----------
@@ -1006,14 +1380,24 @@ def udf(
        Explicit input fields in the exact order of the callable parameters.
        Must be provided together with ``output_schema``.
    output_schema : pyarrow.DataType, pyarrow.Field, or pyarrow.Schema, optional
-        Explicit scalar or named-struct output. Must be non-nullable and be
+        Explicit scalar or named-struct output. Scalar outputs must be
-        provided together with ``input_schema``.
+        non-nullable. Must be provided together with ``input_schema``.
    pip : sequence of str, optional
        Pip requirements for the remote environment.
    conda : sequence of str, optional
        Conda packages for the remote environment, instead of ``pip``.
    conda_channels : sequence of str, optional
        Conda channels in priority order; requires ``conda``.
    env : mapping of str to str, optional
-        Environment variables included in the Function definition.
+        Environment variables included in the Function definition. Not for
        credentials -- these are ordinary configuration, stored with the
        Function and visible wherever it is.
    python_version : str, optional
        Remote Python major/minor version. Defaults to the client version.
    gpu : bool, default False
        Whether every remote execution requires a GPU. The execution platform
        selects one compatible GPU for each worker. The requirement is part of
        the immutable Function version.
    The packaged artifact is a snapshot: the function source plus exactly
    the module-level names it references (modules as imports, importable
@@ -1038,6 +1422,11 @@ def udf(
    ...     return value * 2
    >>> score(1.5)
    3.0
    >>> @udf(pip=["cupy-cuda12x"], gpu=True)
    ... def gpu_score(value: int) -> int:
    ...     return value * 2
    >>> gpu_score.registration_request.runtime.gpu
    True
    """
    def decorate(target: Callable[..., Any]) -> UdfDefinition:
@@ -1049,6 +1438,9 @@ def udf(
            pip=tuple(pip),
            env={} if env is None else env,
            python_version=python_version,
            gpu=gpu,
            conda=tuple(conda),
            conda_channels=tuple(conda_channels),
        )
    if function is None:
@@ -1057,6 +1449,7 @@ def udf(
 __all__ = [
    "AssignmentMapping",
    "ApplicationInput",
    "FunctionApplication",
    "FunctionArtifact",
@@ -7,6 +7,7 @@ from typing import List, Literal, Optional
 from ._lancedb import (
    IndexConfig,
 )
 from .query import DocumentGranularity
 from .types import BaseTokenizerType
 lang_mapping = {
@@ -121,6 +122,11 @@ class FTS:
    >>> config = FTS(block_size=256)
    Create an index that treats each deepest-list element as one document:
    >>> from lancedb.query import DocumentGranularity
    >>> config = FTS(document_granularity=DocumentGranularity.LIST_ELEMENT)
    Attributes
    ----------
    with_position : bool, default False
@@ -172,6 +178,11 @@ class FTS:
        roughly half of the available CPU cores. The effective value is
        limited by the available compute capacity. This build-only setting is
        not persisted with the index and does not apply to remote tables.
    document_granularity : DocumentGranularity, default ROW
        ``ROW`` treats the selected text in one table row as one document.
        ``LIST_ELEMENT`` treats each element of the deepest list on the indexed
        field path as one document and returns its physical coordinates in
        ``_doc_index`` for matching queries.
    Notes
    -----
@@ -196,6 +207,7 @@ class FTS:
    custom_stop_words: Optional[List[str]] = None
    memory_limit: Optional[int] = None
    num_workers: Optional[int] = None
    document_granularity: DocumentGranularity = DocumentGranularity.ROW
@dataclass
@@ -4,15 +4,27 @@
 """Handles to operations a server may run asynchronously."""
 import asyncio
 import json
 from datetime import timedelta
 from typing import Any, Callable, Generic, Optional, TypeVar, cast
 import pyarrow as pa
 from lancedb.background_loop import LOOP
 from . import _lancedb
 from ._lancedb import JobDescription, JobFailureInfo, JobInfo
 T = TypeVar("T")
 __all__ = [
    "AsyncJob",
    "Job",
    "JobDescription",
    "JobFailureInfo",
    "JobInfo",
 ]
 class AsyncJob(Generic[T]):
    """A handle to an operation that may still be running.
@@ -78,6 +90,149 @@ class AsyncJob(Generic[T]):
            return
        await self._inner.cancel()
    async def refresh(self) -> None:
        """Ask the backend for this job's current state, and for a server-side
        job its full record, then cache it for the properties below.
        The properties are all `None` until this runs, because submitting an
        operation returns only a job id. `status` fetches the whole record too;
        `wait` records only the terminal state it establishes.
        """
        if self._inner is None:
            return
        await self._inner.refresh()
    @property
    def state(self) -> Optional[str]:
        """The last observed lifecycle state, without contacting the backend.
        `None` until the handle has talked to it. See :meth:`AsyncJob.refresh`.
        """
        if self._inner is None:
            return "finished"
        return self._inner._state
    @property
    def job_type(self) -> Optional[str]:
        """The job's type, as the server names it.
        `None` for an in-process job, which has no server-side record.
        """
        return self._field("job_type")
    @property
    def creation_ms(self) -> Optional[int]:
        """When the job was created, in milliseconds since the epoch."""
        return self._field("creation_ms")
    @property
    def spec(self) -> Optional[Any]:
        """The job-type-specific specification it was submitted with."""
        return self._field("spec")
    @property
    def result(self) -> Optional[Any]:
        """The job-type-specific terminal result, as reported data rather than
        the typed model :meth:`AsyncJob.wait` returns.
        `None` until the job succeeds, so a job that never terminates reports
        its progress through :meth:`AsyncJob.events` instead.
        """
        return self._field("result")
    @property
    def failure(self) -> Optional[JobFailureInfo]:
        """Why the job failed, when it failed and the server reports a reason."""
        return self._field("failure")
    @property
    def _spec_json(self) -> Optional[str]:
        return self._field("_spec_json")
    @property
    def _result_json(self) -> Optional[str]:
        return self._field("_result_json")
    def _field(self, name: str) -> Optional[Any]:
        description = self._inner._description if self._inner is not None else None
        return getattr(description, name) if description is not None else None
    async def events(
        self,
        *,
        limit: Optional[int] = None,
        filter: Optional[str] = None,
    ) -> "pa.Table":
        """This job's recorded lifecycle events.
        Where the properties above report a terminal result only once the job
        reaches one, events are written as the job runs and outlive the workers
        that produced them. A distributed job records a `claim`/`claim_complete`
        pair per unit of work, each carrying `rows_processed`, so a job that
        never finishes still accounts for what it did.
        Parameters
        ----------
        limit: int, optional
            Maximum event rows to return. The server caps results at 1000 by
            default and 10,000 at most, and truncates without saying so, so
            pass this for a job that emits an event per fragment.
        filter: str, optional
            SQL-like expression over the `state`, `updated_by`, `emitted_from`,
            `emitted_by`, and `claim_entity` columns, such as
            ``state = 'claim_complete'``.
        """
        if self._inner is None:
            raise NotImplementedError(
                "job event history is only available for server-side jobs"
            )
        return await self._inner.events(limit=limit, filter=filter)
    def __repr__(self) -> str:
        return _job_repr("AsyncJob", self)
 _REPR_INDENT = " " * 4
 def _repr_payload(value: Any) -> str:
    """Render a job payload as indented JSON, aligned under its field."""
    try:
        rendered = json.dumps(value, indent=4)
    except TypeError:
        return repr(value)
    return rendered.replace("\n", "\n" + _REPR_INDENT)
 def _job_repr(kind: str, job: Any) -> str:
    """Render every field the handle currently knows, omitting the rest.
    One field per line, with the JSON payloads indented, because a refresh
    job's spec and result are the point of printing it.
    """
    state = job.state
    if state is None:
        # Nothing has been fetched yet, so there is nothing to lay out.
        known = f"id={job.id!r}, " if job.id is not None else ""
        return f"{kind}({known}not refreshed)"
    fields = []
    if job.id is not None:
        fields.append(f"id={job.id!r}")
    fields.append(f"state={state!r}")
    for name in ("job_type", "creation_ms"):
        value = getattr(job, name)
        if value is not None:
            fields.append(f"{name}={value!r}")
    for name in ("spec", "result"):
        value = getattr(job, name)
        if value is not None:
            fields.append(f"{name}={_repr_payload(value)}")
    if job.failure is not None:
        fields.append(f"failure={job.failure!r}")
    body = "".join(f"\n{_REPR_INDENT}{field}," for field in fields)
    return f"{kind}({body}\n)"
 class Job(Generic[T]):
    """Synchronous counterpart of `AsyncJob` with the same result type."""
@@ -122,6 +277,75 @@ class Job(Generic[T]):
            return
        LOOP.run(self._inner.cancel())
    def refresh(self) -> None:
        """Ask the backend for this job's current state and record.
        See :meth:`AsyncJob.refresh`.
        """
        if self._inner is None:
            return
        LOOP.run(self._inner.refresh())
    @property
    def state(self) -> Optional[str]:
        """The last observed lifecycle state. See :attr:`AsyncJob.state`."""
        return self._inner.state if self._inner is not None else "finished"
    @property
    def job_type(self) -> Optional[str]:
        """The job's type. See :attr:`AsyncJob.job_type`."""
        return self._field("job_type")
    @property
    def creation_ms(self) -> Optional[int]:
        """When the job was created. See :attr:`AsyncJob.creation_ms`."""
        return self._field("creation_ms")
    @property
    def spec(self) -> Optional[Any]:
        """The job's specification. See :attr:`AsyncJob.spec`."""
        return self._field("spec")
    @property
    def result(self) -> Optional[Any]:
        """The job's terminal result. See :attr:`AsyncJob.result`."""
        return self._field("result")
    @property
    def failure(self) -> Optional[JobFailureInfo]:
        """Why the job failed. See :attr:`AsyncJob.failure`."""
        return self._field("failure")
    @property
    def _spec_json(self) -> Optional[str]:
        return self._field("_spec_json")
    @property
    def _result_json(self) -> Optional[str]:
        return self._field("_result_json")
    def _field(self, name: str) -> Optional[Any]:
        return getattr(self._inner, name) if self._inner is not None else None
    def events(
        self,
        *,
        limit: Optional[int] = None,
        filter: Optional[str] = None,
    ) -> "pa.Table":
        """This job's recorded lifecycle events.
        See :meth:`AsyncJob.events`.
        """
        if self._inner is None:
            raise NotImplementedError(
                "job event history is only available for server-side jobs"
            )
        return LOOP.run(self._inner.events(limit=limit, filter=filter))
    def __repr__(self) -> str:
        return _job_repr("Job", self)
 def _typed_job(
    inner: "_lancedb.Job", result_decoder: Callable[[str], T]
@@ -42,6 +42,8 @@ class MaterializedViewDefinition:
    """Cap on the number of rows the view holds."""
    inputs: List[str] = field(default_factory=list)
    """Source columns the projections and filter read."""
    source_namespace: List[str] = field(default_factory=list)
    """Namespace holding the source table; empty is the root namespace."""
 def _definition_from_schema(
@@ -53,7 +55,8 @@ def _definition_from_schema(
        raise ValueError(f"Table '{name}' is not a materialized view")
    value = json.loads(raw)
    kind = value.get("kind")
-    if kind != "select":
+    # "namespaced_select" keeps older readers from resolving the source at root.
    if kind not in ("select", "namespaced_select"):
        raise NotImplementedError(
            f"materialized view '{name}' is defined by '{kind}', which this "
            "version of lancedb cannot refresh"
@@ -66,6 +69,7 @@ def _definition_from_schema(
        filter=value.get("filter"),
        limit=value.get("limit"),
        inputs=value.get("inputs", []),
        source_namespace=value.get("source_namespace", []),
    )
@@ -12,6 +12,7 @@ from __future__ import annotations
 import sys
 from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
 from uuid import UUID
 if sys.version_info >= (3, 12):
    from typing import override
@@ -48,8 +49,11 @@ from lancedb._lancedb import (
    connect_namespace_client as _connect_namespace_client,
 )
 from lancedb.background_loop import LOOP
 from lancedb.arrow import AsyncRecordBatchReader
 from lancedb.db import AsyncConnection, DBConnection
 from lancedb.job import AsyncJob, Job
 from lancedb.sql import AsyncQuery as AsyncSqlQuery
 from lancedb.sql import QueryDescription
 from lance_namespace import (
    LanceNamespace,
    connect as namespace_connect,
@@ -1447,6 +1451,37 @@ class AsyncLanceNamespaceDBConnection:
            namespace_path=namespace_path, page_token=page_token, limit=limit
        )
    async def execute_query(
        self,
        query: str,
        *,
        default_namespace_path: Optional[List[str]] = None,
    ) -> AsyncRecordBatchReader:
        """Execute SQL when supported by the underlying connection."""
        return await self._inner.execute_query(
            query,
            default_namespace_path=default_namespace_path,
        )
    async def execute_query_async(
        self,
        query: str,
        *,
        default_namespace_path: Optional[List[str]] = None,
    ) -> AsyncSqlQuery:
        """Start executing SQL when supported by the underlying connection.
        Namespace-backed local connections do not support SQL.
        """
        return await self._inner.execute_query_async(
            query,
            default_namespace_path=default_namespace_path,
        )
    async def describe_query(self, query_id: UUID) -> QueryDescription:
        """Describe a submitted SQL query when supported."""
        return await self._inner.describe_query(query_id)
    async def namespace_client(self) -> LanceNamespace:
        """Get the namespace client for this connection.
@@ -109,6 +109,7 @@ def _query_is_plain_scan(query: Query) -> bool:
    return (
        query.vector is None
        and query.full_text_query is None
        and query.take_offsets is None
        and not query.postfilter
        and not query.order_by
    )
@@ -167,6 +168,12 @@ def _projection_to_scanner_kwargs(columns: QueryProjection) -> Dict[str, Any]:
    return {"columns": projection}
 def _query_request_projection(req: "PyQueryRequest") -> QueryProjection:
    if req.select_source_columns is not None:
        return req.select_source_columns
    return req.select
 def _scanner_kwargs_for_query(
    query: Query,
    blob_mode: BlobMode,
@@ -375,6 +382,13 @@ class FullTextOperator(str, Enum):
    OR = "OR"
 class DocumentGranularity(str, Enum):
    """The unit treated as one full-text-search document."""
    ROW = "row"
    LIST_ELEMENT = "list_element"
 class Occur(str, Enum):
    SHOULD = "SHOULD"
    MUST = "MUST"
@@ -478,6 +492,10 @@ class MatchQuery(FullTextQuery):
    prefix_length : int, optional
        The number of beginning characters being unchanged for fuzzy matching.
        This is useful to achieve prefix matching.
    document_granularity : DocumentGranularity, optional
        Explicitly select row or deepest-list-element documents. If omitted,
        the indexed granularity is inferred. When both granularities are indexed
        for the field, this must be specified. With no index, row granularity is used.
    """
    query: str
@@ -487,6 +505,9 @@ class MatchQuery(FullTextQuery):
    max_expansions: int = pydantic.Field(50, kw_only=True)
    operator: FullTextOperator = pydantic.Field(FullTextOperator.OR, kw_only=True)
    prefix_length: int = pydantic.Field(0, kw_only=True)
    document_granularity: Optional[DocumentGranularity] = pydantic.Field(
        None, kw_only=True
    )
    def query_type(self) -> FullTextQueryType:
        return FullTextQueryType.MATCH
@@ -503,11 +524,20 @@ class PhraseQuery(FullTextQuery):
        The query string to match against.
    column : str
        The name of the column to match against.
    slop : int, default 0
        The maximum number of intervening positions permitted in the phrase.
    document_granularity : DocumentGranularity, optional
        Explicitly select row or deepest-list-element documents. If omitted,
        the indexed granularity is inferred. When both granularities are indexed
        for the field, this must be specified. With no index, row granularity is used.
    """
    query: str
    column: str
    slop: int = pydantic.Field(0, kw_only=True)
    document_granularity: Optional[DocumentGranularity] = pydantic.Field(
        None, kw_only=True
    )
    def query_type(self) -> FullTextQueryType:
        return FullTextQueryType.MATCH_PHRASE
@@ -775,6 +805,10 @@ class Query(pydantic.BaseModel):
    # offset to start fetching results from
    offset: Optional[int] = None
    # Dataset offsets whose duplicate occurrences must be restored after lookup.
    # This is populated when a take query is converted to this serializable form.
    take_offsets: Optional[List[int]] = None
    # if true, will only search the indexed data
    fast_search: Optional[bool] = None
@@ -796,6 +830,7 @@ class Query(pydantic.BaseModel):
        query = cls()
        query.limit = req.limit
        query.offset = req.offset
        query.take_offsets = req.take_offsets
        query.filter = req.filter
        query.full_text_query = req.full_text_search
        query.columns = req.select
@@ -2776,15 +2811,16 @@ class AsyncQueryBase(object):
        req = self._inner.to_query_request()
        schema = await self._table.schema()
        projection = _query_request_projection(req)
        self._blob_auto_row_id = blob_auto_row_id_for_scan(
            schema,
-            req.select,
+            projection,
            with_row_id=self._with_row_id,
        )
        if not self._blob_auto_row_id:
            self._blob_paths = ()
            return
-        self._blob_paths = tuple(blob_v2_projection_sources(schema, req.select).keys())
+        self._blob_paths = tuple(blob_v2_projection_sources(schema, projection).keys())
        self._inner.with_row_id()
    def select(self, columns: Union[List[str], dict[str, str]]) -> Self:
@@ -3378,9 +3414,10 @@ class AsyncQuery(AsyncStandardQuery):
        pass in multiple vectors. When multiple vectors are passed in, if the vector
        column is with multivector type, then the vectors will be treated as a single
        query. Or the vectors will be treated as multiple queries, this can be useful
-        if you want to find the nearest vectors to multiple query vectors.
+        if you want to find the nearest vectors to multiple query vectors. Flat
-        This is not expected to be faster than making multiple queries concurrently;
+        searches share one table scan across the query vectors, avoiding the scan
-        it is just a convenience method. If multiple vectors are passed in then
+        and memory amplification of making multiple queries concurrently. If
        multiple vectors are passed in then
        an additional column `query_index` will be added to the results. This column
        will contain the index of the query vector that the result is nearest to.
        """
@@ -3509,8 +3546,8 @@ class AsyncFTSQuery(AsyncStandardQuery):
        Typically, a single vector is passed in as the query. However, you can also
        pass in multiple vectors.  This can be useful if you want to find the nearest
-        vectors to multiple query vectors. This is not expected to be faster than
+        vectors to multiple query vectors. Flat searches share one table scan across
-        making multiple queries concurrently; it is just a convenience method.
+        the query vectors instead of issuing concurrent full scans.
        If multiple vectors are passed in then an additional column `query_index`
        will be added to the results.  This column will contain the index of the
        query vector that the result is nearest to.
@@ -3870,14 +3907,15 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
        blob_paths: tuple[str, ...] = ()
        if self._table is not None:
            schema = await self._table.schema()
            projection = _query_request_projection(req)
            blob_auto_row_id = blob_auto_row_id_for_scan(
                schema,
-                req.select,
+                projection,
                with_row_id=self._with_row_id,
            )
            if blob_auto_row_id:
                blob_paths = tuple(
-                    blob_v2_projection_sources(schema, req.select).keys()
+                    blob_v2_projection_sources(schema, projection).keys()
                )
        self._blob_auto_row_id = blob_auto_row_id
        self._blob_paths = blob_paths
@@ -7,8 +7,18 @@ import json
 import logging
 from concurrent.futures import ThreadPoolExecutor
 import sys
-from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
+from typing import (
    TYPE_CHECKING,
    Any,
    Dict,
    Iterable,
    List,
    Optional,
    Sequence,
    Union,
 )
 from urllib.parse import urlparse
 from uuid import UUID
 import warnings
 if sys.version_info >= (3, 12):
@@ -25,10 +35,13 @@ from ..common import DATA
 from ..db import DBConnection, LOOP
 from ..functions import FunctionVersion, UdfDefinition
 from ..job import AsyncJob, Job
 from ..sql import Query as SqlQuery
 from ..sql import QueryDescription
 from ..materialized_view import MaterializedView, SelectArg
 from ..secrets import EnvVarSecret, SecretInfo
 if TYPE_CHECKING:
-    from .._lancedb import JobDescription, JobInfo
+    from .._lancedb import JobInfo
 from ..embeddings import EmbeddingFunctionConfig
 from lance_namespace import (
    LanceNamespace,
@@ -116,6 +129,7 @@ class RemoteDBConnection(DBConnection):
        read_timeout: Optional[float] = None,
        storage_options: Optional[Dict[str, str]] = None,
        read_consistency_interval: Optional[timedelta] = None,
        sql_host_override: Optional[str] = None,
    ):
        """Connect to a remote LanceDB database."""
        if isinstance(client_config, dict):
@@ -161,6 +175,7 @@ class RemoteDBConnection(DBConnection):
        self.api_key = api_key
        self.region = region
        self.host_override = host_override
        self.sql_host_override = sql_host_override
        self.storage_options = storage_options
        self.db_name = parsed.netloc
@@ -175,6 +190,7 @@ class RemoteDBConnection(DBConnection):
                api_key=api_key,
                region=region,
                host_override=host_override,
                sql_host_override=sql_host_override,
                client_config=client_config,
                storage_options=storage_options,
                read_consistency_interval=read_consistency_interval,
@@ -193,6 +209,7 @@ class RemoteDBConnection(DBConnection):
                "api_key": self.api_key,
                "region": self.region,
                "host_override": self.host_override,
                "sql_host_override": self.sql_host_override,
                "client_config": _client_config_to_dict(self.client_config),
                "storage_options": self.storage_options,
            }
@@ -732,36 +749,59 @@ class RemoteDBConnection(DBConnection):
        )
    @override
-    def job(self, job_id: str) -> Job:
+    def open_job(self, job_id: str) -> Job:
-        """A [Job][lancedb.job.Job] handle for a server-side job by id.
+        """Open a server-side job by id. See
-
+        [DBConnection.open_job][lancedb.db.DBConnection.open_job].
        The handle is constructed without a server round trip; an unknown id
        surfaces when the handle is used. Dropping the handle has no effect
        on the job itself.
        """
-        return Job(self._conn.job(job_id))
+        return Job(LOOP.run(self._conn.open_job(job_id)))
    @override
-    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
+    def create_function_async(
-        return Job(LOOP.run(self._conn.create_function_async(definition)))
+        self,
        definition: UdfDefinition,
        *,
        secrets: Optional[Sequence[EnvVarSecret]] = None,
    ) -> Job[FunctionVersion]:
        job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets))
        return Job(job)
    @override
    def get_function(self, name: str, *, version: str) -> FunctionVersion:
        return LOOP.run(self._conn.get_function(name, version=version))
    @override
    def list_functions(self) -> List[FunctionVersion]:
        return LOOP.run(self._conn.list_functions())
    @override
    def drop_function(self, name: str, *, version: str) -> bool:
        return LOOP.run(self._conn.drop_function(name, version=version))
    @override
    def create_secret(self, name: str, value: str) -> None:
        LOOP.run(self._conn.create_secret(name, value))
    @override
    def alter_secret(self, name: str, value: str) -> None:
        LOOP.run(self._conn.alter_secret(name, value))
    @override
    def describe_secret(self, name: str) -> SecretInfo:
        return LOOP.run(self._conn.describe_secret(name))
    @override
    def list_secrets(self) -> List[str]:
        return LOOP.run(self._conn.list_secrets())
    @override
    def drop_secret(self, name: str) -> None:
        LOOP.run(self._conn.drop_secret(name))
    @override
    def list_jobs(self) -> List["JobInfo"]:
        """List server-side jobs across the database's tables."""
        return LOOP.run(self._conn.list_jobs())
    @override
    def get_job(self, job_id: str) -> Optional["JobDescription"]:
        """Describe a single server-side job by id.
        Returns None when the server has no such job.
        """
        return LOOP.run(self._conn.get_job(job_id))
    @override
    def cancel_job(self, job_id: str) -> bool:
        """Request cancellation of a server-side job by id.
@@ -773,12 +813,35 @@ class RemoteDBConnection(DBConnection):
        return LOOP.run(self._conn.cancel_job(job_id))
    @override
-    def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
+    def execute_query_async(
-        """The lifecycle event history of a server-side job, as Arrow batches.
+        self,
        query: str,
        *,
        default_namespace_path: Optional[List[str]] = None,
    ) -> SqlQuery:
        """Start executing SQL through this remote connection.
-        Lists history across all jobs when `job_id` is None.
+        Unqualified tables use this connection's database and the
        ``["public"]`` namespace by default. Fully qualified table names may
        reference other databases available to the same deployment.
        """
-        return LOOP.run(self._conn.job_history(job_id))
+        return SqlQuery(
            LOOP.run(
                self._conn.execute_query_async(
                    query,
                    default_namespace_path=default_namespace_path,
                )
            )
        )
    @override
    def describe_query(self, query_id: UUID) -> QueryDescription:
        """Describe a submitted SQL query by its connection-scoped id."""
        return LOOP.run(
            self._conn.describe_query(
                query_id,
            )
        )
    @override
    def namespace_client(self) -> LanceNamespace:
@@ -177,4 +177,7 @@ class OAuthProvider(HeaderProvider):
        if not self._current_token:
            raise RuntimeError("Failed to obtain OAuth token")
-        return {"Authorization": f"Bearer {self._current_token}"}
+        return {
            "Authorization": f"Bearer {self._current_token}",
            "x-lancedb-credential-type": "oidc",
        }
@@ -36,6 +36,7 @@ from lancedb._lancedb import (
    UpdateResult,
 )
 from lancedb.embeddings.base import EmbeddingFunctionConfig
 from lancedb.expr import Expr
 from lancedb.index import (
    FTS,
    BTree,
@@ -61,11 +62,20 @@ from lancedb.table import _normalize_progress
 from ..query import (
    AnalyzePlanDistributedMetrics,
    DocumentGranularity,
    LanceQueryBuilder,
    LanceTakeQueryBuilder,
    LanceVectorQueryBuilder,
 )
-from ..table import AsyncTable, BlobMode, Branches, IndexStatistics, Query, Table, Tags
+from ..table import (
    AsyncTable,
    BlobMode,
    Branches,
    IndexStatistics,
    Query,
    Table,
    Tags,
 )
 from ..types import BaseTokenizerType
@@ -349,6 +359,7 @@ class RemoteTable(Table):
        ngram_max_length: int = 3,
        prefix_only: bool = False,
        block_size: int = 128,
        document_granularity: DocumentGranularity = DocumentGranularity.ROW,
        name: Optional[str] = None,
    ):
        """Create a full-text search index on a column.
@@ -371,6 +382,7 @@ class RemoteTable(Table):
            ngram_max_length=ngram_max_length,
            prefix_only=prefix_only,
            block_size=block_size,
            document_granularity=document_granularity,
        )
        LOOP.run(
            self._table.create_index(
@@ -536,6 +548,7 @@ class RemoteTable(Table):
        LOOP.run(
            self._table.create_index(
                column,
                replace=replace,
                config=config,
                wait_timeout=wait_timeout,
                name=name,
@@ -610,6 +623,7 @@ class RemoteTable(Table):
        fill_value: float = 0.0,
        progress: Optional[Union[bool, Callable, Any]] = None,
        write_parallelism: Optional[int] = None,
        allow_external_blob_outside_bases: bool = False,
    ) -> AddResult:
        """Add more data to the [Table][lancedb.table.Table].
@@ -642,6 +656,8 @@ class RemoteTable(Table):
            data in flight. Defaults to an estimate based on the data size,
            capped at the number of CPU cores. Lower this if bulk ingestion is
            using too much memory.
        allow_external_blob_outside_bases: bool, default False
            Not supported on LanceDB Cloud. Setting this raises.
        Returns
        -------
@@ -658,6 +674,7 @@ class RemoteTable(Table):
                    fill_value=fill_value,
                    progress=progress,
                    write_parallelism=write_parallelism,
                    allow_external_blob_outside_bases=allow_external_blob_outside_bases,
                )
            )
        finally:
@@ -856,7 +873,7 @@ class RemoteTable(Table):
    def update(
        self,
-        where: Optional[str] = None,
+        where: Optional[Union[str, Expr]] = None,
        values: Optional[dict] = None,
        *,
        values_sql: Optional[Dict[str, str]] = None,
@@ -867,9 +884,11 @@ class RemoteTable(Table):
        Parameters
        ----------
-        where: str, optional
+        where: str or [Expr][lancedb.expr.Expr], optional
-            The SQL where clause to use when updating rows. For example, 'x = 2'
+            The filter condition. Can be a SQL string or a type-safe
-            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
+            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
            [lit][lancedb.expr.lit]. The filter must not be empty, or it will
            error.
        values: dict, optional
            The values to update. The keys are the column names and the values
            are the values to set.
@@ -4,30 +4,34 @@
 """Schema helpers for Lance blob columns."""
 import importlib
 from typing import TYPE_CHECKING
 import pyarrow as pa
 import pyarrow.ipc
 if TYPE_CHECKING:
    from lance.blob import BlobType as BlobType
 _BLOB_EXTENSION_NAME = "lance.blob.v2"
 _BLOB_V1_KEY = "lance-encoding:blob"
 _ARROW_EXT_NAME_KEY = "ARROW:extension:name"
 _BLOB_V2_STORAGE_TYPE = pa.struct(
    [
        pa.field("data", pa.large_binary(), nullable=True),
        pa.field("uri", pa.utf8(), nullable=True),
        pa.field("position", pa.uint64(), nullable=True),
        pa.field("size", pa.uint64(), nullable=True),
    ]
 )
 _resolved_blob_type = None
-class BlobType(pa.ExtensionType):
+class _FallbackBlobType(pa.ExtensionType):
-    """PyArrow extension type for a Lance blob v2 column.
+    """lance.blob.v2 extension type used when pylance is not installed."""
    Queries return descriptors; call :meth:`~lancedb.table.Table.fetch_blob_files`
    for lazy reads or :meth:`~lancedb.table.Table.fetch_blobs` for eager bytes.
    """
    def __init__(self) -> None:
-        storage_type = pa.struct(
+        pa.ExtensionType.__init__(self, _BLOB_V2_STORAGE_TYPE, _BLOB_EXTENSION_NAME)
            [
                pa.field("data", pa.large_binary(), nullable=True),
                pa.field("uri", pa.utf8(), nullable=True),
                pa.field("position", pa.uint64(), nullable=True),
                pa.field("size", pa.uint64(), nullable=True),
            ]
        )
        super().__init__(storage_type, _BLOB_EXTENSION_NAME)
    def __arrow_ext_serialize__(self) -> bytes:
        return b""
@@ -35,23 +39,16 @@ class BlobType(pa.ExtensionType):
    @classmethod
    def __arrow_ext_deserialize__(
        cls, storage_type: pa.DataType, serialized: bytes
-    ) -> "BlobType":
+    ) -> "_FallbackBlobType":
        return cls()
    def __reduce__(self):
        # Ensure pickle round-trips on older pyarrow (apache/arrow#35599).
        return type(self).__arrow_ext_deserialize__, (
            self.storage_type,
            self.__arrow_ext_serialize__(),
        )
 try:
    pa.register_extension_type(BlobType())  # type: ignore[arg-type]
 except pa.ArrowKeyError:
    pass
 def _metadata_value(metadata: dict, key: str):
    return metadata.get(key.encode()) or metadata.get(key)
@@ -92,43 +89,105 @@ def is_blob_like_field(field: pa.Field) -> bool:
    return is_blob_v2_field(field) or _metadata_marks_legacy_blob(field.metadata or {})
-def _collect_blob_paths(schema: pa.Schema, is_blob) -> list[str]:
+def _collect_blob_paths(schema: pa.Schema, is_blob) -> list[tuple[str, bool]]:
-    paths: list[str] = []
+    """Walk the schema and return (path, has_list_ancestor) for each blob field."""
    paths: list[tuple[str, bool]] = []
-    def walk(fields, prefix: str) -> None:
+    def walk(fields, prefix: str, has_list_ancestor: bool) -> None:
        for field in fields:
            path = f"{prefix}.{field.name}" if prefix else field.name
            if is_blob(field):
-                paths.append(path)
+                paths.append((path, has_list_ancestor))
            elif pa.types.is_struct(field.type):
-                walk(field.type, path)
+                walk(field.type, path, has_list_ancestor)
            elif (
                pa.types.is_list(field.type)
                or pa.types.is_large_list(field.type)
                or pa.types.is_fixed_size_list(field.type)
            ):
-                walk([field.type.value_field], path)
+                walk([field.type.value_field], path, True)
-    walk(schema, "")
+    walk(schema, "", False)
    return paths
 def blob_column_paths(schema: pa.Schema) -> list[str]:
    """Dotted paths of blob-like columns (v2 extension or legacy metadata)."""
-    return _collect_blob_paths(schema, is_blob_like_field)
+    return [path for path, _ in _collect_blob_paths(schema, is_blob_like_field)]
 def blob_v2_column_paths(schema: pa.Schema) -> list[str]:
-    return _collect_blob_paths(schema, is_blob_v2_field)
+    return [path for path, _ in _collect_blob_paths(schema, is_blob_v2_field)]
 def row_addressable_blob_v2_paths(schema: pa.Schema) -> list[str]:
    """Blob v2 paths with one blob addressable by table row id.
    ``fetch_blobs`` and the descriptor row-id ride-along address one blob per
    row, so a blob inside a list container has no row-id slot and no fetch
    path. Those columns still store and query as raw descriptors.
    """
    return [
        path
        for path, has_list_ancestor in _collect_blob_paths(schema, is_blob_v2_field)
        if not has_list_ancestor
    ]
 def schema_has_blob_field(schema: pa.Schema) -> bool:
    return bool(blob_column_paths(schema))
 def _deserialize_registered_type(extension_type: pa.ExtensionType) -> pa.DataType:
    """Return the type Arrow reconstructs for this extension name."""
    schema = pa.schema([pa.field("value", extension_type)])
    restored = pa.ipc.read_schema(schema.serialize())
    return restored.field("value").type
 def _resolve_blob_type():
    """Return the BlobType class this process should use.
    pylance's class when it owns the lance.blob.v2 registry entry,
    otherwise LanceDB's fallback. A different registered class is an error.
    """
    global _resolved_blob_type
    if _resolved_blob_type is not None:
        return _resolved_blob_type
    try:
        blob_module = importlib.import_module("lance.blob")
    except ModuleNotFoundError as err:
        if err.name not in ("lance", "lance.blob"):
            raise
    else:
        blob_type = getattr(blob_module, "BlobType", None)
        if blob_type is not None:
            registered_type = _deserialize_registered_type(blob_type())
            if type(registered_type) is not blob_type:
                registered_cls = type(registered_type)
                raise ValueError(
                    "lance.blob.v2 is already registered by "
                    f"{registered_cls.__module__}.{registered_cls.__qualname__}"
                )
            _resolved_blob_type = blob_type
            return blob_type
    try:
        pa.register_extension_type(_FallbackBlobType())  # type: ignore[arg-type]
    except pa.ArrowKeyError as err:
        raise ValueError(
            "lance.blob.v2 is already registered by another extension class"
        ) from err
    _resolved_blob_type = _FallbackBlobType
    return _resolved_blob_type
 def blob(name: str, nullable: bool = True) -> pa.Field:
-    """Create a Lance blob v2 column field."""
+    """Create a Lance blob v2 column field.
-    return pa.field(name, BlobType(), nullable=nullable)
+
    When pylance is installed this is ``lance.blob.BlobType``.
    """
    blob_type = _resolve_blob_type()
    return pa.field(name, blob_type(), nullable=nullable)
 def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataType:
@@ -155,3 +214,11 @@ def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataTyp
    ... ])
    """
    return pa.list_(value_type, dimension)
 def __getattr__(name: str):
    if name == "BlobType":
        blob_type = _resolve_blob_type()
        globals()["BlobType"] = blob_type
        return blob_type
    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -0,0 +1,169 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 """Named Secrets, and the bindings that deliver them to Functions.
 A Secret is a database-scoped named credential. Nothing in this module holds a
 value: :class:`EnvVarSecret` names one and says which environment variable it
 should arrive in, and the value is resolved by the remote service when a
 Function bound to it runs. No API returns a stored credential, by construction
 rather than by policy -- there is no code path that could.
 """
 from __future__ import annotations
 import re
 # The same characters LanceDB already admits in a namespace or table name, and
 # no positional rule on top of them: a segment may begin with `_`, `-` or `.`
 # today, so anything narrower would put Secrets out of reach inside namespaces
 # that already exist. Matches the service, which admits the same set.
 _SECRET_NAME = re.compile(r"^[A-Za-z0-9_.-]{1,255}$")
 _ENV_VARIABLE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
 def validate_secret_name(name: str) -> str:
    """Check a Secret name locally and return it unchanged."""
    if not isinstance(name, str):
        raise TypeError(f"Secret name must be a string, not {type(name).__name__}")
    if not _SECRET_NAME.fullmatch(name):
        raise ValueError(f"invalid Secret name: {name!r}")
    return name
 def validate_env_variable(name: str) -> str:
    """Check an environment variable name locally and return it unchanged."""
    if not isinstance(name, str):
        raise TypeError(
            f"environment variable name must be a string, not {type(name).__name__}"
        )
    if not _ENV_VARIABLE.fullmatch(name):
        raise ValueError(f"invalid environment variable name: {name!r}")
    return name
 class EnvVarSecret:
    """A Secret bound to the environment variable a Function's library reads.
    Pass these in the ``secrets`` sequence of
    [DBConnection.create_function][lancedb.db.DBConnection.create_function]. The
    Function's source is unchanged by binding: it reads ``OPENAI_API_KEY`` the
    way it always did, and the binding is what puts a value there.
    This is a local value. Constructing it contacts no server, so it always
    succeeds and says nothing about whether the Secret exists; that is checked
    at registration, where a mistyped Secret name surfaces as a clear "does not
    exist" naming both the Secret and the variable bound to it. A mistyped
    *variable* name cannot be caught anywhere -- nothing knows which variables a
    Function reads -- so it surfaces on the first rows instead.
    The type exists so a credential cannot be passed by accident. A bare string
    in the same position is a plausible-looking mistake with the opposite
    meaning, and it reads identically in a diff.
    Parameters
    ----------
    secret : str
        The Secret's database-scoped name.
    env_variable : str
        The environment variable the Function reads it from.
    Examples
    --------
    >>> from lancedb import EnvVarSecret
    >>> binding = EnvVarSecret(secret="openai-prod", env_variable="OPENAI_API_KEY")
    >>> binding.secret, binding.env_variable
    ('openai-prod', 'OPENAI_API_KEY')
    """
    __slots__ = ("_secret", "_env_variable")
    def __init__(self, secret: str, env_variable: str):
        self._secret = validate_secret_name(secret)
        self._env_variable = validate_env_variable(env_variable)
    @property
    def secret(self) -> str:
        """The Secret's database-scoped name."""
        return self._secret
    @property
    def env_variable(self) -> str:
        """The environment variable the value is delivered in."""
        return self._env_variable
    def __repr__(self) -> str:
        return (
            f"EnvVarSecret(secret={self._secret!r}, "
            f"env_variable={self._env_variable!r})"
        )
    def __eq__(self, other: object) -> bool:
        return (
            isinstance(other, EnvVarSecret)
            and other._secret == self._secret
            and other._env_variable == self._env_variable
        )
    def __hash__(self) -> int:
        return hash((EnvVarSecret, self._secret, self._env_variable))
 class SecretInfo:
    """What a database records about a Secret. Never its value.
    Returned by
    [DBConnection.describe_secret][lancedb.db.DBConnection.describe_secret].
    """
    __slots__ = ("_name", "_created_at", "_updated_at")
    def __init__(self, name: str, created_at: str, updated_at: str):
        self._name = name
        self._created_at = created_at
        self._updated_at = updated_at
    @property
    def name(self) -> str:
        """The Secret's database-scoped name."""
        return self._name
    @property
    def created_at(self) -> str:
        """When the Secret was created, as an RFC 3339 timestamp."""
        return self._created_at
    @property
    def updated_at(self) -> str:
        """When the Secret's value was last rotated, as an RFC 3339 timestamp."""
        return self._updated_at
    @classmethod
    def from_json(cls, value: dict) -> "SecretInfo":
        return cls(
            name=value["name"],
            created_at=value["created_at"],
            updated_at=value["updated_at"],
        )
    def __repr__(self) -> str:
        return (
            f"SecretInfo(name={self._name!r}, created_at={self._created_at!r}, "
            f"updated_at={self._updated_at!r})"
        )
    def __eq__(self, other: object) -> bool:
        return (
            isinstance(other, SecretInfo)
            and other._name == self._name
            and other._created_at == self._created_at
            and other._updated_at == self._updated_at
        )
 __all__ = [
    "EnvVarSecret",
    "SecretInfo",
    "validate_env_variable",
    "validate_secret_name",
 ]
@@ -0,0 +1,88 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 """Handles to SQL queries running on a remote database."""
 from uuid import UUID
 import pyarrow as pa
 from lancedb.background_loop import LOOP
 from . import _lancedb
 from .arrow import AsyncRecordBatchReader
 QueryDescription = _lancedb.QueryDescription
 class AsyncQuery:
    """A handle to a submitted SQL query on an asynchronous connection."""
    def __init__(self, inner: "_lancedb.SqlQuery"):
        self._inner = inner
    @property
    def id(self) -> UUID:
        """The stable identifier scoped to the connection that submitted it."""
        return self._inner.id
    async def describe(self) -> QueryDescription:
        """Get a point-in-time description of the query."""
        return await self._inner.describe()
    async def reader(self) -> AsyncRecordBatchReader:
        """Wait for the initial result stream and return its Arrow reader.
        Results are single-consumer. Calling this method more than once on the
        same query raises an error. Later batches are streamed as they become
        available without waiting for the full query to finish.
        """
        return AsyncRecordBatchReader(await self._inner.reader())
    async def cancel(self) -> None:
        """Request cancellation of the query."""
        await self._inner.cancel()
 class Query:
    """Synchronous counterpart of :class:`AsyncQuery`."""
    def __init__(self, inner: AsyncQuery):
        self._inner = inner
    @property
    def id(self) -> UUID:
        """The stable identifier scoped to the connection that submitted it."""
        return self._inner.id
    def describe(self) -> QueryDescription:
        """Get a point-in-time description of the query."""
        return LOOP.run(self._inner.describe())
    def reader(self) -> pa.RecordBatchReader:
        """Wait for the initial result stream and return a blocking reader.
        Results are single-consumer. Calling this method more than once on the
        same query raises an error. Later batches block only until they become
        available, without waiting for the full query to finish.
        """
        reader = LOOP.run(self._inner.reader())
        def next_batch():
            try:
                return LOOP.run(reader.__anext__())
            except StopAsyncIteration:
                return None
        def batches():
            while (batch := next_batch()) is not None:
                yield batch
        return pa.RecordBatchReader.from_batches(reader.schema, batches())
    def cancel(self) -> None:
        """Request cancellation of the query."""
        LOOP.run(self._inner.cancel())
 __all__ = ["AsyncQuery", "Query", "QueryDescription"]
@@ -85,6 +85,7 @@ from .query import (
    AsyncQuery,
    AsyncTakeQuery,
    AsyncVectorQuery,
    DocumentGranularity,
    FullTextQuery,
    LanceEmptyQueryBuilder,
    LanceFtsQueryBuilder,
@@ -103,7 +104,12 @@ from .util import (
    value_to_sql,
 )
 from .index import lang_mapping
-from .schema import blob_v2_column_paths, schema_has_blob_field
+from .schema import (
    blob_v2_column_paths,
    is_blob_v2_field,
    row_addressable_blob_v2_paths,
    schema_has_blob_field,
 )
 def _should_push_down_query_table(
@@ -425,6 +431,7 @@ def _cast_to_target_schema(
    def gen():
        for batch in reader:
            batch = _coerce_blob_write_columns(batch, reordered_schema)
            # Table but not RecordBatch has cast.
            cast_batches = (
                pa.Table.from_batches([batch]).cast(reordered_schema).to_batches()
@@ -437,6 +444,166 @@ def _cast_to_target_schema(
    return pa.RecordBatchReader.from_batches(reordered_schema, gen())
 def _coerce_blob_write_columns(
    batch: pa.RecordBatch, target_schema: pa.Schema
 ) -> pa.RecordBatch:
    """Materialize blob storage structs before the stream leaves Python.
    merge_insert requires its source reader to already match the table's
    physical schema. Unlike add and insert, it does not pass through
    LanceDB's Rust blob coercion, so preserving binary input here would
    reach Lance as binary and fail the schema check.
    """
    columns = []
    fields = []
    changed = False
    for field, column in zip(batch.schema, batch.columns):
        target_field = target_schema.field(field.name)
        coerced = _coerce_blob_value(column, target_field)
        if coerced is not column:
            column = coerced
            field = pa.field(
                field.name,
                coerced.type,
                field.nullable,
                target_field.metadata,
            )
            changed = True
        columns.append(column)
        fields.append(field)
    if not changed:
        return batch
    return pa.RecordBatch.from_arrays(
        columns, schema=pa.schema(fields, metadata=batch.schema.metadata)
    )
 def _coerce_blob_value(column: pa.Array, target_field: pa.Field) -> pa.Array:
    if is_blob_v2_field(target_field) and _can_coerce_to_blob(column.type):
        return _coerce_value_to_blob(column, target_field)
    target_type = target_field.type
    if pa.types.is_struct(target_type) and pa.types.is_struct(column.type):
        children = []
        fields = []
        changed = False
        for source_field in column.type:
            source_column = column.field(source_field.name)
            nested_target = next(
                (field for field in target_type if field.name == source_field.name),
                None,
            )
            if nested_target is None:
                children.append(source_column)
                fields.append(source_field)
                continue
            coerced = _coerce_blob_value(source_column, nested_target)
            if coerced is not source_column:
                changed = True
            child_array, child_type = _physical_array_and_type(coerced)
            children.append(child_array)
            fields.append(
                pa.field(
                    source_field.name,
                    child_type,
                    source_field.nullable,
                    nested_target.metadata,
                )
            )
        if not changed:
            return column
        return pa.StructArray.from_arrays(
            children,
            fields=fields,
            mask=column.is_null() if column.null_count else None,
        )
    if _is_list_like(target_type) and _is_list_like(column.type):
        return _coerce_blob_list_values(column, target_type.value_field)
    return column
 def _coerce_blob_list_values(
    column: pa.Array, target_value_field: pa.Field
 ) -> pa.Array:
    """Coerce blob values inside a list column, preserving offsets and nulls.
    Works on the raw child values window instead of ``pc.list_flatten`` because
    flatten drops values spanned by null slots, which would misalign offsets.
    """
    mask = column.is_null() if column.null_count else None
    if pa.types.is_fixed_size_list(column.type):
        list_size = column.type.list_size
        values = column.values.slice(column.offset * list_size, len(column) * list_size)
        coerced = _coerce_blob_value(values, target_value_field)
        if coerced is values:
            return column
        physical_values, _ = _physical_array_and_type(coerced)
        return pa.FixedSizeListArray.from_arrays(physical_values, list_size, mask=mask)
    offsets = column.offsets
    first_offset = offsets[0].as_py()
    values = column.values.slice(
        first_offset,
        offsets[-1].as_py() - first_offset,
    )
    coerced = _coerce_blob_value(values, target_value_field)
    if coerced is values:
        return column
    physical_values, _ = _physical_array_and_type(coerced)
    if first_offset:
        offsets = pc.subtract(offsets, pa.scalar(first_offset, offsets.type))
    if pa.types.is_large_list(column.type):
        return pa.LargeListArray.from_arrays(offsets, physical_values, mask=mask)
    return pa.ListArray.from_arrays(offsets, physical_values, mask=mask)
 def _coerce_value_to_blob(values: pa.Array, target_field: pa.Field) -> pa.Array:
    if pa.types.is_null(values.type):
        data = pa.nulls(len(values), type=pa.large_binary())
    elif pa.types.is_large_binary(values.type):
        data = values
    else:
        data = values.cast(pa.large_binary())
    length = len(values)
    storage_type = target_field.type
    if isinstance(storage_type, pa.ExtensionType):
        storage_type = storage_type.storage_type
    storage_fields = list(storage_type)
    children = []
    for storage_field in storage_fields:
        if storage_field.name == "data":
            children.append(data)
        else:
            children.append(pa.nulls(length, type=storage_field.type))
    storage = pa.StructArray.from_arrays(
        children,
        fields=storage_fields,
        mask=values.is_null() if values.null_count else None,
    )
    if isinstance(target_field.type, pa.ExtensionType):
        return pa.ExtensionArray.from_storage(target_field.type, storage)
    return storage
 def _physical_array_and_type(array: pa.Array) -> tuple[pa.Array, pa.DataType]:
    if isinstance(array.type, pa.ExtensionType):
        return array.storage, array.type.storage_type
    return array, array.type
 def _can_coerce_to_blob(data_type: pa.DataType) -> bool:
    return _is_binary_like(data_type) or pa.types.is_null(data_type)
 def _is_binary_like(data_type: pa.DataType) -> bool:
    return (
        pa.types.is_binary(data_type)
        or pa.types.is_large_binary(data_type)
        or pa.types.is_binary_view(data_type)
    )
 def _field_extension_name(field: pa.Field) -> Optional[str]:
    extension_name = getattr(field.type, "extension_name", None)
    if extension_name is not None:
@@ -463,63 +630,71 @@ def _align_field_types(
        target_field = next((f for f in target_fields if f.name == field.name), None)
        if target_field is None:
            raise ValueError(f"Field '{field.name}' not found in target schema")
-        # Preserve arrow.json input until it reaches Lance. LanceDB exposes stored
+        new_fields.append(_align_field(field, target_field))
-        # JSON columns as lance.json (JSONB-backed LargeBinary), but casting the
+    return new_fields
-        # input to that storage type here merely relabels the raw JSON bytes as
+
-        # JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
+
-        if (
+def _align_list_value_field(
-            _field_extension_name(field) == "arrow.json"
+    value_field: pa.Field, target_value_field: pa.Field
-            and _field_extension_name(target_field) == "lance.json"
+) -> pa.Field:
-        ):
+    # A list has exactly one child, so the inferred child name ("item") aligns
-            new_fields.append(field)
+    # positionally and adopts the table's child name; pa.Table.cast renames it.
-            continue
+    return _align_field(value_field, target_value_field).with_name(
-        if pa.types.is_struct(target_field.type):
+        target_value_field.name
-            if pa.types.is_struct(field.type):
+    )
-                new_type = pa.struct(
+
-                    _align_field_types(
+
-                        field.type.fields,
+def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field:
-                        target_field.type.fields,
+    # Preserve arrow.json input until it reaches Lance. LanceDB exposes stored
-                    )
+    # JSON columns as lance.json (JSONB-backed LargeBinary), but casting the
    # input to that storage type here merely relabels the raw JSON bytes as
    # JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
    if (
        _field_extension_name(field) == "arrow.json"
        and _field_extension_name(target_field) == "lance.json"
    ):
        return field
    if pa.types.is_struct(target_field.type):
        if pa.types.is_struct(field.type):
            new_type = pa.struct(
                _align_field_types(
                    field.type.fields,
                    target_field.type.fields,
                )
-            else:
+            )
                new_type = target_field.type
        elif pa.types.is_list(target_field.type):
            if _is_list_like(field.type):
                new_type = pa.list_(
                    _align_field_types(
                        [field.type.value_field],
                        [target_field.type.value_field],
                    )[0]
                )
            else:
                new_type = target_field.type
        elif pa.types.is_large_list(target_field.type):
            if _is_list_like(field.type):
                new_type = pa.large_list(
                    _align_field_types(
                        [field.type.value_field],
                        [target_field.type.value_field],
                    )[0]
                )
            else:
                new_type = target_field.type
        elif pa.types.is_fixed_size_list(target_field.type):
            if _is_list_like(field.type):
                new_type = pa.list_(
                    _align_field_types(
                        [field.type.value_field],
                        [target_field.type.value_field],
                    )[0],
                    target_field.type.list_size,
                )
            else:
                new_type = target_field.type
        else:
            new_type = target_field.type
-        new_fields.append(
+    elif pa.types.is_list(target_field.type):
-            pa.field(field.name, new_type, field.nullable, target_field.metadata)
+        if _is_list_like(field.type):
-        )
+            new_type = pa.list_(
-    return new_fields
+                _align_list_value_field(
                    field.type.value_field, target_field.type.value_field
                )
            )
        else:
            new_type = target_field.type
    elif pa.types.is_large_list(target_field.type):
        if _is_list_like(field.type):
            new_type = pa.large_list(
                _align_list_value_field(
                    field.type.value_field, target_field.type.value_field
                )
            )
        else:
            new_type = target_field.type
    elif pa.types.is_fixed_size_list(target_field.type):
        if _is_list_like(field.type):
            new_type = pa.list_(
                _align_list_value_field(
                    field.type.value_field, target_field.type.value_field
                ),
                target_field.type.list_size,
            )
        else:
            new_type = target_field.type
    else:
        new_type = target_field.type
    return pa.field(field.name, new_type, field.nullable, target_field.metadata)
 def _infer_subschema(
@@ -588,7 +763,7 @@ def sanitize_create_table(
        schema = data.schema
    else:
        if schema is not None:
-            data = pa.Table.from_pylist([], schema)
+            data = pa.Table.from_batches([], schema=schema)
    if schema is None:
        if data is None:
            raise ValueError("Either data or schema must be provided")
@@ -1168,6 +1343,7 @@ class Table(ABC):
        ngram_max_length: int = 3,
        prefix_only: bool = False,
        block_size: int = 128,
        document_granularity: DocumentGranularity = DocumentGranularity.ROW,
        wait_timeout: Optional[timedelta] = None,
        name: Optional[str] = None,
    ):
@@ -1246,6 +1422,11 @@ class Table(ABC):
            The number of documents per compressed posting block. Must be 128
            or 256. A value of 256 uses the experimental FTS V3 format and
            may introduce breaking changes.
        document_granularity: DocumentGranularity, default ROW
            ``ROW`` treats the selected text in one table row as one document.
            ``LIST_ELEMENT`` treats each element of the deepest list on the field
            path as one document and returns its physical coordinates in
            ``_doc_index`` for matching queries.
        wait_timeout: timedelta, optional
            The timeout to wait if indexing is asynchronous.
        name: str, optional
@@ -1269,6 +1450,7 @@ class Table(ABC):
        fill_value: float = 0.0,
        progress: Optional[Union[bool, Callable, Any]] = None,
        write_parallelism: Optional[int] = None,
        allow_external_blob_outside_bases: bool = False,
    ) -> AddResult:
        """Add more data to the [Table][lancedb.table.Table].
@@ -1320,6 +1502,10 @@ class Table(ABC):
            data in flight. Defaults to an estimate based on the data size,
            capped at the number of CPU cores. Lower this if bulk ingestion is
            using too much memory.
        allow_external_blob_outside_bases: bool, default False
            Store blob URIs that sit outside registered blob bases. The row
            keeps a reference, so the object has to stay readable. Local
            tables only.
        Returns
        -------
@@ -1361,7 +1547,9 @@ class Table(ABC):
        on: Union[str, Iterable[str]]
            A column (or columns) to join on.  This is how records from the
            source table and target table are matched.  Typically this is some
-            kind of key or id column.
+            kind of key or id column.  Passing several columns matches on the
            composite key: a source row updates a target row only when it
            agrees on every one of them.
        Examples
        --------
@@ -1492,9 +1680,9 @@ class Table(ABC):
        Offsets are mostly useful for sampling as the set of all valid offsets is easily
        known in advance to be [0, len(table)).
-        No guarantees are made regarding the order in which results are returned.  If
+        No guarantees are made regarding the order in which results are returned.
-        you desire an output order that matches the order of the given offsets, you will
+        Repeated offsets produce repeated rows, which makes this method suitable for
-        need to add the row offset column to the output and align it yourself.
+        sampling with replacement.
        Parameters
        ----------
@@ -1605,6 +1793,9 @@ class Table(ABC):
        The result has the same length and order as ``row_ids``. Null blobs
        produce null slots; valid empty blobs produce ``b""``.
        ``_rowid`` values stay valid after compaction when the table has stable
        row ids.
        Convenience for small payloads. For large values use
        :meth:`fetch_blob_files`.
        """
@@ -1622,6 +1813,9 @@ class Table(ABC):
        The result has the same length and order as ``requests``; null blobs
        produce null slots and empty ranges on non-null blobs produce ``b""``.
        ``_rowid`` values stay valid after compaction when the table has stable
        row ids.
        Row IDs can be obtained from a query with ``with_row_id(True)``. This
        API is currently supported only by local tables.
        """
@@ -1637,6 +1831,9 @@ class Table(ABC):
        ``_rowid`` or a ``_lance_row_id`` field on the blob descriptor. Null
        rows are ``None``. Remote tables require LanceDB Cloud server 0.5.0 or
        newer.
        ``_rowid`` values stay valid after compaction when the table has stable
        row ids.
        """
    @abstractmethod
@@ -1732,7 +1929,7 @@ class Table(ABC):
    @abstractmethod
    def update(
        self,
-        where: Optional[str] = None,
+        where: Optional[Union[str, Expr]] = None,
        values: Optional[dict] = None,
        *,
        values_sql: Optional[Dict[str, str]] = None,
@@ -1747,9 +1944,11 @@ class Table(ABC):
        Parameters
        ----------
-        where: str, optional
+        where: str or [Expr][lancedb.expr.Expr], optional
-            The SQL where clause to use when updating rows. For example, 'x = 2'
+            The filter condition. Can be a SQL string or a type-safe
-            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
+            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
            [lit][lancedb.expr.lit]. The filter must not be empty, or it will
            error.
        values: dict, optional
            The values to update. The keys are the column names and the values
            are the values to set.
@@ -1767,6 +1966,7 @@ class Table(ABC):
        Examples
        --------
        >>> import lancedb
        >>> from lancedb.expr import col
        >>> import pandas as pd
        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1.0, 2], [3, 4], [5, 6]]})
        >>> db = lancedb.connect("./.lancedb")
@@ -1776,7 +1976,7 @@ class Table(ABC):
        0  1  [1.0, 2.0]
        1  2  [3.0, 4.0]
        2  3  [5.0, 6.0]
-        >>> table.update(where="x = 2", values={"vector": [10.0, 10]})
+        >>> table.update(where=col("x") == 2, values={"vector": [10.0, 10]})
        UpdateResult(rows_updated=1, version=2)
        >>> table.to_pandas()
           x        vector
@@ -1976,9 +2176,11 @@ class Table(ABC):
            Function columns are supported only on LanceDB Cloud and
            Enterprise.
        computed: Dict[str, str], optional
-            A map of column name to a SQL expression defining the column. The
+            A mapping from output column names to SQL expressions derives each
-            column's type and inputs are derived from the expression, so no
+            output field from its expression. A direct projection of a Blob v2
-            data type is supplied.
+            field inherits Blob v2 semantics; other expressions derive their
            ordinary Arrow type. Mapping order is declaration and dependency
            order.
            Unlike ``transforms``, the expression is stored rather than
            evaluated now: the column is committed with no values, and rows get
@@ -2115,12 +2317,25 @@ class Table(ABC):
        ----------
        updates : dict
            One or more dicts, each with:
            - "path": str — dot-path to the field (e.g. "embedding" or "a.b.c").
            - "metadata": dict[str, str | None] — keys to set; a value of ``None``
              deletes that key.
            - "replace": bool, optional — replace the field's whole metadata map
              instead of merging (default False).
            The following keys are treated specially, by convention, and should
            be used when appropriate:
            - "lancedb:description": for a human-readable description of a field.
            - ``"lancedb:tag:<name>"`` for a user-defined key-value tag, where the
                suffix names the tag category; e.g. "lancedb:tag:model": "clip".
            - "lancedb:logical-column" for a column grouping; e.g. "feature_v1"
                and "feature_v2" might be in the same logical column.
            - "lancedb:status" for status options ("production", "candidate",
                "deprecated", "archived") to designate the current life cycle
                state of this column.
        Returns
        -------
        UpdateFieldMetadataResult
@@ -2670,7 +2885,7 @@ class LanceTable(Table):
            arrow_tbl = self.to_arrow()
            if blob_mode == "descriptions":
                arrow_tbl = strip_auto_row_ids(
-                    arrow_tbl, blob_v2_column_paths(self.schema)
+                    arrow_tbl, row_addressable_blob_v2_paths(self.schema)
                )
            return arrow_tbl.to_pandas(**kwargs)
@@ -3268,6 +3483,7 @@ class LanceTable(Table):
        ngram_max_length: int = 3,
        prefix_only: bool = False,
        block_size: int = 128,
        document_granularity: DocumentGranularity = DocumentGranularity.ROW,
        name: Optional[str] = None,
    ):
        """Create a full-text search index on a column.
@@ -3319,7 +3535,11 @@ class LanceTable(Table):
            tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)
            tokenizer_configs["custom_stop_words"] = custom_stop_words
-        config = FTS(block_size=block_size, **tokenizer_configs)
+        config = FTS(
            block_size=block_size,
            document_granularity=document_granularity,
            **tokenizer_configs,
        )
        try:
            LOOP.run(
@@ -3409,6 +3629,7 @@ class LanceTable(Table):
        fill_value: float = 0.0,
        progress: Optional[Union[bool, Callable, Any]] = None,
        write_parallelism: Optional[int] = None,
        allow_external_blob_outside_bases: bool = False,
    ) -> AddResult:
        """Add data to the table.
        If vector columns are missing and the table
@@ -3436,6 +3657,9 @@ class LanceTable(Table):
            data in flight. Defaults to an estimate based on the data size,
            capped at the number of CPU cores. Lower this if bulk ingestion is
            using too much memory.
        allow_external_blob_outside_bases: bool, default False
            Allow blob URIs outside registered bases. See :meth:`Table.add`.
            Local tables only.
        Returns
        -------
@@ -3452,6 +3676,7 @@ class LanceTable(Table):
                    fill_value=fill_value,
                    progress=progress,
                    write_parallelism=write_parallelism,
                    allow_external_blob_outside_bases=allow_external_blob_outside_bases,
                )
            )
        finally:
@@ -3806,7 +4031,7 @@ class LanceTable(Table):
    def update(
        self,
-        where: Optional[str] = None,
+        where: Optional[Union[str, Expr]] = None,
        values: Optional[dict] = None,
        *,
        values_sql: Optional[Dict[str, str]] = None,
@@ -3817,9 +4042,11 @@ class LanceTable(Table):
        Parameters
        ----------
-        where: str, optional
+        where: str or [Expr][lancedb.expr.Expr], optional
-            The SQL where clause to use when updating rows. For example, 'x = 2'
+            The filter condition. Can be a SQL string or a type-safe
-            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
+            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
            [lit][lancedb.expr.lit]. The filter must not be empty, or it will
            error.
        values: dict, optional
            The values to update. The keys are the column names and the values
            are the values to set.
@@ -3837,6 +4064,7 @@ class LanceTable(Table):
        Examples
        --------
        >>> import lancedb
        >>> from lancedb.expr import col
        >>> import pandas as pd
        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1.0, 2], [3, 4], [5, 6]]})
        >>> db = lancedb.connect("./.lancedb")
@@ -3846,7 +4074,7 @@ class LanceTable(Table):
        0  1  [1.0, 2.0]
        1  2  [3.0, 4.0]
        2  3  [5.0, 6.0]
-        >>> table.update(where="x = 2", values={"vector": [10.0, 10]})
+        >>> table.update(where=col("x") == 2, values={"vector": [10.0, 10]})
        UpdateResult(rows_updated=1, version=2)
        >>> table.to_pandas()
           x        vector
@@ -3873,6 +4101,7 @@ class LanceTable(Table):
            )
            and not self._route_pushdown_to_rust
            and self.current_branch() is None
            and query.take_offsets is None
        ):
            from lancedb.namespace import _execute_server_side_query
@@ -5061,7 +5290,9 @@ class AsyncTable:
        if blob_mode == "descriptions" or not schema_has_blob_field(schema):
            arrow_tbl = await self.to_arrow()
            if blob_mode == "descriptions":
-                arrow_tbl = strip_auto_row_ids(arrow_tbl, blob_v2_column_paths(schema))
+                arrow_tbl = strip_auto_row_ids(
                    arrow_tbl, row_addressable_blob_v2_paths(schema)
                )
            return arrow_tbl.to_pandas(**kwargs)
        if blob_mode == "lazy" and get_uri_scheme(await self.uri()) == "memory":
@@ -5365,6 +5596,7 @@ class AsyncTable:
        fill_value: Optional[float] = None,
        progress: Optional[Union[bool, Callable, Any]] = None,
        write_parallelism: Optional[int] = None,
        allow_external_blob_outside_bases: bool = False,
    ) -> AddResult:
        """Add more data to the [AsyncTable][lancedb.table.AsyncTable].
@@ -5395,6 +5627,9 @@ class AsyncTable:
            data in flight. Defaults to an estimate based on the data size,
            capped at the number of CPU cores. Lower this if bulk ingestion is
            using too much memory.
        allow_external_blob_outside_bases: bool, default False
            Allow blob URIs outside registered bases. See :meth:`Table.add`.
            Local tables only.
        """
        schema = await self.schema()
@@ -5431,6 +5666,7 @@ class AsyncTable:
                mode or "append",
                progress=progress,
                write_parallelism=write_parallelism,
                allow_external_blob_outside_bases=allow_external_blob_outside_bases,
            )
        except RuntimeError as e:
            if "Cast error" in str(e):
@@ -5476,7 +5712,9 @@ class AsyncTable:
        on: Union[str, Iterable[str]]
            A column (or columns) to join on.  This is how records from the
            source table and target table are matched.  Typically this is some
-            kind of key or id column.
+            kind of key or id column.  Passing several columns matches on the
            composite key: a source row updates a target row only when it
            agrees on every one of them.
        Examples
        --------
@@ -5759,7 +5997,23 @@ class AsyncTable:
    def _sync_query_to_async(
        self, query: Query
-    ) -> AsyncHybridQuery | AsyncFTSQuery | AsyncVectorQuery | AsyncQuery:
+    ) -> (
        AsyncHybridQuery
        | AsyncFTSQuery
        | AsyncVectorQuery
        | AsyncQuery
        | AsyncTakeQuery
    ):
        if query.take_offsets is not None:
            take_query = self.take_offsets(query.take_offsets)
            if query.columns:
                take_query = take_query.select(query.columns)
            if query.use_lsm is not None:
                take_query = take_query.use_lsm(query.use_lsm)
            if query.with_row_id:
                take_query = take_query.with_row_id()
            return take_query
        async_query = self.query()
        if query.limit is not None:
            async_query = async_query.limit(query.limit)
@@ -5824,6 +6078,7 @@ class AsyncTable:
                self._namespace_client, self._pushdown_operations
            )
            and not self._route_pushdown_to_rust
            and query.take_offsets is None
        ):
            from lancedb.namespace import _execute_server_side_query
@@ -5955,7 +6210,7 @@ class AsyncTable:
        self,
        updates: Optional[Dict[str, Any]] = None,
        *,
-        where: Optional[str] = None,
+        where: Optional[Union[str, Expr]] = None,
        updates_sql: Optional[Dict[str, str]] = None,
    ) -> UpdateResult:
        """
@@ -5970,9 +6225,11 @@ class AsyncTable:
            The updates to apply.  The keys should be the name of the column to
            update.  The values should be the new values to assign.  This is
            required unless updates_sql is supplied.
-        where: str, optional
+        where: str or [Expr][lancedb.expr.Expr], optional
-            An SQL filter that controls which rows are updated. For example, 'x = 2'
+            The filter condition. Can be a SQL string or a type-safe
-            or 'x IN (1, 2, 3)'.  Only rows that satisfy this filter will be udpated.
+            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
            [lit][lancedb.expr.lit]. Only rows that satisfy this filter will
            be updated.
        updates_sql: dict, optional
            The updates to apply, expressed as SQL expression strings.  The keys should
            be column names. The values should be SQL expressions.  These can be SQL
@@ -5990,13 +6247,14 @@ class AsyncTable:
        --------
        >>> import asyncio
        >>> import lancedb
        >>> from lancedb.expr import col
        >>> import pandas as pd
        >>> async def demo_update():
        ...     data = pd.DataFrame({"x": [1, 2], "vector": [[1, 2], [3, 4]]})
        ...     db = await lancedb.connect_async("./.lancedb")
        ...     table = await db.create_table("my_table", data)
        ...     # x is [1, 2], vector is [[1, 2], [3, 4]]
-        ...     await table.update({"vector": [10, 10]}, where="x = 2")
+        ...     await table.update({"vector": [10, 10]}, where=col("x") == 2)
        ...     # x is [1, 2], vector is [[1, 2], [10, 10]]
        ...     await table.update(updates_sql={"x": "x + 1"})
        ...     # x is [2, 3], vector is [[1, 2], [10, 10]]
@@ -6010,7 +6268,8 @@ class AsyncTable:
        if updates is not None:
            updates_sql = {k: value_to_sql(v) for k, v in updates.items()}
-        return await self._inner.update(updates_sql, where)
+        predicate = where.to_sql() if isinstance(where, Expr) else where
        return await self._inner.update(updates_sql, predicate)
    async def add_columns(
        self,
@@ -6042,8 +6301,11 @@ class AsyncTable:
            Function columns are supported only on LanceDB Cloud and
            Enterprise.
        computed: Dict[str, str], optional
-            A map of column name to a SQL expression defining the column. The
+            A mapping from output column names to SQL expressions derives each
-            column's type and inputs are derived from the expression.
+            output field from its expression. A direct projection of a Blob v2
            field inherits Blob v2 semantics; other expressions derive their
            ordinary Arrow type. Mapping order is declaration and dependency
            order.
            Unlike ``transforms``, the expression is stored rather than
            evaluated now: the column is committed with no values, and rows get
@@ -6314,6 +6576,9 @@ class AsyncTable:
        Offsets are mostly useful for sampling as the set of all valid offsets is easily
        known in advance to be [0, len(table)).
        No guarantees are made regarding the order in which results are returned.
        Repeated offsets produce repeated rows.
        Parameters
        ----------
        offsets: list[int]
@@ -105,7 +105,7 @@ def test_quickstart(tmp_path):
    tbl.create_index(num_sub_vectors=1)
    # --8<-- [end:create_index]
    # --8<-- [start:delete_rows]
-    tbl.delete('item = "fizz"')
+    tbl.delete("item = 'fizz'")
    # --8<-- [end:delete_rows]
    # --8<-- [start:drop_table]
    db.drop_table("my_table")
@@ -201,7 +201,7 @@ async def test_quickstart_async(tmp_path):
    await tbl.create_index("vector")
    # --8<-- [end:create_index_async]
    # --8<-- [start:delete_rows_async]
-    await tbl.delete('item = "fizz"')
+    await tbl.delete("item = 'fizz'")
    # --8<-- [end:delete_rows_async]
    # --8<-- [start:drop_table_async]
    await db.drop_table("my_table_async")
@@ -266,7 +266,7 @@ def test_table():
    tbl.add(pydantic_model_items)
    # --8<-- [end:add_table_from_pydantic]
    # --8<-- [start:delete_row]
-    tbl.delete('item = "fizz"')
+    tbl.delete("item = 'fizz'")
    # --8<-- [end:delete_row]
    # --8<-- [start:delete_specific_row]
    data = [
@@ -538,7 +538,7 @@ async def test_table_async():
    await async_tbl.add(pydantic_model_items)
    # --8<-- [end:add_table_async_from_pydantic]
    # --8<-- [start:delete_row_async]
-    await async_tbl.delete('item = "fizz"')
+    await async_tbl.delete("item = 'fizz'")
    # --8<-- [end:delete_row_async]
    # --8<-- [start:delete_specific_row_async]
    data = [
@@ -2,17 +2,41 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 import io
 import subprocess
 import sys
 import textwrap
 import lance
 import pyarrow as pa
 import pyarrow.compute as pc
 import pytest
 from lance.blob import BlobType as LanceBlobType
 import lancedb
-from lancedb._blob import read_row_ids_from_hits, stash_auto_row_ids
+from lancedb._blob import (
    blob_v2_projection_sources,
    read_row_ids_from_hits,
    stash_auto_row_ids,
 )
 from lancedb.expr import col
 from lancedb.index import FTS
 from lancedb.schema import blob_column_paths, blob_v2_column_paths
 _HIDE_LANCE_BLOB = """\
 import importlib.abc
 import sys
 class _MissingLanceBlob(importlib.abc.MetaPathFinder):
    def find_spec(self, fullname, path, target=None):
        if fullname == "lance.blob" or fullname.startswith("lance.blob."):
            raise ModuleNotFoundError(fullname, name="lance.blob")
 sys.modules.pop("lance.blob", None)
 sys.meta_path.insert(0, _MissingLanceBlob())
 """
 def _blob_table(name, rows):
    db = lancedb.connect("memory:///")
    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
@@ -42,10 +66,204 @@ def _row_ids_by_id(table):
    return dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
 def _assert_missing_blob_row_ids(exc_info):
    message = str(exc_info.value)
    assert "row ids" in message
    assert "rowaddr" not in message
    assert "fragment" not in message
 def _assert_fetch_apis_reject_missing_row_ids(table, row_ids):
    with pytest.raises(ValueError) as exc_info:
        table.fetch_blobs("image", row_ids)
    _assert_missing_blob_row_ids(exc_info)
    with pytest.raises(ValueError) as exc_info:
        table.fetch_blob_files("image", row_ids)
    _assert_missing_blob_row_ids(exc_info)
    with pytest.raises(ValueError) as exc_info:
        table.fetch_blob_ranges("image", [(row_id, 0, 1) for row_id in row_ids])
    _assert_missing_blob_row_ids(exc_info)
 def test_blob_factory_declares_v2_field():
    field = lancedb.blob("image")
    assert isinstance(field.type, pa.ExtensionType)
    assert field.type.extension_name == "lance.blob.v2"
    assert lancedb.BlobType is LanceBlobType
    assert type(field.type) is LanceBlobType
 def test_blob_type_works_without_pylance():
    script = _HIDE_LANCE_BLOB + textwrap.dedent(
        """\
        import lancedb
        import pyarrow as pa
        field = lancedb.blob("image")
        if not isinstance(field.type, pa.ExtensionType):
            raise SystemExit("expected an extension type")
        if field.type.extension_name != "lance.blob.v2":
            raise SystemExit(field.type.extension_name)
        if lancedb.BlobType is not type(field.type):
            raise SystemExit("BlobType is not the field type class")
        if lancedb.BlobType.__module__ != "lancedb.schema":
            raise SystemExit(lancedb.BlobType.__module__)
        db = lancedb.connect("memory:///")
        table = db.create_table(
            "images",
            schema=pa.schema([pa.field("id", pa.int64()), field]),
        )
        table.add([{"id": 1, "image": b"hello"}])
        result = (
            table.merge_insert("id")
            .when_matched_update_all()
            .when_not_matched_insert_all()
            .execute([{"id": 1, "image": b"updated"}, {"id": 2, "image": b"inserted"}])
        )
        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
            raise SystemExit(
                f"merge_insert rows updated={result.num_updated_rows} "
                f"inserted={result.num_inserted_rows}"
            )
        """
    )
    result = subprocess.run(
        [sys.executable, "-c", script],
        capture_output=True,
        text=True,
        check=False,
    )
    assert result.returncode == 0, result.stderr
 def test_blob_resolves_pylance_type_without_eager_import():
    script = textwrap.dedent(
        """\
        import sys
        import lancedb
        if "lance.blob" in sys.modules:
            raise SystemExit("import lancedb imported lance.blob")
        field = lancedb.blob("image")
        from lance.blob import BlobType
        if type(field.type) is not BlobType:
            raise SystemExit(f"{type(field.type)} is not {BlobType}")
        import lance
        image = lance.blob_array([b"x"])
        if type(image.type) is not BlobType:
            raise SystemExit("blob_array used a different class")
        if type(image.type) is not type(field.type):
            raise SystemExit("field and array classes differ")
        """
    )
    result = subprocess.run(
        [sys.executable, "-c", script],
        capture_output=True,
        text=True,
        check=False,
    )
    assert result.returncode == 0, result.stderr
 def test_blob_fallback_fails_if_name_already_registered():
    script = _HIDE_LANCE_BLOB + textwrap.dedent(
        """\
        import pyarrow as pa
        class OtherBlobType(pa.ExtensionType):
            def __init__(self):
                super().__init__(
                    pa.struct([pa.field("data", pa.large_binary())]),
                    "lance.blob.v2",
                )
            def __arrow_ext_serialize__(self):
                return b""
            @classmethod
            def __arrow_ext_deserialize__(cls, storage_type, serialized):
                return cls()
        pa.register_extension_type(OtherBlobType())
        import lancedb
        try:
            lancedb.blob("image")
        except ValueError as err:
            if "already registered" not in str(err):
                raise SystemExit(err)
        else:
            raise SystemExit("expected ValueError")
        """
    )
    result = subprocess.run(
        [sys.executable, "-c", script],
        capture_output=True,
        text=True,
        check=False,
    )
    assert result.returncode == 0, result.stderr
 def test_blob_type_rejects_competing_registration_with_pylance():
    script = textwrap.dedent(
        """\
        import pyarrow as pa
        import pyarrow.ipc
        class OtherBlobType(pa.ExtensionType):
            def __init__(self):
                super().__init__(
                    pa.struct(
                        [
                            pa.field("data", pa.large_binary()),
                            pa.field("uri", pa.utf8()),
                            pa.field("position", pa.uint64()),
                            pa.field("size", pa.uint64()),
                        ]
                    ),
                    "lance.blob.v2",
                )
            def __arrow_ext_serialize__(self):
                return b""
            @classmethod
            def __arrow_ext_deserialize__(cls, storage_type, serialized):
                return cls()
        pa.register_extension_type(OtherBlobType())
        from lance.blob import BlobType
        if BlobType is OtherBlobType:
            raise SystemExit("pylance BlobType was replaced")
        schema = pa.schema([pa.field("value", BlobType())])
        restored = pa.ipc.read_schema(schema.serialize())
        if type(restored.field("value").type) is not OtherBlobType:
            raise SystemExit(type(restored.field("value").type))
        import lancedb
        try:
            lancedb.blob("image")
        except ValueError as err:
            if "__main__.OtherBlobType" not in str(err):
                raise SystemExit(err)
        else:
            raise SystemExit("expected ValueError")
        """
    )
    result = subprocess.run(
        [sys.executable, "-c", script],
        capture_output=True,
        text=True,
        check=False,
    )
    assert result.returncode == 0, result.stderr
 def test_blob_v2_column_paths_include_list_children():
@@ -70,6 +288,14 @@ def test_blob_v2_column_paths_include_list_children():
    ]
 def test_blob_v2_projection_sources_use_typed_column_name():
    schema = pa.schema([lancedb.blob("blob")])
    assert blob_v2_projection_sources(schema, {"blob_alias": col("blob")}) == {
        "blob_alias": "blob"
    }
 def _legacy_v1_table(name):
    db = lancedb.connect("memory:///")
    schema = pa.schema(
@@ -166,6 +392,20 @@ async def test_async_table_to_pandas_descriptions_mode_omits_row_id():
    assert set(descriptor.keys()) == {"kind", "position", "size", "blob_id", "blob_uri"}
@pytest.mark.asyncio
 async def test_async_typed_blob_projection_preserves_source_column():
    db = await lancedb.connect_async("memory:///typed_blob_projection")
    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("blob")])
    table = await db.create_table("typed_blob_projection", schema=schema)
    await table.add([{"id": 1, "blob": b"alpha"}])
    hits = await table.query().select({"blob_alias": col("blob")}).to_arrow()
    assert "_lance_row_id" in hits.schema.field("blob_alias").type.names
    blobs = await table.fetch_blobs("blob", hits)
    assert blobs.to_pylist() == [b"alpha"]
 def test_fetch_blobs_round_trip():
    table = _blob_table(
        "round_trip",
@@ -176,6 +416,292 @@ def test_fetch_blobs_round_trip():
    assert [blobs[0].as_py(), blobs[1].as_py()] == [b"alpha", b"beta"]
 def test_merge_insert_writes_python_bytes():
    table = _blob_table("merge_bytes", [{"id": 1, "image": b"before"}])
    result = (
        table.merge_insert("id")
        .when_matched_update_all()
        .when_not_matched_insert_all()
        .execute([{"id": 1, "image": b"updated"}, {"id": 2, "image": b"inserted"}])
    )
    assert result.num_updated_rows == 1
    assert result.num_inserted_rows == 1
    by_id = _row_ids_by_id(table)
    blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
    assert blobs.to_pylist() == [b"updated", b"inserted"]
 def test_merge_insert_bytes_after_reopen_without_touching_blob_type(tmp_path):
    db = lancedb.connect(tmp_path)
    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
    table = db.create_table("images", schema=schema)
    table.add([{"id": 1, "image": b"hello"}])
    script = textwrap.dedent(
        f"""\
        import lancedb
        db = lancedb.connect({str(tmp_path)!r})
        table = db.open_table("images")
        image_type = table.schema.field("image").type
        if type(image_type).__name__ != "StructType":
            raise SystemExit(f"expected StructType, got {{type(image_type)}}")
        result = (
            table.merge_insert("id")
            .when_matched_update_all()
            .when_not_matched_insert_all()
            .execute(
                [{{"id": 1, "image": b"updated"}}, {{"id": 2, "image": b"inserted"}}]
            )
        )
        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
            raise SystemExit(
                f"rows updated={{result.num_updated_rows}} "
                f"inserted={{result.num_inserted_rows}}"
            )
        hits = table.search().with_row_id(True).limit(10).to_arrow()
        by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
        blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
        if blobs.to_pylist() != [b"updated", b"inserted"]:
            raise SystemExit(blobs.to_pylist())
        """
    )
    result = subprocess.run(
        [sys.executable, "-c", script],
        capture_output=True,
        text=True,
        check=False,
    )
    assert result.returncode == 0, result.stderr
 def test_merge_insert_bytes_after_reopen_without_pylance(tmp_path):
    db = lancedb.connect(tmp_path)
    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
    table = db.create_table("images", schema=schema)
    table.add([{"id": 1, "image": b"hello"}])
    script = _HIDE_LANCE_BLOB + textwrap.dedent(
        f"""\
        import lancedb
        db = lancedb.connect({str(tmp_path)!r})
        table = db.open_table("images")
        image_type = table.schema.field("image").type
        if type(image_type).__name__ != "StructType":
            raise SystemExit(f"expected StructType, got {{type(image_type)}}")
        result = (
            table.merge_insert("id")
            .when_matched_update_all()
            .when_not_matched_insert_all()
            .execute(
                [{{"id": 1, "image": b"updated"}}, {{"id": 2, "image": b"inserted"}}]
            )
        )
        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
            raise SystemExit(
                f"rows updated={{result.num_updated_rows}} "
                f"inserted={{result.num_inserted_rows}}"
            )
        hits = table.search().with_row_id(True).limit(10).to_arrow()
        by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
        blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
        if blobs.to_pylist() != [b"updated", b"inserted"]:
            raise SystemExit(blobs.to_pylist())
        """
    )
    result = subprocess.run(
        [sys.executable, "-c", script],
        capture_output=True,
        text=True,
        check=False,
    )
    assert result.returncode == 0, result.stderr
 def test_merge_insert_blob_array_into_reopened_unregistered_table(tmp_path):
    db = lancedb.connect(tmp_path)
    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
    table = db.create_table("images", schema=schema)
    table.add([{"id": 1, "image": b"before"}])
    script = textwrap.dedent(
        f"""\
        import pyarrow as pa
        import lancedb
        db = lancedb.connect({str(tmp_path)!r})
        table = db.open_table("images")
        image_type = table.schema.field("image").type
        if type(image_type).__name__ != "StructType":
            raise SystemExit(
                f"expected StructType before lance import, got {{type(image_type)}}"
            )
        import lance
        updates = pa.Table.from_arrays(
            [
                pa.array([1, 2], type=pa.int64()),
                lance.blob_array([b"updated", b"inserted"]),
            ],
            names=["id", "image"],
        )
        result = (
            table.merge_insert("id")
            .when_matched_update_all()
            .when_not_matched_insert_all()
            .execute(updates)
        )
        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
            raise SystemExit(
                f"rows updated={{result.num_updated_rows}} "
                f"inserted={{result.num_inserted_rows}}"
            )
        hits = table.search().with_row_id(True).limit(10).to_arrow()
        by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
        blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
        if blobs.to_pylist() != [b"updated", b"inserted"]:
            raise SystemExit(blobs.to_pylist())
        """
    )
    result = subprocess.run(
        [sys.executable, "-c", script],
        capture_output=True,
        text=True,
        check=False,
    )
    assert result.returncode == 0, result.stderr
 def test_add_all_null_blob_column():
    db = lancedb.connect("memory:///")
    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
    table = db.create_table("all_null", schema=schema)
    table.add([{"id": 1, "image": None}, {"id": 2, "image": None}])
    by_id = _row_ids_by_id(table)
    blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
    assert blobs.to_pylist() == [None, None]
 def test_create_table_nested_blob_schema_without_rows():
    db = lancedb.connect("memory:///")
    schema = pa.schema(
        [
            pa.field("id", pa.int64()),
            pa.field("info", pa.struct([lancedb.blob("blob")])),
            pa.field("images", pa.list_(lancedb.blob("image"))),
        ]
    )
    table = db.create_table("nested_empty", schema=schema)
    assert table.count_rows() == 0
 def test_merge_insert_nested_blob_dicts():
    db = lancedb.connect("memory:///")
    info = pa.StructArray.from_arrays(
        [
            pa.array(["first"], type=pa.string()),
            _blob_array("blob", [b"before"]),
        ],
        names=["name", "blob"],
    )
    data = pa.Table.from_arrays(
        [pa.array([1], type=pa.int64()), info],
        names=["id", "info"],
    )
    table = db.create_table("nested_merge", data=data)
    result = (
        table.merge_insert("id")
        .when_matched_update_all()
        .execute([{"id": 1, "info": {"name": "first", "blob": b"after"}}])
    )
    assert result.num_updated_rows == 1
    by_id = _row_ids_by_id(table)
    blobs = table.fetch_blobs("info.blob", [by_id[1]])
    assert blobs.to_pylist() == [b"after"]
 def _list_blob_table(name):
    db = lancedb.connect("memory:///")
    blob_field = lancedb.blob("image")
    images = pa.ListArray.from_arrays(
        pa.array([0, 1], type=pa.int32()), _blob_array("image", [b"before"])
    )
    data = pa.Table.from_arrays(
        [pa.array([1], type=pa.int64()), images],
        schema=pa.schema(
            [pa.field("id", pa.int64()), pa.field("images", pa.list_(blob_field))]
        ),
    )
    return db.create_table(name, data=data)
 def test_merge_insert_list_blob_dicts():
    table = _list_blob_table("list_merge")
    result = (
        table.merge_insert("id")
        .when_matched_update_all()
        .when_not_matched_insert_all()
        .execute([{"id": 1, "images": [b"one", b"two"]}, {"id": 2, "images": None}])
    )
    assert result.num_updated_rows == 1
    assert result.num_inserted_rows == 1
    hits = table.search().limit(10).to_arrow()
    sizes = {
        row["id"]: None if row["images"] is None else [d["size"] for d in row["images"]]
        for row in hits.to_pylist()
    }
    assert sizes == {1: [3, 3], 2: None}
 def test_list_blob_column_queries_as_raw_descriptors():
    table = _list_blob_table("list_query")
    hits = table.search().limit(10).to_arrow()
    element = hits.schema.field("images").type.value_type
    assert pa.types.is_struct(element)
    assert "_lance_row_id" not in element.names
    with pytest.raises(ValueError, match="expected struct before segment"):
        table.fetch_blobs("images.image", [0])
 def test_row_addressable_paths_exclude_list_children():
    from lancedb.schema import row_addressable_blob_v2_paths
    schema = pa.schema(
        [
            pa.field("id", pa.int64()),
            pa.field("info", pa.struct([lancedb.blob("blob")])),
            pa.field("images", pa.list_(lancedb.blob("image"))),
        ]
    )
    assert blob_v2_column_paths(schema) == ["info.blob", "images.image"]
    assert row_addressable_blob_v2_paths(schema) == ["info.blob"]
 def test_merge_insert_writes_pylance_blob_array():
    table = _blob_table("merge_pylance", [{"id": 1, "image": b"before"}])
    image = lance.blob_array([b"updated", b"inserted"])
    assert type(image.type) is LanceBlobType
    assert type(image.type) is type(lancedb.BlobType())
    updates = pa.Table.from_arrays(
        [pa.array([1, 2], type=pa.int64()), image], names=["id", "image"]
    )
    result = (
        table.merge_insert("id")
        .when_matched_update_all()
        .when_not_matched_insert_all()
        .execute(updates)
    )
    assert result.num_updated_rows == 1
    assert result.num_inserted_rows == 1
    by_id = _row_ids_by_id(table)
    blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
    assert blobs.to_pylist() == [b"updated", b"inserted"]
 def test_fetch_blobs_accepts_query_result():
    table = _blob_table("from_result", [{"id": 1, "image": b"gamma"}])
    hits = table.search().limit(10).to_arrow()
@@ -184,6 +710,25 @@ def test_fetch_blobs_accepts_query_result():
    assert {blobs[i].as_py() for i in range(len(blobs))} == {b"gamma"}
 def test_fetch_blobs_after_compact_with_stable_row_ids(tmp_path):
    db = lancedb.connect(
        tmp_path, storage_options={"new_table_enable_stable_row_ids": "true"}
    )
    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
    table = db.create_table("t", schema=schema)
    table.add([{"id": 1, "image": b"frag-one"}])
    table.add([{"id": 2, "image": b"frag-two"}])
    by_id = _row_ids_by_id(table)
    ids = [by_id[1], by_id[2]]
    table.optimize()
    blobs = table.fetch_blobs("image", ids)
    assert blobs.to_pylist() == [b"frag-one", b"frag-two"]
    ranges = table.fetch_blob_ranges("image", [(ids[0], 5, 3), (ids[1], 5, 3)])
    assert ranges.to_pylist() == [b"one", b"two"]
 def test_fetch_blobs_preserves_null_and_empty_values():
    table = _blob_table(
        "nulls",
@@ -232,8 +777,25 @@ def test_fetch_blob_ranges_validates_requests():
    with pytest.raises(ValueError, match="offset \\+ length overflowed"):
        table.fetch_blob_ranges("image", [(row_id, 2**64 - 1, 1)])
-    with pytest.raises(ValueError, match="row IDs"):
+    with pytest.raises(ValueError) as exc_info:
        table.fetch_blob_ranges("image", [(2**64 - 1, 0, 1)])
    _assert_missing_blob_row_ids(exc_info)
 def test_fetch_blob_apis_reject_missing_fragment_row_addr():
    table = _blob_table("missing_frag", [{"id": 1, "image": b"x"}])
    live = _row_ids_by_id(table)[1]
    _assert_fetch_apis_reject_missing_row_ids(table, [1 << 32, live])
 def test_fetch_blob_apis_reject_deleted_row_ids():
    table = _blob_table(
        "deleted_rows",
        [{"id": 1, "image": b"one"}, {"id": 2, "image": b"two"}],
    )
    by_id = _row_ids_by_id(table)
    table.delete("id = 2")
    _assert_fetch_apis_reject_missing_row_ids(table, [by_id[2], by_id[1]])
 def test_fetch_blob_ranges_empty_requests_returns_empty_array():
@@ -403,6 +965,50 @@ async def test_blob_v2_hybrid_fetch_blobs_async():
    assert {blobs[i].as_py() for i in range(len(blobs))} == {b"alpha", b"beta"}
@pytest.mark.asyncio
 async def test_async_hybrid_typed_blob_projection_preserves_source_column():
    db = await lancedb.connect_async("memory:///hybrid_typed_blob")
    schema = pa.schema(
        [
            pa.field("id", pa.int64()),
            pa.field("text", pa.utf8()),
            pa.field("vector", pa.list_(pa.float32(), list_size=2)),
            lancedb.blob("blob"),
        ]
    )
    table = await db.create_table("hybrid_typed_blob", schema=schema)
    await table.add(
        [
            {
                "id": 1,
                "text": "hello alpha",
                "vector": [1.0, 0.0],
                "blob": b"alpha",
            },
            {
                "id": 2,
                "text": "hello beta",
                "vector": [0.9, 0.1],
                "blob": b"beta",
            },
        ]
    )
    await table.create_index("text", config=FTS(with_position=False))
    hits = await (
        table.query()
        .nearest_to([1.0, 0.0])
        .nearest_to_text("hello")
        .select({"blob_alias": col("blob")})
        .limit(2)
        .to_arrow()
    )
    assert "_lance_row_id" in hits.schema.field("blob_alias").type.names
    blobs = await table.fetch_blobs("blob", hits)
    assert {blobs[i].as_py() for i in range(len(blobs))} == {b"alpha", b"beta"}
 def test_blob_file_seek_read_and_read_range():
    payload = _identifiable_payload(1024)
    table = _blob_table("seek_read", [{"id": 1, "image": payload}])
@@ -617,3 +1223,71 @@ def test_fetch_blobs_nested_path_survives_sort_after_query():
 def _identifiable_payload(size: int) -> bytes:
    block = 256
    return b"".join(bytes([i % 256]) * block for i in range(size // block))
 def _external_uri_blob_array(uris):
    blob_type = lancedb.blob("image").type
    storage_type = blob_type.storage_type
    child_names = [field.name for field in storage_type]
    assert "uri" in child_names, "blob layout no longer has a uri child"
    children = [
        pa.array(uris if field.name == "uri" else [None] * len(uris), type=field.type)
        for field in storage_type
    ]
    storage = pa.StructArray.from_arrays(children, fields=list(storage_type))
    return pa.ExtensionArray.from_storage(blob_type, storage)
 def _external_uri_table_and_rows(name, uris):
    db = lancedb.connect("memory:///")
    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
    table = db.create_table(name, schema=schema)
    rows = pa.Table.from_arrays(
        [
            pa.array(range(len(uris)), type=pa.int64()),
            _external_uri_blob_array(uris),
        ],
        schema=schema,
    )
    return table, rows
 def test_add_external_uri_struct_round_trips_with_flag(tmp_path):
    payload = b"external-uri-bytes"
    blob_path = tmp_path / "payload.bin"
    blob_path.write_bytes(payload)
    table, rows = _external_uri_table_and_rows("external_struct", [blob_path.as_uri()])
    table.add(rows, allow_external_blob_outside_bases=True)
    hits = table.search().to_arrow()
    blobs = table.fetch_blobs("image", hits)
    assert blobs[0].as_py() == payload
 def test_add_external_uri_without_flag_raises(tmp_path):
    blob_path = tmp_path / "payload.bin"
    blob_path.write_bytes(b"unreachable")
    table, rows = _external_uri_table_and_rows("external_no_flag", [blob_path.as_uri()])
    with pytest.raises(ValueError, match="allow_external_blob_outside_bases"):
        table.add(rows)
    assert table.count_rows() == 0
 def test_add_external_uri_string_round_trips_with_flag(tmp_path):
    payload = b"external-uri-bytes"
    blob_path = tmp_path / "payload.bin"
    blob_path.write_bytes(payload)
    db = lancedb.connect("memory:///")
    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
    table = db.create_table("external_string", schema=schema)
    table.add(
        [{"id": 1, "image": blob_path.as_uri()}],
        allow_external_blob_outside_bases=True,
    )
    hits = table.search().to_arrow()
    blobs = table.fetch_blobs("image", hits)
    assert blobs[0].as_py() == payload
--- a/Show More
+++ b/Show More