chore: retain computed column support in beta.4 update

`add_columns().computed()` accepted several columns in one call but bound
each against the table's schema as it stood before the call, so `a` and
`b = a + 1` had to be two commits. A server staging declarations behind
other schema work has no atomic way to do that, and a caller reading the
builder's plural signature reasonably expects the batch to be one.

Each accepted column now joins the schema the next one resolves against,
so the batch is planned and committed as one. Order is the dependency
order; reading ahead is still an unknown column. `validate_declarations`
exposes the schema-level checks -- the Function-binding guard and the
planning -- without a commit, for callers that must reject before earlier
work in the same request lands; LSM state is table state and stays a
commit-time check.

Dependent columns need refresh to be dependency-aware, or `b =
coalesce(a, 0)` refreshed before `a` bakes zeros from `a`'s placeholder
null and the fill-once contract keeps them. Refresh now walks the
dependency graph once and fills each reachable column once in dependency
order, each as its own commit, then the requested column. Every fill in
the pass covers only the fragments of the snapshot the pass started from:
a commit may rebase over a concurrent append, and the fragment that admits
carries placeholder nulls no earlier fill covered, so it waits for a later
refresh and is reported as `rows_remaining`. Two concurrent fills of one
input collide on its field in lance's conflict check, so a dependent fill
can only commit over inputs that were durable when it read them.
`rows_filled` counts the requested column only; the async result's
`published_version` is the last commit of the pass, inputs included.

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.38.0-beta.12"
+current_version = "0.38.0-beta.11"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

.github/dependabot.yml

@@ -44,27 +44,3 @@ updates:
       python-deps:
         patterns:
           - "*"
-
-  # The npm ecosystem covers pnpm lockfiles. There are two separate installs:
-  # the bindings themselves and the examples, which have their own lockfile.
-  # As with cargo and pip above, only bump the lockfile — the version ranges
-  # in package.json are our consumers' constraints, not ours.
-  - package-ecosystem: npm
-    directory: /nodejs
-    schedule:
-      interval: weekly
-    versioning-strategy: lockfile-only
-    groups:
-      nodejs-deps:
-        patterns:
-          - "*"
-
-  - package-ecosystem: npm
-    directory: /nodejs/examples
-    schedule:
-      interval: weekly
-    versioning-strategy: lockfile-only
-    groups:
-      nodejs-examples-deps:
-        patterns:
-          - "*"

.github/workflows/dev.yml

@@ -29,14 +29,12 @@ jobs:
     steps:
       - uses: actions/setup-node@v6
         with:
-          node-version: "24"
-      - uses: pnpm/action-setup@v6
-        with:
-          version: 11.1.1
+          node-version: "18"
       # These rules are disabled because Github will always ensure there
       # is a blank line between the title and the body and Github will
       # word wrap the description field to ensure a reasonable max line
       # length.
+      - run: npm install @commitlint/config-conventional
       - run: >
           echo 'module.exports = {
             "rules": {
@@ -45,11 +43,7 @@ jobs:
               "body-leading-blank": [0, "always"]
             }
           }' > .commitlintrc.js
-      - run: >
-          pnpm dlx
-          --package @commitlint/cli@21.2.2
-          --package @commitlint/config-conventional@21.2.2
-          commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
+      - run: npx commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
         env:
           COMMIT_MSG: >
             ${{ github.event.pull_request.title }}
@@ -60,7 +54,7 @@ jobs:
         with:
           script: |
             const message = `**ACTION NEEDED**
-
+              
               Lance follows the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) for release automation.
 
               The PR title and description are used as the merge commit message.\

.github/workflows/docs-link-check.yml

@@ -56,7 +56,7 @@ jobs:
         uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0
         with:
           # Restricted to http(s) on purpose. Much of docs/src is generated
-          # API reference (the js/ tree comes from `pnpm run docs` in nodejs)
+          # API reference (the js/ tree comes from `npm run docs` in nodejs)
           # and the hand-written pages use mkdocstrings cross-references and
           # nav-relative paths that only resolve in the site mkdocs builds,
           # not in this checkout, so relative links would be reported as

.github/workflows/docs.yml

@@ -55,7 +55,9 @@ jobs:
       - name: Set up node
         uses: actions/setup-node@v6
         with:
-          node-version: 24
+          node-version: 20
+          cache: 'npm'
+          cache-dependency-path: docs/package-lock.json
       - name: Install node dependencies
         working-directory: nodejs
         run: |

.github/workflows/nodejs.yml

@@ -47,8 +47,9 @@ jobs:
         version: 11.1.1
     - uses: actions/setup-node@v6
       with:
-        # Build on a supported LTS; the matrix job below covers every
-        # Node version the library claims to support.
+        # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+        # in October. The library itself still supports Node >= 18
+        # (see test matrix below).
         node-version: 24
         cache: 'pnpm'
         cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -83,7 +84,7 @@ jobs:
     timeout-minutes: 30
     strategy:
       matrix:
-        node-version: [ "22", "24", "26" ]
+        node-version: [ "18", "20" ]
     runs-on: "ubuntu-22.04"
     defaults:
       run:
@@ -100,9 +101,9 @@ jobs:
     - uses: actions/setup-node@v6
       name: Setup Node.js 24 for build
       with:
-        # Build and install once on a fixed version so the generated docs
-        # are identical across matrix legs; the tests below then run on each
-        # supported Node version.
+        # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+        # in October. Build/install runs on Node 24; tests run on the
+        # matrix version below using direct jest invocation.
         node-version: 24
         cache: 'pnpm'
         cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -151,9 +152,9 @@ jobs:
         S3_TEST: "1"
         # Newer @smithy/core uses dynamic ESM imports.
         NODE_OPTIONS: "--experimental-vm-modules"
-      # Invoke the installed jest binary directly; the pnpm shim is set up
-      # against the build-phase Node, not the version selected above.
-      run: node_modules/.bin/jest --verbose
+      # Invoke jest directly because pnpm 11 itself requires Node 22+
+      # while the matrix tests on older Node versions.
+      run: npx jest --verbose
     - name: Test examples
       working-directory: ./
       env:
@@ -163,7 +164,7 @@ jobs:
       run: |
         python ci/mock_openai.py &
         cd nodejs/examples
-        node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose
+        npx jest --testEnvironment jest-environment-node-single-context --verbose
   macos:
     timeout-minutes: 30
     # macos-15 ships a newer linker; the older macos-14 linker fails to insert
@@ -184,7 +185,8 @@ jobs:
         version: 11.1.1
     - uses: actions/setup-node@v6
       with:
-        # pnpm 11 requires Node >= 22.13.
+        # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+        # in October.
         node-version: 24
         cache: 'pnpm'
         cache-dependency-path: nodejs/pnpm-lock.yaml

.github/workflows/npm-publish.yml

@@ -40,31 +40,40 @@ jobs:
           - target: aarch64-apple-darwin
             host: macos-latest
             features: fp16kernels
-            # Fat LTO was ~111 of this job's ~113 minutes.
-            lto: thin
-            codegen_units: 16
             pre_build: |-
               brew install protobuf
+              # Fat LTO (the workspace default in .cargo/config.toml) is
+              # single-threaded and is the peak-memory step of the build. On
+              # this runner it accounted for ~111 of the job's ~113 minutes,
+              # making it the critical path of the entire publish pipeline.
+              # ThinLTO parallelizes it across the runner's cores, for a few
+              # percent of runtime performance.
+              export CARGO_PROFILE_RELEASE_LTO=thin
+              export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
           - target: x86_64-pc-windows-msvc
             host: windows-2025
             features: ","
-            # The lower peak also keeps this on the standard 4-core runner.
-            lto: thin
-            codegen_units: 16
             pre_build: |-
               choco install --no-progress protoc ninja nasm
               tail -n 1000 /c/ProgramData/chocolatey/logs/chocolatey.log
               # There is an issue where choco doesn't add nasm to the path
               export PATH="$PATH:/c/Program Files/NASM"
               nasm -v
+              # See the ThinLTO note on aarch64-apple-darwin above. Keeping
+              # peak memory down is also what lets this run on the standard
+              # 4-core runner: the 8-core larger runner was only needed to
+              # stop fat LTO from OOMing rustc-LLVM.
+              export CARGO_PROFILE_RELEASE_LTO=thin
+              export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
           - target: aarch64-pc-windows-msvc
             host: windows-2025
             features: ","
-            lto: thin
-            codegen_units: 16
             pre_build: |-
                 choco install --no-progress protoc
                 rustup target add aarch64-pc-windows-msvc
+                # See the ThinLTO note on aarch64-apple-darwin above.
+                export CARGO_PROFILE_RELEASE_LTO=thin
+                export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
           - target: x86_64-unknown-linux-gnu
             host: ubuntu-latest
             features: fp16kernels
@@ -94,14 +103,6 @@ jobs:
             # https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
             docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
             features: "fp16kernels"
-            # Fat LTO OOM-killed rustc every nightly; even with lld it peaked
-            # at 31391 MiB of the runner's 32 GiB.
-            lto: thin
-            codegen_units: 16
-            # arm64 Linux links through GNU `ld` where x86_64 defaults to
-            # `rust-lld`, which is why only arm64 OOM'd. lld cut the largest
-            # linker process 7.0 -> 4.0 GiB (lancedb/sophon#7313).
-            linker: /tmp/aarch64-lld-clang
             pre_build: |-
               set -e &&
               apt-get update &&
@@ -111,30 +112,9 @@ jobs:
               # AT_HWCAP2 (added in Linux 3.17). Define it for aws-lc-sys.
               export CFLAGS="$CFLAGS -DAT_HWCAP2=26" &&
               rustup target add aarch64-unknown-linux-gnu
-              # Not `&&`-chained: in dash, errexit does not fire for a
-              # non-final command in an `&&` list, so failures were ignored.
-              #
-              # A wrapper rather than `-C link-arg` because the per-target
-              # rustflags variable does not reach every unit that links, while
-              # the linker variable does. `clang` because GCC silently ignores
-              # `-fuse-ld=lld` unless built with lld support. Two echoes
-              # because printf's newline escape gets rewritten to `;` between
-              # here and the container.
-              echo '#!/bin/sh' > /tmp/aarch64-lld-clang
-              echo 'exec clang --target=aarch64-unknown-linux-gnu --sysroot=/usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot --gcc-toolchain=/usr/aarch64-unknown-linux-gnu -fuse-ld=lld "$@"' >> /tmp/aarch64-lld-clang
-              chmod 0755 /tmp/aarch64-lld-clang
-              # Fail now, not at the cdylib link ~30 minutes later. Linking at
-              # all also proves lld resolved; clang errors out when it cannot.
-              echo 'int main(void){return 0;}' > /tmp/probe.c
-              /tmp/aarch64-lld-clang /tmp/probe.c -o /tmp/probe
-              readelf -h /tmp/probe | grep AArch64
           - target: aarch64-unknown-linux-musl
             host: ubuntu-2404-8x-x64
             features: ","
-            # Fat LTO took the whole runner down. lld cannot help: it died
-            # inside rustc's LLVM, before any linker was spawned.
-            lto: thin
-            codegen_units: 16
             pre_build: |-
               set -e &&
               sudo apt-get update &&
@@ -143,19 +123,6 @@ jobs:
               export EXTRA_ARGS="-x"
     name: build - ${{ matrix.settings.target }}
     runs-on: ${{ matrix.settings.host }}
-    # On the job, not exported from `pre_build`: `Swatinem/rust-cache` hashes
-    # `CARGO_*` into its cache key before any step runs, so a step-local export
-    # leaves the key unchanged while cargo still rebuilds cold. The ThinLTO
-    # legs had been doing that every run.
-    #
-    # Not `RUSTFLAGS`: setting it, even to "", discards every config-file
-    # rustflag, silently dropping .cargo/config.toml's `target-cpu` and
-    # `target-feature` from the published binaries.
-    env:
-      CARGO_PROFILE_RELEASE_LTO: ${{ matrix.settings.lto || 'fat' }}
-      CARGO_PROFILE_RELEASE_CODEGEN_UNITS: ${{ matrix.settings.codegen_units || '1' }}
-      # Empty elsewhere: a per-target variable is only read for that triple.
-      CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: ${{ matrix.settings.linker }}
     defaults:
       run:
         working-directory: nodejs
@@ -168,7 +135,8 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v6
         with:
-          # pnpm 11 requires Node >= 22.13.
+          # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+          # in October.
           node-version: 24
           cache: pnpm
           cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -201,15 +169,19 @@ jobs:
           # creating ref). The nightly cadence also keeps entries inside
           # GitHub's 7-day eviction window, which a tag-only trigger would not.
           save-if: ${{ github.ref == 'refs/heads/main' }}
-      # Docker builds can use rust-cache too: the workspace is bind-mounted, so
-      # `target/` lives on the host and rust-cache's prune keeps the entry
-      # small.
+      # Docker builds can use rust-cache too. `target/` already lives on the
+      # host because the whole workspace is bind-mounted into the container, and
+      # rust-cache's prune and save run host-side, so they can manage it -- which
+      # is what keeps the entry to dependency artifacts rather than a multi-GB
+      # copy of everything.
       #
       # Two differences from the native builds. The container's CARGO_HOME is
-      # bind-mounted from `.cargo-cache` rather than ~/.cargo, so that is cached
-      # explicitly. And the key uses the *host* rustc version, not the compiler
-      # that built these artifacts -- safe, since cargo fingerprints the real
-      # one; a base-image bump just costs one cold build.
+      # bind-mounted from `.cargo-cache` rather than the host's ~/.cargo, so that
+      # has to be cached explicitly. And the key is derived from the *host* rustc
+      # version, which is not the compiler that produced these artifacts; that is
+      # safe because cargo fingerprints the real compiler and rebuilds on a
+      # mismatch, it just means a base-image toolchain bump costs one cold build
+      # instead of invalidating the key.
       - name: Cache cargo (docker builds)
         uses: Swatinem/rust-cache@v2
         if: ${{ matrix.settings.docker }}
@@ -238,19 +210,14 @@ jobs:
           # cache step above saves. Previously the registry mounts pointed at
           # `.cargo/...`, a path nothing cached, so the container re-downloaded
           # the whole crate registry on every run.
-          #
-          # `docker run` inherits nothing; `-e NAME` carries the job's `env:` in.
           options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
                     -v ${{ github.workspace }}/.cargo-cache/registry/cache:/usr/local/cargo/registry/cache \
                     -v ${{ github.workspace }}/.cargo-cache/registry/index:/usr/local/cargo/registry/index \
-                    -e CARGO_PROFILE_RELEASE_LTO \
-                    -e CARGO_PROFILE_RELEASE_CODEGEN_UNITS \
-                    -e CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER \
                     -v ${{ github.workspace }}:/build -w /build/nodejs"
           run: |
             set -e
             ${{ matrix.settings.pre_build }}
-            node_modules/.bin/napi build --platform --release \
+            npx napi build --platform --release \
               --features ${{ matrix.settings.features }} \
               --target ${{ matrix.settings.target }} \
               --dts ../lancedb/native.d.ts \
@@ -270,7 +237,7 @@ jobs:
       - name: Build
         run: |
           ${{ matrix.settings.pre_build }}
-          node_modules/.bin/napi build --platform --release \
+          npx napi build --platform --release \
               --features ${{ matrix.settings.features }} \
               --target ${{ matrix.settings.target }} \
               --dts ../lancedb/native.d.ts \
@@ -289,18 +256,6 @@ jobs:
         if: always()
         run: df -h
         shell: bash
-      - name: Report peak memory
-        if: always() && runner.os == 'Linux'
-        shell: bash
-        run: |
-          peak=$(find /sys/fs/cgroup -name memory.peak -readable \
-                 -exec cat {} + 2>/dev/null | sort -n | tail -1)
-          if [ -n "$peak" ]; then
-            echo "peak memory: $((peak / 1024 / 1024)) MiB"
-          else
-            echo "peak memory: unavailable (no readable cgroup v2 memory.peak)"
-          fi
-          free -g || true
       - name: Upload artifact
         uses: actions/upload-artifact@v7
         with:
@@ -338,7 +293,7 @@ jobs:
           - target: aarch64-unknown-linux-gnu
             host: ubuntu-2404-8x-arm64
         node:
-          - '22'
+          - '20'
     runs-on: ${{ matrix.settings.host }}
     defaults:
       run:
@@ -384,9 +339,9 @@ jobs:
       - name: Move built files
         run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
       - name: Test bindings
-        # Invoke the installed jest binary directly; the pnpm shim is set up
-        # against the install-phase Node, not the version selected above.
-        run: node_modules/.bin/jest --verbose
+        # Invoke jest directly because pnpm 11 itself requires Node 22+
+        # while the matrix tests on older Node versions.
+        run: npx jest --verbose
   publish:
     name: Publish
     runs-on: ubuntu-latest

.github/workflows/update_package_lock_run.yml

@@ -0,0 +1,22 @@
+name: Update package-lock.json
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          ref: main
+          persist-credentials: false
+          fetch-depth: 0
+          lfs: true
+      - uses: ./.github/workflows/update_package_lock
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.github/workflows/update_package_lock_run_nodejs.yml

@@ -0,0 +1,22 @@
+name: Update NodeJs package-lock.json
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          ref: main
+          persist-credentials: false
+          fetch-depth: 0
+          lfs: true
+      - uses: ./.github/workflows/update_package_lock_nodejs
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.pre-commit-config.yaml

@@ -20,10 +20,7 @@ repos:
     hooks:
       - id: local-biome-check
         name: biome check
-        # Use the biome from nodejs/package.json rather than a separately
-        # pinned one: the two drifted apart and disagreed on formatting, so
-        # this hook rejected code that `pnpm lint` accepted.
-        entry: nodejs/node_modules/.bin/biome check --config-path nodejs/biome.json nodejs/
+        entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
         language: system
         types: [text]
         files: "nodejs/.*"

AGENTS.md

@@ -38,7 +38,7 @@ Before committing changes, run formatting for every language you touched. At min
 * Rust changes: run `cargo fmt --all`.
 * Python changes: run `ruff format .` and `ruff check .` from the repository root,
   and run targeted tests through `cd python && uv run ...`.
-* TypeScript changes: run the relevant `pnpm` lint, format, build, and docs commands in `nodejs`.
+* TypeScript changes: run the relevant `npm`/`pnpm` lint, format, build, and docs commands in `nodejs`.
 
 Before creating a PR, the exact value passed to `gh pr create --title` must follow
 Conventional Commits, such as `fix: support nested field paths in native index creation`
@@ -101,12 +101,12 @@ Python bindings changes:
 TypeScript bindings changes:
 
 1. Add napi-rs method binding on `Table` in `nodejs/src/table.rs`.
-2. Run `pnpm build` to generate TypeScript definitions.
+2. Run `npm run build` to generate TypeScript definitions.
 3. Add typescript method on abstract class `Table` in `nodejs/src/table.ts`.
 4. Add concrete method on `LocalTable` class in `nodejs/src/native_table.ts`.
     * Note: despite the name, this class is also used for remote tables.
 5. Add test in `nodejs/__test__/table.test.ts`.
-6. Run `pnpm run docs` to generate TypeScript documentation.
+6. Run `npm run docs` to generate TypeScript documentation.
 
 ## Python API reference
 

Cargo.lock

@@ -1597,9 +1597,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
 
 [[package]]
 name = "chacha20"
-version = "0.10.2"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
+checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601"
 dependencies = [
  "cfg-if 1.0.4",
  "cpufeatures 0.3.0",
@@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arrow-array",
  "rand 0.9.5",
@@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 
 [[package]]
 name = "lance"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -4888,8 +4888,8 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4911,7 +4911,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4925,7 +4925,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -4934,8 +4934,8 @@ dependencies = [
 
 [[package]]
 name = "lance-bitpacking"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arrayref",
  "crunchy",
@@ -4945,8 +4945,8 @@ dependencies = [
 
 [[package]]
 name = "lance-core"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4983,8 +4983,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datafusion"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5013,8 +5013,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5031,8 +5031,8 @@ dependencies = [
 
 [[package]]
 name = "lance-derive"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5041,8 +5041,8 @@ dependencies = [
 
 [[package]]
 name = "lance-encoding"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -5075,8 +5075,8 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -5107,8 +5107,8 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -5172,8 +5172,8 @@ dependencies = [
 
 [[package]]
 name = "lance-index-core"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5195,8 +5195,8 @@ dependencies = [
 
 [[package]]
 name = "lance-io"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5236,8 +5236,8 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5251,8 +5251,8 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arrow",
  "async-trait",
@@ -5264,8 +5264,8 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-impls"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -5318,8 +5318,8 @@ dependencies = [
 
 [[package]]
 name = "lance-select"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5333,8 +5333,8 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5374,8 +5374,8 @@ dependencies = [
 
 [[package]]
 name = "lance-testing"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5388,8 +5388,8 @@ dependencies = [
 
 [[package]]
 name = "lance-tokenizer"
-version = "12.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
+version = "12.0.0-beta.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
 dependencies = [
  "frostem",
  "icu_segmenter",
@@ -5402,7 +5402,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb"
-version = "0.38.0-beta.12"
+version = "0.38.0-beta.11"
 dependencies = [
  "ahash",
  "anyhow",
@@ -5490,7 +5490,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb-nodejs"
-version = "0.38.0-beta.12"
+version = "0.38.0-beta.11"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5515,7 +5515,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb-python"
-version = "0.38.0-beta.12"
+version = "0.38.0-beta.11"
 dependencies = [
  "arrow",
  "async-trait",

Cargo.toml

@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"
 
 [workspace.dependencies]
-lance = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=12.0.0-beta.4", default-features = false, "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=12.0.0-beta.4", default-features = false, "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=12.0.0-beta.4", default-features = false, "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
 lancedb = { path = "rust/lancedb", default-features = false }
 ahash = "0.8"
 # Note that this one does not include pyarrow

Makefile

@@ -5,5 +5,5 @@ licenses:
 	cd python && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
 	cd python && uv sync --all-extras && uv tool run pip-licenses --python .venv/bin/python --format=markdown --with-urls --output-file=PYTHON_THIRD_PARTY_LICENSES.md
 	cd nodejs && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
-	cd nodejs && pnpm dlx license-checker@25 --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
+	cd nodejs && npx license-checker --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
 	cd java && ./mvnw license:aggregate-add-third-party -q

ci/update_lockfiles.sh

@@ -12,12 +12,16 @@ done
 # This updates the lockfile without building
 cargo metadata --quiet > /dev/null
 
+pushd nodejs || exit 1
+npm install --package-lock-only --silent
+popd
+
 if git diff --quiet --exit-code; then
   echo "No lockfile changes to commit; skipping amend."
 elif $AMEND; then
-  git add Cargo.lock
+  git add Cargo.lock nodejs/package-lock.json
   git commit --amend --no-edit
 else
-  git add Cargo.lock
+  git add Cargo.lock nodejs/package-lock.json
   git commit -m "Update lockfiles"
 fi

deny.toml

@@ -131,13 +131,18 @@ allow = [
     "BSD-3-Clause",
     "ISC",
     "Unicode-3.0",
+    "Unicode-DFS-2016",
     "Zlib",
     "CC0-1.0",
     "MPL-2.0",
     "BSL-1.0",
+    "OpenSSL",
     # 0BSD ("BSD Zero Clause") is effectively public domain — no attribution
     # required. Pulled in by `mock_instant`.
     "0BSD",
+    # bzip2-1.0.6 is the permissive upstream bzip2 license (BSD-like). Pulled
+    # in by `libbz2-rs-sys`, the pure-Rust bzip2 implementation.
+    "bzip2-1.0.6",
     # CDLA-Permissive-2.0 is a permissive data license used by `webpki-roots`
     # for the Mozilla CA root bundle. Data-only, distribution-compatible.
     "CDLA-Permissive-2.0",
@@ -145,7 +150,12 @@ allow = [
 confidence-threshold = 0.8
 # Per-crate license exceptions: allow a license for a specific crate only,
 # rather than globally via the `allow` list above.
-exceptions = []
+exceptions = [
+    # CDDL-1.0 (copyleft) is pulled in only as a dev/profiling dependency via
+    # `inferno` -> `pprof` -> `lance-testing`; it is a test dependency that we
+    # do not distribute, so scope the allowance to `inferno` alone.
+    { allow = ["CDDL-1.0"], crate = "inferno" },
+]
 # Crates whose license cannot be determined from Cargo metadata but whose
 # license we've manually confirmed from upstream. Keep this list minimal.
 [[licenses.clarify]]

docs/README.md

@@ -47,24 +47,22 @@ pytest -vv python/tests/docs
 
 ### Checking typescript examples
 
-The examples depend on `@lancedb/lancedb` at `file:../dist`, so the package must be
-built before running the tests. This uses pnpm; see the
-[Typescript contributing guide](../nodejs/CONTRIBUTING.md) for the toolchain setup.
+The `@lancedb/lancedb` package must be built before running the tests:
 
 ```shell
 pushd nodejs
-pnpm install
-pnpm build
+npm ci
+npm run build
 popd
 ```
 
-Then you can run the examples by going to the `nodejs/examples` directory, which is a
-separate pnpm package with its own lockfile:
+Then you can run the examples by going to the `nodejs/examples` directory and
+running the tests like a normal npm package:
 
 ```shell
 pushd nodejs/examples
-pnpm install
-pnpm test
+npm ci
+npm test
 popd
 ```
 
@@ -86,7 +84,6 @@ The new files should be checked into the repository.
 
 ```shell
 pushd nodejs
-# `pnpm docs` would invoke pnpm's built-in `docs` command, not the script.
-pnpm run docs
+npm run docs
 popd
 ```

docs/package-lock.json

@@ -0,0 +1,135 @@
+{
+  "name": "lancedb-docs-test",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "lancedb-docs-test",
+      "version": "1.0.0",
+      "license": "Apache 2",
+      "dependencies": {
+        "apache-arrow": "file:../node/node_modules/apache-arrow",
+        "vectordb": "file:../node"
+      },
+      "devDependencies": {
+        "@types/node": "^20.11.8",
+        "typescript": "^5.3.3"
+      }
+    },
+    "../node": {
+      "name": "vectordb",
+      "version": "0.21.2-beta.0",
+      "cpu": [
+        "x64",
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "os": [
+        "darwin",
+        "linux",
+        "win32"
+      ],
+      "dependencies": {
+        "@neon-rs/load": "^0.0.74",
+        "axios": "^1.4.0"
+      },
+      "devDependencies": {
+        "@neon-rs/cli": "^0.0.160",
+        "@types/chai": "^4.3.4",
+        "@types/chai-as-promised": "^7.1.5",
+        "@types/mocha": "^10.0.1",
+        "@types/node": "^18.16.2",
+        "@types/sinon": "^10.0.15",
+        "@types/temp": "^0.9.1",
+        "@types/uuid": "^9.0.3",
+        "@typescript-eslint/eslint-plugin": "^5.59.1",
+        "apache-arrow-old": "npm:apache-arrow@13.0.0",
+        "cargo-cp-artifact": "^0.1",
+        "chai": "^4.3.7",
+        "chai-as-promised": "^7.1.1",
+        "eslint": "^8.39.0",
+        "eslint-config-standard-with-typescript": "^34.0.1",
+        "eslint-plugin-import": "^2.26.0",
+        "eslint-plugin-n": "^15.7.0",
+        "eslint-plugin-promise": "^6.1.1",
+        "mocha": "^10.2.0",
+        "openai": "^4.24.1",
+        "sinon": "^15.1.0",
+        "temp": "^0.9.4",
+        "ts-node": "^10.9.1",
+        "ts-node-dev": "^2.0.0",
+        "typedoc": "^0.24.7",
+        "typedoc-plugin-markdown": "^3.15.3",
+        "typescript": "^5.1.0",
+        "uuid": "^9.0.0"
+      },
+      "optionalDependencies": {
+        "@lancedb/vectordb-darwin-arm64": "0.21.2-beta.0",
+        "@lancedb/vectordb-darwin-x64": "0.21.2-beta.0",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.21.2-beta.0",
+        "@lancedb/vectordb-linux-x64-gnu": "0.21.2-beta.0",
+        "@lancedb/vectordb-win32-x64-msvc": "0.21.2-beta.0"
+      },
+      "peerDependencies": {
+        "@apache-arrow/ts": "^14.0.2",
+        "apache-arrow": "^14.0.2"
+      }
+    },
+    "../node/node_modules/apache-arrow": {
+      "version": "14.0.2",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@types/command-line-args": "5.2.0",
+        "@types/command-line-usage": "5.0.2",
+        "@types/node": "20.3.0",
+        "@types/pad-left": "2.1.1",
+        "command-line-args": "5.2.1",
+        "command-line-usage": "7.0.1",
+        "flatbuffers": "23.5.26",
+        "json-bignum": "^0.0.3",
+        "pad-left": "^2.1.0",
+        "tslib": "^2.5.3"
+      },
+      "bin": {
+        "arrow2csv": "bin/arrow2csv.js"
+      }
+    },
+    "node_modules/@types/node": {
+      "version": "20.11.8",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.8.tgz",
+      "integrity": "sha512-i7omyekpPTNdv4Jb/Rgqg0RU8YqLcNsI12quKSDkRXNfx7Wxdm6HhK1awT3xTgEkgxPn3bvnSpiEAc7a7Lpyow==",
+      "dev": true,
+      "dependencies": {
+        "undici-types": "~5.26.4"
+      }
+    },
+    "node_modules/apache-arrow": {
+      "resolved": "../node/node_modules/apache-arrow",
+      "link": true
+    },
+    "node_modules/typescript": {
+      "version": "5.3.3",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz",
+      "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==",
+      "dev": true,
+      "bin": {
+        "tsc": "bin/tsc",
+        "tsserver": "bin/tsserver"
+      },
+      "engines": {
+        "node": ">=14.17"
+      }
+    },
+    "node_modules/undici-types": {
+      "version": "5.26.5",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
+      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
+      "dev": true
+    },
+    "node_modules/vectordb": {
+      "resolved": "../node",
+      "link": true
+    }
+  }
+}

docs/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "lancedb-docs-test",
+  "version": "1.0.0",
+  "description": "auto-generated tests from doc",
+  "author": "dev@lancedb.com",
+  "license": "Apache 2",
+  "dependencies": {
+    "apache-arrow": "file:../node/node_modules/apache-arrow",
+    "vectordb": "file:../node"
+  },
+  "scripts": {
+    "build": "tsc -b && cd ../node && npm run build-release",
+    "example": "npm run build && node",
+    "test": "npm run build && ls dist/*.js | xargs -n 1 node"
+  },
+  "devDependencies": {
+    "@types/node": "^20.11.8",
+    "typescript": "^5.3.3"
+  }
+}

docs/src/java/java.md

@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-core</artifactId>
-    <version>0.38.0-beta.12</version>
+    <version>0.38.0-beta.11</version>
 </dependency>
 ```
 

docs/src/python/python.md

@@ -223,14 +223,10 @@ tokens = list(
 Blob columns store large binary values out of line so they can be read lazily
 instead of being materialized with the rest of the row.
 
-`lancedb.BlobType` is `lance.blob.BlobType` when pylance is installed. Without
-pylance, LanceDB uses a matching `lance.blob.v2` extension type so blob columns
-still work. Queries return descriptors. Call
-[`fetch_blob_files`][lancedb.table.Table.fetch_blob_files] for lazy reads or
-[`fetch_blobs`][lancedb.table.Table.fetch_blobs] for eager bytes.
-
 ::: lancedb.blob
 
+::: lancedb.BlobType
+
 ::: lancedb._blob.BlobFile
     options:
       show_root_full_path: false

docs/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "include": [
+    "src/*.ts",
+  ],
+  "compilerOptions": {
+    "target": "es2022",
+    "module": "nodenext",
+    "declaration": true,
+    "outDir": "./dist",
+    "strict": true,
+    "allowJs": true,
+    "resolveJsonModule": true,
+  },
+  "exclude": [
+    "./dist/*",
+  ]
+}

java/lancedb-core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
       <groupId>com.lancedb</groupId>
       <artifactId>lancedb-parent</artifactId>
-      <version>0.38.0-beta.12</version>
+      <version>0.38.0-beta.11</version>
       <relativePath>../pom.xml</relativePath>
     </parent>
 

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.38.0-beta.12</version>
+    <version>0.38.0-beta.11</version>
     <packaging>pom</packaging>
     <name>${project.artifactId}</name>
     <description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>12.0.0-beta.2</lance-core.version>
+        <lance-core.version>12.0.0-beta.4</lance-core.version>
         <spotless.skip>false</spotless.skip>
         <spotless.version>2.30.0</spotless.version>
         <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.38.0-beta.12"
+version = "0.38.0-beta.11"
 publish = false
 license.workspace = true
 description.workspace = true

nodejs/__test__/package.test.ts

@@ -5,8 +5,8 @@ import packageJson = require("../package.json");
 
 describe("package metadata", () => {
   it("requires Node.js type declarations compatible with the runtime", () => {
-    expect(packageJson.engines.node).toBe(">= 22");
-    expect(packageJson.peerDependencies["@types/node"]).toBe(">=22");
+    expect(packageJson.engines.node).toBe(">= 18");
+    expect(packageJson.peerDependencies["@types/node"]).toBe(">=18");
     expect(packageJson.peerDependenciesMeta["@types/node"]).toEqual({
       optional: true,
     });

nodejs/__test__/remote.test.ts

@@ -3,7 +3,6 @@
 
 import * as http from "http";
 import { RequestListener } from "http";
-import packageJson = require("../package.json");
 import {
   ClientConfig,
   Connection,
@@ -71,13 +70,7 @@ async function withMockDatabase(
   try {
     await callback(db);
   } finally {
-    // `close()` alone leaves the port bound until keep-alive sockets drain, so
-    // a single failing test would cascade into EADDRINUSE for every test after
-    // it. Destroy the connections and wait for the port to actually be free.
-    await new Promise<void>((resolve) => {
-      server.closeAllConnections();
-      server.close(() => resolve());
-    });
+    server.close();
   }
 }
 
@@ -138,7 +131,7 @@ describe("remote connection", () => {
       (req, res) => {
         expect(req.headers["x-api-key"]).toEqual("fake");
         expect(req.headers["user-agent"]).toEqual(
-          `LanceDB-Node-Client/${packageJson.version}`,
+          `LanceDB-Node-Client/${process.env.npm_package_version}`,
         );
 
         const body = JSON.stringify({ tables: [] });

nodejs/examples/package.json

@@ -8,8 +8,7 @@
     "//1": "--experimental-vm-modules is needed to run jest with sentence-transformers",
     "//2": "--testEnvironment is needed to run jest with sentence-transformers",
     "//3": "See: https://github.com/huggingface/transformers.js/issues/57",
-    "//4": "jest is invoked by its JS entry, not node_modules/.bin/jest: under pnpm that path is a shell shim, which `node` cannot execute",
-    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testEnvironment jest-environment-node-single-context --verbose",
+    "test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
     "lint": "biome check *.ts && biome format *.ts",
     "lint-ci": "biome ci .",
     "lint-fix": "biome check --write *.ts && pnpm format",

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.38.0-beta.12",
+	"version": "0.38.0-beta.11",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.38.0-beta.12",
+	"version": "0.38.0-beta.11",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.38.0-beta.12",
+	"version": "0.38.0-beta.11",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.38.0-beta.12",
+	"version": "0.38.0-beta.11",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.38.0-beta.12",
+	"version": "0.38.0-beta.11",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.38.0-beta.12",
+  "version": "0.38.0-beta.11",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.38.0-beta.12",
+	"version": "0.38.0-beta.11",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.38.0-beta.12",
+  "version": "0.38.0-beta.11",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -67,7 +67,7 @@
     "timeout": "3m"
   },
   "engines": {
-    "node": ">= 22"
+    "node": ">= 18"
   },
   "packageManager": "pnpm@11.1.1",
   "cpu": ["x64", "arm64"],
@@ -101,7 +101,7 @@
     "openai": "4.29.2"
   },
   "peerDependencies": {
-    "@types/node": ">=22",
+    "@types/node": ">=18",
     "apache-arrow": ">=15.0.0 <=18.1.0"
   },
   "peerDependenciesMeta": {

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.38.0-beta.12"
+version = "0.38.0-beta.11"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"

python/python/lancedb/__init__.py

@@ -6,7 +6,7 @@ import importlib.metadata
 import os
 from concurrent.futures import ThreadPoolExecutor
 from datetime import timedelta
-from typing import Dict, Optional, Union, Any, List, Iterable, TYPE_CHECKING
+from typing import Dict, Optional, Union, Any, List, Iterable
 
 __version__ = importlib.metadata.version("lancedb")
 
@@ -20,7 +20,7 @@ from .db import AsyncConnection, DBConnection, LanceDBConnection
 from .remote import ClientConfig
 from .remote.db import RemoteDBConnection
 from .expr import Expr, col, lit, func
-from .schema import blob, vector
+from .schema import blob, vector, BlobType
 from .job import AsyncJob, Job
 from .functions import (
     FunctionArtifactRequest as FunctionArtifactRequest,
@@ -49,19 +49,6 @@ from .namespace import (
 )
 
 
-if TYPE_CHECKING:
-    from lance.blob import BlobType as BlobType
-
-
-def __getattr__(name: str):
-    if name == "BlobType":
-        from .schema import BlobType
-
-        globals()["BlobType"] = BlobType
-        return BlobType
-    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
-
-
 def _check_s3_bucket_with_dots(
     uri: str, storage_options: Optional[Dict[str, str]]
 ) -> None:

python/python/lancedb/_blob.py

@@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Optional, Union
 import pyarrow as pa
 
 from .expr import Expr
-from .schema import row_addressable_blob_v2_paths
+from .schema import blob_v2_column_paths
 from .types import BlobMode, QueryProjection, QueryProjectionSpec
 
 if TYPE_CHECKING:
@@ -119,7 +119,7 @@ def blob_v2_projection_sources(
     schema: pa.Schema,
     projection: QueryProjection,
 ) -> dict[str, str]:
-    blob_columns = row_addressable_blob_v2_paths(schema)
+    blob_columns = blob_v2_column_paths(schema)
     if not blob_columns:
         return {}
     columns = set(blob_columns)
@@ -140,9 +140,7 @@ def v2_projection_needs_row_id(
 ) -> bool:
     if with_row_id:
         return False
-    return projection_includes_blob_column(
-        projection, row_addressable_blob_v2_paths(schema)
-    )
+    return projection_includes_blob_column(projection, blob_v2_column_paths(schema))
 
 
 def blob_auto_row_id_for_scan(

python/python/lancedb/remote/table.py

@@ -67,15 +67,7 @@ from ..query import (
     LanceTakeQueryBuilder,
     LanceVectorQueryBuilder,
 )
-from ..table import (
-    AsyncTable,
-    BlobMode,
-    Branches,
-    IndexStatistics,
-    Query,
-    Table,
-    Tags,
-)
+from ..table import AsyncTable, BlobMode, Branches, IndexStatistics, Query, Table, Tags
 from ..types import BaseTokenizerType
 
 

python/python/lancedb/schema.py

@@ -4,34 +4,30 @@
 
 """Schema helpers for Lance blob columns."""
 
-import importlib
-from typing import TYPE_CHECKING
-
 import pyarrow as pa
-import pyarrow.ipc
-
-if TYPE_CHECKING:
-    from lance.blob import BlobType as BlobType
 
 _BLOB_EXTENSION_NAME = "lance.blob.v2"
 _BLOB_V1_KEY = "lance-encoding:blob"
 _ARROW_EXT_NAME_KEY = "ARROW:extension:name"
-_BLOB_V2_STORAGE_TYPE = pa.struct(
-    [
-        pa.field("data", pa.large_binary(), nullable=True),
-        pa.field("uri", pa.utf8(), nullable=True),
-        pa.field("position", pa.uint64(), nullable=True),
-        pa.field("size", pa.uint64(), nullable=True),
-    ]
-)
-_resolved_blob_type = None
 
 
-class _FallbackBlobType(pa.ExtensionType):
-    """lance.blob.v2 extension type used when pylance is not installed."""
+class BlobType(pa.ExtensionType):
+    """PyArrow extension type for a Lance blob v2 column.
+
+    Queries return descriptors; call :meth:`~lancedb.table.Table.fetch_blob_files`
+    for lazy reads or :meth:`~lancedb.table.Table.fetch_blobs` for eager bytes.
+    """
 
     def __init__(self) -> None:
-        pa.ExtensionType.__init__(self, _BLOB_V2_STORAGE_TYPE, _BLOB_EXTENSION_NAME)
+        storage_type = pa.struct(
+            [
+                pa.field("data", pa.large_binary(), nullable=True),
+                pa.field("uri", pa.utf8(), nullable=True),
+                pa.field("position", pa.uint64(), nullable=True),
+                pa.field("size", pa.uint64(), nullable=True),
+            ]
+        )
+        super().__init__(storage_type, _BLOB_EXTENSION_NAME)
 
     def __arrow_ext_serialize__(self) -> bytes:
         return b""
@@ -39,16 +35,23 @@ class _FallbackBlobType(pa.ExtensionType):
     @classmethod
     def __arrow_ext_deserialize__(
         cls, storage_type: pa.DataType, serialized: bytes
-    ) -> "_FallbackBlobType":
+    ) -> "BlobType":
         return cls()
 
     def __reduce__(self):
+        # Ensure pickle round-trips on older pyarrow (apache/arrow#35599).
         return type(self).__arrow_ext_deserialize__, (
             self.storage_type,
             self.__arrow_ext_serialize__(),
         )
 
 
+try:
+    pa.register_extension_type(BlobType())  # type: ignore[arg-type]
+except pa.ArrowKeyError:
+    pass
+
+
 def _metadata_value(metadata: dict, key: str):
     return metadata.get(key.encode()) or metadata.get(key)
 
@@ -89,105 +92,43 @@ def is_blob_like_field(field: pa.Field) -> bool:
     return is_blob_v2_field(field) or _metadata_marks_legacy_blob(field.metadata or {})
 
 
-def _collect_blob_paths(schema: pa.Schema, is_blob) -> list[tuple[str, bool]]:
-    """Walk the schema and return (path, has_list_ancestor) for each blob field."""
-    paths: list[tuple[str, bool]] = []
+def _collect_blob_paths(schema: pa.Schema, is_blob) -> list[str]:
+    paths: list[str] = []
 
-    def walk(fields, prefix: str, has_list_ancestor: bool) -> None:
+    def walk(fields, prefix: str) -> None:
         for field in fields:
             path = f"{prefix}.{field.name}" if prefix else field.name
             if is_blob(field):
-                paths.append((path, has_list_ancestor))
+                paths.append(path)
             elif pa.types.is_struct(field.type):
-                walk(field.type, path, has_list_ancestor)
+                walk(field.type, path)
             elif (
                 pa.types.is_list(field.type)
                 or pa.types.is_large_list(field.type)
                 or pa.types.is_fixed_size_list(field.type)
             ):
-                walk([field.type.value_field], path, True)
+                walk([field.type.value_field], path)
 
-    walk(schema, "", False)
+    walk(schema, "")
     return paths
 
 
 def blob_column_paths(schema: pa.Schema) -> list[str]:
     """Dotted paths of blob-like columns (v2 extension or legacy metadata)."""
-    return [path for path, _ in _collect_blob_paths(schema, is_blob_like_field)]
+    return _collect_blob_paths(schema, is_blob_like_field)
 
 
 def blob_v2_column_paths(schema: pa.Schema) -> list[str]:
-    return [path for path, _ in _collect_blob_paths(schema, is_blob_v2_field)]
-
-
-def row_addressable_blob_v2_paths(schema: pa.Schema) -> list[str]:
-    """Blob v2 paths with one blob addressable by table row id.
-
-    ``fetch_blobs`` and the descriptor row-id ride-along address one blob per
-    row, so a blob inside a list container has no row-id slot and no fetch
-    path. Those columns still store and query as raw descriptors.
-    """
-    return [
-        path
-        for path, has_list_ancestor in _collect_blob_paths(schema, is_blob_v2_field)
-        if not has_list_ancestor
-    ]
+    return _collect_blob_paths(schema, is_blob_v2_field)
 
 
 def schema_has_blob_field(schema: pa.Schema) -> bool:
     return bool(blob_column_paths(schema))
 
 
-def _deserialize_registered_type(extension_type: pa.ExtensionType) -> pa.DataType:
-    """Return the type Arrow reconstructs for this extension name."""
-    schema = pa.schema([pa.field("value", extension_type)])
-    restored = pa.ipc.read_schema(schema.serialize())
-    return restored.field("value").type
-
-
-def _resolve_blob_type():
-    """Return the BlobType class this process should use.
-
-    pylance's class when it owns the lance.blob.v2 registry entry,
-    otherwise LanceDB's fallback. A different registered class is an error.
-    """
-    global _resolved_blob_type
-    if _resolved_blob_type is not None:
-        return _resolved_blob_type
-    try:
-        blob_module = importlib.import_module("lance.blob")
-    except ModuleNotFoundError as err:
-        if err.name not in ("lance", "lance.blob"):
-            raise
-    else:
-        blob_type = getattr(blob_module, "BlobType", None)
-        if blob_type is not None:
-            registered_type = _deserialize_registered_type(blob_type())
-            if type(registered_type) is not blob_type:
-                registered_cls = type(registered_type)
-                raise ValueError(
-                    "lance.blob.v2 is already registered by "
-                    f"{registered_cls.__module__}.{registered_cls.__qualname__}"
-                )
-            _resolved_blob_type = blob_type
-            return blob_type
-    try:
-        pa.register_extension_type(_FallbackBlobType())  # type: ignore[arg-type]
-    except pa.ArrowKeyError as err:
-        raise ValueError(
-            "lance.blob.v2 is already registered by another extension class"
-        ) from err
-    _resolved_blob_type = _FallbackBlobType
-    return _resolved_blob_type
-
-
 def blob(name: str, nullable: bool = True) -> pa.Field:
-    """Create a Lance blob v2 column field.
-
-    When pylance is installed this is ``lance.blob.BlobType``.
-    """
-    blob_type = _resolve_blob_type()
-    return pa.field(name, blob_type(), nullable=nullable)
+    """Create a Lance blob v2 column field."""
+    return pa.field(name, BlobType(), nullable=nullable)
 
 
 def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataType:
@@ -214,11 +155,3 @@ def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataTyp
     ... ])
     """
     return pa.list_(value_type, dimension)
-
-
-def __getattr__(name: str):
-    if name == "BlobType":
-        blob_type = _resolve_blob_type()
-        globals()["BlobType"] = blob_type
-        return blob_type
-    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

python/python/lancedb/table.py

@@ -104,12 +104,7 @@ from .util import (
     value_to_sql,
 )
 from .index import lang_mapping
-from .schema import (
-    blob_v2_column_paths,
-    is_blob_v2_field,
-    row_addressable_blob_v2_paths,
-    schema_has_blob_field,
-)
+from .schema import blob_v2_column_paths, schema_has_blob_field
 
 
 def _should_push_down_query_table(
@@ -431,7 +426,6 @@ def _cast_to_target_schema(
 
     def gen():
         for batch in reader:
-            batch = _coerce_blob_write_columns(batch, reordered_schema)
             # Table but not RecordBatch has cast.
             cast_batches = (
                 pa.Table.from_batches([batch]).cast(reordered_schema).to_batches()
@@ -444,166 +438,6 @@ def _cast_to_target_schema(
     return pa.RecordBatchReader.from_batches(reordered_schema, gen())
 
 
-def _coerce_blob_write_columns(
-    batch: pa.RecordBatch, target_schema: pa.Schema
-) -> pa.RecordBatch:
-    """Materialize blob storage structs before the stream leaves Python.
-
-    merge_insert requires its source reader to already match the table's
-    physical schema. Unlike add and insert, it does not pass through
-    LanceDB's Rust blob coercion, so preserving binary input here would
-    reach Lance as binary and fail the schema check.
-    """
-    columns = []
-    fields = []
-    changed = False
-    for field, column in zip(batch.schema, batch.columns):
-        target_field = target_schema.field(field.name)
-        coerced = _coerce_blob_value(column, target_field)
-        if coerced is not column:
-            column = coerced
-            field = pa.field(
-                field.name,
-                coerced.type,
-                field.nullable,
-                target_field.metadata,
-            )
-            changed = True
-        columns.append(column)
-        fields.append(field)
-    if not changed:
-        return batch
-    return pa.RecordBatch.from_arrays(
-        columns, schema=pa.schema(fields, metadata=batch.schema.metadata)
-    )
-
-
-def _coerce_blob_value(column: pa.Array, target_field: pa.Field) -> pa.Array:
-    if is_blob_v2_field(target_field) and _can_coerce_to_blob(column.type):
-        return _coerce_value_to_blob(column, target_field)
-
-    target_type = target_field.type
-    if pa.types.is_struct(target_type) and pa.types.is_struct(column.type):
-        children = []
-        fields = []
-        changed = False
-        for source_field in column.type:
-            source_column = column.field(source_field.name)
-            nested_target = next(
-                (field for field in target_type if field.name == source_field.name),
-                None,
-            )
-            if nested_target is None:
-                children.append(source_column)
-                fields.append(source_field)
-                continue
-            coerced = _coerce_blob_value(source_column, nested_target)
-            if coerced is not source_column:
-                changed = True
-            child_array, child_type = _physical_array_and_type(coerced)
-            children.append(child_array)
-            fields.append(
-                pa.field(
-                    source_field.name,
-                    child_type,
-                    source_field.nullable,
-                    nested_target.metadata,
-                )
-            )
-        if not changed:
-            return column
-        return pa.StructArray.from_arrays(
-            children,
-            fields=fields,
-            mask=column.is_null() if column.null_count else None,
-        )
-
-    if _is_list_like(target_type) and _is_list_like(column.type):
-        return _coerce_blob_list_values(column, target_type.value_field)
-
-    return column
-
-
-def _coerce_blob_list_values(
-    column: pa.Array, target_value_field: pa.Field
-) -> pa.Array:
-    """Coerce blob values inside a list column, preserving offsets and nulls.
-
-    Works on the raw child values window instead of ``pc.list_flatten`` because
-    flatten drops values spanned by null slots, which would misalign offsets.
-    """
-    mask = column.is_null() if column.null_count else None
-    if pa.types.is_fixed_size_list(column.type):
-        list_size = column.type.list_size
-        values = column.values.slice(column.offset * list_size, len(column) * list_size)
-        coerced = _coerce_blob_value(values, target_value_field)
-        if coerced is values:
-            return column
-        physical_values, _ = _physical_array_and_type(coerced)
-        return pa.FixedSizeListArray.from_arrays(physical_values, list_size, mask=mask)
-    offsets = column.offsets
-    first_offset = offsets[0].as_py()
-    values = column.values.slice(
-        first_offset,
-        offsets[-1].as_py() - first_offset,
-    )
-    coerced = _coerce_blob_value(values, target_value_field)
-    if coerced is values:
-        return column
-    physical_values, _ = _physical_array_and_type(coerced)
-    if first_offset:
-        offsets = pc.subtract(offsets, pa.scalar(first_offset, offsets.type))
-    if pa.types.is_large_list(column.type):
-        return pa.LargeListArray.from_arrays(offsets, physical_values, mask=mask)
-    return pa.ListArray.from_arrays(offsets, physical_values, mask=mask)
-
-
-def _coerce_value_to_blob(values: pa.Array, target_field: pa.Field) -> pa.Array:
-    if pa.types.is_null(values.type):
-        data = pa.nulls(len(values), type=pa.large_binary())
-    elif pa.types.is_large_binary(values.type):
-        data = values
-    else:
-        data = values.cast(pa.large_binary())
-    length = len(values)
-    storage_type = target_field.type
-    if isinstance(storage_type, pa.ExtensionType):
-        storage_type = storage_type.storage_type
-    storage_fields = list(storage_type)
-    children = []
-    for storage_field in storage_fields:
-        if storage_field.name == "data":
-            children.append(data)
-        else:
-            children.append(pa.nulls(length, type=storage_field.type))
-    storage = pa.StructArray.from_arrays(
-        children,
-        fields=storage_fields,
-        mask=values.is_null() if values.null_count else None,
-    )
-    if isinstance(target_field.type, pa.ExtensionType):
-        return pa.ExtensionArray.from_storage(target_field.type, storage)
-    return storage
-
-
-def _physical_array_and_type(array: pa.Array) -> tuple[pa.Array, pa.DataType]:
-    if isinstance(array.type, pa.ExtensionType):
-        return array.storage, array.type.storage_type
-    return array, array.type
-
-
-def _can_coerce_to_blob(data_type: pa.DataType) -> bool:
-    return _is_binary_like(data_type) or pa.types.is_null(data_type)
-
-
-def _is_binary_like(data_type: pa.DataType) -> bool:
-    return (
-        pa.types.is_binary(data_type)
-        or pa.types.is_large_binary(data_type)
-        or pa.types.is_binary_view(data_type)
-    )
-
-
 def _field_extension_name(field: pa.Field) -> Optional[str]:
     extension_name = getattr(field.type, "extension_name", None)
     if extension_name is not None:
@@ -630,73 +464,65 @@ def _align_field_types(
         target_field = next((f for f in target_fields if f.name == field.name), None)
         if target_field is None:
             raise ValueError(f"Field '{field.name}' not found in target schema")
-        new_fields.append(_align_field(field, target_field))
+        # Preserve arrow.json input until it reaches Lance. LanceDB exposes stored
+        # JSON columns as lance.json (JSONB-backed LargeBinary), but casting the
+        # input to that storage type here merely relabels the raw JSON bytes as
+        # JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
+        if (
+            _field_extension_name(field) == "arrow.json"
+            and _field_extension_name(target_field) == "lance.json"
+        ):
+            new_fields.append(field)
+            continue
+        if pa.types.is_struct(target_field.type):
+            if pa.types.is_struct(field.type):
+                new_type = pa.struct(
+                    _align_field_types(
+                        field.type.fields,
+                        target_field.type.fields,
+                    )
+                )
+            else:
+                new_type = target_field.type
+        elif pa.types.is_list(target_field.type):
+            if _is_list_like(field.type):
+                new_type = pa.list_(
+                    _align_field_types(
+                        [field.type.value_field],
+                        [target_field.type.value_field],
+                    )[0]
+                )
+            else:
+                new_type = target_field.type
+        elif pa.types.is_large_list(target_field.type):
+            if _is_list_like(field.type):
+                new_type = pa.large_list(
+                    _align_field_types(
+                        [field.type.value_field],
+                        [target_field.type.value_field],
+                    )[0]
+                )
+            else:
+                new_type = target_field.type
+        elif pa.types.is_fixed_size_list(target_field.type):
+            if _is_list_like(field.type):
+                new_type = pa.list_(
+                    _align_field_types(
+                        [field.type.value_field],
+                        [target_field.type.value_field],
+                    )[0],
+                    target_field.type.list_size,
+                )
+            else:
+                new_type = target_field.type
+        else:
+            new_type = target_field.type
+        new_fields.append(
+            pa.field(field.name, new_type, field.nullable, target_field.metadata)
+        )
     return new_fields
 
 
-def _align_list_value_field(
-    value_field: pa.Field, target_value_field: pa.Field
-) -> pa.Field:
-    # A list has exactly one child, so the inferred child name ("item") aligns
-    # positionally and adopts the table's child name; pa.Table.cast renames it.
-    return _align_field(value_field, target_value_field).with_name(
-        target_value_field.name
-    )
-
-
-def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field:
-    # Preserve arrow.json input until it reaches Lance. LanceDB exposes stored
-    # JSON columns as lance.json (JSONB-backed LargeBinary), but casting the
-    # input to that storage type here merely relabels the raw JSON bytes as
-    # JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
-    if (
-        _field_extension_name(field) == "arrow.json"
-        and _field_extension_name(target_field) == "lance.json"
-    ):
-        return field
-    if pa.types.is_struct(target_field.type):
-        if pa.types.is_struct(field.type):
-            new_type = pa.struct(
-                _align_field_types(
-                    field.type.fields,
-                    target_field.type.fields,
-                )
-            )
-        else:
-            new_type = target_field.type
-    elif pa.types.is_list(target_field.type):
-        if _is_list_like(field.type):
-            new_type = pa.list_(
-                _align_list_value_field(
-                    field.type.value_field, target_field.type.value_field
-                )
-            )
-        else:
-            new_type = target_field.type
-    elif pa.types.is_large_list(target_field.type):
-        if _is_list_like(field.type):
-            new_type = pa.large_list(
-                _align_list_value_field(
-                    field.type.value_field, target_field.type.value_field
-                )
-            )
-        else:
-            new_type = target_field.type
-    elif pa.types.is_fixed_size_list(target_field.type):
-        if _is_list_like(field.type):
-            new_type = pa.list_(
-                _align_list_value_field(
-                    field.type.value_field, target_field.type.value_field
-                ),
-                target_field.type.list_size,
-            )
-        else:
-            new_type = target_field.type
-    else:
-        new_type = target_field.type
-    return pa.field(field.name, new_type, field.nullable, target_field.metadata)
-
-
 def _infer_subschema(
     schema: List[pa.Field],
     reference_fields: List[pa.Field],
@@ -763,7 +589,7 @@ def sanitize_create_table(
         schema = data.schema
     else:
         if schema is not None:
-            data = pa.Table.from_batches([], schema=schema)
+            data = pa.Table.from_pylist([], schema)
     if schema is None:
         if data is None:
             raise ValueError("Either data or schema must be provided")
@@ -2165,11 +1991,9 @@ class Table(ABC):
             Function columns are supported only on LanceDB Cloud and
             Enterprise.
         computed: Dict[str, str], optional
-            A mapping from output column names to SQL expressions derives each
-            output field from its expression. A direct projection of a Blob v2
-            field inherits Blob v2 semantics; other expressions derive their
-            ordinary Arrow type. Mapping order is declaration and dependency
-            order.
+            A map of column name to a SQL expression defining the column. The
+            column's type and inputs are derived from the expression, so no
+            data type is supplied.
 
             Unlike ``transforms``, the expression is stored rather than
             evaluated now: the column is committed with no values, and rows get
@@ -2874,7 +2698,7 @@ class LanceTable(Table):
             arrow_tbl = self.to_arrow()
             if blob_mode == "descriptions":
                 arrow_tbl = strip_auto_row_ids(
-                    arrow_tbl, row_addressable_blob_v2_paths(self.schema)
+                    arrow_tbl, blob_v2_column_paths(self.schema)
                 )
             return arrow_tbl.to_pandas(**kwargs)
 
@@ -5278,9 +5102,7 @@ class AsyncTable:
         if blob_mode == "descriptions" or not schema_has_blob_field(schema):
             arrow_tbl = await self.to_arrow()
             if blob_mode == "descriptions":
-                arrow_tbl = strip_auto_row_ids(
-                    arrow_tbl, row_addressable_blob_v2_paths(schema)
-                )
+                arrow_tbl = strip_auto_row_ids(arrow_tbl, blob_v2_column_paths(schema))
             return arrow_tbl.to_pandas(**kwargs)
 
         if blob_mode == "lazy" and get_uri_scheme(await self.uri()) == "memory":
@@ -6270,11 +6092,8 @@ class AsyncTable:
             Function columns are supported only on LanceDB Cloud and
             Enterprise.
         computed: Dict[str, str], optional
-            A mapping from output column names to SQL expressions derives each
-            output field from its expression. A direct projection of a Blob v2
-            field inherits Blob v2 semantics; other expressions derive their
-            ordinary Arrow type. Mapping order is declaration and dependency
-            order.
+            A map of column name to a SQL expression defining the column. The
+            column's type and inputs are derived from the expression.
 
             Unlike ``transforms``, the expression is stored rather than
             evaluated now: the column is committed with no values, and rows get

python/python/tests/test_blob.py

@@ -2,15 +2,10 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 import io
-import subprocess
-import sys
-import textwrap
 
-import lance
 import pyarrow as pa
 import pyarrow.compute as pc
 import pytest
-from lance.blob import BlobType as LanceBlobType
 
 import lancedb
 from lancedb._blob import (
@@ -23,20 +18,6 @@ from lancedb.index import FTS
 from lancedb.schema import blob_column_paths, blob_v2_column_paths
 
 
-_HIDE_LANCE_BLOB = """\
-import importlib.abc
-import sys
-
-class _MissingLanceBlob(importlib.abc.MetaPathFinder):
-    def find_spec(self, fullname, path, target=None):
-        if fullname == "lance.blob" or fullname.startswith("lance.blob."):
-            raise ModuleNotFoundError(fullname, name="lance.blob")
-
-sys.modules.pop("lance.blob", None)
-sys.meta_path.insert(0, _MissingLanceBlob())
-"""
-
-
 def _blob_table(name, rows):
     db = lancedb.connect("memory:///")
     schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
@@ -70,181 +51,6 @@ def test_blob_factory_declares_v2_field():
     field = lancedb.blob("image")
     assert isinstance(field.type, pa.ExtensionType)
     assert field.type.extension_name == "lance.blob.v2"
-    assert lancedb.BlobType is LanceBlobType
-    assert type(field.type) is LanceBlobType
-
-
-def test_blob_type_works_without_pylance():
-    script = _HIDE_LANCE_BLOB + textwrap.dedent(
-        """\
-        import lancedb
-        import pyarrow as pa
-
-        field = lancedb.blob("image")
-        if not isinstance(field.type, pa.ExtensionType):
-            raise SystemExit("expected an extension type")
-        if field.type.extension_name != "lance.blob.v2":
-            raise SystemExit(field.type.extension_name)
-        if lancedb.BlobType is not type(field.type):
-            raise SystemExit("BlobType is not the field type class")
-        if lancedb.BlobType.__module__ != "lancedb.schema":
-            raise SystemExit(lancedb.BlobType.__module__)
-
-        db = lancedb.connect("memory:///")
-        table = db.create_table(
-            "images",
-            schema=pa.schema([pa.field("id", pa.int64()), field]),
-        )
-        table.add([{"id": 1, "image": b"hello"}])
-        result = (
-            table.merge_insert("id")
-            .when_matched_update_all()
-            .when_not_matched_insert_all()
-            .execute([{"id": 1, "image": b"updated"}, {"id": 2, "image": b"inserted"}])
-        )
-        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
-            raise SystemExit(
-                f"merge_insert rows updated={result.num_updated_rows} "
-                f"inserted={result.num_inserted_rows}"
-            )
-        """
-    )
-    result = subprocess.run(
-        [sys.executable, "-c", script],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    assert result.returncode == 0, result.stderr
-
-
-def test_blob_resolves_pylance_type_without_eager_import():
-    script = textwrap.dedent(
-        """\
-        import sys
-        import lancedb
-
-        if "lance.blob" in sys.modules:
-            raise SystemExit("import lancedb imported lance.blob")
-        field = lancedb.blob("image")
-        from lance.blob import BlobType
-
-        if type(field.type) is not BlobType:
-            raise SystemExit(f"{type(field.type)} is not {BlobType}")
-        import lance
-
-        image = lance.blob_array([b"x"])
-        if type(image.type) is not BlobType:
-            raise SystemExit("blob_array used a different class")
-        if type(image.type) is not type(field.type):
-            raise SystemExit("field and array classes differ")
-        """
-    )
-    result = subprocess.run(
-        [sys.executable, "-c", script],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    assert result.returncode == 0, result.stderr
-
-
-def test_blob_fallback_fails_if_name_already_registered():
-    script = _HIDE_LANCE_BLOB + textwrap.dedent(
-        """\
-        import pyarrow as pa
-
-        class OtherBlobType(pa.ExtensionType):
-            def __init__(self):
-                super().__init__(
-                    pa.struct([pa.field("data", pa.large_binary())]),
-                    "lance.blob.v2",
-                )
-
-            def __arrow_ext_serialize__(self):
-                return b""
-
-            @classmethod
-            def __arrow_ext_deserialize__(cls, storage_type, serialized):
-                return cls()
-
-        pa.register_extension_type(OtherBlobType())
-        import lancedb
-
-        try:
-            lancedb.blob("image")
-        except ValueError as err:
-            if "already registered" not in str(err):
-                raise SystemExit(err)
-        else:
-            raise SystemExit("expected ValueError")
-        """
-    )
-    result = subprocess.run(
-        [sys.executable, "-c", script],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    assert result.returncode == 0, result.stderr
-
-
-def test_blob_type_rejects_competing_registration_with_pylance():
-    script = textwrap.dedent(
-        """\
-        import pyarrow as pa
-        import pyarrow.ipc
-
-        class OtherBlobType(pa.ExtensionType):
-            def __init__(self):
-                super().__init__(
-                    pa.struct(
-                        [
-                            pa.field("data", pa.large_binary()),
-                            pa.field("uri", pa.utf8()),
-                            pa.field("position", pa.uint64()),
-                            pa.field("size", pa.uint64()),
-                        ]
-                    ),
-                    "lance.blob.v2",
-                )
-
-            def __arrow_ext_serialize__(self):
-                return b""
-
-            @classmethod
-            def __arrow_ext_deserialize__(cls, storage_type, serialized):
-                return cls()
-
-        pa.register_extension_type(OtherBlobType())
-
-        from lance.blob import BlobType
-
-        if BlobType is OtherBlobType:
-            raise SystemExit("pylance BlobType was replaced")
-        schema = pa.schema([pa.field("value", BlobType())])
-        restored = pa.ipc.read_schema(schema.serialize())
-        if type(restored.field("value").type) is not OtherBlobType:
-            raise SystemExit(type(restored.field("value").type))
-
-        import lancedb
-
-        try:
-            lancedb.blob("image")
-        except ValueError as err:
-            if "__main__.OtherBlobType" not in str(err):
-                raise SystemExit(err)
-        else:
-            raise SystemExit("expected ValueError")
-        """
-    )
-    result = subprocess.run(
-        [sys.executable, "-c", script],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    assert result.returncode == 0, result.stderr
 
 
 def test_blob_v2_column_paths_include_list_children():
@@ -397,292 +203,6 @@ def test_fetch_blobs_round_trip():
     assert [blobs[0].as_py(), blobs[1].as_py()] == [b"alpha", b"beta"]
 
 
-def test_merge_insert_writes_python_bytes():
-    table = _blob_table("merge_bytes", [{"id": 1, "image": b"before"}])
-    result = (
-        table.merge_insert("id")
-        .when_matched_update_all()
-        .when_not_matched_insert_all()
-        .execute([{"id": 1, "image": b"updated"}, {"id": 2, "image": b"inserted"}])
-    )
-    assert result.num_updated_rows == 1
-    assert result.num_inserted_rows == 1
-    by_id = _row_ids_by_id(table)
-    blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
-    assert blobs.to_pylist() == [b"updated", b"inserted"]
-
-
-def test_merge_insert_bytes_after_reopen_without_touching_blob_type(tmp_path):
-    db = lancedb.connect(tmp_path)
-    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
-    table = db.create_table("images", schema=schema)
-    table.add([{"id": 1, "image": b"hello"}])
-
-    script = textwrap.dedent(
-        f"""\
-        import lancedb
-
-        db = lancedb.connect({str(tmp_path)!r})
-        table = db.open_table("images")
-        image_type = table.schema.field("image").type
-        if type(image_type).__name__ != "StructType":
-            raise SystemExit(f"expected StructType, got {{type(image_type)}}")
-        result = (
-            table.merge_insert("id")
-            .when_matched_update_all()
-            .when_not_matched_insert_all()
-            .execute(
-                [{{"id": 1, "image": b"updated"}}, {{"id": 2, "image": b"inserted"}}]
-            )
-        )
-        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
-            raise SystemExit(
-                f"rows updated={{result.num_updated_rows}} "
-                f"inserted={{result.num_inserted_rows}}"
-            )
-        hits = table.search().with_row_id(True).limit(10).to_arrow()
-        by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
-        blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
-        if blobs.to_pylist() != [b"updated", b"inserted"]:
-            raise SystemExit(blobs.to_pylist())
-        """
-    )
-    result = subprocess.run(
-        [sys.executable, "-c", script],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    assert result.returncode == 0, result.stderr
-
-
-def test_merge_insert_bytes_after_reopen_without_pylance(tmp_path):
-    db = lancedb.connect(tmp_path)
-    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
-    table = db.create_table("images", schema=schema)
-    table.add([{"id": 1, "image": b"hello"}])
-
-    script = _HIDE_LANCE_BLOB + textwrap.dedent(
-        f"""\
-        import lancedb
-
-        db = lancedb.connect({str(tmp_path)!r})
-        table = db.open_table("images")
-        image_type = table.schema.field("image").type
-        if type(image_type).__name__ != "StructType":
-            raise SystemExit(f"expected StructType, got {{type(image_type)}}")
-        result = (
-            table.merge_insert("id")
-            .when_matched_update_all()
-            .when_not_matched_insert_all()
-            .execute(
-                [{{"id": 1, "image": b"updated"}}, {{"id": 2, "image": b"inserted"}}]
-            )
-        )
-        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
-            raise SystemExit(
-                f"rows updated={{result.num_updated_rows}} "
-                f"inserted={{result.num_inserted_rows}}"
-            )
-        hits = table.search().with_row_id(True).limit(10).to_arrow()
-        by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
-        blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
-        if blobs.to_pylist() != [b"updated", b"inserted"]:
-            raise SystemExit(blobs.to_pylist())
-        """
-    )
-    result = subprocess.run(
-        [sys.executable, "-c", script],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    assert result.returncode == 0, result.stderr
-
-
-def test_merge_insert_blob_array_into_reopened_unregistered_table(tmp_path):
-    db = lancedb.connect(tmp_path)
-    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
-    table = db.create_table("images", schema=schema)
-    table.add([{"id": 1, "image": b"before"}])
-
-    script = textwrap.dedent(
-        f"""\
-        import pyarrow as pa
-        import lancedb
-
-        db = lancedb.connect({str(tmp_path)!r})
-        table = db.open_table("images")
-        image_type = table.schema.field("image").type
-        if type(image_type).__name__ != "StructType":
-            raise SystemExit(
-                f"expected StructType before lance import, got {{type(image_type)}}"
-            )
-
-        import lance
-
-        updates = pa.Table.from_arrays(
-            [
-                pa.array([1, 2], type=pa.int64()),
-                lance.blob_array([b"updated", b"inserted"]),
-            ],
-            names=["id", "image"],
-        )
-        result = (
-            table.merge_insert("id")
-            .when_matched_update_all()
-            .when_not_matched_insert_all()
-            .execute(updates)
-        )
-        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
-            raise SystemExit(
-                f"rows updated={{result.num_updated_rows}} "
-                f"inserted={{result.num_inserted_rows}}"
-            )
-        hits = table.search().with_row_id(True).limit(10).to_arrow()
-        by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
-        blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
-        if blobs.to_pylist() != [b"updated", b"inserted"]:
-            raise SystemExit(blobs.to_pylist())
-        """
-    )
-    result = subprocess.run(
-        [sys.executable, "-c", script],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    assert result.returncode == 0, result.stderr
-
-
-def test_add_all_null_blob_column():
-    db = lancedb.connect("memory:///")
-    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
-    table = db.create_table("all_null", schema=schema)
-    table.add([{"id": 1, "image": None}, {"id": 2, "image": None}])
-    by_id = _row_ids_by_id(table)
-    blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
-    assert blobs.to_pylist() == [None, None]
-
-
-def test_create_table_nested_blob_schema_without_rows():
-    db = lancedb.connect("memory:///")
-    schema = pa.schema(
-        [
-            pa.field("id", pa.int64()),
-            pa.field("info", pa.struct([lancedb.blob("blob")])),
-            pa.field("images", pa.list_(lancedb.blob("image"))),
-        ]
-    )
-    table = db.create_table("nested_empty", schema=schema)
-    assert table.count_rows() == 0
-
-
-def test_merge_insert_nested_blob_dicts():
-    db = lancedb.connect("memory:///")
-    info = pa.StructArray.from_arrays(
-        [
-            pa.array(["first"], type=pa.string()),
-            _blob_array("blob", [b"before"]),
-        ],
-        names=["name", "blob"],
-    )
-    data = pa.Table.from_arrays(
-        [pa.array([1], type=pa.int64()), info],
-        names=["id", "info"],
-    )
-    table = db.create_table("nested_merge", data=data)
-    result = (
-        table.merge_insert("id")
-        .when_matched_update_all()
-        .execute([{"id": 1, "info": {"name": "first", "blob": b"after"}}])
-    )
-    assert result.num_updated_rows == 1
-    by_id = _row_ids_by_id(table)
-    blobs = table.fetch_blobs("info.blob", [by_id[1]])
-    assert blobs.to_pylist() == [b"after"]
-
-
-def _list_blob_table(name):
-    db = lancedb.connect("memory:///")
-    blob_field = lancedb.blob("image")
-    images = pa.ListArray.from_arrays(
-        pa.array([0, 1], type=pa.int32()), _blob_array("image", [b"before"])
-    )
-    data = pa.Table.from_arrays(
-        [pa.array([1], type=pa.int64()), images],
-        schema=pa.schema(
-            [pa.field("id", pa.int64()), pa.field("images", pa.list_(blob_field))]
-        ),
-    )
-    return db.create_table(name, data=data)
-
-
-def test_merge_insert_list_blob_dicts():
-    table = _list_blob_table("list_merge")
-    result = (
-        table.merge_insert("id")
-        .when_matched_update_all()
-        .when_not_matched_insert_all()
-        .execute([{"id": 1, "images": [b"one", b"two"]}, {"id": 2, "images": None}])
-    )
-    assert result.num_updated_rows == 1
-    assert result.num_inserted_rows == 1
-    hits = table.search().limit(10).to_arrow()
-    sizes = {
-        row["id"]: None if row["images"] is None else [d["size"] for d in row["images"]]
-        for row in hits.to_pylist()
-    }
-    assert sizes == {1: [3, 3], 2: None}
-
-
-def test_list_blob_column_queries_as_raw_descriptors():
-    table = _list_blob_table("list_query")
-    hits = table.search().limit(10).to_arrow()
-    element = hits.schema.field("images").type.value_type
-    assert pa.types.is_struct(element)
-    assert "_lance_row_id" not in element.names
-    with pytest.raises(ValueError, match="expected struct before segment"):
-        table.fetch_blobs("images.image", [0])
-
-
-def test_row_addressable_paths_exclude_list_children():
-    from lancedb.schema import row_addressable_blob_v2_paths
-
-    schema = pa.schema(
-        [
-            pa.field("id", pa.int64()),
-            pa.field("info", pa.struct([lancedb.blob("blob")])),
-            pa.field("images", pa.list_(lancedb.blob("image"))),
-        ]
-    )
-    assert blob_v2_column_paths(schema) == ["info.blob", "images.image"]
-    assert row_addressable_blob_v2_paths(schema) == ["info.blob"]
-
-
-def test_merge_insert_writes_pylance_blob_array():
-    table = _blob_table("merge_pylance", [{"id": 1, "image": b"before"}])
-    image = lance.blob_array([b"updated", b"inserted"])
-    assert type(image.type) is LanceBlobType
-    assert type(image.type) is type(lancedb.BlobType())
-    updates = pa.Table.from_arrays(
-        [pa.array([1, 2], type=pa.int64()), image], names=["id", "image"]
-    )
-
-    result = (
-        table.merge_insert("id")
-        .when_matched_update_all()
-        .when_not_matched_insert_all()
-        .execute(updates)
-    )
-
-    assert result.num_updated_rows == 1
-    assert result.num_inserted_rows == 1
-    by_id = _row_ids_by_id(table)
-    blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
-    assert blobs.to_pylist() == [b"updated", b"inserted"]
-
-
 def test_fetch_blobs_accepts_query_result():
     table = _blob_table("from_result", [{"id": 1, "image": b"gamma"}])
     hits = table.search().limit(10).to_arrow()

python/python/tests/test_table.py

@@ -4087,29 +4087,6 @@ def test_computed_column_rejects_transforms_and_computed_together(tmp_path):
         table.add_columns({"a": "x + 1"}, computed={"b": "x * 2"})
 
 
-def test_computed_column_blob_projection_inherits_semantics(tmp_path):
-    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
-    db = lancedb.connect(tmp_path)
-    table = db.create_table("computed_column_blob", schema=schema)
-    table.add(
-        [
-            {"id": 1, "image": b"hello"},
-            {"id": 2, "image": b""},
-            {"id": 3, "image": None},
-        ]
-    )
-
-    table.add_columns(computed={"image_copy": "image", "second_copy": "image_copy"})
-    assert table.refresh_column("image_copy").rows_filled == 2
-    assert table.refresh_column("second_copy").rows_filled == 2
-    assert table.blob_columns() == ["image", "image_copy", "second_copy"]
-
-    hits = table.search().with_row_id(True).limit(10).to_arrow()
-    rows = sorted(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
-    copied = table.fetch_blobs("second_copy", [row_id for _, row_id in rows])
-    assert copied.to_pylist() == [b"hello", b"", None]
-
-
 @pytest.mark.asyncio
 async def test_computed_column_async(tmp_path):
     db = await lancedb.connect_async(tmp_path)

python/python/tests/test_util.py

@@ -7,7 +7,6 @@ import pathlib
 from typing import Optional
 
 import lance
-from lance.blob import BlobType as LanceBlobType
 from lancedb.conftest import MockTextEmbeddingFunction
 from lancedb.embeddings.base import EmbeddingFunctionConfig
 from lancedb.embeddings.registry import EmbeddingFunctionRegistry
@@ -908,165 +907,6 @@ def test_cast_to_target_schema():
     assert output == expected
 
 
-def test_cast_to_target_schema_coerces_binary_to_blob_v2():
-    data = pa.table({"image": pa.array([b"hello", None], type=pa.binary())})
-    target = pa.schema([lancedb.blob("image")])
-
-    output = _cast_to_target_schema(data.to_reader(), target).read_all()
-
-    image = output["image"].chunk(0)
-    assert type(image.type) is lancedb.BlobType
-    assert image.storage.to_pylist() == [
-        {"data": b"hello", "uri": None, "position": None, "size": None},
-        None,
-    ]
-
-
-def test_cast_to_target_schema_coerces_binary_to_metadata_blob_struct():
-    storage = lancedb.blob("image").type.storage_type
-    target = pa.schema(
-        [
-            pa.field(
-                "image",
-                storage,
-                metadata={
-                    b"ARROW:extension:name": b"lance.blob.v2",
-                    b"ARROW:extension:metadata": b"",
-                },
-            )
-        ]
-    )
-    data = pa.table({"image": pa.array([b"hello", None], type=pa.binary())})
-
-    output = _cast_to_target_schema(data.to_reader(), target).read_all()
-
-    image = output["image"].chunk(0)
-    assert not isinstance(image.type, pa.ExtensionType)
-    assert image.to_pylist() == [
-        {"data": b"hello", "uri": None, "position": None, "size": None},
-        None,
-    ]
-
-
-def test_cast_to_target_schema_coerces_nested_binary_blob():
-    data = pa.table(
-        {
-            "info": pa.array(
-                [{"blob": b"hello"}, {"blob": None}],
-                type=pa.struct([pa.field("blob", pa.binary())]),
-            )
-        }
-    )
-    target = pa.schema([pa.field("info", pa.struct([lancedb.blob("blob")]))])
-
-    output = _cast_to_target_schema(data.to_reader(), target).read_all()
-
-    blob = output["info"].chunk(0).field("blob")
-    assert type(blob.type) is lancedb.BlobType
-    assert blob.storage.to_pylist() == [
-        {"data": b"hello", "uri": None, "position": None, "size": None},
-        None,
-    ]
-
-
-def test_cast_to_target_schema_coerces_list_binary_blob_with_inferred_child_name():
-    data = pa.table(
-        {"images": pa.array([[b"a", b"b"], None], type=pa.list_(pa.binary()))}
-    )
-    target = pa.schema([pa.field("images", pa.list_(lancedb.blob("image")))])
-
-    output = _cast_to_target_schema(data.to_reader(), target).read_all()
-
-    images = output["images"].chunk(0)
-    assert images.type.value_field.name == "image"
-    assert type(images.type.value_type) is lancedb.BlobType
-    assert images.to_pylist()[1] is None
-    assert images.values.storage.to_pylist() == [
-        {"data": b"a", "uri": None, "position": None, "size": None},
-        {"data": b"b", "uri": None, "position": None, "size": None},
-    ]
-
-
-def test_list_blob_coercion_preserves_null_slots_with_nonzero_extent():
-    child = pa.field("image", pa.binary())
-    source = pa.ListArray.from_arrays(
-        pa.array([0, 2, 4], type=pa.int32()),
-        pa.array([b"a", b"b", b"dead", b"beef"], type=pa.binary()),
-        mask=pa.array([False, True]),
-    ).cast(pa.list_(child))
-    target = pa.schema([pa.field("images", pa.list_(lancedb.blob("image")))])
-
-    output = _cast_to_target_schema(
-        pa.table({"images": source}).to_reader(), target
-    ).read_all()
-
-    images = output["images"].chunk(0)
-    assert images.to_pylist()[1] is None
-    assert [b["data"] for b in images.to_pylist()[0]] == [b"a", b"b"]
-
-
-def test_fixed_size_list_blob_coercion_keeps_null_rows():
-    child = pa.field("frame", pa.binary())
-    source = (
-        pa.FixedSizeListArray.from_arrays(
-            pa.array([b"a", b"b", b"c", b"d"], type=pa.binary()), 2
-        )
-        .take(pa.array([0, None], type=pa.int32()))
-        .cast(pa.list_(child, 2))
-    )
-    target = pa.schema([pa.field("frames", pa.list_(lancedb.blob("frame"), 2))])
-
-    output = _cast_to_target_schema(
-        pa.table({"frames": source}).to_reader(), target
-    ).read_all()
-
-    frames = output["frames"].chunk(0)
-    assert frames.to_pylist()[1] is None
-    assert [b["data"] for b in frames.to_pylist()[0]] == [b"a", b"b"]
-
-
-def test_cast_to_target_schema_accepts_pylance_blob_v2():
-    target_type = lancedb.BlobType()
-    source = lance.blob_array([b"hello", None])
-    assert type(source.type) is LanceBlobType
-    assert type(source.type) is type(target_type)
-    data = pa.table({"image": source})
-    target = pa.schema([pa.field("image", target_type)])
-
-    output = _cast_to_target_schema(data.to_reader(), target).read_all()
-
-    image = output["image"].chunk(0)
-    assert type(image.type) is LanceBlobType
-    assert image.type == target_type
-    assert image.storage.to_pylist() == [
-        {"data": b"hello", "uri": None, "position": None, "size": None},
-        None,
-    ]
-
-
-def test_cast_to_target_schema_rejects_different_blob_v2_class():
-    class OtherBlobType(pa.ExtensionType):
-        def __init__(self):
-            super().__init__(lancedb.BlobType().storage_type, "lance.blob.v2")
-
-        def __arrow_ext_serialize__(self) -> bytes:
-            return b""
-
-        @classmethod
-        def __arrow_ext_deserialize__(
-            cls, storage_type: pa.DataType, serialized: bytes
-        ) -> "OtherBlobType":
-            return cls()
-
-    storage = lance.blob_array([b"hello"]).storage
-    source = pa.ExtensionArray.from_storage(OtherBlobType(), storage)
-    data = pa.table({"image": source})
-    target = pa.schema([lancedb.blob("image")])
-
-    with pytest.raises(pa.ArrowTypeError, match="different extension type"):
-        _cast_to_target_schema(data.to_reader(), target).read_all()
-
-
 def test_sanitize_data_stream():
     # Make sure we don't collect the whole stream when running sanitize_data
     schema = pa.schema({"a": pa.int32()})

rust/lancedb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.38.0-beta.12"
+version = "0.38.0-beta.11"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true

rust/lancedb/src/database/listing.rs

@@ -13,7 +13,7 @@ use lance::dataset::{ReadParams, WriteMode, builder::DatasetBuilder};
 use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore};
 use lance_datafusion::utils::StreamingWriteSource;
 use lance_file::version::LanceFileVersion;
-use lance_io::object_store::{ReadDirOptions, StorageOptionsAccessor, StorageOptionsProvider};
+use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
 use lance_table::io::commit::commit_handler_from_url;
 use object_store::local::LocalFileSystem;
 use snafu::ResultExt;
@@ -281,22 +281,6 @@ impl std::fmt::Display for ListingDatabase {
 }
 
 const LANCE_EXTENSION: &str = "lance";
-
-/// The table a listed child of the database names, or `None` if the child is not a table.
-///
-/// A table is the directory `<name>.lance`; a loose file or any other directory under the
-/// database prefix belongs to something else. `dir_suffix` is `.lance`, built once by the
-/// caller rather than per child.
-/// The table a listed child directory holds, or `None` if it is not a table at all.
-///
-/// Only directories are considered, so a loose object named like a table is not one.
-fn table_name(location: &object_store::path::Path, dir_suffix: &str) -> Option<String> {
-    location
-        .filename()?
-        .strip_suffix(dir_suffix)
-        .map(String::from)
-        .filter(|name| !name.is_empty())
-}
 const ENGINE: &str = "engine";
 const MIRRORED_STORE: &str = "mirroredStore";
 
@@ -960,72 +944,51 @@ impl Database for ListingDatabase {
         Ok(f)
     }
 
-    /// List the tables in the database, a page at a time.
-    ///
-    /// The page_token is opaque, unlike the `start_after` parameter of [`Self::table_names()`].
-    ///
-    /// When there are no more results, the returned page_token will be None.
-    ///
-    /// `limit` is the maximum number of tables to return in the response. But it is possible
-    /// for the response to contain fewer than `limit` tables, even when there are more tables
-    /// to return. Clients should check the returned page_token to determine if there are
-    /// more results, rather than relying on the number of tables returned.
-    ///
-    /// The order that results are returned in not guaranteed to be stable across calls,
-    /// so clients should not rely on it.
     async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
         if request.id.as_ref().map(|v| !v.is_empty()).unwrap_or(false) {
             return self.namespace_database().list_tables(request).await;
         }
-        let limit = request.limit.map(|limit| limit.max(0) as usize);
-        let dir_suffix = format!(".{LANCE_EXTENSION}");
-        let mut tables = Vec::new();
-        let mut page_token = request.page_token.filter(|token| !token.is_empty());
+        let mut f = self
+            .object_store
+            .read_dir(self.base_path.clone())
+            .await?
+            .iter()
+            .map(Path::new)
+            .filter(|path| {
+                let is_lance = path
+                    .extension()
+                    .and_then(|e| e.to_str())
+                    .map(|e| e == LANCE_EXTENSION);
+                is_lance.unwrap_or(false)
+            })
+            .filter_map(|p| p.file_stem().and_then(|s| s.to_str().map(String::from)))
+            .collect::<Vec<String>>();
+        f.sort();
 
-        // A page of nothing: the store rejects a limit of zero, and no table was handed over
-        // for a token to resume after.
-        if limit == Some(0) {
-            return Ok(ListTablesResponse {
-                context: None,
-                tables,
-                page_token: None,
-            });
+        // Handle pagination with page_token
+        if let Some(ref page_token) = request.page_token {
+            let index = f
+                .iter()
+                .position(|name| name.as_str() > page_token.as_str())
+                .unwrap_or(f.len());
+            f.drain(0..index);
         }
 
-        loop {
-            // Ask only for what the page still has room for, so a database holding more
-            // than one page costs one request per page rather than one per table.
-            let listing = self
-                .object_store
-                .read_dir_page(
-                    self.base_path.clone(),
-                    ReadDirOptions {
-                        page_token: page_token.take(),
-                        limit: limit.map(|limit| limit - tables.len()),
-                    },
-                )
-                .await?;
-            page_token = listing.page_token;
-            // Only child directories can be tables, and the store already separates them
-            // out, so the objects in the page are not looked at.
-            tables.extend(
-                listing
-                    .result
-                    .common_prefixes
-                    .iter()
-                    .filter_map(|location| table_name(location, &dir_suffix)),
-            );
-            // Children that are not tables leave the page short of the limit, so keep
-            // going until the page is full or the database runs out.
-            if page_token.is_none() || limit.is_none_or(|limit| tables.len() >= limit) {
-                break;
+        // Determine if there's a next page. The token is the last name of this page,
+        // not the first of the next one: the next page resumes strictly after the
+        // token, so naming the next page's first entry would skip it.
+        let next_page_token = match request.limit {
+            Some(limit) if f.len() > limit as usize => {
+                f.truncate(limit as usize);
+                f.last().cloned()
             }
-        }
+            _ => None,
+        };
 
         Ok(ListTablesResponse {
             context: None,
-            tables,
-            page_token,
+            tables: f,
+            page_token: next_page_token,
         })
     }
 
@@ -1521,182 +1484,6 @@ mod tests {
     use tokio::sync::Barrier;
     use tokio::time::timeout;
 
-    async fn create_tables(db: &ListingDatabase, names: &[&str]) {
-        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
-        for name in names {
-            db.create_table(CreateTableRequest {
-                name: name.to_string(),
-                namespace_path: vec![],
-                data: Box::new(RecordBatch::new_empty(schema.clone())) as Box<dyn Scannable>,
-                mode: CreateTableMode::Create,
-                write_options: Default::default(),
-                location: None,
-                namespace_client: None,
-            })
-            .await
-            .unwrap();
-        }
-    }
-
-    /// Every table in the database, taken `limit` at a time, which is how a caller walks a
-    /// listing: the token ends the walk, never a short page.
-    async fn walk(db: &ListingDatabase, limit: Option<i32>) -> Vec<String> {
-        let mut seen = Vec::new();
-        let mut page_token = None;
-        loop {
-            let page = db
-                .list_tables(ListTablesRequest {
-                    limit,
-                    page_token,
-                    ..Default::default()
-                })
-                .await
-                .unwrap();
-            seen.extend(page.tables);
-            page_token = page.page_token;
-            if page_token.is_none() {
-                return seen;
-            }
-            assert!(
-                seen.len() < 100,
-                "the walk is serving tables more than once"
-            );
-        }
-    }
-
-    /// Paging with the returned token has to visit every table exactly once, whatever the
-    /// page size, with nothing lost or repeated at a boundary.
-    #[rstest::rstest]
-    #[tokio::test]
-    async fn test_list_tables_pages_over_every_table_once(#[values(1, 2, 3, 5, 10)] limit: i32) {
-        let (_tempdir, db) = setup_database().await;
-        create_tables(&db, &["a", "b", "c", "d", "e"]).await;
-
-        assert_eq!(walk(&db, Some(limit)).await, vec!["a", "b", "c", "d", "e"]);
-    }
-
-    /// The token is opaque: it is whatever resumes the store the database sits on, not a
-    /// table name. Callers hand it back and nothing else.
-    ///
-    /// Nothing validates a token, so one invented by a caller is read as a position rather
-    /// than refused — which is why the token has to come back from a previous page.
-    #[tokio::test]
-    async fn test_the_page_token_is_not_a_table_name() {
-        let (_tempdir, db) = setup_database().await;
-        create_tables(&db, &["a", "b", "c"]).await;
-
-        let page = db
-            .list_tables(ListTablesRequest {
-                limit: Some(1),
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-
-        assert_eq!(page.tables, vec!["a"]);
-        let token = page.page_token.expect("two tables are still to come");
-        assert_ne!(token, "a");
-
-        // Handing it back is the only thing a caller does with it, and it resumes.
-        let rest = db
-            .list_tables(ListTablesRequest {
-                page_token: Some(token),
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-        assert_eq!(rest.tables, vec!["b", "c"]);
-    }
-
-    /// A limit the listing does not fill leaves no token behind, so a caller paging by token
-    /// stops without asking for an empty page.
-    #[tokio::test]
-    async fn test_a_listing_that_runs_out_has_no_token() {
-        let (_tempdir, db) = setup_database().await;
-        create_tables(&db, &["a", "b"]).await;
-
-        let page = db
-            .list_tables(ListTablesRequest {
-                limit: Some(10),
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-
-        assert_eq!(page.tables, vec!["a", "b"]);
-        assert_eq!(page.page_token, None);
-    }
-
-    /// An empty page token means "from the start", which is how a client looping on a token
-    /// spells its first request.
-    #[tokio::test]
-    async fn test_an_empty_page_token_lists_from_the_start() {
-        let (_tempdir, db) = setup_database().await;
-        create_tables(&db, &["a", "b"]).await;
-
-        let page = db
-            .list_tables(ListTablesRequest {
-                page_token: Some(String::new()),
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-
-        assert_eq!(page.tables, vec!["a", "b"]);
-    }
-
-    /// Listing follows the order the object store lists directories in, so a name that
-    /// extends another comes first: the `-` of `users-archive.lance` sorts below the `.` of
-    /// `users.lance`. Pagination pushes its cursor into the list request, so it cannot report
-    /// an order other than the one it resumes in.
-    #[tokio::test]
-    async fn test_listing_order_follows_the_store_not_the_table_name() {
-        let (_tempdir, db) = setup_database().await;
-        create_tables(&db, &["users", "users-archive", "users.old"]).await;
-
-        assert_eq!(
-            walk(&db, None).await,
-            vec!["users-archive", "users", "users.old"]
-        );
-        // And paging reports the same order, so a walk sees each table once.
-        assert_eq!(
-            walk(&db, Some(1)).await,
-            vec!["users-archive", "users", "users.old"]
-        );
-    }
-
-    /// Only directories named `<name>.lance` are tables; loose files and other directories
-    /// under the database prefix are not. A page spent on them is filled from the next one,
-    /// so a page holding only non-tables does not read as an empty database.
-    #[tokio::test]
-    async fn test_listing_ignores_non_table_children() {
-        let (tempdir, db) = setup_database().await;
-        create_tables(&db, &["real"]).await;
-        std::fs::write(tempdir.path().join("aaa-loose.lance"), b"not a table").unwrap();
-        create_dir_all(tempdir.path().join("aaa-scratch")).unwrap();
-
-        let page = db
-            .list_tables(ListTablesRequest {
-                limit: Some(1),
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-
-        assert_eq!(page.tables, vec!["real"]);
-    }
-
-    #[tokio::test]
-    async fn listing_ignores_empty_table_name() {
-        let (tempdir, db) = setup_database().await;
-        create_dir_all(tempdir.path().join(".lance")).unwrap();
-        let page = db.list_tables(ListTablesRequest::default()).await.unwrap();
-        assert!(
-            page.tables.is_empty(),
-            "invalid empty table name was listed"
-        );
-    }
-
     async fn setup_database() -> (tempfile::TempDir, ListingDatabase) {
         let tempdir = tempdir().unwrap();
         let uri = tempdir.path().to_str().unwrap();

rust/lancedb/src/remote/table.rs

@@ -3180,8 +3180,8 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
             self.schema().await?.as_ref(),
             "schema evolution",
         )?;
-        // The server plans the declaration against its table schema, including
-        // Blob v2 semantics inherited by a direct field projection.
+        // The server plans the declaration: expression validation, type
+        // inference and the persisted binding all happen there.
         let entries = columns
             .iter()
             .map(
@@ -5734,9 +5734,8 @@ mod tests {
             ))
             .execute()
             .await;
-        let err = match result {
-            Ok(_) => panic!("legacy remote query unexpectedly succeeded"),
-            Err(err) => err,
+        let Err(err) = result else {
+            panic!("legacy remote query unexpectedly succeeded")
         };
 
         assert!(
@@ -7388,8 +7387,8 @@ mod tests {
         assert_eq!(result.version, if old_server { 0 } else { 43 });
     }
 
-    /// A declaration is sent as `{name, computed}` for the server to plan; the
-    /// client never types the expression itself.
+    /// A declaration is sent as `{name, computed}` entries for the server to
+    /// plan; the client never types the expression itself.
     #[tokio::test]
     async fn test_add_computed_columns_sends_the_expression() {
         let table = Table::new_with_handler("my_table", |request| match request.url().path() {

rust/lancedb/src/table.rs

@@ -750,8 +750,8 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
     /// Declare computed columns, each defined by a SQL expression.
     ///
     /// Where the declaration is planned depends on the backend: a local table
-    /// validates and types the expression itself, while a remote one sends the
-    /// expression for the server to plan.
+    /// validates and types the expression itself, a remote one sends the text
+    /// for the server to plan.
     async fn add_computed_columns(
         &self,
         _columns: &[(String, String)],

rust/lancedb/src/table/computed_columns.rs

@@ -9,35 +9,29 @@
 //! refresh fills the rows.
 //!
 //! The rule is tagged by kind ([`ComputedColumnKind`]) because kinds differ in
-//! where the column's type and inputs come from. A SQL expression determines
-//! its inputs and physical result type. A direct projection of a Blob v2 field
-//! also inherits that field's semantic type while execution continues to use
-//! `LargeBinary`. A kind resolved through a registry cannot be typed without
-//! consulting it.
-//! Registered Functions use an exact remote version plus a schema-level
-//! Function binding; unknown newer kinds remain readable and fail closed
-//! before mutation.
+//! where the column's type and inputs come from. A SQL expression is
+//! self-describing -- both are derived from the expression, so a caller writes
+//! neither -- while a kind resolved through a registry cannot be typed without
+//! consulting it. Registered Functions use an exact remote version plus a
+//! schema-level Function binding; unknown newer kinds remain readable and fail
+//! closed before mutation.
 //!
 //! [`computed_columns`] and [`computed_column_from_field`] read declarations
 //! back off a schema.
 
-use std::collections::{BTreeSet, HashMap, HashSet};
+use std::collections::{BTreeSet, HashMap};
 use std::sync::Arc;
 
 use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema, SchemaRef};
-use datafusion_common::{ScalarValue, tree_node::TreeNode};
-use datafusion_expr::Expr;
+use datafusion_common::tree_node::TreeNode;
 use datafusion_physical_plan::PhysicalExpr;
 use lance::dataset::NewColumnTransform;
-use lance_arrow::FieldExt;
-use lance_core::datatypes::{BLOB_V2_DESC_FIELD, format_field_path_minimal, parse_field_path};
 use lance_datafusion::planner::Planner;
 use lance_namespace::models::{JsonArrowDataType, JsonArrowField, JsonArrowSchema};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 
 use crate::function::{FunctionApplication, FunctionBinding};
-use crate::utils::resolve_arrow_field_path;
 use crate::{Error, Result};
 
 /// Field metadata key marking a column as computed. The value is `"true"`.
@@ -1112,20 +1106,15 @@ pub(crate) fn ensure_no_foreign_declarations<'a>(
     fields: impl IntoIterator<Item = &'a Arc<ArrowField>>,
 ) -> Result<()> {
     for field in fields {
-        ensure_no_foreign_declaration(field)?;
-    }
-    Ok(())
-}
-
-fn ensure_no_foreign_declaration(field: &ArrowField) -> Result<()> {
-    if field.metadata().keys().any(|k| is_declaration_key(k)) {
-        return Err(Error::InvalidInput {
-            message: format!(
-                "field '{}' carries computed-column metadata; declare computed columns \
-                 with add_columns().computed()",
-                field.name()
-            ),
-        });
+        if field.metadata().keys().any(|k| is_declaration_key(k)) {
+            return Err(Error::InvalidInput {
+                message: format!(
+                    "field '{}' carries computed-column metadata; declare computed columns \
+                     with add_columns().computed()",
+                    field.name()
+                ),
+            });
+        }
     }
     Ok(())
 }
@@ -1173,154 +1162,15 @@ pub(crate) struct BoundExpression {
     /// The columns the expression names, as written; nested inputs keep
     /// their dotted path.
     pub inputs: Vec<String>,
-    /// The top-level columns evaluation reads, in physical-expression order.
-    /// A nested input appears through its root.
+    /// The top-level columns evaluation reads, in [`Self::read_schema`]
+    /// order. A nested input appears through its root.
     pub roots: Vec<String>,
+    /// The projected schema evaluation runs against.
+    pub read_schema: SchemaRef,
     /// The compiled expression.
     pub physical: Arc<dyn PhysicalExpr>,
     /// The type the expression yields.
     pub data_type: DataType,
-    /// Blob v2 leaves the scan must materialize as `LargeBinary`.
-    pub blob_paths: Vec<String>,
-    /// A directly projected Blob v2 field whose semantics the output inherits.
-    projected_blob_field: Option<ArrowField>,
-}
-
-fn is_direct_field_projection(expr: &Expr) -> bool {
-    match expr {
-        Expr::Column(_) => true,
-        Expr::ScalarFunction(function)
-            if function.name() == "get_field" && function.args.len() == 2 =>
-        {
-            is_direct_field_projection(&function.args[0])
-                && matches!(
-                    &function.args[1],
-                    Expr::Literal(ScalarValue::Utf8(Some(_)), _)
-                )
-        }
-        _ => false,
-    }
-}
-
-fn projected_blob_field(schema: &ArrowSchema, expr: &Expr) -> Result<Option<ArrowField>> {
-    if !is_direct_field_projection(expr) {
-        return Ok(None);
-    }
-    let paths = Planner::column_names_in_expr(expr);
-    let [path] = paths.as_slice() else {
-        return Ok(None);
-    };
-    let (_, field) = resolve_arrow_field_path(schema, path)?;
-    Ok(field.is_blob_v2().then_some(field))
-}
-
-fn collect_blob_paths(field: &ArrowField, parent: &[String], paths: &mut Vec<Vec<String>>) {
-    let mut path = parent.to_vec();
-    path.push(field.name().clone());
-    if field.is_blob_v2() {
-        paths.push(path);
-        return;
-    }
-    match field.data_type() {
-        DataType::Struct(children) => {
-            for child in children {
-                collect_blob_paths(child, &path, paths);
-            }
-        }
-        DataType::List(child)
-        | DataType::LargeList(child)
-        | DataType::FixedSizeList(child, _)
-        | DataType::Map(child, _) => collect_blob_paths(child, &path, paths),
-        _ => {}
-    }
-}
-
-fn schema_blob_paths(schema: &ArrowSchema) -> Vec<Vec<String>> {
-    let mut paths = Vec::new();
-    for field in schema.fields() {
-        collect_blob_paths(field, &[], &mut paths);
-    }
-    paths
-}
-
-fn transform_blob_field(
-    field: &ArrowField,
-    parent: &[String],
-    materialized: &HashSet<Vec<String>>,
-) -> ArrowField {
-    let mut path = parent.to_vec();
-    path.push(field.name().clone());
-    if field.is_blob_v2() {
-        if materialized.contains(&path) {
-            return ArrowField::new(field.name(), DataType::LargeBinary, field.is_nullable());
-        }
-        return ArrowField::new(
-            field.name(),
-            BLOB_V2_DESC_FIELD.data_type().clone(),
-            field.is_nullable(),
-        )
-        .with_metadata(BLOB_V2_DESC_FIELD.metadata().clone());
-    }
-
-    let data_type = match field.data_type() {
-        DataType::Struct(children) => DataType::Struct(
-            children
-                .iter()
-                .map(|child| Arc::new(transform_blob_field(child, &path, materialized)))
-                .collect(),
-        ),
-        DataType::List(child) => {
-            DataType::List(Arc::new(transform_blob_field(child, &path, materialized)))
-        }
-        DataType::LargeList(child) => {
-            DataType::LargeList(Arc::new(transform_blob_field(child, &path, materialized)))
-        }
-        DataType::FixedSizeList(child, size) => DataType::FixedSizeList(
-            Arc::new(transform_blob_field(child, &path, materialized)),
-            *size,
-        ),
-        DataType::Map(child, sorted) => DataType::Map(
-            Arc::new(transform_blob_field(child, &path, materialized)),
-            *sorted,
-        ),
-        _ => return field.clone(),
-    };
-    ArrowField::new(field.name(), data_type, field.is_nullable())
-        .with_metadata(field.metadata().clone())
-}
-
-fn blob_runtime_schema(schema: &ArrowSchema, materialized: &HashSet<Vec<String>>) -> SchemaRef {
-    Arc::new(ArrowSchema::new_with_metadata(
-        schema
-            .fields()
-            .iter()
-            .map(|field| Arc::new(transform_blob_field(field, &[], materialized)))
-            .collect::<Fields>(),
-        schema.metadata().clone(),
-    ))
-}
-
-fn referenced_blob_paths(schema: &ArrowSchema, inputs: &[String]) -> Result<Vec<Vec<String>>> {
-    let input_paths = inputs
-        .iter()
-        .map(|input| {
-            parse_field_path(input).map_err(|error| Error::InvalidInput {
-                message: format!("invalid computed-column input path '{input}': {error}"),
-            })
-        })
-        .collect::<Result<Vec<_>>>()?;
-    Ok(schema_blob_paths(schema)
-        .into_iter()
-        .filter(|blob_path| {
-            input_paths.iter().any(|input_path| {
-                input_path.len() <= blob_path.len()
-                    && input_path
-                        .iter()
-                        .zip(blob_path)
-                        .all(|(input, blob)| input == blob)
-            })
-        })
-        .collect())
 }
 
 /// Parse, resolve and compile `expression` against `schema`.
@@ -1335,18 +1185,10 @@ pub(crate) fn bind(schema: SchemaRef, column: &str, expression: &str) -> Result<
         message,
     };
 
-    // Blob v2 is a semantic type whose runtime expression ABI is
-    // `LargeBinary`. Parse against that ABI first so a direct Blob reference
-    // is not mistaken for its storage descriptor struct.
-    let all_blob_paths = schema_blob_paths(schema.as_ref())
-        .into_iter()
-        .collect::<HashSet<_>>();
-    let parsing_schema = blob_runtime_schema(schema.as_ref(), &all_blob_paths);
-    let planner = Planner::new(parsing_schema);
+    let planner = Planner::new(schema.clone());
     let parsed = planner
         .parse_expr(expression)
         .map_err(|e| invalid(e.to_string()))?;
-    let projected_blob_field = projected_blob_field(schema.as_ref(), &parsed)?;
 
     // A declaration is evaluated more than once -- staging and writing are
     // separate passes, and a refresh years later replays the same text -- so
@@ -1376,19 +1218,13 @@ pub(crate) fn bind(schema: SchemaRef, column: &str, expression: &str) -> Result<
     inputs.sort();
     inputs.dedup();
 
-    let blob_paths = referenced_blob_paths(schema.as_ref(), &inputs)?;
-    let runtime_schema = blob_runtime_schema(
-        schema.as_ref(),
-        &blob_paths.iter().cloned().collect::<HashSet<_>>(),
-    );
-
     // A nested input is recorded by its path but read through its root
     // column; Schema::index_of resolves top-level names only. Resolved here
     // rather than left to the planner so an unknown column names itself in
     // the error instead of surfacing as a plan failure.
     let mut indices = Vec::with_capacity(inputs.len());
     for input in &inputs {
-        let index = runtime_schema
+        let index = schema
             .index_of(root(input))
             .map_err(|_| invalid(format!("unknown column '{input}'")))?;
         if !indices.contains(&index) {
@@ -1401,7 +1237,7 @@ pub(crate) fn bind(schema: SchemaRef, column: &str, expression: &str) -> Result<
     // compiles the expression has to be built on the projected schema
     // evaluation will actually read.
     let read_schema = Arc::new(
-        runtime_schema
+        schema
             .project(&indices)
             .map_err(|e| invalid(e.to_string()))?,
     );
@@ -1411,8 +1247,7 @@ pub(crate) fn bind(schema: SchemaRef, column: &str, expression: &str) -> Result<
         .map(|field| field.name().clone())
         .collect();
 
-    let runtime_planner = Planner::new(runtime_schema);
-    let optimized = runtime_planner
+    let optimized = planner
         .optimize_expr(parsed)
         .map_err(|e| invalid(e.to_string()))?;
     let physical = Planner::new(read_schema.clone())
@@ -1425,16 +1260,9 @@ pub(crate) fn bind(schema: SchemaRef, column: &str, expression: &str) -> Result<
     Ok(BoundExpression {
         inputs,
         roots,
+        read_schema,
         physical,
         data_type,
-        blob_paths: blob_paths
-            .iter()
-            .map(|path| {
-                let segments = path.iter().map(String::as_str).collect::<Vec<_>>();
-                format_field_path_minimal(&segments)
-            })
-            .collect(),
-        projected_blob_field,
     })
 }
 
@@ -1447,10 +1275,10 @@ pub(crate) fn bind(schema: SchemaRef, column: &str, expression: &str) -> Result<
 /// is one a refresh can always act on.
 ///
 /// Each accepted column joins the schema the next one resolves against, so a
-/// batch may declare `a` and then `b = a + 1` in one commit. Refresh order
-/// then matters, and refresh enforces it: `b` is refused while `a` still has
-/// unfilled rows.
-fn plan_declarations(schema: SchemaRef, columns: &[(String, String)]) -> Result<Vec<ArrowField>> {
+/// batch may declare `a` and then `b = a + 1` in one commit. Refresh fills a
+/// column's computed inputs before the column, so the order of refresh calls
+/// does not matter.
+pub(crate) fn plan(schema: SchemaRef, columns: &[(String, String)]) -> Result<Vec<ArrowField>> {
     if columns.is_empty() {
         return Err(Error::InvalidInput {
             message: "at least one computed column is required".into(),
@@ -1462,28 +1290,15 @@ fn plan_declarations(schema: SchemaRef, columns: &[(String, String)]) -> Result<
 
     for (name, expression) in columns {
         if schema.field_with_name(name).is_ok() {
-            return Err(Error::ColumnAlreadyExists {
-                name: name.to_string(),
-            });
+            return Err(Error::ColumnAlreadyExists { name: name.clone() });
         }
 
         let bound = bind(schema.clone(), name, expression)?;
 
         // Declared columns start entirely null, so nullability is a property
         // of the declaration rather than of what the expression yields.
-        let computed_metadata = computed_column_metadata(expression, &bound.inputs);
-        let field = match bound.projected_blob_field {
-            Some(source) => {
-                let mut metadata = source.metadata().clone();
-                metadata.retain(|key, _| !is_declaration_key(key));
-                metadata.extend(computed_metadata);
-                source
-                    .with_name(name)
-                    .with_nullable(true)
-                    .with_metadata(metadata)
-            }
-            None => ArrowField::new(name, bound.data_type, true).with_metadata(computed_metadata),
-        };
+        let field = ArrowField::new(name, bound.data_type, true)
+            .with_metadata(computed_column_metadata(expression, &bound.inputs));
         schema = Arc::new(ArrowSchema::new_with_metadata(
             schema
                 .fields()
@@ -1499,10 +1314,6 @@ fn plan_declarations(schema: SchemaRef, columns: &[(String, String)]) -> Result<
     Ok(fields)
 }
 
-pub(crate) fn plan(schema: SchemaRef, columns: &[(String, String)]) -> Result<Vec<ArrowField>> {
-    plan_declarations(schema, columns)
-}
-
 /// Run the schema-level checks of
 /// [`AddColumnsBuilder::computed`](super::AddColumnsBuilder::computed) against
 /// `schema` without committing: the Function-binding guard and the planning of
@@ -1541,7 +1352,7 @@ pub(crate) fn declare(
     schema: SchemaRef,
     columns: &[(String, String)],
 ) -> Result<NewColumnTransform> {
-    let fields = plan_declarations(schema, columns)?;
+    let fields = plan(schema, columns)?;
     Ok(NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new(
         fields,
     ))))
@@ -1667,44 +1478,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_direct_blob_projection_inherits_semantics() {
-        let schema = Arc::new(ArrowSchema::new(vec![crate::blob("image", false)]));
-        let fields = plan(
-            schema,
-            &[
-                ("first".to_string(), "image".to_string()),
-                ("second".to_string(), "first".to_string()),
-            ],
-        )
-        .unwrap();
-
-        for field in &fields {
-            assert!(field.is_blob_v2());
-            assert!(field.is_nullable());
-        }
-        assert_eq!(
-            fields[1]
-                .metadata()
-                .get(EXPRESSION_META_KEY)
-                .map(String::as_str),
-            Some("first")
-        );
-    }
-
-    #[test]
-    fn test_blob_expression_transformation_does_not_inherit_semantics() {
-        let schema = Arc::new(ArrowSchema::new(vec![crate::blob("image", true)]));
-        let fields = plan(
-            schema,
-            &[("payload".to_string(), "coalesce(image, image)".to_string())],
-        )
-        .unwrap();
-
-        assert!(!fields[0].is_blob_v2());
-        assert_eq!(fields[0].data_type(), &DataType::LargeBinary);
-    }
-
     /// The binding reaches the schema only if `AllNulls` carries per-field
     /// metadata through the commit. The whole representation rests on it.
     #[tokio::test]

rust/lancedb/src/table/datafusion/blob_coerce.rs

@@ -36,14 +36,6 @@ pub(super) fn coerce_blob_expr(
     };
 
     let input_shape = match input_field.data_type() {
-        DataType::Null => {
-            let expr: Arc<dyn PhysicalExpr> = Arc::new(CastExpr::new(
-                input_expr,
-                table_field.data_type().clone(),
-                None,
-            ));
-            return Ok((expr, table_field.clone()));
-        }
         DataType::Binary | DataType::LargeBinary | DataType::BinaryView => BlobInputShape::Bytes,
         DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => BlobInputShape::String,
         DataType::Struct(children) => {
@@ -163,7 +155,7 @@ mod tests {
     use crate::blob::blob;
     use arrow_array::{
         Array, ArrayRef, BinaryArray, BinaryViewArray, Int32Array, Int64Array, LargeBinaryArray,
-        NullArray, RecordBatch, StringArray, StringViewArray, StructArray, UInt8Array, UInt64Array,
+        RecordBatch, StringArray, StringViewArray, StructArray, UInt8Array, UInt64Array,
     };
     use arrow_schema::Schema;
     use datafusion::prelude::SessionContext;
@@ -287,18 +279,6 @@ mod tests {
         assert_eq!(data.value(0), b"view");
     }
 
-    #[tokio::test]
-    async fn null_column_coerces_to_all_null_blob_struct() {
-        let batch = batch_with_image(
-            Field::new("image", DataType::Null, true),
-            Arc::new(NullArray::new(2)),
-        );
-        let coerced = coerce(batch, &blob_table_schema()).await;
-        let image = image_struct(&coerced);
-        assert!(image.is_null(0));
-        assert!(image.is_null(1));
-    }
-
     #[tokio::test]
     async fn binary_nulls_stay_null_after_coercion() {
         let batch = batch_with_image(