Merge remote-tracking branch 'origin/main' into yang/udf-secret-api

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.39.0-beta.5"
+current_version = "0.38.0-beta.11"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

.github/dependabot.yml

@@ -44,27 +44,3 @@ updates:
       python-deps:
         patterns:
           - "*"
-
-  # The npm ecosystem covers pnpm lockfiles. There are two separate installs:
-  # the bindings themselves and the examples, which have their own lockfile.
-  # As with cargo and pip above, only bump the lockfile — the version ranges
-  # in package.json are our consumers' constraints, not ours.
-  - package-ecosystem: npm
-    directory: /nodejs
-    schedule:
-      interval: weekly
-    versioning-strategy: lockfile-only
-    groups:
-      nodejs-deps:
-        patterns:
-          - "*"
-
-  - package-ecosystem: npm
-    directory: /nodejs/examples
-    schedule:
-      interval: weekly
-    versioning-strategy: lockfile-only
-    groups:
-      nodejs-examples-deps:
-        patterns:
-          - "*"

.github/workflows/dev.yml

@@ -29,14 +29,12 @@ jobs:
     steps:
       - uses: actions/setup-node@v6
         with:
-          node-version: "24"
-      - uses: pnpm/action-setup@v6
-        with:
-          version: 11.1.1
+          node-version: "18"
       # These rules are disabled because Github will always ensure there
       # is a blank line between the title and the body and Github will
       # word wrap the description field to ensure a reasonable max line
       # length.
+      - run: npm install @commitlint/config-conventional
       - run: >
           echo 'module.exports = {
             "rules": {
@@ -45,11 +43,7 @@ jobs:
               "body-leading-blank": [0, "always"]
             }
           }' > .commitlintrc.js
-      - run: >
-          pnpm dlx
-          --package @commitlint/cli@21.2.2
-          --package @commitlint/config-conventional@21.2.2
-          commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
+      - run: npx commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
         env:
           COMMIT_MSG: >
             ${{ github.event.pull_request.title }}
@@ -60,7 +54,7 @@ jobs:
         with:
           script: |
             const message = `**ACTION NEEDED**
-
+              
               Lance follows the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) for release automation.
 
               The PR title and description are used as the merge commit message.\

.github/workflows/docs-link-check.yml

@@ -56,7 +56,7 @@ jobs:
         uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0
         with:
           # Restricted to http(s) on purpose. Much of docs/src is generated
-          # API reference (the js/ tree comes from `pnpm run docs` in nodejs)
+          # API reference (the js/ tree comes from `npm run docs` in nodejs)
           # and the hand-written pages use mkdocstrings cross-references and
           # nav-relative paths that only resolve in the site mkdocs builds,
           # not in this checkout, so relative links would be reported as

.github/workflows/docs.yml

@@ -55,7 +55,9 @@ jobs:
       - name: Set up node
         uses: actions/setup-node@v6
         with:
-          node-version: 24
+          node-version: 20
+          cache: 'npm'
+          cache-dependency-path: docs/package-lock.json
       - name: Install node dependencies
         working-directory: nodejs
         run: |

.github/workflows/nodejs.yml

@@ -47,8 +47,9 @@ jobs:
         version: 11.1.1
     - uses: actions/setup-node@v6
       with:
-        # Build on a supported LTS; the matrix job below covers every
-        # Node version the library claims to support.
+        # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+        # in October. The library itself still supports Node >= 18
+        # (see test matrix below).
         node-version: 24
         cache: 'pnpm'
         cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -83,7 +84,7 @@ jobs:
     timeout-minutes: 30
     strategy:
       matrix:
-        node-version: [ "22", "24", "26" ]
+        node-version: [ "18", "20" ]
     runs-on: "ubuntu-22.04"
     defaults:
       run:
@@ -100,9 +101,9 @@ jobs:
     - uses: actions/setup-node@v6
       name: Setup Node.js 24 for build
       with:
-        # Build and install once on a fixed version so the generated docs
-        # are identical across matrix legs; the tests below then run on each
-        # supported Node version.
+        # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+        # in October. Build/install runs on Node 24; tests run on the
+        # matrix version below using direct jest invocation.
         node-version: 24
         cache: 'pnpm'
         cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -151,9 +152,9 @@ jobs:
         S3_TEST: "1"
         # Newer @smithy/core uses dynamic ESM imports.
         NODE_OPTIONS: "--experimental-vm-modules"
-      # Invoke the installed jest binary directly; the pnpm shim is set up
-      # against the build-phase Node, not the version selected above.
-      run: node_modules/.bin/jest --verbose
+      # Invoke jest directly because pnpm 11 itself requires Node 22+
+      # while the matrix tests on older Node versions.
+      run: npx jest --verbose
     - name: Test examples
       working-directory: ./
       env:
@@ -163,7 +164,7 @@ jobs:
       run: |
         python ci/mock_openai.py &
         cd nodejs/examples
-        node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose
+        npx jest --testEnvironment jest-environment-node-single-context --verbose
   macos:
     timeout-minutes: 30
     # macos-15 ships a newer linker; the older macos-14 linker fails to insert
@@ -184,7 +185,8 @@ jobs:
         version: 11.1.1
     - uses: actions/setup-node@v6
       with:
-        # pnpm 11 requires Node >= 22.13.
+        # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+        # in October.
         node-version: 24
         cache: 'pnpm'
         cache-dependency-path: nodejs/pnpm-lock.yaml

.github/workflows/npm-publish.yml

@@ -40,31 +40,40 @@ jobs:
           - target: aarch64-apple-darwin
             host: macos-latest
             features: fp16kernels
-            # Fat LTO was ~111 of this job's ~113 minutes.
-            lto: thin
-            codegen_units: 16
             pre_build: |-
               brew install protobuf
+              # Fat LTO (the workspace default in .cargo/config.toml) is
+              # single-threaded and is the peak-memory step of the build. On
+              # this runner it accounted for ~111 of the job's ~113 minutes,
+              # making it the critical path of the entire publish pipeline.
+              # ThinLTO parallelizes it across the runner's cores, for a few
+              # percent of runtime performance.
+              export CARGO_PROFILE_RELEASE_LTO=thin
+              export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
           - target: x86_64-pc-windows-msvc
             host: windows-2025
             features: ","
-            # The lower peak also keeps this on the standard 4-core runner.
-            lto: thin
-            codegen_units: 16
             pre_build: |-
               choco install --no-progress protoc ninja nasm
               tail -n 1000 /c/ProgramData/chocolatey/logs/chocolatey.log
               # There is an issue where choco doesn't add nasm to the path
               export PATH="$PATH:/c/Program Files/NASM"
               nasm -v
+              # See the ThinLTO note on aarch64-apple-darwin above. Keeping
+              # peak memory down is also what lets this run on the standard
+              # 4-core runner: the 8-core larger runner was only needed to
+              # stop fat LTO from OOMing rustc-LLVM.
+              export CARGO_PROFILE_RELEASE_LTO=thin
+              export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
           - target: aarch64-pc-windows-msvc
             host: windows-2025
             features: ","
-            lto: thin
-            codegen_units: 16
             pre_build: |-
                 choco install --no-progress protoc
                 rustup target add aarch64-pc-windows-msvc
+                # See the ThinLTO note on aarch64-apple-darwin above.
+                export CARGO_PROFILE_RELEASE_LTO=thin
+                export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
           - target: x86_64-unknown-linux-gnu
             host: ubuntu-latest
             features: fp16kernels
@@ -94,14 +103,6 @@ jobs:
             # https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
             docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
             features: "fp16kernels"
-            # Fat LTO OOM-killed rustc every nightly; even with lld it peaked
-            # at 31391 MiB of the runner's 32 GiB.
-            lto: thin
-            codegen_units: 16
-            # arm64 Linux links through GNU `ld` where x86_64 defaults to
-            # `rust-lld`, which is why only arm64 OOM'd. lld cut the largest
-            # linker process 7.0 -> 4.0 GiB (lancedb/sophon#7313).
-            linker: /tmp/aarch64-lld-clang
             pre_build: |-
               set -e &&
               apt-get update &&
@@ -111,30 +112,9 @@ jobs:
               # AT_HWCAP2 (added in Linux 3.17). Define it for aws-lc-sys.
               export CFLAGS="$CFLAGS -DAT_HWCAP2=26" &&
               rustup target add aarch64-unknown-linux-gnu
-              # Not `&&`-chained: in dash, errexit does not fire for a
-              # non-final command in an `&&` list, so failures were ignored.
-              #
-              # A wrapper rather than `-C link-arg` because the per-target
-              # rustflags variable does not reach every unit that links, while
-              # the linker variable does. `clang` because GCC silently ignores
-              # `-fuse-ld=lld` unless built with lld support. Two echoes
-              # because printf's newline escape gets rewritten to `;` between
-              # here and the container.
-              echo '#!/bin/sh' > /tmp/aarch64-lld-clang
-              echo 'exec clang --target=aarch64-unknown-linux-gnu --sysroot=/usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot --gcc-toolchain=/usr/aarch64-unknown-linux-gnu -fuse-ld=lld "$@"' >> /tmp/aarch64-lld-clang
-              chmod 0755 /tmp/aarch64-lld-clang
-              # Fail now, not at the cdylib link ~30 minutes later. Linking at
-              # all also proves lld resolved; clang errors out when it cannot.
-              echo 'int main(void){return 0;}' > /tmp/probe.c
-              /tmp/aarch64-lld-clang /tmp/probe.c -o /tmp/probe
-              readelf -h /tmp/probe | grep AArch64
           - target: aarch64-unknown-linux-musl
             host: ubuntu-2404-8x-x64
             features: ","
-            # Fat LTO took the whole runner down. lld cannot help: it died
-            # inside rustc's LLVM, before any linker was spawned.
-            lto: thin
-            codegen_units: 16
             pre_build: |-
               set -e &&
               sudo apt-get update &&
@@ -143,19 +123,6 @@ jobs:
               export EXTRA_ARGS="-x"
     name: build - ${{ matrix.settings.target }}
     runs-on: ${{ matrix.settings.host }}
-    # On the job, not exported from `pre_build`: `Swatinem/rust-cache` hashes
-    # `CARGO_*` into its cache key before any step runs, so a step-local export
-    # leaves the key unchanged while cargo still rebuilds cold. The ThinLTO
-    # legs had been doing that every run.
-    #
-    # Not `RUSTFLAGS`: setting it, even to "", discards every config-file
-    # rustflag, silently dropping .cargo/config.toml's `target-cpu` and
-    # `target-feature` from the published binaries.
-    env:
-      CARGO_PROFILE_RELEASE_LTO: ${{ matrix.settings.lto || 'fat' }}
-      CARGO_PROFILE_RELEASE_CODEGEN_UNITS: ${{ matrix.settings.codegen_units || '1' }}
-      # Empty elsewhere: a per-target variable is only read for that triple.
-      CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: ${{ matrix.settings.linker }}
     defaults:
       run:
         working-directory: nodejs
@@ -168,7 +135,8 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v6
         with:
-          # pnpm 11 requires Node >= 22.13.
+          # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+          # in October.
           node-version: 24
           cache: pnpm
           cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -201,15 +169,19 @@ jobs:
           # creating ref). The nightly cadence also keeps entries inside
           # GitHub's 7-day eviction window, which a tag-only trigger would not.
           save-if: ${{ github.ref == 'refs/heads/main' }}
-      # Docker builds can use rust-cache too: the workspace is bind-mounted, so
-      # `target/` lives on the host and rust-cache's prune keeps the entry
-      # small.
+      # Docker builds can use rust-cache too. `target/` already lives on the
+      # host because the whole workspace is bind-mounted into the container, and
+      # rust-cache's prune and save run host-side, so they can manage it -- which
+      # is what keeps the entry to dependency artifacts rather than a multi-GB
+      # copy of everything.
       #
       # Two differences from the native builds. The container's CARGO_HOME is
-      # bind-mounted from `.cargo-cache` rather than ~/.cargo, so that is cached
-      # explicitly. And the key uses the *host* rustc version, not the compiler
-      # that built these artifacts -- safe, since cargo fingerprints the real
-      # one; a base-image bump just costs one cold build.
+      # bind-mounted from `.cargo-cache` rather than the host's ~/.cargo, so that
+      # has to be cached explicitly. And the key is derived from the *host* rustc
+      # version, which is not the compiler that produced these artifacts; that is
+      # safe because cargo fingerprints the real compiler and rebuilds on a
+      # mismatch, it just means a base-image toolchain bump costs one cold build
+      # instead of invalidating the key.
       - name: Cache cargo (docker builds)
         uses: Swatinem/rust-cache@v2
         if: ${{ matrix.settings.docker }}
@@ -238,19 +210,14 @@ jobs:
           # cache step above saves. Previously the registry mounts pointed at
           # `.cargo/...`, a path nothing cached, so the container re-downloaded
           # the whole crate registry on every run.
-          #
-          # `docker run` inherits nothing; `-e NAME` carries the job's `env:` in.
           options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
                     -v ${{ github.workspace }}/.cargo-cache/registry/cache:/usr/local/cargo/registry/cache \
                     -v ${{ github.workspace }}/.cargo-cache/registry/index:/usr/local/cargo/registry/index \
-                    -e CARGO_PROFILE_RELEASE_LTO \
-                    -e CARGO_PROFILE_RELEASE_CODEGEN_UNITS \
-                    -e CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER \
                     -v ${{ github.workspace }}:/build -w /build/nodejs"
           run: |
             set -e
             ${{ matrix.settings.pre_build }}
-            node_modules/.bin/napi build --platform --release \
+            npx napi build --platform --release \
               --features ${{ matrix.settings.features }} \
               --target ${{ matrix.settings.target }} \
               --dts ../lancedb/native.d.ts \
@@ -270,7 +237,7 @@ jobs:
       - name: Build
         run: |
           ${{ matrix.settings.pre_build }}
-          node_modules/.bin/napi build --platform --release \
+          npx napi build --platform --release \
               --features ${{ matrix.settings.features }} \
               --target ${{ matrix.settings.target }} \
               --dts ../lancedb/native.d.ts \
@@ -289,18 +256,6 @@ jobs:
         if: always()
         run: df -h
         shell: bash
-      - name: Report peak memory
-        if: always() && runner.os == 'Linux'
-        shell: bash
-        run: |
-          peak=$(find /sys/fs/cgroup -name memory.peak -readable \
-                 -exec cat {} + 2>/dev/null | sort -n | tail -1)
-          if [ -n "$peak" ]; then
-            echo "peak memory: $((peak / 1024 / 1024)) MiB"
-          else
-            echo "peak memory: unavailable (no readable cgroup v2 memory.peak)"
-          fi
-          free -g || true
       - name: Upload artifact
         uses: actions/upload-artifact@v7
         with:
@@ -338,7 +293,7 @@ jobs:
           - target: aarch64-unknown-linux-gnu
             host: ubuntu-2404-8x-arm64
         node:
-          - '22'
+          - '20'
     runs-on: ${{ matrix.settings.host }}
     defaults:
       run:
@@ -384,9 +339,9 @@ jobs:
       - name: Move built files
         run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
       - name: Test bindings
-        # Invoke the installed jest binary directly; the pnpm shim is set up
-        # against the install-phase Node, not the version selected above.
-        run: node_modules/.bin/jest --verbose
+        # Invoke jest directly because pnpm 11 itself requires Node 22+
+        # while the matrix tests on older Node versions.
+        run: npx jest --verbose
   publish:
     name: Publish
     runs-on: ubuntu-latest

.github/workflows/rust.yml

@@ -232,10 +232,7 @@ jobs:
           ALL_FEATURES=`cargo metadata --format-version=1 --no-deps \
             | jq -r '.packages[] | .features | keys | .[]' \
             | grep -v s3-test | sort | uniq | paste -s -d "," -`
-          # Run doctests before test binaries fill the runner disk. Examples are
-          # already built by the Linux job, so avoid retaining them here.
-          cargo test --profile ci --features $ALL_FEATURES --locked --doc
-          cargo test --profile ci --features $ALL_FEATURES --locked --lib --tests
+          cargo test --profile ci --features $ALL_FEATURES --locked
 
   windows:
     strategy:

.github/workflows/update_package_lock_run.yml

@@ -0,0 +1,22 @@
+name: Update package-lock.json
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          ref: main
+          persist-credentials: false
+          fetch-depth: 0
+          lfs: true
+      - uses: ./.github/workflows/update_package_lock
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.github/workflows/update_package_lock_run_nodejs.yml

@@ -0,0 +1,22 @@
+name: Update NodeJs package-lock.json
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          ref: main
+          persist-credentials: false
+          fetch-depth: 0
+          lfs: true
+      - uses: ./.github/workflows/update_package_lock_nodejs
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.pre-commit-config.yaml

@@ -20,10 +20,7 @@ repos:
     hooks:
       - id: local-biome-check
         name: biome check
-        # Use the biome from nodejs/package.json rather than a separately
-        # pinned one: the two drifted apart and disagreed on formatting, so
-        # this hook rejected code that `pnpm lint` accepted.
-        entry: nodejs/node_modules/.bin/biome check --config-path nodejs/biome.json nodejs/
+        entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
         language: system
         types: [text]
         files: "nodejs/.*"

AGENTS.md

@@ -38,7 +38,7 @@ Before committing changes, run formatting for every language you touched. At min
 * Rust changes: run `cargo fmt --all`.
 * Python changes: run `ruff format .` and `ruff check .` from the repository root,
   and run targeted tests through `cd python && uv run ...`.
-* TypeScript changes: run the relevant `pnpm` lint, format, build, and docs commands in `nodejs`.
+* TypeScript changes: run the relevant `npm`/`pnpm` lint, format, build, and docs commands in `nodejs`.
 
 Before creating a PR, the exact value passed to `gh pr create --title` must follow
 Conventional Commits, such as `fix: support nested field paths in native index creation`
@@ -101,12 +101,12 @@ Python bindings changes:
 TypeScript bindings changes:
 
 1. Add napi-rs method binding on `Table` in `nodejs/src/table.rs`.
-2. Run `pnpm build` to generate TypeScript definitions.
+2. Run `npm run build` to generate TypeScript definitions.
 3. Add typescript method on abstract class `Table` in `nodejs/src/table.ts`.
 4. Add concrete method on `LocalTable` class in `nodejs/src/native_table.ts`.
     * Note: despite the name, this class is also used for remote tables.
 5. Add test in `nodejs/__test__/table.test.ts`.
-6. Run `pnpm run docs` to generate TypeScript documentation.
+6. Run `npm run docs` to generate TypeScript documentation.
 
 ## Python API reference
 

Cargo.lock

@@ -332,34 +332,6 @@ dependencies = [
  "num-traits",
 ]
 
-[[package]]
-name = "arrow-flight"
-version = "58.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2dbe34824c639e43136af8f106992792ab456540d54b880bc320a3192502d2e"
-dependencies = [
- "arrow-arith",
- "arrow-array",
- "arrow-buffer",
- "arrow-cast",
- "arrow-data",
- "arrow-ipc",
- "arrow-ord",
- "arrow-row",
- "arrow-schema",
- "arrow-select",
- "arrow-string",
- "base64 0.22.1",
- "bytes",
- "futures",
- "once_cell",
- "paste",
- "prost",
- "prost-types",
- "tonic",
- "tonic-prost",
-]
-
 [[package]]
 name = "arrow-ipc"
 version = "58.4.0"
@@ -563,9 +535,9 @@ dependencies = [
 
 [[package]]
 name = "async-trait"
-version = "0.1.92"
+version = "0.1.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
+checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1157,7 +1129,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
 dependencies = [
  "async-trait",
- "axum-core 0.4.5",
+ "axum-core",
  "bytes",
  "futures-util",
  "http 1.5.0",
@@ -1166,7 +1138,7 @@ dependencies = [
  "hyper 1.9.0",
  "hyper-util",
  "itoa",
- "matchit 0.7.3",
+ "matchit",
  "memchr",
  "mime",
  "percent-encoding",
@@ -1184,31 +1156,6 @@ dependencies = [
  "tracing",
 ]
 
-[[package]]
-name = "axum"
-version = "0.8.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"
-dependencies = [
- "axum-core 0.5.6",
- "bytes",
- "futures-util",
- "http 1.5.0",
- "http-body 1.1.0",
- "http-body-util",
- "itoa",
- "matchit 0.8.4",
- "memchr",
- "mime",
- "percent-encoding",
- "pin-project-lite",
- "serde_core",
- "sync_wrapper",
- "tower",
- "tower-layer",
- "tower-service",
-]
-
 [[package]]
 name = "axum-core"
 version = "0.4.5"
@@ -1230,24 +1177,6 @@ dependencies = [
  "tracing",
 ]
 
-[[package]]
-name = "axum-core"
-version = "0.5.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1"
-dependencies = [
- "bytes",
- "futures-core",
- "http 1.5.0",
- "http-body 1.1.0",
- "http-body-util",
- "mime",
- "pin-project-lite",
- "sync_wrapper",
- "tower-layer",
- "tower-service",
-]
-
 [[package]]
 name = "backoff"
 version = "0.4.0"
@@ -1514,9 +1443,9 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
 
 [[package]]
 name = "bytemuck"
-version = "1.25.2"
+version = "1.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
+checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
 dependencies = [
  "bytemuck_derive",
 ]
@@ -1668,9 +1597,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
 
 [[package]]
 name = "chacha20"
-version = "0.10.2"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
+checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601"
 dependencies = [
  "cfg-if 1.0.4",
  "cpufeatures 0.3.0",
@@ -3526,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arrow-array",
  "rand 0.9.5",
@@ -4886,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 
 [[package]]
 name = "lance"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -4959,8 +4888,8 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4982,7 +4911,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4996,7 +4925,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5005,8 +4934,8 @@ dependencies = [
 
 [[package]]
 name = "lance-bitpacking"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arrayref",
  "crunchy",
@@ -5016,8 +4945,8 @@ dependencies = [
 
 [[package]]
 name = "lance-core"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5054,8 +4983,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datafusion"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5071,7 +5000,6 @@ dependencies = [
  "datafusion-functions",
  "datafusion-physical-expr",
  "futures",
- "half",
  "jsonb",
  "lance-arrow",
  "lance-core",
@@ -5085,8 +5013,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5103,8 +5031,8 @@ dependencies = [
 
 [[package]]
 name = "lance-derive"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5113,8 +5041,8 @@ dependencies = [
 
 [[package]]
 name = "lance-encoding"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -5147,8 +5075,8 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -5179,8 +5107,8 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -5244,8 +5172,8 @@ dependencies = [
 
 [[package]]
 name = "lance-index-core"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5267,8 +5195,8 @@ dependencies = [
 
 [[package]]
 name = "lance-io"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5308,8 +5236,8 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5323,29 +5251,27 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arrow",
  "async-trait",
  "bytes",
  "lance-core",
  "lance-namespace-reqwest-client",
- "serde",
- "serde_json",
  "snafu 0.9.0",
 ]
 
 [[package]]
 name = "lance-namespace-impls"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arrow",
  "arrow-ipc",
  "arrow-schema",
  "async-trait",
- "axum 0.7.9",
+ "axum",
  "base64 0.22.1",
  "bytes",
  "chrono",
@@ -5378,9 +5304,9 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-reqwest-client"
-version = "0.12.0"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8d23e54b1634d5bbb434f8dd33dc3c05f6e58d876a9a27b3b4aef58ddbe11af"
+checksum = "0a030196da1c994b63a96a4f0bf5b0cfa459fe6dadc9e962320246ca328da22a"
 dependencies = [
  "reqwest 0.12.28",
  "serde",
@@ -5392,8 +5318,8 @@ dependencies = [
 
 [[package]]
 name = "lance-select"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5407,8 +5333,8 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5448,8 +5374,8 @@ dependencies = [
 
 [[package]]
 name = "lance-testing"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5462,8 +5388,8 @@ dependencies = [
 
 [[package]]
 name = "lance-tokenizer"
-version = "12.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.14#a8a101774a1c9647065cc60137094feadbe55296"
+version = "12.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.2#dafa4642658d996b3e31dde91e02f72db7860d7e"
 dependencies = [
  "frostem",
  "icu_segmenter",
@@ -5476,7 +5402,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb"
-version = "0.39.0-beta.4"
+version = "0.38.0-beta.11"
 dependencies = [
  "ahash",
  "anyhow",
@@ -5485,7 +5411,6 @@ dependencies = [
  "arrow-buffer",
  "arrow-cast",
  "arrow-data",
- "arrow-flight",
  "arrow-ipc",
  "arrow-ord",
  "arrow-schema",
@@ -5541,7 +5466,6 @@ dependencies = [
  "polars",
  "polars-arrow",
  "pprof 0.14.1",
- "prost",
  "rand 0.9.5",
  "random_word",
  "regex",
@@ -5558,7 +5482,6 @@ dependencies = [
  "test-log",
  "tokenizers",
  "tokio",
- "tonic",
  "url",
  "urlencoding",
  "uuid",
@@ -5567,7 +5490,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb-nodejs"
-version = "0.39.0-beta.4"
+version = "0.38.0-beta.11"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5592,7 +5515,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb-python"
-version = "0.39.0-beta.4"
+version = "0.38.0-beta.11"
 dependencies = [
  "arrow",
  "async-trait",
@@ -5616,7 +5539,6 @@ dependencies = [
  "serde_json",
  "snafu 0.8.9",
  "tokio",
- "uuid",
 ]
 
 [[package]]
@@ -5826,9 +5748,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.34"
+version = "0.4.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
+checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
 
 [[package]]
 name = "loom"
@@ -5937,12 +5859,6 @@ version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
 
-[[package]]
-name = "matchit"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
-
 [[package]]
 name = "matrixmultiply"
 version = "0.3.10"
@@ -6085,9 +6001,9 @@ dependencies = [
 
 [[package]]
 name = "moka"
-version = "0.12.16"
+version = "0.12.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9"
+checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046"
 dependencies = [
  "async-lock",
  "crossbeam-channel",
@@ -6181,15 +6097,14 @@ dependencies = [
 
 [[package]]
 name = "napi"
-version = "3.12.2"
+version = "3.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58c5f4d5375213fdb7be2655e152386e82f026f9a5ba36a75556e11359aafe09"
+checksum = "de33522036981030a75c231829566bc63414e08101a6f5ff4ac6cef19c8e0941"
 dependencies = [
  "bitflags 2.11.1",
  "chrono",
  "ctor 1.0.12",
  "futures",
- "libc",
  "napi-build",
  "napi-sys",
  "nohash-hasher",
@@ -6201,15 +6116,15 @@ dependencies = [
 
 [[package]]
 name = "napi-build"
-version = "2.4.1"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60fdf9b392c50e7c4170fa633bd909490ed7835cea4c046776d1a4dd8d2ae0ab"
+checksum = "5282704fbe8d49b0cf8b08e3f33233416a528658f205c7e5ace63b582de0b11c"
 
 [[package]]
 name = "napi-derive"
-version = "3.6.3"
+version = "3.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fa55ea69990c90b888e9e77044410e304ce7f35de599dc6d0b5c1923d2e59af"
+checksum = "4d5c9c02556ea6dc99dffd36c1ce60141411657438501a125b675776d011ce92"
 dependencies = [
  "convert_case",
  "ctor 1.0.12",
@@ -6221,9 +6136,9 @@ dependencies = [
 
 [[package]]
 name = "napi-derive-backend"
-version = "6.1.2"
+version = "6.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df4056ac7c18e4438ccf0edaed4340ca0d269278c8ec19284f7b23cb039fd0ae"
+checksum = "d60b5d773ad46c698c8cc2cd9fde0b283d39cbb7f71c04bee633c7bdba4423bd"
 dependencies = [
  "convert_case",
  "proc-macro2",
@@ -7818,7 +7733,6 @@ dependencies = [
  "pyo3-build-config",
  "pyo3-ffi",
  "pyo3-macros",
- "uuid",
 ]
 
 [[package]]
@@ -8687,9 +8601,9 @@ dependencies = [
 
 [[package]]
 name = "roaring"
-version = "0.11.5"
+version = "0.11.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1"
+checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9"
 dependencies = [
  "bytemuck",
  "byteorder",
@@ -9149,9 +9063,9 @@ dependencies = [
 
 [[package]]
 name = "serde_with"
-version = "3.22.0"
+version = "3.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a"
+checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c"
 dependencies = [
  "base64 0.22.1",
  "bs58",
@@ -9159,7 +9073,6 @@ dependencies = [
  "hex",
  "indexmap 1.9.3",
  "indexmap 2.14.0",
- "jiff",
  "schemars 0.9.0",
  "schemars 1.2.1",
  "serde_core",
@@ -9170,9 +9083,9 @@ dependencies = [
 
 [[package]]
 name = "serde_with_macros"
-version = "3.22.0"
+version = "3.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46"
+checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660"
 dependencies = [
  "darling 0.23.0",
  "proc-macro2",
@@ -10171,7 +10084,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef"
 dependencies = [
  "async-trait",
- "axum 0.8.9",
  "base64 0.22.1",
  "bytes",
  "h2 0.4.16",
@@ -10183,11 +10095,9 @@ dependencies = [
  "hyper-util",
  "percent-encoding",
  "pin-project",
- "rustls-native-certs",
  "socket2 0.6.3",
  "sync_wrapper",
  "tokio",
- "tokio-rustls 0.26.4",
  "tokio-stream",
  "tower",
  "tower-layer",
@@ -10542,9 +10452,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "uuid"
-version = "1.26.0"
+version = "1.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812"
+checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
 dependencies = [
  "getrandom 0.4.2",
  "js-sys",

Cargo.toml

@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"
 
 [workspace.dependencies]
-lance = { "version" = "=12.0.0-beta.14", default-features = false, "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=12.0.0-beta.14", default-features = false, "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=12.0.0-beta.14", default-features = false, "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=12.0.0-beta.14", "tag" = "v12.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=12.0.0-beta.2", default-features = false, "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=12.0.0-beta.2", "tag" = "v12.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
 lancedb = { path = "rust/lancedb", default-features = false }
 ahash = "0.8"
 # Note that this one does not include pyarrow
@@ -39,7 +39,6 @@ arrow-ord = "58.0.0"
 arrow-schema = "58.0.0"
 arrow-select = "58.0.0"
 arrow-cast = "58.0.0"
-arrow-flight = { version = "58.0.0", features = ["flight-sql-experimental"] }
 async-trait = "0"
 bytes = "1"
 datafusion = { version = "54.0.0", default-features = false }
@@ -72,8 +71,7 @@ serde = "1"
 serde_json = "1"
 tempfile = "3.5.0"
 tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
-tonic = { version = "0.14", features = ["tls-native-roots", "tls-ring"] }
-uuid = { version = "1.7.0", features = ["v4", "v7"] }
+uuid = { version = "1.7.0", features = ["v4"] }
 chrono = { version = "0.4", default-features = false, features = ["clock"] }
 
 [profile.ci]

Makefile

@@ -5,5 +5,5 @@ licenses:
 	cd python && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
 	cd python && uv sync --all-extras && uv tool run pip-licenses --python .venv/bin/python --format=markdown --with-urls --output-file=PYTHON_THIRD_PARTY_LICENSES.md
 	cd nodejs && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
-	cd nodejs && pnpm dlx license-checker@25 --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
+	cd nodejs && npx license-checker --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
 	cd java && ./mvnw license:aggregate-add-third-party -q

ci/update_lockfiles.sh

@@ -12,12 +12,16 @@ done
 # This updates the lockfile without building
 cargo metadata --quiet > /dev/null
 
+pushd nodejs || exit 1
+npm install --package-lock-only --silent
+popd
+
 if git diff --quiet --exit-code; then
   echo "No lockfile changes to commit; skipping amend."
 elif $AMEND; then
-  git add Cargo.lock
+  git add Cargo.lock nodejs/package-lock.json
   git commit --amend --no-edit
 else
-  git add Cargo.lock
+  git add Cargo.lock nodejs/package-lock.json
   git commit -m "Update lockfiles"
 fi

deny.toml

@@ -131,13 +131,18 @@ allow = [
     "BSD-3-Clause",
     "ISC",
     "Unicode-3.0",
+    "Unicode-DFS-2016",
     "Zlib",
     "CC0-1.0",
     "MPL-2.0",
     "BSL-1.0",
+    "OpenSSL",
     # 0BSD ("BSD Zero Clause") is effectively public domain — no attribution
     # required. Pulled in by `mock_instant`.
     "0BSD",
+    # bzip2-1.0.6 is the permissive upstream bzip2 license (BSD-like). Pulled
+    # in by `libbz2-rs-sys`, the pure-Rust bzip2 implementation.
+    "bzip2-1.0.6",
     # CDLA-Permissive-2.0 is a permissive data license used by `webpki-roots`
     # for the Mozilla CA root bundle. Data-only, distribution-compatible.
     "CDLA-Permissive-2.0",
@@ -145,7 +150,12 @@ allow = [
 confidence-threshold = 0.8
 # Per-crate license exceptions: allow a license for a specific crate only,
 # rather than globally via the `allow` list above.
-exceptions = []
+exceptions = [
+    # CDDL-1.0 (copyleft) is pulled in only as a dev/profiling dependency via
+    # `inferno` -> `pprof` -> `lance-testing`; it is a test dependency that we
+    # do not distribute, so scope the allowance to `inferno` alone.
+    { allow = ["CDDL-1.0"], crate = "inferno" },
+]
 # Crates whose license cannot be determined from Cargo metadata but whose
 # license we've manually confirmed from upstream. Keep this list minimal.
 [[licenses.clarify]]

docs/README.md

@@ -47,24 +47,22 @@ pytest -vv python/tests/docs
 
 ### Checking typescript examples
 
-The examples depend on `@lancedb/lancedb` at `file:../dist`, so the package must be
-built before running the tests. This uses pnpm; see the
-[Typescript contributing guide](../nodejs/CONTRIBUTING.md) for the toolchain setup.
+The `@lancedb/lancedb` package must be built before running the tests:
 
 ```shell
 pushd nodejs
-pnpm install
-pnpm build
+npm ci
+npm run build
 popd
 ```
 
-Then you can run the examples by going to the `nodejs/examples` directory, which is a
-separate pnpm package with its own lockfile:
+Then you can run the examples by going to the `nodejs/examples` directory and
+running the tests like a normal npm package:
 
 ```shell
 pushd nodejs/examples
-pnpm install
-pnpm test
+npm ci
+npm test
 popd
 ```
 
@@ -86,7 +84,6 @@ The new files should be checked into the repository.
 
 ```shell
 pushd nodejs
-# `pnpm docs` would invoke pnpm's built-in `docs` command, not the script.
-pnpm run docs
+npm run docs
 popd
 ```

docs/openapi.yml

@@ -446,15 +446,6 @@ paths:
               properties:
                 column:
                   type: string
-                name:
-                  type: string
-                  description: Optional name for the created index.
-                replace:
-                  type: boolean
-                  default: true
-                  description: |
-                    Whether to replace an existing index with the same resolved
-                    name. Defaults to true.
                 metric_type:
                   type: string
                   nullable: false

docs/package-lock.json

@@ -0,0 +1,135 @@
+{
+  "name": "lancedb-docs-test",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "lancedb-docs-test",
+      "version": "1.0.0",
+      "license": "Apache 2",
+      "dependencies": {
+        "apache-arrow": "file:../node/node_modules/apache-arrow",
+        "vectordb": "file:../node"
+      },
+      "devDependencies": {
+        "@types/node": "^20.11.8",
+        "typescript": "^5.3.3"
+      }
+    },
+    "../node": {
+      "name": "vectordb",
+      "version": "0.21.2-beta.0",
+      "cpu": [
+        "x64",
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "os": [
+        "darwin",
+        "linux",
+        "win32"
+      ],
+      "dependencies": {
+        "@neon-rs/load": "^0.0.74",
+        "axios": "^1.4.0"
+      },
+      "devDependencies": {
+        "@neon-rs/cli": "^0.0.160",
+        "@types/chai": "^4.3.4",
+        "@types/chai-as-promised": "^7.1.5",
+        "@types/mocha": "^10.0.1",
+        "@types/node": "^18.16.2",
+        "@types/sinon": "^10.0.15",
+        "@types/temp": "^0.9.1",
+        "@types/uuid": "^9.0.3",
+        "@typescript-eslint/eslint-plugin": "^5.59.1",
+        "apache-arrow-old": "npm:apache-arrow@13.0.0",
+        "cargo-cp-artifact": "^0.1",
+        "chai": "^4.3.7",
+        "chai-as-promised": "^7.1.1",
+        "eslint": "^8.39.0",
+        "eslint-config-standard-with-typescript": "^34.0.1",
+        "eslint-plugin-import": "^2.26.0",
+        "eslint-plugin-n": "^15.7.0",
+        "eslint-plugin-promise": "^6.1.1",
+        "mocha": "^10.2.0",
+        "openai": "^4.24.1",
+        "sinon": "^15.1.0",
+        "temp": "^0.9.4",
+        "ts-node": "^10.9.1",
+        "ts-node-dev": "^2.0.0",
+        "typedoc": "^0.24.7",
+        "typedoc-plugin-markdown": "^3.15.3",
+        "typescript": "^5.1.0",
+        "uuid": "^9.0.0"
+      },
+      "optionalDependencies": {
+        "@lancedb/vectordb-darwin-arm64": "0.21.2-beta.0",
+        "@lancedb/vectordb-darwin-x64": "0.21.2-beta.0",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.21.2-beta.0",
+        "@lancedb/vectordb-linux-x64-gnu": "0.21.2-beta.0",
+        "@lancedb/vectordb-win32-x64-msvc": "0.21.2-beta.0"
+      },
+      "peerDependencies": {
+        "@apache-arrow/ts": "^14.0.2",
+        "apache-arrow": "^14.0.2"
+      }
+    },
+    "../node/node_modules/apache-arrow": {
+      "version": "14.0.2",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@types/command-line-args": "5.2.0",
+        "@types/command-line-usage": "5.0.2",
+        "@types/node": "20.3.0",
+        "@types/pad-left": "2.1.1",
+        "command-line-args": "5.2.1",
+        "command-line-usage": "7.0.1",
+        "flatbuffers": "23.5.26",
+        "json-bignum": "^0.0.3",
+        "pad-left": "^2.1.0",
+        "tslib": "^2.5.3"
+      },
+      "bin": {
+        "arrow2csv": "bin/arrow2csv.js"
+      }
+    },
+    "node_modules/@types/node": {
+      "version": "20.11.8",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.8.tgz",
+      "integrity": "sha512-i7omyekpPTNdv4Jb/Rgqg0RU8YqLcNsI12quKSDkRXNfx7Wxdm6HhK1awT3xTgEkgxPn3bvnSpiEAc7a7Lpyow==",
+      "dev": true,
+      "dependencies": {
+        "undici-types": "~5.26.4"
+      }
+    },
+    "node_modules/apache-arrow": {
+      "resolved": "../node/node_modules/apache-arrow",
+      "link": true
+    },
+    "node_modules/typescript": {
+      "version": "5.3.3",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz",
+      "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==",
+      "dev": true,
+      "bin": {
+        "tsc": "bin/tsc",
+        "tsserver": "bin/tsserver"
+      },
+      "engines": {
+        "node": ">=14.17"
+      }
+    },
+    "node_modules/undici-types": {
+      "version": "5.26.5",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
+      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
+      "dev": true
+    },
+    "node_modules/vectordb": {
+      "resolved": "../node",
+      "link": true
+    }
+  }
+}

docs/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "lancedb-docs-test",
+  "version": "1.0.0",
+  "description": "auto-generated tests from doc",
+  "author": "dev@lancedb.com",
+  "license": "Apache 2",
+  "dependencies": {
+    "apache-arrow": "file:../node/node_modules/apache-arrow",
+    "vectordb": "file:../node"
+  },
+  "scripts": {
+    "build": "tsc -b && cd ../node && npm run build-release",
+    "example": "npm run build && node",
+    "test": "npm run build && ls dist/*.js | xargs -n 1 node"
+  },
+  "devDependencies": {
+    "@types/node": "^20.11.8",
+    "typescript": "^5.3.3"
+  }
+}

docs/src/java/java.md

@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-core</artifactId>
-    <version>0.39.0-beta.5</version>
+    <version>0.38.0-beta.11</version>
 </dependency>
 ```
 

docs/src/js/classes/Connection.md

@@ -448,6 +448,26 @@ on the returned job to know when cleanup has finished.
 
 ***
 
+### getJob()
+
+```ts
+abstract getJob(jobId): Promise<null | JobDescription>
+```
+
+Describe a single server-side job by id.
+
+Resolves to `null` when the server has no such job.
+
+#### Parameters
+
+* **jobId**: `string`
+
+#### Returns
+
+`Promise`&lt;`null` \| [`JobDescription`](../interfaces/JobDescription.md)&gt;
+
+***
+
 ### isOpen()
 
 ```ts
@@ -462,6 +482,48 @@ Return true if the connection has not been closed
 
 ***
 
+### job()
+
+```ts
+abstract job(jobId): Job
+```
+
+A [Job](Job.md) handle for a server-side job by id.
+
+The handle is constructed without a server round trip; an unknown id
+surfaces when the handle is used. Dropping the handle has no effect on
+the job itself.
+
+#### Parameters
+
+* **jobId**: `string`
+
+#### Returns
+
+[`Job`](Job.md)
+
+***
+
+### jobHistory()
+
+```ts
+abstract jobHistory(jobId?): Promise<Table<any>>
+```
+
+The lifecycle event history of a server-side job, as an Arrow table.
+
+Lists history across all jobs when `jobId` is omitted.
+
+#### Parameters
+
+* **jobId?**: `string`
+
+#### Returns
+
+`Promise`&lt;`Table`&lt;`any`&gt;&gt;
+
+***
+
 ### listJobs()
 
 ```ts
@@ -586,30 +648,6 @@ A page of table names and an
 
 ***
 
-### openJob()
-
-```ts
-abstract openJob(jobId): Promise<Job>
-```
-
-Open a server-side job by id, returning a handle with its record already
-populated. Rejects when the server has no such job, the way
-[Connection.openTable](Connection.md#opentable) does for a missing table.
-
-The returned [Job](Job.md) answers for its own state, specification,
-result, failure and event history, so there is no separate
-connection-level call for any of them.
-
-#### Parameters
-
-* **jobId**: `string`
-
-#### Returns
-
-`Promise`&lt;[`Job`](Job.md)&gt;
-
-***
-
 ### openMaterializedView()
 
 ```ts

docs/src/js/classes/Job.md

@@ -8,116 +8,28 @@
 
 A handle to an operation that may still be running.
 
-The operation may already be complete when the handle is created.
+## Constructors
 
-The detail getters read what the handle last observed. Submitting an
-operation returns only a job id, so populating them eagerly would cost an
-extra round trip on every call:
+### new Job()
 
-- [Job.refresh](Job.md#refresh) and [Job.status](Job.md#status) fetch the whole record.
-- [Job.wait](Job.md#wait) records the terminal state it establishes, but not the
-  rest of the record.
-- Everything is null until one of those runs.
+```ts
+new Job(): Job
+```
+
+#### Returns
+
+[`Job`](Job.md)
 
 ## Accessors
 
-### creationMs
-
-```ts
-get creationMs(): null | number
-```
-
-When the job was created, in milliseconds since the epoch.
-
-#### Returns
-
-`null` \| `number`
-
-***
-
-### failure
-
-```ts
-get failure(): null | JobFailureInfo
-```
-
-Why the job failed, when it failed and the server reports a reason.
-
-#### Returns
-
-`null` \| [`JobFailureInfo`](../interfaces/JobFailureInfo.md)
-
-***
-
 ### id
 
 ```ts
 get id(): null | string
 ```
 
-Identifies the operation on the server that is running it.
-
-Operations that run in this process have no server id. The value is
-opaque: parsing it or storing it to resume the job later is not supported.
-
-#### Returns
-
-`null` \| `string`
-
-***
-
-### jobType
-
-```ts
-get jobType(): null | string
-```
-
-The job's type, as the server names it. Null for an in-process job, which
-has no server-side record.
-
-#### Returns
-
-`null` \| `string`
-
-***
-
-### result
-
-```ts
-get result(): any
-```
-
-The job-type-specific terminal result. Null until the job succeeds, so a
-job that never terminates reports its progress through [Job.events](Job.md#events)
-instead.
-
-#### Returns
-
-`any`
-
-***
-
-### spec
-
-```ts
-get spec(): any
-```
-
-The job-type-specific specification it was submitted with.
-
-#### Returns
-
-`any`
-
-***
-
-### state
-
-```ts
-get state(): null | string
-```
-
-The last observed lifecycle state, without contacting the backend.
+Identifies the operation on the server that is running it. Operations
+that run in this process have no server id. The value is opaque.
 
 #### Returns
 
@@ -139,61 +51,18 @@ Request cancellation. Cancelling a finished operation is a no-op.
 
 ***
 
-### events()
-
-```ts
-events(options?): Promise<Table<any>>
-```
-
-This job's recorded lifecycle events.
-
-Where the getters above report a terminal result only once the job reaches
-one, events are written as the job runs and outlive the workers that
-produced them. A distributed job records a `claim`/`claim_complete` pair
-per unit of work, each carrying `rows_processed`, so a job that never
-finishes still accounts for what it did.
-
-The server caps results at 1000 rows by default and 10,000 at most, and
-truncates without saying so, so pass `limit` for a job that emits an event
-per fragment. `filter` is a SQL-like expression over the `state`,
-`updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
-
-#### Parameters
-
-* **options?**: [`JobEventsOptions`](../interfaces/JobEventsOptions.md)
-
-#### Returns
-
-`Promise`&lt;`Table`&lt;`any`&gt;&gt;
-
-***
-
-### refresh()
-
-```ts
-refresh(): Promise<void>
-```
-
-Ask the backend for this job's current state, and for a server-side job
-its full record, then cache it for the getters above.
-
-#### Returns
-
-`Promise`&lt;`void`&gt;
-
-***
-
 ### status()
 
 ```ts
 status(): Promise<string>
 ```
 
-The operation's current lifecycle state: "running", "finished", "failed",
-or "cancelled".
+The operation's current lifecycle state: "running", "finished",
+"failed", or "cancelled".
 
-A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject on a
-terminal failure state. Also refreshes the getters above.
+A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject
+on a terminal failure state. States a newer server reports that this
+client version does not know pass through as-is.
 
 #### Returns
 
@@ -201,22 +70,6 @@ terminal failure state. Also refreshes the getters above.
 
 ***
 
-### toString()
-
-```ts
-toString(): string
-```
-
-Every field the handle currently knows, one per line, with the JSON
-payloads indented -- a refresh job's spec and result are the point of
-printing it.
-
-#### Returns
-
-`string`
-
-***
-
 ### wait()
 
 ```ts

docs/src/js/classes/Table.md

@@ -676,17 +676,9 @@ List all the versions of the table
 abstract mergeInsert(on): MergeInsertBuilder
 ```
 
-Create a [MergeInsertBuilder](MergeInsertBuilder.md), which combines new data with the
-existing table in a single transaction — inserting, updating and deleting
-rows depending on how they match.
-
 #### Parameters
 
 * **on**: `string` \| `string`[]
-    The column, or columns, to match source rows against target
-    rows on. Typically a key or id column. Several columns match on the
-    composite key: a source row updates a target row only when it agrees on
-    every one of them.
 
 #### Returns
 

docs/src/js/globals.md

@@ -96,7 +96,7 @@
 - [IvfFlatOptions](interfaces/IvfFlatOptions.md)
 - [IvfPqOptions](interfaces/IvfPqOptions.md)
 - [IvfRqOptions](interfaces/IvfRqOptions.md)
-- [JobEventsOptions](interfaces/JobEventsOptions.md)
+- [JobDescription](interfaces/JobDescription.md)
 - [JobFailureInfo](interfaces/JobFailureInfo.md)
 - [JobInfo](interfaces/JobInfo.md)
 - [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)

docs/src/js/interfaces/JobDescription.md

@@ -0,0 +1,66 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / JobDescription
+
+# Interface: JobDescription
+
+A described job from `Connection.getJob`.
+
+## Properties
+
+### creationMs
+
+```ts
+creationMs: number;
+```
+
+When the job was created, in milliseconds since the epoch.
+
+***
+
+### failure?
+
+```ts
+optional failure: JobFailureInfo;
+```
+
+Why the job failed, when the job is failed and the server reports a
+reason.
+
+***
+
+### jobId
+
+```ts
+jobId: string;
+```
+
+***
+
+### jobType
+
+```ts
+jobType: string;
+```
+
+***
+
+### specJson?
+
+```ts
+optional specJson: string;
+```
+
+The job-type-specific specification as a JSON string, when present.
+
+***
+
+### state
+
+```ts
+state: string;
+```
+
+Lifecycle state: "running", "finished", "failed", or "cancelled".

docs/src/js/interfaces/JobEventsOptions.md

@@ -1,29 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / JobEventsOptions
-
-# Interface: JobEventsOptions
-
-Which of a job's events [Job.events](../classes/Job.md#events) returns.
-
-## Properties
-
-### filter?
-
-```ts
-optional filter: string;
-```
-
-SQL-like filter over the event columns.
-
-***
-
-### limit?
-
-```ts
-optional limit: number;
-```
-
-Maximum event rows to return, up to the server maximum of 10,000.

docs/src/js/interfaces/JobInfo.md

@@ -26,7 +26,7 @@ When the job was created, in milliseconds since the epoch.
 jobId: string;
 ```
 
-The job id -- what `Connection.openJob` and `Connection.cancelJob`
+The job id -- what `Connection.getJob` and `Connection.cancelJob`
 accept.
 
 ***

docs/src/js/interfaces/MaterializedViewDefinition.md

@@ -50,16 +50,6 @@ projections: [string, string][];
 
 ***
 
-### sourceNamespace
-
-```ts
-sourceNamespace: string[];
-```
-
-Namespace holding the source table; empty is the root namespace.
-
-***
-
 ### sourceTable
 
 ```ts

docs/src/python/python.md

@@ -28,59 +28,6 @@ is also an [asynchronous API client](#connections-asynchronous).
 
 ::: lancedb.Session
 
-## Remote SQL
-
-Submit SQL against a remote LanceDB database through the connection.
-The connected database and `default_namespace_path=["public"]` are used for
-unqualified tables. Fully qualified references can still query other databases
-and namespaces available to the same deployment. `execute_query` returns a
-reader as soon as its initial result stream is available. `execute_query_async`
-returns a query handle immediately; use it to inspect progress, open a reader,
-or cancel the query. The SQL client is initialized by the first query and
-retained for the lifetime of the remote connection. Query ids are random,
-connection-scoped references rather than encoded SQL or durable resume tokens:
-
-```python
-import lancedb
-
-db = lancedb.connect(
-    "db://analytics",
-    api_key="ldb_...",
-    host_override="https://api.example.com",
-    sql_host_override="grpc+tls://sql.example.com:10026",
-)
-reader = db.execute_query(
-    """
-    SELECT events.id, accounts.name
-    FROM analytics.public.events AS events
-    JOIN users.public.accounts AS accounts ON events.user_id = accounts.id
-    """,
-    default_namespace_path=["public"],
-)
-for batch in reader:
-    print(batch.num_rows)
-
-query = db.execute_query_async("SELECT * FROM events")
-print(query.id)
-print(query.describe().status)
-for batch in query.reader():
-    print(batch.num_rows)
-
-# The async connection exposes the same lifecycle without blocking:
-# async_db = await lancedb.connect_async(
-#     "db://analytics",
-#     api_key="ldb_...",
-#     host_override="https://api.example.com",
-#     sql_host_override="grpc+tls://sql.example.com:10026",
-# )
-# reader = await async_db.execute_query("SELECT * FROM events")
-# query = await async_db.execute_query_async("SELECT * FROM events")
-# description = await async_db.describe_query(query.id)
-# async for batch in await query.reader():
-#     print(batch.num_rows)
-# await query.cancel()
-```
-
 ## Namespaces (Synchronous)
 
 A namespace-backed connection resolves tables through a
@@ -147,8 +94,6 @@ listing a storage directory.
 
 ::: lancedb.functions.OutputMapping
 
-::: lancedb.functions.AssignmentMapping
-
 ::: lancedb.functions.FunctionBinding
 
 ::: lancedb.functions.RefreshColumnResult
@@ -157,18 +102,6 @@ listing a storage directory.
 
 ::: lancedb.job.AsyncJob
 
-::: lancedb.job.JobInfo
-
-::: lancedb.job.JobDescription
-
-::: lancedb.job.JobFailureInfo
-
-::: lancedb.sql.Query
-
-::: lancedb.sql.AsyncQuery
-
-::: lancedb.sql.QueryDescription
-
 ## Materialized Views (Synchronous)
 
 ::: lancedb.materialized_view.MaterializedView
@@ -316,12 +249,6 @@ still work. Queries return descriptors. Call
 
 ::: lancedb.exceptions.MissingColumnError
 
-::: lancedb.exceptions.JobNotFoundError
-
-::: lancedb.exceptions.JobFailedError
-
-::: lancedb.exceptions.JobCancelledError
-
 ## Integrations
 
 ## Pydantic

docs/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "include": [
+    "src/*.ts",
+  ],
+  "compilerOptions": {
+    "target": "es2022",
+    "module": "nodenext",
+    "declaration": true,
+    "outDir": "./dist",
+    "strict": true,
+    "allowJs": true,
+    "resolveJsonModule": true,
+  },
+  "exclude": [
+    "./dist/*",
+  ]
+}

java/lancedb-core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
       <groupId>com.lancedb</groupId>
       <artifactId>lancedb-parent</artifactId>
-      <version>0.39.0-beta.5</version>
+      <version>0.38.0-beta.11</version>
       <relativePath>../pom.xml</relativePath>
     </parent>
 

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.39.0-beta.5</version>
+    <version>0.38.0-beta.11</version>
     <packaging>pom</packaging>
     <name>${project.artifactId}</name>
     <description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>12.0.0-beta.14</lance-core.version>
+        <lance-core.version>12.0.0-beta.2</lance-core.version>
         <spotless.skip>false</spotless.skip>
         <spotless.version>2.30.0</spotless.version>
         <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.39.0-beta.5"
+version = "0.38.0-beta.11"
 publish = false
 license.workspace = true
 description.workspace = true

nodejs/__test__/materialized_view.test.ts

@@ -48,28 +48,6 @@ describe("materialized views", () => {
     expect(definitionFromMetadata(safe, "v").limit).toBe(42);
   });
 
-  it("reads the namespaced select kind and refuses unknown kinds", () => {
-    // "namespaced_select" is the namespaced form of "select": same shape, a
-    // separate kind so readers that predate it refuse instead of resolving
-    // the source at the root.
-    const namespaced = new Map([
-      [
-        DEFINITION_META_KEY,
-        '{"kind":"namespaced_select","source_table":"people","source_namespace":["ns"]}',
-      ],
-    ]);
-    const definition = definitionFromMetadata(namespaced, "v");
-    expect(definition.sourceTable).toBe("people");
-    expect(definition.sourceNamespace).toEqual(["ns"]);
-
-    const unknown = new Map([
-      [DEFINITION_META_KEY, '{"kind":"select_v3","source_table":"people"}'],
-    ]);
-    expect(() => definitionFromMetadata(unknown, "v")).toThrow(
-      /cannot refresh/,
-    );
-  });
-
   it("creates, refreshes and queries a view", async () => {
     const view = await db.createMaterializedView("adults", "people", {
       select: ["name", ["shout", "upper(name)"]],

nodejs/__test__/package.test.ts

@@ -5,8 +5,8 @@ import packageJson = require("../package.json");
 
 describe("package metadata", () => {
   it("requires Node.js type declarations compatible with the runtime", () => {
-    expect(packageJson.engines.node).toBe(">= 22");
-    expect(packageJson.peerDependencies["@types/node"]).toBe(">=22");
+    expect(packageJson.engines.node).toBe(">= 18");
+    expect(packageJson.peerDependencies["@types/node"]).toBe(">=18");
     expect(packageJson.peerDependenciesMeta["@types/node"]).toEqual({
       optional: true,
     });

nodejs/__test__/remote.test.ts

@@ -3,7 +3,6 @@
 
 import * as http from "http";
 import { RequestListener } from "http";
-import packageJson = require("../package.json");
 import {
   ClientConfig,
   Connection,
@@ -71,13 +70,7 @@ async function withMockDatabase(
   try {
     await callback(db);
   } finally {
-    // `close()` alone leaves the port bound until keep-alive sockets drain, so
-    // a single failing test would cascade into EADDRINUSE for every test after
-    // it. Destroy the connections and wait for the port to actually be free.
-    await new Promise<void>((resolve) => {
-      server.closeAllConnections();
-      server.close(() => resolve());
-    });
+    server.close();
   }
 }
 
@@ -138,7 +131,7 @@ describe("remote connection", () => {
       (req, res) => {
         expect(req.headers["x-api-key"]).toEqual("fake");
         expect(req.headers["user-agent"]).toEqual(
-          `LanceDB-Node-Client/${packageJson.version}`,
+          `LanceDB-Node-Client/${process.env.npm_package_version}`,
         );
 
         const body = JSON.stringify({ tables: [] });
@@ -939,7 +932,6 @@ describe("remote connection jobs surface", () => {
     const { tableFromArrays, tableToIPC } = await import("apache-arrow");
     const eventsTable = tableFromArrays({ state: ["created", "succeeded"] });
     const eventsBody = Buffer.from(tableToIPC(eventsTable, "stream"));
-    const queryEventsPayloads: Record<string, unknown>[] = [];
 
     await withMockDatabase(
       (req, res) => {
@@ -968,16 +960,6 @@ describe("remote connection jobs surface", () => {
                 );
             }
           } else if (req.url === "/v1/jobs/describe") {
-            if (payload["job_id"] === "job-2") {
-              res
-                .writeHead(200, { "Content-Type": "application/json" })
-                .end(
-                  '{"job_id": "job-2", "job_type": "refresh_column", ' +
-                    '"job_state": "DONE", "creation_ms": 2000, ' +
-                    '"result": {"rows_assigned": 1000000}}',
-                );
-              return;
-            }
             if (payload["job_id"] !== "job-1") {
               res.writeHead(404).end("no such job");
               return;
@@ -999,7 +981,6 @@ describe("remote connection jobs surface", () => {
               .writeHead(200, { "Content-Type": "application/json" })
               .end('{"job_id": "job-1"}');
           } else if (req.url === "/v1/jobs/query_events") {
-            queryEventsPayloads.push(payload);
             res
               .writeHead(200, {
                 "Content-Type": "application/vnd.apache.arrow.stream",
@@ -1016,65 +997,22 @@ describe("remote connection jobs surface", () => {
         expect(jobs[0].state).toEqual("running");
         expect(jobs[1].state).toEqual("finished");
 
+        const description = await db.getJob("job-1");
+        expect(description?.state).toEqual("failed");
+        expect(JSON.parse(description?.specJson ?? "")).toEqual({
+          column: "vec",
+        });
+        expect(description?.failure?.message).toEqual("worker died");
+        expect(await db.getJob("missing")).toBeNull();
+
         expect(await db.cancelJob("job-1")).toBe(true);
         expect(await db.cancelJob("missing")).toBe(false);
 
-        // Opening a job hands back a populated handle; a missing one rejects.
-        await expect(db.openJob("missing")).rejects.toThrow("not found");
-        const finished = await db.openJob("job-2");
-        expect(finished.state).toEqual("finished");
-        expect(finished.result).toEqual({
-          // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
-          rows_assigned: 1000000,
-        });
+        const history = await db.jobHistory("job-1");
+        expect(history.numRows).toEqual(2);
 
-        const job = await db.openJob("job-1");
+        const job = db.job("job-1");
         expect(job.id).toEqual("job-1");
-
-        // openJob already populated the handle; refresh() re-reads it.
-        expect(job.state).toEqual("failed");
-        await job.refresh();
-        expect(job.state).toEqual("failed");
-        expect(job.jobType).toEqual("create_index");
-        expect(job.creationMs).toEqual(1000);
-        expect(job.spec).toEqual({ column: "vec" });
-        expect(job.result).toBeNull();
-        expect(job.failure?.message).toEqual("worker died");
-
-        // The handle reaches its own events, supplying its job id.
-        const jobEvents = await job.events({
-          limit: 500,
-          filter: "state = 'claim_complete'",
-        });
-        expect(jobEvents.numRows).toEqual(2);
-        expect(queryEventsPayloads.pop()).toEqual({
-          // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
-          job_id: "job-1",
-          limit: 500,
-          filter: "state = 'claim_complete'",
-        });
-
-        // Printing lays every known field out on its own line, with the JSON
-        // payloads indented rather than crammed onto one line.
-        expect(`${job}`).toEqual(
-          [
-            "Job(",
-            '    id="job-1",',
-            '    state="failed",',
-            '    jobType="create_index",',
-            "    creationMs=1000,",
-            "    spec={",
-            '        "column": "vec"',
-            "    },",
-            "    failure={",
-            '        "phase": "execute",',
-            '        "message": "worker died",',
-            '        "retryable": true',
-            "    },",
-            ")",
-          ].join("\n"),
-        );
-
         expect(await job.status()).toEqual("failed");
         await expect(job.wait()).rejects.toThrow("worker died");
       },

nodejs/__test__/table.test.ts

@@ -737,12 +737,11 @@ it("should query documents with LangChain PDF metadata", async () => {
 
 describe("merge insert", () => {
   let tmpDir: tmp.DirResult;
-  let conn: Connection;
   let table: Table;
 
   beforeEach(async () => {
     tmpDir = tmp.dirSync({ unsafeCleanup: true });
-    conn = await connect(tmpDir.name);
+    const conn = await connect(tmpDir.name);
 
     table = await conn.createTable("some_table", [
       { a: 1, b: "a" },
@@ -780,38 +779,6 @@ describe("merge insert", () => {
 
     expect(result.map((row) => ({ ...row }))).toEqual(expected);
   });
-  test("upsert on a composite key", async () => {
-    const composite = await conn.createTable("composite", [
-      { shard: "a", id: 1, val: "x" },
-      { shard: "a", id: 2, val: "y" },
-      { shard: "b", id: 1, val: "z" },
-    ]);
-
-    // ("a", 1) matches an existing row and updates it. ("b", 2) agrees with an
-    // existing row on each key column separately but on neither pair, so it is
-    // an insert.
-    const mergeInsertRes = await composite
-      .mergeInsert(["shard", "id"])
-      .whenMatchedUpdateAll()
-      .whenNotMatchedInsertAll()
-      .execute([
-        { shard: "a", id: 1, val: "X" },
-        { shard: "b", id: 2, val: "W" },
-      ]);
-    expect(mergeInsertRes.numUpdatedRows).toBe(1);
-    expect(mergeInsertRes.numInsertedRows).toBe(1);
-
-    const result = (await composite.toArrow())
-      .toArray()
-      .sort((a, b) => a.shard.localeCompare(b.shard) || a.id - b.id);
-
-    expect(result.map((row) => ({ ...row }))).toEqual([
-      { shard: "a", id: 1, val: "X" },
-      { shard: "a", id: 2, val: "y" },
-      { shard: "b", id: 1, val: "z" },
-      { shard: "b", id: 2, val: "W" },
-    ]);
-  });
   test("conditional update", async () => {
     const newData = [
       { a: 2, b: "x" },

nodejs/examples/package.json

@@ -8,8 +8,7 @@
     "//1": "--experimental-vm-modules is needed to run jest with sentence-transformers",
     "//2": "--testEnvironment is needed to run jest with sentence-transformers",
     "//3": "See: https://github.com/huggingface/transformers.js/issues/57",
-    "//4": "jest is invoked by its JS entry, not node_modules/.bin/jest: under pnpm that path is a shell shim, which `node` cannot execute",
-    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testEnvironment jest-environment-node-single-context --verbose",
+    "test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
     "lint": "biome check *.ts && biome format *.ts",
     "lint-ci": "biome ci .",
     "lint-fix": "biome check --write *.ts && pnpm format",

nodejs/lancedb/connection.ts

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
+import { tableFromIPC } from "apache-arrow";
 import {
   Data,
   SchemaLike,
@@ -15,7 +16,6 @@ import {
   makeEmptyTable,
 } from "./arrow";
 import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
-import { Job } from "./job";
 import {
   MaterializedView,
   MaterializedViewSelect,
@@ -27,6 +27,8 @@ import type {
   CreateNamespaceResponse,
   DescribeNamespaceResponse,
   DropNamespaceResponse,
+  Job,
+  JobDescription,
   JobInfo,
   ListNamespacesResponse,
   ListTablesResponse,
@@ -555,19 +557,24 @@ export abstract class Connection {
   ): Promise<void>;
 
   /**
-   * Open a server-side job by id, returning a handle with its record already
-   * populated. Rejects when the server has no such job, the way
-   * {@link Connection.openTable} does for a missing table.
+   * A {@link Job} handle for a server-side job by id.
    *
-   * The returned {@link Job} answers for its own state, specification,
-   * result, failure and event history, so there is no separate
-   * connection-level call for any of them.
+   * The handle is constructed without a server round trip; an unknown id
+   * surfaces when the handle is used. Dropping the handle has no effect on
+   * the job itself.
    */
-  abstract openJob(jobId: string): Promise<Job>;
+  abstract job(jobId: string): Job;
 
   /** List server-side jobs across the database's tables. */
   abstract listJobs(): Promise<JobInfo[]>;
 
+  /**
+   * Describe a single server-side job by id.
+   *
+   * Resolves to `null` when the server has no such job.
+   */
+  abstract getJob(jobId: string): Promise<JobDescription | null>;
+
   /**
    * Request cancellation of a server-side job by id.
    *
@@ -575,6 +582,13 @@ export abstract class Connection {
    * such job exists. Cancelling an already-terminal job is a no-op success.
    */
   abstract cancelJob(jobId: string): Promise<boolean>;
+
+  /**
+   * The lifecycle event history of a server-side job, as an Arrow table.
+   *
+   * Lists history across all jobs when `jobId` is omitted.
+   */
+  abstract jobHistory(jobId?: string): Promise<ArrowTable>;
 }
 
 /** @hideconstructor */
@@ -855,7 +869,7 @@ export class LocalConnection extends Connection {
   }
 
   async dropTableAsync(name: string, namespacePath?: string[]): Promise<Job> {
-    return new Job(await this.inner.dropTableAsync(name, namespacePath ?? []));
+    return this.inner.dropTableAsync(name, namespacePath ?? []);
   }
 
   async dropAllTables(namespacePath?: string[]): Promise<void> {
@@ -914,17 +928,29 @@ export class LocalConnection extends Connection {
     );
   }
 
-  async openJob(jobId: string): Promise<Job> {
-    return new Job(await this.inner.openJob(jobId));
+  job(jobId: string): Job {
+    return this.inner.job(jobId);
   }
 
   async listJobs(): Promise<JobInfo[]> {
     return this.inner.listJobs();
   }
 
+  async getJob(jobId: string): Promise<JobDescription | null> {
+    return this.inner.getJob(jobId);
+  }
+
   async cancelJob(jobId: string): Promise<boolean> {
     return this.inner.cancelJob(jobId);
   }
+
+  async jobHistory(jobId?: string): Promise<ArrowTable> {
+    const buf = await this.inner.jobHistory(jobId);
+    if (buf.length === 0) {
+      return new ArrowTable();
+    }
+    return tableFromIPC(buf);
+  }
 }
 
 /**

nodejs/lancedb/index.ts

@@ -94,9 +94,13 @@ export {
   RenameTableOptions,
 } from "./connection";
 
-export { JobFailureInfo, JobInfo, Session } from "./native.js";
-
-export { Job, JobEventsOptions } from "./job";
+export {
+  Job,
+  JobDescription,
+  JobFailureInfo,
+  JobInfo,
+  Session,
+} from "./native.js";
 
 export {
   AutoQuery,

nodejs/lancedb/job.ts

@@ -1,188 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-import { Table as ArrowTable, tableFromIPC } from "apache-arrow";
-import { JobFailureInfo, Job as NativeJob } from "./native";
-
-/** Which of a job's events {@link Job.events} returns. */
-export interface JobEventsOptions {
-  /** Maximum event rows to return, up to the server maximum of 10,000. */
-  limit?: number;
-  /** SQL-like filter over the event columns. */
-  filter?: string;
-}
-
-/**
- * A handle to an operation that may still be running.
- *
- * The operation may already be complete when the handle is created.
- *
- * The detail getters read what the handle last observed. Submitting an
- * operation returns only a job id, so populating them eagerly would cost an
- * extra round trip on every call:
- *
- * - {@link Job.refresh} and {@link Job.status} fetch the whole record.
- * - {@link Job.wait} records the terminal state it establishes, but not the
- *   rest of the record.
- * - Everything is null until one of those runs.
- *
- * @hideconstructor
- */
-export class Job {
-  private readonly inner: NativeJob;
-
-  constructor(inner: NativeJob) {
-    this.inner = inner;
-  }
-
-  /**
-   * Identifies the operation on the server that is running it.
-   *
-   * Operations that run in this process have no server id. The value is
-   * opaque: parsing it or storing it to resume the job later is not supported.
-   */
-  get id(): string | null {
-    return this.inner.id ?? null;
-  }
-
-  /** The last observed lifecycle state, without contacting the backend. */
-  get state(): string | null {
-    return this.inner.state ?? null;
-  }
-
-  /**
-   * The job's type, as the server names it. Null for an in-process job, which
-   * has no server-side record.
-   */
-  get jobType(): string | null {
-    return this.inner.jobType ?? null;
-  }
-
-  /** When the job was created, in milliseconds since the epoch. */
-  get creationMs(): number | null {
-    return this.inner.creationMs ?? null;
-  }
-
-  /** The job-type-specific specification it was submitted with. */
-  // biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
-  get spec(): any | null {
-    return parseJson(this.inner.specJson);
-  }
-
-  /**
-   * The job-type-specific terminal result. Null until the job succeeds, so a
-   * job that never terminates reports its progress through {@link Job.events}
-   * instead.
-   */
-  // biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
-  get result(): any | null {
-    return parseJson(this.inner.resultJson);
-  }
-
-  /** Why the job failed, when it failed and the server reports a reason. */
-  get failure(): JobFailureInfo | null {
-    return this.inner.failure ?? null;
-  }
-
-  /**
-   * The operation's current lifecycle state: "running", "finished", "failed",
-   * or "cancelled".
-   *
-   * A point snapshot; unlike {@link Job.wait} it does not block or reject on a
-   * terminal failure state. Also refreshes the getters above.
-   */
-  async status(): Promise<string> {
-    return this.inner.status();
-  }
-
-  /** Wait until the operation reaches a terminal state. */
-  async wait(): Promise<void> {
-    return this.inner.wait();
-  }
-
-  /** Request cancellation. Cancelling a finished operation is a no-op. */
-  async cancel(): Promise<void> {
-    return this.inner.cancel();
-  }
-
-  /**
-   * Ask the backend for this job's current state, and for a server-side job
-   * its full record, then cache it for the getters above.
-   */
-  async refresh(): Promise<void> {
-    return this.inner.refresh();
-  }
-
-  /**
-   * This job's recorded lifecycle events.
-   *
-   * Where the getters above report a terminal result only once the job reaches
-   * one, events are written as the job runs and outlive the workers that
-   * produced them. A distributed job records a `claim`/`claim_complete` pair
-   * per unit of work, each carrying `rows_processed`, so a job that never
-   * finishes still accounts for what it did.
-   *
-   * The server caps results at 1000 rows by default and 10,000 at most, and
-   * truncates without saying so, so pass `limit` for a job that emits an event
-   * per fragment. `filter` is a SQL-like expression over the `state`,
-   * `updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
-   */
-  async events(options?: JobEventsOptions): Promise<ArrowTable> {
-    const buf = await this.inner.events(options?.limit, options?.filter);
-    if (buf.length === 0) {
-      return new ArrowTable();
-    }
-    return tableFromIPC(buf);
-  }
-
-  /**
-   * Every field the handle currently knows, one per line, with the JSON
-   * payloads indented -- a refresh job's spec and result are the point of
-   * printing it.
-   */
-  toString(): string {
-    if (this.state === null) {
-      const known = this.id === null ? "" : `id=${JSON.stringify(this.id)}, `;
-      return `Job(${known}not refreshed)`;
-    }
-    const fields: string[] = [];
-    if (this.id !== null) {
-      fields.push(`id=${JSON.stringify(this.id)}`);
-    }
-    fields.push(`state=${JSON.stringify(this.state)}`);
-    if (this.jobType !== null) {
-      fields.push(`jobType=${JSON.stringify(this.jobType)}`);
-    }
-    if (this.creationMs !== null) {
-      fields.push(`creationMs=${this.creationMs}`);
-    }
-    for (const [name, value] of [
-      ["spec", this.spec],
-      ["result", this.result],
-    ] as const) {
-      if (value !== null) {
-        fields.push(`${name}=${indentJson(value)}`);
-      }
-    }
-    if (this.failure !== null) {
-      fields.push(`failure=${indentJson(this.failure)}`);
-    }
-    return `Job(${fields.map((field) => `\n${REPR_INDENT}${field},`).join("")}\n)`;
-  }
-
-  [Symbol.for("nodejs.util.inspect.custom")](): string {
-    return this.toString();
-  }
-}
-
-const REPR_INDENT = "    ";
-
-// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
-function indentJson(value: any): string {
-  return JSON.stringify(value, null, 4).replace(/\n/g, `\n${REPR_INDENT}`);
-}
-
-// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
-function parseJson(raw: string | null | undefined): any | null {
-  return raw === null || raw === undefined ? null : JSON.parse(raw);
-}

nodejs/lancedb/materialized_view.ts

@@ -19,8 +19,6 @@ export interface MaterializedViewDefinition {
   limit?: number;
   /** Source columns the projections and filter read. */
   inputs: string[];
-  /** Namespace holding the source table; empty is the root namespace. */
-  sourceNamespace: string[];
 }
 
 /**
@@ -80,8 +78,7 @@ export function definitionFromMetadata(
   }
   // biome-ignore lint/suspicious/noExplicitAny: raw JSON
   const value: any = JSON.parse(raw);
-  // "namespaced_select" keeps older readers from resolving the source at root.
-  if (value.kind !== "select" && value.kind !== "namespaced_select") {
+  if (value.kind !== "select") {
     throw new Error(
       `materialized view '${name}' is defined by '${value.kind}', which this ` +
         "version of lancedb cannot refresh",
@@ -106,7 +103,6 @@ export function definitionFromMetadata(
     filter: value.filter ?? undefined,
     limit,
     inputs: value.inputs ?? [],
-    sourceNamespace: value.source_namespace ?? [],
   };
 }
 

nodejs/lancedb/table.ts

@@ -19,7 +19,6 @@ import {
 
 import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
 import { IndexOptions } from "./indices";
-import { Job } from "./job";
 import { MergeInsertBuilder } from "./merge";
 import {
   AddColumnsResult,
@@ -31,6 +30,7 @@ import {
   DropColumnsResult,
   IndexConfig,
   IndexStatistics,
+  Job,
   LsmStats,
   Branches as NativeBranches,
   OptimizeStats,
@@ -919,16 +919,6 @@ export abstract class Table {
   /** Return the table as an arrow table */
   abstract toArrow(): Promise<ArrowTable>;
 
-  /**
-   * Create a {@link MergeInsertBuilder}, which combines new data with the
-   * existing table in a single transaction — inserting, updating and deleting
-   * rows depending on how they match.
-   *
-   * @param on - The column, or columns, to match source rows against target
-   * rows on. Typically a key or id column. Several columns match on the
-   * composite key: a source row updates a target row only when it agrees on
-   * every one of them.
-   */
   abstract mergeInsert(on: string | string[]): MergeInsertBuilder;
 
   /** List all the stats of a specified index
@@ -1124,15 +1114,13 @@ export class LocalTable extends Table {
   ): Promise<Job> {
     // biome-ignore lint/suspicious/noExplicitAny: skip
     const nativeIndex = (options?.config as any)?.inner;
-    return new Job(
-      await this.inner.createIndexAsync(
-        nativeIndex,
-        column,
-        options?.replace,
-        options?.waitTimeoutSeconds,
-        options?.name,
-        options?.train,
-      ),
+    return await this.inner.createIndexAsync(
+      nativeIndex,
+      column,
+      options?.replace,
+      options?.waitTimeoutSeconds,
+      options?.name,
+      options?.train,
     );
   }
 
@@ -1315,7 +1303,7 @@ export class LocalTable extends Table {
   }
 
   async refreshColumnAsync(column: string): Promise<Job> {
-    return new Job(await this.inner.refreshColumnAsync(column));
+    return await this.inner.refreshColumnAsync(column);
   }
 
   async refreshMaterializedView(

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.39.0-beta.5",
+	"version": "0.38.0-beta.11",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.39.0-beta.5",
+	"version": "0.38.0-beta.11",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.39.0-beta.5",
+	"version": "0.38.0-beta.11",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.39.0-beta.5",
+	"version": "0.38.0-beta.11",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.39.0-beta.5",
+	"version": "0.38.0-beta.11",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.39.0-beta.5",
+  "version": "0.38.0-beta.11",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.39.0-beta.5",
+	"version": "0.38.0-beta.11",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.39.0-beta.5",
+  "version": "0.38.0-beta.11",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -44,7 +44,7 @@
     "@biomejs/biome": "^1.7.3",
     "@jest/globals": "^29.7.0",
     "@napi-rs/cli": "3.7.0",
-    "@opentelemetry/sdk-metrics": "^2.10.0",
+    "@opentelemetry/sdk-metrics": "^1.30.0",
     "@types/axios": "^0.14.0",
     "@types/jest": "^29.1.2",
     "@types/node": "22.7.4",
@@ -56,7 +56,7 @@
     "eslint": "^8.57.0",
     "jest": "^29.7.0",
     "shx": "^0.3.4",
-    "tmp": "^0.2.7",
+    "tmp": "^0.2.3",
     "ts-jest": "^29.1.2",
     "typedoc": "0.26.4",
     "typedoc-plugin-markdown": "4.2.1",
@@ -67,7 +67,7 @@
     "timeout": "3m"
   },
   "engines": {
-    "node": ">= 22"
+    "node": ">= 18"
   },
   "packageManager": "pnpm@11.1.1",
   "cpu": ["x64", "arm64"],
@@ -101,7 +101,7 @@
     "openai": "4.29.2"
   },
   "peerDependencies": {
-    "@types/node": ">=22",
+    "@types/node": ">=18",
     "apache-arrow": ">=15.0.0 <=18.1.0"
   },
   "peerDependenciesMeta": {

nodejs/pnpm-workspace.yaml

@@ -16,41 +16,3 @@ allowBuilds:
   onnxruntime-node: true
   protobufjs: true
   sharp: true
-
-minimumReleaseAgeExclude:
-  - protobufjs@7.5.8
-  - tmp@0.2.6
-  - form-data@4.0.6
-  - tar@7.5.16
-  - markdown-it@14.1.2
-  - linkify-it@5.0.1
-  - js-yaml@3.15.0
-  - js-yaml@4.1.2
-  - protobufjs@7.6.1
-  - protobufjs@7.6.3
-  - '@babel/core@7.29.1'
-  - axios@1.18.0
-  - brace-expansion@2.1.2
-  - brace-expansion@1.1.16
-  - js-yaml@4.3.0
-  - tar@7.5.18
-  - tar@7.5.19
-  - tar@7.5.17
-  - protobufjs@7.6.5
-  - linkify-it@5.0.2
-  - sharp@0.35.0
-  - brace-expansion@1.1.17
-  - brace-expansion@2.1.3
-  - brace-expansion@2.1.4
-  - brace-expansion@1.1.18
-  - js-yaml@3.15.1
-  - js-yaml@4.3.1
-  - tar@7.5.21
-  - '@opentelemetry/core@2.8.0'
-
-# @huggingface/transformers pins sharp ^0.33.5 and no released version has moved
-# past ^0.34.5, all of which inherit the libvips CVEs in GHSA-f88m-g3jw-g9cj.
-# Force the patched line. sharp is only reached by transformers' image pipeline,
-# which LanceDB's text embedding function never uses.
-overrides:
-  sharp: ^0.35.4

nodejs/src/connection.rs

@@ -442,15 +442,13 @@ impl Connection {
         self.get_inner()?.drop_all_tables(&ns).await.default_error()
     }
 
-    /// Open a server-side job by id, returning a handle with its record
-    /// already populated. Rejects when the server has no such job.
+    /// A `Job` handle for a server-side job by id.
     ///
-    /// The returned handle answers for its own state, specification, result,
-    /// failure and event history, so there is no separate connection-level
-    /// call for any of them.
-    #[napi(catch_unwind)]
-    pub async fn open_job(&self, job_id: String) -> napi::Result<crate::job::Job> {
-        let job = self.get_inner()?.open_job(&job_id).await.default_error()?;
+    /// The handle is constructed without a server round trip; an unknown id
+    /// surfaces when the handle is used.
+    #[napi]
+    pub fn job(&self, job_id: String) -> napi::Result<crate::job::Job> {
+        let job = self.get_inner()?.job(job_id).default_error()?;
         Ok(crate::job::Job::new(job))
     }
 
@@ -461,6 +459,17 @@ impl Connection {
         Ok(jobs.into_iter().map(Into::into).collect())
     }
 
+    /// Describe a single server-side job by id. `null` when the server has
+    /// no such job.
+    #[napi(catch_unwind)]
+    pub async fn get_job(
+        &self,
+        job_id: String,
+    ) -> napi::Result<Option<crate::job::JobDescription>> {
+        let description = self.get_inner()?.get_job(&job_id).await.default_error()?;
+        Ok(description.map(Into::into))
+    }
+
     /// Request cancellation of a server-side job by id. Returns true if the
     /// server accepted the cancellation, false if no such job exists.
     #[napi(catch_unwind)]
@@ -468,6 +477,34 @@ impl Connection {
         self.get_inner()?.cancel_job(&job_id).await.default_error()
     }
 
+    /// The lifecycle event history of a server-side job (all jobs when
+    /// `job_id` is null), as an Arrow IPC stream buffer. Empty when there is
+    /// no history.
+    #[napi(catch_unwind)]
+    pub async fn job_history(&self, job_id: Option<String>) -> napi::Result<Buffer> {
+        let batches = self
+            .get_inner()?
+            .job_history(job_id.as_deref())
+            .await
+            .default_error()?;
+        let Some(first) = batches.first() else {
+            return Ok(Buffer::from(Vec::<u8>::new()));
+        };
+        let mut out = Vec::new();
+        let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut out, &first.schema())
+            .map_err(|e| napi::Error::from_reason(e.to_string()))?;
+        for batch in &batches {
+            writer
+                .write(batch)
+                .map_err(|e| napi::Error::from_reason(e.to_string()))?;
+        }
+        writer
+            .finish()
+            .map_err(|e| napi::Error::from_reason(e.to_string()))?;
+        drop(writer);
+        Ok(Buffer::from(out))
+    }
+
     #[napi(catch_unwind)]
     /// Describe a namespace and return its properties.
     pub async fn describe_namespace(

nodejs/src/job.rs

@@ -3,9 +3,6 @@
 
 use std::sync::Arc;
 
-use arrow_array::RecordBatch;
-use lancedb::job::JobEventsRequest;
-use napi::bindgen_prelude::Buffer;
 use napi_derive::napi;
 
 use crate::error::NapiErrorExt;
@@ -58,98 +55,12 @@ impl Job {
     pub async fn cancel(&self) -> napi::Result<()> {
         self.inner.cancel().await.default_error()
     }
-
-    /// Ask the backend for this job's current state, and for a server-side job
-    /// its full record, then cache it for the getters below.
-    ///
-    /// They are all null until this runs, because submitting an operation
-    /// returns only a job id. {@link Job.status} fetches the whole record too;
-    /// {@link Job.wait} records only the terminal state it establishes.
-    #[napi(catch_unwind)]
-    pub async fn refresh(&self) -> napi::Result<()> {
-        self.inner.refresh().await.default_error()
-    }
-
-    /// The last observed lifecycle state, without contacting the backend.
-    #[napi(getter)]
-    pub fn state(&self) -> Option<String> {
-        self.inner.state()
-    }
-
-    /// The job's type, as the server names it. Null for an in-process job,
-    /// which has no server-side record.
-    #[napi(getter)]
-    pub fn job_type(&self) -> Option<String> {
-        self.inner.job_type()
-    }
-
-    /// When the job was created, in milliseconds since the epoch.
-    #[napi(getter)]
-    pub fn creation_ms(&self) -> Option<i64> {
-        self.inner.creation_ms()
-    }
-
-    /// The job-type-specific specification as a JSON string, when present.
-    #[napi(getter)]
-    pub fn spec_json(&self) -> Option<String> {
-        self.inner.spec().map(|spec| spec.to_string())
-    }
-
-    /// The job-type-specific terminal result as a JSON string. Null until the
-    /// job succeeds, so a job that never terminates reports its progress
-    /// through {@link Job.events} instead.
-    #[napi(getter)]
-    pub fn result_json(&self) -> Option<String> {
-        self.inner.result().map(|result| result.to_string())
-    }
-
-    /// Why the job failed, when it failed and the server reports a reason.
-    #[napi(getter)]
-    pub fn failure(&self) -> Option<JobFailureInfo> {
-        self.inner.failure().map(|failure| JobFailureInfo {
-            phase: failure.phase,
-            message: failure.message,
-            retryable: failure.retryable,
-        })
-    }
-
-    /// This job's recorded lifecycle events, as an Arrow IPC stream buffer.
-    /// The TypeScript wrapper turns it into an Arrow table.
-    #[napi(catch_unwind)]
-    pub async fn events(&self, limit: Option<u32>, filter: Option<String>) -> napi::Result<Buffer> {
-        let batches = self
-            .inner
-            .events(JobEventsRequest { limit, filter })
-            .await
-            .default_error()?;
-        batches_to_ipc_buffer(&batches)
-    }
-}
-
-/// Serialise Arrow batches as a single IPC stream for the TypeScript layer.
-fn batches_to_ipc_buffer(batches: &[RecordBatch]) -> napi::Result<Buffer> {
-    let Some(first) = batches.first() else {
-        return Ok(Buffer::from(Vec::<u8>::new()));
-    };
-    let mut out = Vec::new();
-    let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut out, &first.schema())
-        .map_err(|e| napi::Error::from_reason(e.to_string()))?;
-    for batch in batches {
-        writer
-            .write(batch)
-            .map_err(|e| napi::Error::from_reason(e.to_string()))?;
-    }
-    writer
-        .finish()
-        .map_err(|e| napi::Error::from_reason(e.to_string()))?;
-    drop(writer);
-    Ok(Buffer::from(out))
 }
 
 /// A row from `Connection.listJobs`: one server-side job.
 #[napi(object)]
 pub struct JobInfo {
-    /// The job id -- what `Connection.openJob` and `Connection.cancelJob`
+    /// The job id -- what `Connection.getJob` and `Connection.cancelJob`
     /// accept.
     pub job_id: String,
     /// The table the job runs against, without URI or namespace.
@@ -180,3 +91,36 @@ pub struct JobFailureInfo {
     pub message: Option<String>,
     pub retryable: Option<bool>,
 }
+
+/// A described job from `Connection.getJob`.
+#[napi(object)]
+pub struct JobDescription {
+    pub job_id: String,
+    pub job_type: String,
+    /// Lifecycle state: "running", "finished", "failed", or "cancelled".
+    pub state: String,
+    /// When the job was created, in milliseconds since the epoch.
+    pub creation_ms: i64,
+    /// The job-type-specific specification as a JSON string, when present.
+    pub spec_json: Option<String>,
+    /// Why the job failed, when the job is failed and the server reports a
+    /// reason.
+    pub failure: Option<JobFailureInfo>,
+}
+
+impl From<lancedb::database::JobDescription> for JobDescription {
+    fn from(description: lancedb::database::JobDescription) -> Self {
+        Self {
+            job_id: description.job_id,
+            job_type: description.job_type,
+            state: description.state,
+            creation_ms: description.creation_ms,
+            spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
+            failure: description.failure.map(|failure| JobFailureInfo {
+                phase: failure.phase,
+                message: failure.message,
+                retryable: failure.retryable,
+            }),
+        }
+    }
+}

nodejs/src/query.rs

@@ -664,11 +664,7 @@ impl JsFullTextQuery {
 }
 
 fn parse_fts_query(query: Object) -> napi::Result<FullTextSearchQuery> {
-    // `&JsFullTextQuery` recovers a native class reference through napi's borrow-tracked
-    // path, which is only usable from generated `#[napi]` argument conversion. This is a
-    // manual lookup on a nested `Object` property instead, so use `ClassInstance`, which
-    // unwraps the class without requiring a borrow scope.
-    if let Ok(Some(query)) = query.get::<ClassInstance<JsFullTextQuery>>("query") {
+    if let Ok(Some(query)) = query.get::<&JsFullTextQuery>("query") {
         Ok(FullTextSearchQuery::new_query(query.inner.clone()))
     } else if let Ok(Some(query_text)) = query.get::<String>("query") {
         let mut query_text = query_text;

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.39.0-beta.5"
+version = "0.38.0-beta.11"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"
@@ -28,7 +28,7 @@ env_logger.workspace = true
 log.workspace = true
 # Maturin enables extension-module mode for Python builds. Keeping it out of
 # Cargo features lets Rust unit tests link against libpython.
-pyo3 = { version = "0.28", features = ["abi3-py310", "chrono", "uuid"] }
+pyo3 = { version = "0.28", features = ["abi3-py310", "chrono"] }
 chrono.workspace = true
 pyo3-async-runtimes = { version = "0.28", features = [
     "attributes",
@@ -40,7 +40,6 @@ serde.workspace = true
 serde_json.workspace = true
 snafu.workspace = true
 tokio.workspace = true
-uuid.workspace = true
 libc = "0.2"
 
 [build-dependencies]

python/pyproject.toml

@@ -139,7 +139,6 @@ include = [
     "python/lancedb/exceptions.py",
     "python/lancedb/background_loop.py",
     "python/lancedb/schema.py",
-    "python/lancedb/sql.py",
     "python/lancedb/remote/__init__.py",
     "python/lancedb/remote/errors.py",
     "python/lancedb/embeddings/__init__.py",

python/python/lancedb/__init__.py

@@ -22,11 +22,7 @@ from .remote.db import RemoteDBConnection
 from .expr import Expr, col, lit, func
 from .schema import blob, vector
 from .job import AsyncJob, Job
-from .sql import AsyncQuery as AsyncSqlQuery
-from .sql import Query as SqlQuery
-from .sql import QueryDescription
 from .functions import (
-    AssignmentMapping as AssignmentMapping,
     FunctionArtifactRequest as FunctionArtifactRequest,
     FunctionApplication as FunctionApplication,
     FunctionBinding as FunctionBinding,
@@ -105,7 +101,6 @@ def connect(
     api_key: Optional[str] = None,
     region: str = "us-east-1",
     host_override: Optional[str] = None,
-    sql_host_override: Optional[str] = None,
     read_consistency_interval: Optional[timedelta] = None,
     request_thread_pool: Optional[Union[int, ThreadPoolExecutor]] = None,
     client_config: Union[ClientConfig, Dict[str, Any], None] = None,
@@ -134,9 +129,6 @@ def connect(
         The region to use for LanceDB Cloud.
     host_override: str, optional
         The override url for LanceDB Cloud.
-    sql_host_override: str, optional
-        The remote SQL service endpoint override. The client connects lazily when SQL
-        is first executed and retains that connection.
     read_consistency_interval: timedelta, default None
         The interval at which to check for updates to the table from other
         processes. If None, then consistency is not checked. For performance
@@ -278,7 +270,6 @@ def connect(
             api_key,
             region,
             host_override,
-            sql_host_override=sql_host_override,
             # TODO: remove this (deprecation warning downstream)
             request_thread_pool=request_thread_pool,
             client_config=client_config,
@@ -421,7 +412,6 @@ def deserialize_conn(
             parsed["api_key"],
             parsed.get("region", "us-east-1"),
             host_override=parsed.get("host_override"),
-            sql_host_override=parsed.get("sql_host_override"),
             client_config=parsed.get("client_config"),
             storage_options=storage_options,
         )
@@ -435,7 +425,6 @@ async def connect_async(
     api_key: Optional[str] = None,
     region: str = "us-east-1",
     host_override: Optional[str] = None,
-    sql_host_override: Optional[str] = None,
     read_consistency_interval: Optional[timedelta] = None,
     client_config: Optional[Union[ClientConfig, Dict[str, Any]]] = None,
     storage_options: Optional[Dict[str, str]] = None,
@@ -458,9 +447,6 @@ async def connect_async(
         The region to use for LanceDB Cloud.
     host_override: str, optional
         The override url for LanceDB Cloud.
-    sql_host_override: str, optional
-        The remote SQL service endpoint override. The client connects lazily when SQL
-        is first executed and retains that connection.
     read_consistency_interval: timedelta, default None
         The interval at which to check for updates to the table from other
         processes. If None, then consistency is not checked. For performance
@@ -548,7 +534,6 @@ async def connect_async(
             api_key,
             region,
             host_override,
-            sql_host_override,
             read_consistency_interval_secs,
             client_config,
             storage_options,
@@ -571,7 +556,6 @@ __all__ = [
     "connect_namespace_async",
     "AsyncConnection",
     "AsyncJob",
-    "AsyncSqlQuery",
     "AsyncLanceNamespaceDBConnection",
     "AsyncTable",
     "FtsToken",
@@ -586,8 +570,6 @@ __all__ = [
     "vector",
     "DBConnection",
     "Job",
-    "QueryDescription",
-    "SqlQuery",
     "LanceDBConnection",
     "LanceNamespaceDBConnection",
     "LsmWriteSpec",

python/python/lancedb/_lancedb.pyi

@@ -1,7 +1,6 @@
 from datetime import date, datetime, timedelta
 from decimal import Decimal
 from typing import Dict, List, Optional, Tuple, Any, TypedDict, Union, Literal
-from uuid import UUID
 
 import pyarrow as pa
 
@@ -148,20 +147,15 @@ class Connection(object):
         start_after: Optional[str],
         limit: Optional[int],
     ) -> list[str]: ...  # Deprecated: Use list_tables instead
-    async def open_job(self, job_id: str) -> Job: ...
+    def job(self, job_id: str) -> Job: ...
     async def create_function_async(self, request_json: str) -> Job: ...
     async def get_function(self, name: str, version: str) -> str: ...
-    async def list_functions(self) -> List[str]: ...
-    async def drop_function(self, name: str, version: str) -> bool: ...
     async def list_jobs(self) -> List[JobInfo]: ...
+    async def get_job(self, job_id: str) -> Optional[JobDescription]: ...
     async def cancel_job(self, job_id: str) -> bool: ...
-    async def execute_query_async(
-        self,
-        query: str,
-        *,
-        default_namespace_path: Optional[List[str]] = None,
-    ) -> SqlQuery: ...
-    async def describe_query(self, query_id: UUID) -> QueryDescription: ...
+    async def job_history(
+        self, job_id: Optional[str] = None
+    ) -> List[pa.RecordBatch]: ...
     async def create_table(
         self,
         name: str,
@@ -240,20 +234,9 @@ class BlobFile:
 class Job:
     @property
     def id(self) -> Optional[str]: ...
-    @property
-    def _state(self) -> Optional[str]: ...
-    @property
-    def _description(self) -> Optional[JobDescription]: ...
     async def status(self) -> str: ...
     async def wait(self) -> Optional[str]: ...
     async def cancel(self) -> None: ...
-    async def refresh(self) -> None: ...
-    async def events(
-        self,
-        *,
-        limit: Optional[int] = None,
-        filter: Optional[str] = None,
-    ) -> pa.Table: ...
 
 class JobInfo:
     @property
@@ -285,33 +268,10 @@ class JobDescription:
     @property
     def creation_ms(self) -> int: ...
     @property
-    def _spec_json(self) -> Optional[str]: ...
-    @property
-    def _result_json(self) -> Optional[str]: ...
-    @property
-    def spec(self) -> Optional[Any]: ...
-    @property
-    def result(self) -> Optional[Any]: ...
+    def spec_json(self) -> Optional[str]: ...
     @property
     def failure(self) -> Optional[JobFailureInfo]: ...
 
-class SqlQuery:
-    @property
-    def id(self) -> UUID: ...
-    async def describe(self) -> QueryDescription: ...
-    async def reader(self) -> RecordBatchStream: ...
-    async def cancel(self) -> None: ...
-
-class QueryDescription:
-    @property
-    def id(self) -> UUID: ...
-    @property
-    def status(self) -> str: ...
-    @property
-    def progress(self) -> Optional[float]: ...
-    @property
-    def expires_at(self) -> Optional[datetime]: ...
-
 class Table:
     def name(self) -> str: ...
     def __repr__(self) -> str: ...
@@ -490,7 +450,6 @@ async def connect(
     api_key: Optional[str],
     region: Optional[str],
     host_override: Optional[str],
-    sql_host_override: Optional[str],
     read_consistency_interval: Optional[float],
     client_config: Optional[Union[ClientConfig, Dict[str, Any]]],
     storage_options: Optional[Dict[str, str]],
@@ -647,7 +606,6 @@ class FullTextQuery:
 class PyQueryRequest:
     limit: Optional[int]
     offset: Optional[int]
-    take_offsets: Optional[List[int]]
     filter: Optional[Union[str, bytes]]
     full_text_search: Optional[FullTextQuery]
     select: Optional[Union[str, List[str]]]

python/python/lancedb/db.py

@@ -16,10 +16,10 @@ from typing import (
     Iterable,
     List,
     Literal,
+    Mapping,
     Optional,
     Union,
 )
-from uuid import UUID
 
 if sys.version_info >= (3, 12):
     from typing import override
@@ -48,9 +48,6 @@ from . import __version__
 from ._lancedb import connect as lancedb_connect  # type: ignore
 from .functions import FunctionVersion, UdfDefinition
 from .job import AsyncJob, Job, _typed_job
-from .sql import AsyncQuery as AsyncSqlQuery
-from .sql import Query as SqlQuery
-from .sql import QueryDescription
 from .materialized_view import (
     AsyncMaterializedView,
     MaterializedView,
@@ -72,11 +69,10 @@ import deprecation
 
 if TYPE_CHECKING:
     import pyarrow as pa
-    from .arrow import AsyncRecordBatchReader
     from .pydantic import LanceModel
 
     from ._lancedb import Connection as LanceDbConnection
-    from ._lancedb import JobInfo
+    from ._lancedb import JobDescription, JobInfo
     from .common import DATA, URI
     from .embeddings import EmbeddingFunctionConfig
     from ._lancedb import Session
@@ -692,17 +688,35 @@ class DBConnection(EnforceOverrides):
         """
         raise NotImplementedError("serialize is not supported for this connection type")
 
-    def create_function(self, definition: UdfDefinition) -> FunctionVersion:
+    def create_function(
+        self,
+        definition: UdfDefinition,
+        *,
+        secrets: Optional[Mapping[str, str]] = None,
+    ) -> FunctionVersion:
         """Register a scalar Python UDF and wait for its immutable version.
 
+        ``secrets`` must contain exactly the names declared by
+        ``@udf(secrets=[...])``. Values are sent in the create request and
+        stored server-side in the private execution artifact; returned
+        Function and Job metadata contain only the declared names.
         This is the blocking counterpart of :meth:`create_function_async`.
         Local connections raise ``NotImplementedError``.
         """
-        return self.create_function_async(definition).wait()
+        return self.create_function_async(definition, secrets=secrets).wait()
 
-    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
+    def create_function_async(
+        self,
+        definition: UdfDefinition,
+        *,
+        secrets: Optional[Mapping[str, str]] = None,
+    ) -> Job[FunctionVersion]:
         """Register a scalar Python UDF through the remote Function catalog.
 
+        ``secrets`` must contain exactly the names declared by
+        ``@udf(secrets=[...])``. Values are sent in the create request and
+        stored server-side in the private execution artifact; returned
+        Function and Job metadata contain only the declared names.
         Submission returns a typed job. The immutable Function version becomes
         available only when :meth:`Job.wait` succeeds. Local connections raise
         ``NotImplementedError``.
@@ -717,51 +731,26 @@ class DBConnection(EnforceOverrides):
             "Function catalog operations are not supported for this connection type"
         )
 
-    def list_functions(self) -> List[FunctionVersion]:
-        """List every published immutable Function version.
+    def job(self, job_id: str) -> Job:
+        """A [Job][lancedb.job.Job] handle for a server-side job by id.
 
-        Results are ordered by Function name then version. Local connections
-        raise ``NotImplementedError``.
-
-        Examples
-        --------
-        List the identities available to use in Function-backed columns:
-
-        ```python
-        [(function.name, function.version) for function in db.list_functions()]
-        ```
+        The handle is constructed without a server round trip; an unknown id
+        surfaces when the handle is used. Dropping the handle has no effect
+        on the job itself.
         """
-        raise NotImplementedError(
-            "Function catalog operations are not supported for this connection type"
-        )
-
-    def drop_function(self, name: str, *, version: str) -> bool:
-        """Drop one exact immutable Function version from the remote catalog.
-
-        Returns True when the version changed to Dropped and False for an
-        idempotent replay. Local connections raise NotImplementedError.
-        """
-        raise NotImplementedError(
-            "Function catalog operations are not supported for this connection type"
-        )
-
-    def open_job(self, job_id: str) -> Job:
-        """Open a server-side job by id, returning a handle with its record
-        already populated.
-
-        The returned [Job][lancedb.job.Job] answers for its own state,
-        specification, result, failure and event history, so there is no
-        separate connection-level call for any of them.
-
-        Raises `JobNotFoundError` when the server has no such job, the way
-        `open_table` does for a missing table.
-        """
-        raise NotImplementedError("open_job is not supported for this connection type")
+        raise NotImplementedError("job is not supported for this connection type")
 
     def list_jobs(self) -> List[JobInfo]:
         """List server-side jobs across the database's tables."""
         raise NotImplementedError("list_jobs is not supported for this connection type")
 
+    def get_job(self, job_id: str) -> Optional[JobDescription]:
+        """Describe a single server-side job by id.
+
+        Returns None when the server has no such job.
+        """
+        raise NotImplementedError("get_job is not supported for this connection type")
+
     def cancel_job(self, job_id: str) -> bool:
         """Request cancellation of a server-side job by id.
 
@@ -773,38 +762,14 @@ class DBConnection(EnforceOverrides):
             "cancel_job is not supported for this connection type"
         )
 
-    def execute_query(
-        self,
-        query: str,
-        *,
-        default_namespace_path: Optional[List[str]] = None,
-    ) -> pa.RecordBatchReader:
-        """Execute SQL and return a blocking Arrow reader.
+    def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
+        """The lifecycle event history of a server-side job, as Arrow batches.
 
-        This submits through :meth:`execute_query_async` and waits until the
-        initial result stream is readable. It does not wait for the full query
-        to finish.
+        Lists history across all jobs when `job_id` is None.
         """
-        return self.execute_query_async(
-            query,
-            default_namespace_path=default_namespace_path,
-        ).reader()
-
-    def execute_query_async(
-        self,
-        query: str,
-        *,
-        default_namespace_path: Optional[List[str]] = None,
-    ) -> SqlQuery:
-        """Start executing SQL and return its query handle.
-
-        Local connections do not support SQL.
-        """
-        raise NotImplementedError("SQL is not supported for this connection type")
-
-    def describe_query(self, query_id: UUID) -> QueryDescription:
-        """Describe a submitted SQL query by its connection-scoped id."""
-        raise NotImplementedError("SQL is not supported for this connection type")
+        raise NotImplementedError(
+            "job_history is not supported for this connection type"
+        )
 
 
 class LanceDBConnection(DBConnection):
@@ -901,7 +866,6 @@ class LanceDBConnection(DBConnection):
                 None,
                 None,
                 None,
-                None,
                 read_consistency_interval_secs,
                 None,
                 storage_options,
@@ -1450,34 +1414,42 @@ class LanceDBConnection(DBConnection):
         )
 
     @override
-    def open_job(self, job_id: str) -> Job:
-        """Open a server-side job by id. See
-        [DBConnection.open_job][lancedb.db.DBConnection.open_job].
+    def job(self, job_id: str) -> Job:
+        """A [Job][lancedb.job.Job] handle for a server-side job by id.
+
+        The handle is constructed without a server round trip; an unknown id
+        surfaces when the handle is used. Dropping the handle has no effect
+        on the job itself.
         """
-        return Job(LOOP.run(self._conn.open_job(job_id)))
+        return Job(self._conn.job(job_id))
 
     @override
-    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
-        job = LOOP.run(self._conn.create_function_async(definition))
+    def create_function_async(
+        self,
+        definition: UdfDefinition,
+        *,
+        secrets: Optional[Mapping[str, str]] = None,
+    ) -> Job[FunctionVersion]:
+        job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets))
         return Job(job)
 
     @override
     def get_function(self, name: str, *, version: str) -> FunctionVersion:
         return LOOP.run(self._conn.get_function(name, version=version))
 
-    @override
-    def list_functions(self) -> List[FunctionVersion]:
-        return LOOP.run(self._conn.list_functions())
-
-    @override
-    def drop_function(self, name: str, *, version: str) -> bool:
-        return LOOP.run(self._conn.drop_function(name, version=version))
-
     @override
     def list_jobs(self) -> List[JobInfo]:
         """List server-side jobs across the database's tables."""
         return LOOP.run(self._conn.list_jobs())
 
+    @override
+    def get_job(self, job_id: str) -> Optional[JobDescription]:
+        """Describe a single server-side job by id.
+
+        Returns None when the server has no such job.
+        """
+        return LOOP.run(self._conn.get_job(job_id))
+
     @override
     def cancel_job(self, job_id: str) -> bool:
         """Request cancellation of a server-side job by id.
@@ -1488,6 +1460,14 @@ class LanceDBConnection(DBConnection):
         """
         return LOOP.run(self._conn.cancel_job(job_id))
 
+    @override
+    def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
+        """The lifecycle event history of a server-side job, as Arrow batches.
+
+        Lists history across all jobs when `job_id` is None.
+        """
+        return LOOP.run(self._conn.job_history(job_id))
+
     @override
     def namespace_client(self) -> LanceNamespace:
         """Get the equivalent namespace client for this connection.
@@ -2258,24 +2238,35 @@ class AsyncConnection(object):
             namespace_path = []
         await self._inner.drop_all_tables(namespace_path=namespace_path)
 
-    async def open_job(self, job_id: str) -> AsyncJob:
-        """Open a server-side job by id. See
-        [DBConnection.open_job][lancedb.db.DBConnection.open_job].
+    def job(self, job_id: str) -> AsyncJob:
+        """An [AsyncJob][lancedb.job.AsyncJob] handle for a server-side job
+        by id.
+
+        The handle is constructed without a server round trip; an unknown id
+        surfaces when the handle is used. Dropping the handle has no effect
+        on the job itself.
         """
-        return AsyncJob(await self._inner.open_job(job_id))
+        return AsyncJob(self._inner.job(job_id))
 
     async def create_function_async(
-        self, definition: UdfDefinition
+        self,
+        definition: UdfDefinition,
+        *,
+        secrets: Optional[Mapping[str, str]] = None,
     ) -> AsyncJob[FunctionVersion]:
         """Register a scalar Python UDF through the remote Function catalog.
 
+        ``secrets`` must contain exactly the names declared by
+        ``@udf(secrets=[...])``. Values are sent in the create request and
+        stored server-side in the private execution artifact; returned
+        Function and Job metadata contain only the declared names.
         The returned typed job resolves to the immutable Function version.
         Local connections raise ``NotImplementedError``.
         """
         if not isinstance(definition, UdfDefinition):
             raise TypeError("create_function_async requires a @udf definition")
         inner = await self._inner.create_function_async(
-            definition.registration_request.to_canonical_json()
+            definition._submission_json(secrets)
         )
         return _typed_job(inner, FunctionVersion.from_json)
 
@@ -2283,25 +2274,17 @@ class AsyncConnection(object):
         """Open one exact immutable Function version from the remote catalog."""
         return FunctionVersion.from_json(await self._inner.get_function(name, version))
 
-    async def list_functions(self) -> List[FunctionVersion]:
-        """List every published immutable Function version.
-
-        Results are ordered by Function name then version. Local connections
-        raise ``NotImplementedError``.
-        """
-        return [
-            FunctionVersion.from_json(value)
-            for value in await self._inner.list_functions()
-        ]
-
-    async def drop_function(self, name: str, *, version: str) -> bool:
-        """Drop one exact immutable Function version from the remote catalog."""
-        return await self._inner.drop_function(name, version)
-
     async def list_jobs(self) -> List[JobInfo]:
         """List server-side jobs across the database's tables."""
         return await self._inner.list_jobs()
 
+    async def get_job(self, job_id: str) -> Optional[JobDescription]:
+        """Describe a single server-side job by id.
+
+        Returns None when the server has no such job.
+        """
+        return await self._inner.get_job(job_id)
+
     async def cancel_job(self, job_id: str) -> bool:
         """Request cancellation of a server-side job by id.
 
@@ -2311,46 +2294,12 @@ class AsyncConnection(object):
         """
         return await self._inner.cancel_job(job_id)
 
-    async def execute_query(
-        self,
-        query: str,
-        *,
-        default_namespace_path: Optional[List[str]] = None,
-    ) -> AsyncRecordBatchReader:
-        """Execute SQL and return an asynchronous Arrow reader.
+    async def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
+        """The lifecycle event history of a server-side job, as Arrow batches.
 
-        This submits through :meth:`execute_query_async` and waits until the
-        initial result stream is readable. It does not wait for the full query
-        to finish.
+        Lists history across all jobs when `job_id` is None.
         """
-        submitted = await self.execute_query_async(
-            query,
-            default_namespace_path=default_namespace_path,
-        )
-        return await submitted.reader()
-
-    async def execute_query_async(
-        self,
-        query: str,
-        *,
-        default_namespace_path: Optional[List[str]] = None,
-    ) -> AsyncSqlQuery:
-        """Start executing SQL and return its query handle.
-
-        The database from ``connect_async`` is used for unqualified database
-        references. The namespace defaults to ``["public"]``. Local
-        connections raise ``NotImplementedError``.
-        """
-        return AsyncSqlQuery(
-            await self._inner.execute_query_async(
-                query,
-                default_namespace_path=default_namespace_path,
-            )
-        )
-
-    async def describe_query(self, query_id: UUID) -> QueryDescription:
-        """Describe a submitted SQL query by its connection-scoped id."""
-        return await self._inner.describe_query(query_id)
+        return await self._inner.job_history(job_id)
 
     async def namespace_client(self) -> LanceNamespace:
         """Get the equivalent namespace client for this connection.

python/python/lancedb/exceptions.py

@@ -35,9 +35,3 @@ class JobCancelledError(RuntimeError):
     """Exception raised when an asynchronous job was cancelled."""
 
     pass
-
-
-class JobNotFoundError(ValueError):
-    """Exception raised when opening a job the server does not have."""
-
-    pass

python/python/lancedb/functions.py

@@ -4,7 +4,7 @@
 """Canonical Function values exchanged with LanceDB Enterprise services.
 
 These immutable models contain client/wire state only. Catalog persistence,
-environment bake, and execution are owned by Sophon.
+environment bake, secret resolution, and execution are owned by Sophon.
 ``RefreshColumnResult`` is also the backend-neutral result of a local
 expression-backed refresh job.
 """
@@ -49,25 +49,11 @@ from pydantic import (
     model_validator,
 )
 
-from .schema import is_blob_v2_field as _is_blob_v2_field
-
 _Int32 = conint(strict=True, ge=-(2**31), le=2**31 - 1)
 _UInt32 = conint(strict=True, ge=0, le=2**32 - 1)
 _UInt64 = conint(strict=True, ge=0, le=2**64 - 1)
 
 
-def _validate_gpu_wire_marker(value: Any) -> bool:
-    if value is not True:
-        raise ValueError("runtime.gpu must be true")
-    return True
-
-
-def _normalize_gpu_marker(value: bool) -> Optional[bool]:
-    if not isinstance(value, bool):
-        raise ValueError("gpu must be a boolean")
-    return True if value else None
-
-
 class _FrozenDict(dict):
     def _immutable(self, *args, **kwargs):
         raise TypeError("remote canonical values are immutable")
@@ -243,7 +229,7 @@ class PythonEnvironmentSpec(_RemoteValue):
 
 
 class PythonRuntimeSpec(_RemoteValue):
-    """Remote runtime definition with environment values.
+    """Remote runtime definition with non-secret environment values.
 
     V1 supports ``kind="python"``. Newer runtime kinds remain readable, while
     their unknown payload fields are intentionally not retained by the client.
@@ -253,23 +239,6 @@ class PythonRuntimeSpec(_RemoteValue):
     python_version: Optional[str] = None
     environment: Optional[PythonEnvironmentSpec] = None
     env: Optional[Mapping[str, str]] = None
-    gpu: Optional[bool] = None
-
-    @model_validator(mode="before")
-    @classmethod
-    def _discard_unknown_runtime_payload(cls, value):
-        if isinstance(value, Mapping):
-            kind = value.get("kind")
-            if isinstance(kind, str) and kind not in {"python", "python_v2"}:
-                return {"kind": kind}
-        return value
-
-    @field_validator("gpu", mode="before")
-    @classmethod
-    def _validate_gpu_marker(cls, value):
-        if value is None:
-            return None
-        return _validate_gpu_wire_marker(value)
 
     @model_validator(mode="after")
     def _validate_runtime_kind(self):
@@ -278,28 +247,18 @@ class PythonRuntimeSpec(_RemoteValue):
                 raise ValueError("python runtime requires python_version")
             if self.environment is None:
                 raise ValueError("python runtime requires environment")
-            if self.gpu is not None:
-                raise ValueError("python runtime with gpu requires kind='python_v2'")
-        elif self.kind == "python_v2":
-            if self.python_version is None:
-                raise ValueError("python_v2 runtime requires python_version")
-            if self.environment is None:
-                raise ValueError("python_v2 runtime requires environment")
-            if self.gpu is None:
-                raise ValueError("python_v2 runtime requires gpu")
         else:
             object.__setattr__(self, "python_version", None)
             object.__setattr__(self, "environment", None)
             object.__setattr__(self, "env", None)
-            object.__setattr__(self, "gpu", None)
         return self
 
 
 class FunctionVersion(_RemoteValue):
     """An exact immutable Function version returned by Enterprise.
 
-    The GPU execution requirement is part of this identity. CPU and memory sizing,
-    priority, concurrency, and retry policy belong to the execution platform.
+    Scheduling resources, priority, concurrency, and retry policy belong to
+    the submitting Job and are not part of this identity.
     """
 
     name: str
@@ -309,6 +268,7 @@ class FunctionVersion(_RemoteValue):
     runtime: PythonRuntimeSpec
     runtime_digest: str
     environment_digest: str
+    required_secrets: tuple[str, ...] = ()
     created_at: str
 
     def __call__(self, **inputs: Any) -> FunctionApplication:
@@ -370,12 +330,17 @@ class FunctionVersion(_RemoteValue):
 
 
 class FunctionRegistrationRequest(_RemoteValue):
-    """Stable remote registration envelope produced by :func:`udf`."""
+    """Stable remote registration envelope produced by :func:`udf`.
+
+    Only secret names are represented. Secret values are supplied separately
+    when the definition is submitted and are not part of this durable value.
+    """
 
     name: str
     artifact: FunctionArtifactRequest
     signature: FunctionSignature
     runtime: PythonRuntimeSpec
+    required_secrets: tuple[str, ...] = ()
 
 
 class FunctionVersionRef(_OpenRemoteValue):
@@ -470,7 +435,11 @@ class InputBinding(_RemoteValue):
 
 
 class OutputMapping(_RemoteValue):
-    """One stable result-field mapping."""
+    """One stable result-field mapping.
+
+    Assignment state is outside the Slice 1 client contract. During the NULL
+    transition Lance exposes no public cell-flag identifier to persist here.
+    """
 
     result_field: str
     output_name: str
@@ -480,13 +449,6 @@ class OutputMapping(_RemoteValue):
     nullable: bool
 
 
-class AssignmentMapping(_RemoteValue):
-    """Internal physical column preserving flattened struct validity."""
-
-    output_name: str
-    output_field_id: _Int32
-
-
 class FunctionBinding(_RemoteValue):
     """Immutable Function binding persisted by the Enterprise table service."""
 
@@ -494,7 +456,6 @@ class FunctionBinding(_RemoteValue):
     function: FunctionVersionRef
     inputs: tuple[InputBinding, ...]
     outputs: tuple[OutputMapping, ...]
-    assignment: Optional[AssignmentMapping] = None
     input_schema: Optional[Mapping[str, Any]] = None
     output_schema: Optional[Mapping[str, Any]] = None
 
@@ -524,13 +485,27 @@ class RefreshColumnResult(_RemoteValue):
 
 
 _FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$")
-_FUNCTION_BLOB_V2_TYPE = "blob_v2"
-_ARROW_EXTENSION_NAME_KEY = "ARROW:extension:name"
-_BLOB_V2_EXTENSION_NAME = "lance.blob.v2"
-_NESTED_BLOB_COLLECTION_ERROR = (
-    "unsupported Arrow type for Function signature: Blob v2 fields nested under "
-    "collection types are not supported"
-)
+_SECRET_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+# Keep this byte limit aligned with Sophon's MAX_FUNCTION_SECRET_VALUE_BYTES.
+_MAX_FUNCTION_SECRET_VALUE_BYTES = 64 * 1024
+_MAX_FUNCTION_SECRET_VALUES_BYTES = 512 * 1024
+
+
+def _validate_secret_value(name: str, value: Any) -> str:
+    """Validate one secret value before building the create request."""
+    if not isinstance(value, str):
+        raise TypeError(f"Function secret {name!r} value must be a string")
+    if not value:
+        raise ValueError(f"Function secret {name!r} value must be non-empty")
+    if "\0" in value:
+        raise ValueError(f"Function secret {name!r} value must not contain NUL")
+    value_bytes = len(value.encode("utf-8"))
+    if value_bytes > _MAX_FUNCTION_SECRET_VALUE_BYTES:
+        raise ValueError(
+            f"Function secret {name!r} value exceeds the "
+            f"{_MAX_FUNCTION_SECRET_VALUE_BYTES}-byte limit"
+        )
+    return value
 
 
 _GRAMMAR_PRIMITIVES = (
@@ -547,7 +522,6 @@ _GRAMMAR_PRIMITIVES = (
     (pa.float32(), "float32"),
     (pa.float64(), "float64"),
     (pa.string(), "utf8"),
-    (pa.large_string(), "large_utf8"),
     (pa.binary(), "binary"),
     (pa.date32(), "date32"),
     (pa.date64(), "date64"),
@@ -555,258 +529,31 @@ _GRAMMAR_PRIMITIVES = (
 
 
 def _canonical_arrow_type(data_type: pa.DataType) -> str:
-    """The compact Function grammar, or canonical exact JSON for nested types."""
-    grammar = _grammar_arrow_type(data_type)
-    if grammar is not None:
-        return grammar
-    exact = _exact_arrow_type(data_type)
-    return json.dumps(exact, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
-
-
-def _grammar_arrow_type(data_type: pa.DataType) -> Optional[str]:
+    """The server's V1 Function type grammar. Anything outside it is rejected
+    here rather than at registration."""
     for candidate, name in _GRAMMAR_PRIMITIVES:
         if data_type == candidate:
             return name
     if pa.types.is_list(data_type) or pa.types.is_large_list(data_type):
-        item = _grammar_list_item(data_type)
-        if item is None:
-            return None
         prefix = "list" if pa.types.is_list(data_type) else "large_list"
-        return f"{prefix}<{item}>"
+        return f"{prefix}<{_canonical_list_item(data_type)}>"
     if pa.types.is_fixed_size_list(data_type) and data_type.list_size > 0:
-        item = _grammar_list_item(data_type)
-        if item is not None:
-            return f"fixed_size_list<{item}, {data_type.list_size}>"
-    return None
+        return (
+            f"fixed_size_list<{_canonical_list_item(data_type)}, {data_type.list_size}>"
+        )
+    raise TypeError(f"unsupported Arrow type for Function signature: {data_type}")
 
 
-def _grammar_list_item(data_type: pa.DataType) -> Optional[str]:
+def _canonical_list_item(data_type: pa.DataType) -> str:
     """The grammar names only the item type; it always means a non-nullable
-    child called `item`, so other child properties require exact JSON."""
+    child called `item`, so any other child metadata cannot be represented."""
     child = data_type.value_field
     if child.name != "item" or child.nullable or child.metadata:
-        return None
-    return _grammar_arrow_type(child.type)
-
-
-def _validate_exact_arrow_field(field: pa.Field) -> None:
-    if not field.name:
         raise TypeError(
-            "unsupported Arrow type for Function signature: field names "
-            "must not be empty"
+            "unsupported Arrow type for Function signature: list items must be a "
+            f"non-nullable field named 'item', got {child}"
         )
-    if _is_blob_v2_field(field):
-        if not _has_supported_blob_v2_layout(field):
-            raise TypeError(
-                "unsupported Arrow type for Function signature: lance.blob.v2 "
-                f"requires a supported Blob storage layout, got {field}"
-            )
-        metadata = {
-            (key.decode() if isinstance(key, bytes) else key): (
-                value.decode() if isinstance(value, bytes) else value
-            )
-            for key, value in (field.metadata or {}).items()
-        }
-        if metadata and metadata != {
-            _ARROW_EXTENSION_NAME_KEY: _BLOB_V2_EXTENSION_NAME
-        }:
-            raise TypeError(
-                "unsupported Arrow type for Function signature: lance.blob.v2 "
-                "field metadata must contain only its canonical extension marker"
-            )
-    elif field.metadata:
-        raise TypeError(
-            "unsupported Arrow type for Function signature: field metadata "
-            f"is not supported, got {field}"
-        )
-
-
-def _has_supported_blob_v2_layout(field: pa.Field) -> bool:
-    data_type = field.type
-    if isinstance(data_type, pa.ExtensionType):
-        data_type = data_type.storage_type
-    if not pa.types.is_struct(data_type):
-        return False
-
-    fields = tuple(data_type)
-
-    def matches(spec, compare_nullable) -> bool:
-        return len(fields) == len(spec) and all(
-            actual.name == name
-            and actual.type == expected_type
-            and (not check_nullable or actual.nullable == nullable)
-            for actual, (name, expected_type, nullable), check_nullable in zip(
-                fields, spec, compare_nullable
-            )
-        )
-
-    logical_minimal = (
-        ("data", pa.large_binary(), True),
-        ("uri", pa.utf8(), True),
-    )
-    logical_full = logical_minimal + (
-        ("position", pa.uint64(), True),
-        ("size", pa.uint64(), True),
-    )
-    prepared = (
-        ("kind", pa.uint8(), True),
-        ("data", pa.large_binary(), True),
-        ("uri", pa.utf8(), True),
-        ("blob_id", pa.uint32(), True),
-        ("blob_size", pa.uint64(), True),
-        ("position", pa.uint64(), True),
-    )
-    descriptor = (
-        ("kind", pa.uint8(), False),
-        ("position", pa.uint64(), False),
-        ("size", pa.uint64(), False),
-        ("blob_id", pa.uint32(), False),
-        ("blob_uri", pa.utf8(), False),
-    )
-    return (
-        matches(logical_minimal, (True, True))
-        or matches(logical_full, (True, True, False, False))
-        or matches(prepared, (True,) * len(prepared))
-        or matches(descriptor, (False,) * len(descriptor))
-    )
-
-
-def _canonical_arrow_field(field: pa.Field) -> str:
-    _validate_exact_arrow_field(field)
-    if _is_blob_v2_field(field):
-        return _FUNCTION_BLOB_V2_TYPE
-    return _canonical_arrow_type(field.type)
-
-
-def _blob_storage_type(field: pa.Field) -> pa.DataType:
-    data_type = field.type
-    if isinstance(data_type, pa.ExtensionType):
-        return data_type.storage_type
-    return data_type
-
-
-def _exact_blob_storage_type(field: pa.Field) -> dict[str, Any]:
-    storage = _blob_storage_type(field)
-    if not pa.types.is_struct(storage):
-        raise TypeError(
-            "unsupported Arrow type for Function signature: lance.blob.v2 "
-            "requires struct storage"
-        )
-    return {
-        "type": "struct",
-        "fields": [
-            {
-                "name": child.name,
-                "nullable": child.nullable,
-                "type": (
-                    {"type": "large_binary"}
-                    if pa.types.is_large_binary(child.type)
-                    else _exact_arrow_type(child.type)
-                ),
-            }
-            for child in storage
-        ],
-    }
-
-
-def _data_type_has_blob_v2(data_type: pa.DataType) -> bool:
-    if pa.types.is_struct(data_type):
-        return any(
-            _is_blob_v2_field(field) or _data_type_has_blob_v2(field.type)
-            for field in data_type
-        )
-    if (
-        pa.types.is_list(data_type)
-        or pa.types.is_large_list(data_type)
-        or pa.types.is_fixed_size_list(data_type)
-    ):
-        field = data_type.value_field
-        return _is_blob_v2_field(field) or _data_type_has_blob_v2(field.type)
-    if pa.types.is_map(data_type):
-        return any(
-            _is_blob_v2_field(field) or _data_type_has_blob_v2(field.type)
-            for field in (data_type.key_field, data_type.item_field)
-        )
-    return False
-
-
-def _exact_arrow_field(
-    field: pa.Field, *, inside_collection: bool = False
-) -> dict[str, Any]:
-    _validate_exact_arrow_field(field)
-    if _is_blob_v2_field(field):
-        if inside_collection:
-            raise TypeError(_NESTED_BLOB_COLLECTION_ERROR)
-        return {
-            "name": field.name,
-            "nullable": field.nullable,
-            "type": _exact_blob_storage_type(field),
-            "metadata": {
-                _ARROW_EXTENSION_NAME_KEY: _BLOB_V2_EXTENSION_NAME,
-            },
-        }
-    value = {
-        "name": field.name,
-        "nullable": field.nullable,
-        "type": _exact_arrow_type(field.type, inside_collection=inside_collection),
-    }
-    return value
-
-
-def _exact_arrow_type(
-    data_type: pa.DataType, *, inside_collection: bool = False
-) -> dict[str, Any]:
-    for candidate, name in _GRAMMAR_PRIMITIVES:
-        if data_type == candidate:
-            return {"type": name}
-    if pa.types.is_struct(data_type):
-        fields = list(data_type)
-        names = [field.name for field in fields]
-        if not fields or len(set(names)) != len(names):
-            raise TypeError(
-                "unsupported Arrow type for Function signature: structs must have "
-                "non-empty, uniquely named fields"
-            )
-        return {
-            "type": "struct",
-            "fields": [
-                _exact_arrow_field(field, inside_collection=inside_collection)
-                for field in fields
-            ],
-        }
-    if (
-        pa.types.is_list(data_type)
-        or pa.types.is_large_list(data_type)
-        or pa.types.is_fixed_size_list(data_type)
-    ):
-        if pa.types.is_fixed_size_list(data_type):
-            if data_type.value_field.name != "item":
-                raise TypeError(
-                    "unsupported Arrow type for Function signature: fixed-size list "
-                    "items must be named 'item'"
-                )
-            if data_type.list_size <= 0:
-                raise TypeError(
-                    f"unsupported Arrow type for Function signature: {data_type}"
-                )
-        value: dict[str, Any] = {
-            "type": (
-                "list"
-                if pa.types.is_list(data_type)
-                else "large_list"
-                if pa.types.is_large_list(data_type)
-                else "fixed_size_list"
-            ),
-            "fields": [
-                _exact_arrow_field(data_type.value_field, inside_collection=True)
-            ],
-        }
-        if pa.types.is_fixed_size_list(data_type):
-            value["length"] = data_type.list_size
-        return value
-    if pa.types.is_map(data_type) and _data_type_has_blob_v2(data_type):
-        raise TypeError(_NESTED_BLOB_COLLECTION_ERROR)
-    raise TypeError(f"unsupported Arrow type for Function signature: {data_type}")
+    return _canonical_arrow_type(child.type)
 
 
 def _list_of(item: pa.DataType) -> pa.DataType:
@@ -880,15 +627,8 @@ def _callable_parameters(function: Callable[..., Any]) -> tuple[inspect.Paramete
 
 def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutput:
     if isinstance(output, pa.Schema):
-        if output.metadata:
-            raise TypeError("Function output schema metadata is not supported")
         fields = tuple(output)
-    elif (
-        isinstance(output, pa.Field)
-        and not _is_blob_v2_field(output)
-        and pa.types.is_struct(output.type)
-    ):
-        _validate_exact_arrow_field(output)
+    elif isinstance(output, pa.Field) and pa.types.is_struct(output.type):
         if output.nullable:
             raise ValueError("Function output must be non-nullable")
         fields = tuple(output.type)
@@ -904,19 +644,18 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
             raise TypeError(
                 "output_schema must be a PyArrow DataType, Field, or Schema"
             )
-        _validate_exact_arrow_field(field)
         if field.nullable:
             raise ValueError("Function output must be non-nullable")
         return FunctionOutput(
             kind="scalar",
-            arrow_type=_canonical_arrow_field(field),
+            arrow_type=_canonical_arrow_type(field.type),
             nullable=False,
         )
 
     if not fields:
         raise ValueError("named-struct Function output must contain at least one field")
-    for field in fields:
-        _validate_exact_arrow_field(field)
+    if any(field.nullable for field in fields):
+        raise ValueError("Function output fields must be non-nullable")
     names = [field.name for field in fields]
     if len(set(names)) != len(names):
         raise ValueError("Function output field names must be unique")
@@ -925,8 +664,8 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
         fields=tuple(
             FunctionResultField(
                 name=field.name,
-                arrow_type=_canonical_arrow_field(field),
-                nullable=field.nullable,
+                arrow_type=_canonical_arrow_type(field.type),
+                nullable=False,
             )
             for field in fields
         ),
@@ -945,10 +684,6 @@ def _infer_signature(
     if input_schema is not None:
         if not isinstance(input_schema, pa.Schema):
             raise TypeError("input_schema must be a PyArrow Schema")
-        if input_schema.metadata:
-            raise TypeError("Function input schema metadata is not supported")
-        for field in input_schema:
-            _validate_exact_arrow_field(field)
         expected = tuple(parameter.name for parameter in parameters)
         actual = tuple(input_schema.names)
         if actual != expected:
@@ -959,7 +694,7 @@ def _infer_signature(
         inputs = tuple(
             FunctionParameter(
                 name=field.name,
-                arrow_type=_canonical_arrow_field(field),
+                arrow_type=_canonical_arrow_type(field.type),
                 nullable=field.nullable,
             )
             for field in input_schema
@@ -982,9 +717,7 @@ def _infer_signature(
         inputs.append(
             FunctionParameter(
                 name=parameter.name,
-                arrow_type=_canonical_arrow_field(
-                    pa.field(parameter.name, data_type, nullable=nullable)
-                ),
+                arrow_type=_canonical_arrow_type(data_type),
                 nullable=nullable,
             )
         )
@@ -1203,8 +936,8 @@ class UdfDefinition:
         output_schema: Optional[pa.DataType | pa.Field | pa.Schema],
         pip: tuple[str, ...],
         env: Mapping[str, str],
+        secrets: tuple[str, ...],
         python_version: Optional[str],
-        gpu: bool = False,
         conda: tuple[str, ...] = (),
         conda_channels: tuple[str, ...] = (),
     ):
@@ -1230,17 +963,26 @@ class UdfDefinition:
             for key, value in environment.items()
         ):
             raise TypeError("Function env keys and values must be strings")
+        required_secrets = tuple(sorted(set(secrets)))
+        invalid_secrets = [
+            secret for secret in required_secrets if not _SECRET_NAME.fullmatch(secret)
+        ]
+        if invalid_secrets:
+            raise ValueError(f"invalid Function secret names: {invalid_secrets!r}")
+        overlap = set(environment) & set(required_secrets)
+        if overlap:
+            raise ValueError(
+                f"Function env and secret names must be disjoint: {sorted(overlap)!r}"
+            )
         signature = _infer_signature(function, input_schema, output_schema)
         source = _package_source(function)
         digest = f"sha256:{hashlib.sha256(source).hexdigest()}"
-        gpu_marker = _normalize_gpu_marker(gpu)
         runtime = PythonRuntimeSpec(
-            kind="python_v2" if gpu_marker is not None else "python",
+            kind="python",
             python_version=python_version
             or f"{sys.version_info.major}.{sys.version_info.minor}",
             environment=environment_spec,
             env=environment,
-            gpu=gpu_marker,
         )
         self._function = function
         self._request = FunctionRegistrationRequest(
@@ -1260,14 +1002,65 @@ class UdfDefinition:
             ),
             signature=signature,
             runtime=runtime,
+            required_secrets=required_secrets,
         )
         functools.update_wrapper(self, function)
 
     @property
     def registration_request(self) -> FunctionRegistrationRequest:
-        """The immutable request sent by ``create_function_async``."""
+        """The immutable, value-free client model for a Function submission."""
         return self._request
 
+    def _submission_json(self, secrets: Optional[Mapping[str, str]]) -> str:
+        """Build one registration submission without retaining values on self."""
+        if secrets is None:
+            secret_values: Mapping[str, str] = {}
+        elif not isinstance(secrets, Mapping):
+            raise TypeError("Function secrets must be a mapping of names to strings")
+        else:
+            secret_values = secrets
+
+        if any(not isinstance(name, str) for name in secret_values):
+            raise TypeError("Function secret names must be strings")
+        expected = set(self._request.required_secrets)
+        provided = set(secret_values)
+        if provided != expected:
+            missing = sorted(expected - provided)
+            unexpected = sorted(provided - expected)
+            details = []
+            if missing:
+                details.append(f"missing: {missing!r}")
+            if unexpected:
+                details.append(f"unexpected: {unexpected!r}")
+            raise ValueError(
+                "Function secret values must exactly match the declared secrets ("
+                + "; ".join(details)
+                + ")"
+            )
+
+        canonical_values = {}
+        total_bytes = 0
+        for name in sorted(secret_values):
+            value = _validate_secret_value(name, secret_values[name])
+            total_bytes += len(value.encode("utf-8"))
+            if total_bytes > _MAX_FUNCTION_SECRET_VALUES_BYTES:
+                raise ValueError(
+                    "Function secret values exceed the "
+                    f"{_MAX_FUNCTION_SECRET_VALUES_BYTES}-byte request limit"
+                )
+            canonical_values[name] = value
+
+        submission = self._request._known_dict()
+        if canonical_values:
+            submission["secret_values"] = canonical_values
+        return json.dumps(
+            submission,
+            ensure_ascii=False,
+            allow_nan=False,
+            sort_keys=True,
+            separators=(",", ":"),
+        )
+
     def __call__(self, *args, **kwargs):
         return self._function(*args, **kwargs)
 
@@ -1285,8 +1078,8 @@ def udf(
     output_schema: Optional[pa.DataType | pa.Field | pa.Schema] = None,
     pip: tuple[str, ...] | list[str] = (),
     env: Optional[Mapping[str, str]] = None,
+    secrets: tuple[str, ...] | list[str] = (),
     python_version: Optional[str] = None,
-    gpu: bool = False,
     conda: tuple[str, ...] | list[str] = (),
     conda_channels: tuple[str, ...] | list[str] = (),
 ) -> Callable[[Callable[..., Any]], UdfDefinition]: ...
@@ -1300,8 +1093,8 @@ def udf(
     output_schema: Optional[pa.DataType | pa.Field | pa.Schema] = None,
     pip: tuple[str, ...] | list[str] = (),
     env: Optional[Mapping[str, str]] = None,
+    secrets: tuple[str, ...] | list[str] = (),
     python_version: Optional[str] = None,
-    gpu: bool = False,
     conda: tuple[str, ...] | list[str] = (),
     conda_channels: tuple[str, ...] | list[str] = (),
 ):
@@ -1309,9 +1102,8 @@ def udf(
 
     Input and output signatures are inferred from supported annotations. For
     Arrow types annotations cannot express precisely, pass ``input_schema``
-    and ``output_schema`` together. Scalar outputs must be non-nullable. Every
-    named-struct field may be nullable; Enterprise preserves the struct's
-    validity when the result is expanded into sibling columns.
+    and ``output_schema`` together. Nullable outputs are rejected because V1
+    uses physical NULL to represent unassigned computed-column rows.
 
     Parameters
     ----------
@@ -1323,8 +1115,8 @@ def udf(
         Explicit input fields in the exact order of the callable parameters.
         Must be provided together with ``output_schema``.
     output_schema : pyarrow.DataType, pyarrow.Field, or pyarrow.Schema, optional
-        Explicit scalar or named-struct output. Scalar outputs must be
-        non-nullable. Must be provided together with ``input_schema``.
+        Explicit scalar or named-struct output. Must be non-nullable and be
+        provided together with ``input_schema``.
     pip : sequence of str, optional
         Pip requirements for the remote environment.
     conda : sequence of str, optional
@@ -1332,13 +1124,12 @@ def udf(
     conda_channels : sequence of str, optional
         Conda channels in priority order; requires ``conda``.
     env : mapping of str to str, optional
-        Environment variables included in the Function definition.
+        Non-secret environment variables. Use ``secrets`` for credentials.
+    secrets : sequence of str, optional
+        Names of secrets required by the callable. Supply their values separately
+        to ``create_function`` or ``create_function_async``.
     python_version : str, optional
         Remote Python major/minor version. Defaults to the client version.
-    gpu : bool, default False
-        Whether every remote execution requires a GPU. The execution platform
-        selects one compatible GPU for each worker. The requirement is part of
-        the immutable Function version.
 
     The packaged artifact is a snapshot: the function source plus exactly
     the module-level names it references (modules as imports, importable
@@ -1358,16 +1149,15 @@ def udf(
     Examples
     --------
     >>> from lancedb import udf
-    >>> @udf(pip=["numpy==2.2.0"])
+    >>> @udf(pip=["numpy==2.2.0"], secrets=["MODEL_TOKEN"])
     ... def score(value: float) -> float:
     ...     return value * 2
     >>> score(1.5)
     3.0
-    >>> @udf(pip=["cupy-cuda12x"], gpu=True)
-    ... def gpu_score(value: int) -> int:
-    ...     return value * 2
-    >>> gpu_score.registration_request.runtime.gpu
-    True
+    >>> db.create_function(  # doctest: +SKIP
+    ...     score, secrets={"MODEL_TOKEN": "user-secret-value"}
+    ... )
+
     """
 
     def decorate(target: Callable[..., Any]) -> UdfDefinition:
@@ -1378,8 +1168,8 @@ def udf(
             output_schema=output_schema,
             pip=tuple(pip),
             env={} if env is None else env,
+            secrets=tuple(secrets),
             python_version=python_version,
-            gpu=gpu,
             conda=tuple(conda),
             conda_channels=tuple(conda_channels),
         )
@@ -1390,7 +1180,6 @@ def udf(
 
 
 __all__ = [
-    "AssignmentMapping",
     "ApplicationInput",
     "FunctionApplication",
     "FunctionArtifact",

python/python/lancedb/job.py

@@ -4,27 +4,15 @@
 """Handles to operations a server may run asynchronously."""
 
 import asyncio
-import json
 from datetime import timedelta
 from typing import Any, Callable, Generic, Optional, TypeVar, cast
 
-import pyarrow as pa
-
 from lancedb.background_loop import LOOP
 
 from . import _lancedb
-from ._lancedb import JobDescription, JobFailureInfo, JobInfo
 
 T = TypeVar("T")
 
-__all__ = [
-    "AsyncJob",
-    "Job",
-    "JobDescription",
-    "JobFailureInfo",
-    "JobInfo",
-]
-
 
 class AsyncJob(Generic[T]):
     """A handle to an operation that may still be running.
@@ -90,149 +78,6 @@ class AsyncJob(Generic[T]):
             return
         await self._inner.cancel()
 
-    async def refresh(self) -> None:
-        """Ask the backend for this job's current state, and for a server-side
-        job its full record, then cache it for the properties below.
-
-        The properties are all `None` until this runs, because submitting an
-        operation returns only a job id. `status` fetches the whole record too;
-        `wait` records only the terminal state it establishes.
-        """
-        if self._inner is None:
-            return
-        await self._inner.refresh()
-
-    @property
-    def state(self) -> Optional[str]:
-        """The last observed lifecycle state, without contacting the backend.
-
-        `None` until the handle has talked to it. See :meth:`AsyncJob.refresh`.
-        """
-        if self._inner is None:
-            return "finished"
-        return self._inner._state
-
-    @property
-    def job_type(self) -> Optional[str]:
-        """The job's type, as the server names it.
-
-        `None` for an in-process job, which has no server-side record.
-        """
-        return self._field("job_type")
-
-    @property
-    def creation_ms(self) -> Optional[int]:
-        """When the job was created, in milliseconds since the epoch."""
-        return self._field("creation_ms")
-
-    @property
-    def spec(self) -> Optional[Any]:
-        """The job-type-specific specification it was submitted with."""
-        return self._field("spec")
-
-    @property
-    def result(self) -> Optional[Any]:
-        """The job-type-specific terminal result, as reported data rather than
-        the typed model :meth:`AsyncJob.wait` returns.
-
-        `None` until the job succeeds, so a job that never terminates reports
-        its progress through :meth:`AsyncJob.events` instead.
-        """
-        return self._field("result")
-
-    @property
-    def failure(self) -> Optional[JobFailureInfo]:
-        """Why the job failed, when it failed and the server reports a reason."""
-        return self._field("failure")
-
-    @property
-    def _spec_json(self) -> Optional[str]:
-        return self._field("_spec_json")
-
-    @property
-    def _result_json(self) -> Optional[str]:
-        return self._field("_result_json")
-
-    def _field(self, name: str) -> Optional[Any]:
-        description = self._inner._description if self._inner is not None else None
-        return getattr(description, name) if description is not None else None
-
-    async def events(
-        self,
-        *,
-        limit: Optional[int] = None,
-        filter: Optional[str] = None,
-    ) -> "pa.Table":
-        """This job's recorded lifecycle events.
-
-        Where the properties above report a terminal result only once the job
-        reaches one, events are written as the job runs and outlive the workers
-        that produced them. A distributed job records a `claim`/`claim_complete`
-        pair per unit of work, each carrying `rows_processed`, so a job that
-        never finishes still accounts for what it did.
-
-        Parameters
-        ----------
-        limit: int, optional
-            Maximum event rows to return. The server caps results at 1000 by
-            default and 10,000 at most, and truncates without saying so, so
-            pass this for a job that emits an event per fragment.
-        filter: str, optional
-            SQL-like expression over the `state`, `updated_by`, `emitted_from`,
-            `emitted_by`, and `claim_entity` columns, such as
-            ``state = 'claim_complete'``.
-        """
-        if self._inner is None:
-            raise NotImplementedError(
-                "job event history is only available for server-side jobs"
-            )
-        return await self._inner.events(limit=limit, filter=filter)
-
-    def __repr__(self) -> str:
-        return _job_repr("AsyncJob", self)
-
-
-_REPR_INDENT = " " * 4
-
-
-def _repr_payload(value: Any) -> str:
-    """Render a job payload as indented JSON, aligned under its field."""
-    try:
-        rendered = json.dumps(value, indent=4)
-    except TypeError:
-        return repr(value)
-    return rendered.replace("\n", "\n" + _REPR_INDENT)
-
-
-def _job_repr(kind: str, job: Any) -> str:
-    """Render every field the handle currently knows, omitting the rest.
-
-    One field per line, with the JSON payloads indented, because a refresh
-    job's spec and result are the point of printing it.
-    """
-    state = job.state
-    if state is None:
-        # Nothing has been fetched yet, so there is nothing to lay out.
-        known = f"id={job.id!r}, " if job.id is not None else ""
-        return f"{kind}({known}not refreshed)"
-
-    fields = []
-    if job.id is not None:
-        fields.append(f"id={job.id!r}")
-    fields.append(f"state={state!r}")
-    for name in ("job_type", "creation_ms"):
-        value = getattr(job, name)
-        if value is not None:
-            fields.append(f"{name}={value!r}")
-    for name in ("spec", "result"):
-        value = getattr(job, name)
-        if value is not None:
-            fields.append(f"{name}={_repr_payload(value)}")
-    if job.failure is not None:
-        fields.append(f"failure={job.failure!r}")
-    body = "".join(f"\n{_REPR_INDENT}{field}," for field in fields)
-    return f"{kind}({body}\n)"
-
 
 class Job(Generic[T]):
     """Synchronous counterpart of `AsyncJob` with the same result type."""
@@ -277,75 +122,6 @@ class Job(Generic[T]):
             return
         LOOP.run(self._inner.cancel())
 
-    def refresh(self) -> None:
-        """Ask the backend for this job's current state and record.
-
-        See :meth:`AsyncJob.refresh`.
-        """
-        if self._inner is None:
-            return
-        LOOP.run(self._inner.refresh())
-
-    @property
-    def state(self) -> Optional[str]:
-        """The last observed lifecycle state. See :attr:`AsyncJob.state`."""
-        return self._inner.state if self._inner is not None else "finished"
-
-    @property
-    def job_type(self) -> Optional[str]:
-        """The job's type. See :attr:`AsyncJob.job_type`."""
-        return self._field("job_type")
-
-    @property
-    def creation_ms(self) -> Optional[int]:
-        """When the job was created. See :attr:`AsyncJob.creation_ms`."""
-        return self._field("creation_ms")
-
-    @property
-    def spec(self) -> Optional[Any]:
-        """The job's specification. See :attr:`AsyncJob.spec`."""
-        return self._field("spec")
-
-    @property
-    def result(self) -> Optional[Any]:
-        """The job's terminal result. See :attr:`AsyncJob.result`."""
-        return self._field("result")
-
-    @property
-    def failure(self) -> Optional[JobFailureInfo]:
-        """Why the job failed. See :attr:`AsyncJob.failure`."""
-        return self._field("failure")
-
-    @property
-    def _spec_json(self) -> Optional[str]:
-        return self._field("_spec_json")
-
-    @property
-    def _result_json(self) -> Optional[str]:
-        return self._field("_result_json")
-
-    def _field(self, name: str) -> Optional[Any]:
-        return getattr(self._inner, name) if self._inner is not None else None
-
-    def events(
-        self,
-        *,
-        limit: Optional[int] = None,
-        filter: Optional[str] = None,
-    ) -> "pa.Table":
-        """This job's recorded lifecycle events.
-
-        See :meth:`AsyncJob.events`.
-        """
-        if self._inner is None:
-            raise NotImplementedError(
-                "job event history is only available for server-side jobs"
-            )
-        return LOOP.run(self._inner.events(limit=limit, filter=filter))
-
-    def __repr__(self) -> str:
-        return _job_repr("Job", self)
-
 
 def _typed_job(
     inner: "_lancedb.Job", result_decoder: Callable[[str], T]

python/python/lancedb/materialized_view.py

@@ -42,8 +42,6 @@ class MaterializedViewDefinition:
     """Cap on the number of rows the view holds."""
     inputs: List[str] = field(default_factory=list)
     """Source columns the projections and filter read."""
-    source_namespace: List[str] = field(default_factory=list)
-    """Namespace holding the source table; empty is the root namespace."""
 
 
 def _definition_from_schema(
@@ -55,8 +53,7 @@ def _definition_from_schema(
         raise ValueError(f"Table '{name}' is not a materialized view")
     value = json.loads(raw)
     kind = value.get("kind")
-    # "namespaced_select" keeps older readers from resolving the source at root.
-    if kind not in ("select", "namespaced_select"):
+    if kind != "select":
         raise NotImplementedError(
             f"materialized view '{name}' is defined by '{kind}', which this "
             "version of lancedb cannot refresh"
@@ -69,7 +66,6 @@ def _definition_from_schema(
         filter=value.get("filter"),
         limit=value.get("limit"),
         inputs=value.get("inputs", []),
-        source_namespace=value.get("source_namespace", []),
     )
 
 

python/python/lancedb/namespace.py

@@ -12,7 +12,6 @@ from __future__ import annotations
 
 import sys
 from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
-from uuid import UUID
 
 if sys.version_info >= (3, 12):
     from typing import override
@@ -49,11 +48,8 @@ from lancedb._lancedb import (
     connect_namespace_client as _connect_namespace_client,
 )
 from lancedb.background_loop import LOOP
-from lancedb.arrow import AsyncRecordBatchReader
 from lancedb.db import AsyncConnection, DBConnection
 from lancedb.job import AsyncJob, Job
-from lancedb.sql import AsyncQuery as AsyncSqlQuery
-from lancedb.sql import QueryDescription
 from lance_namespace import (
     LanceNamespace,
     connect as namespace_connect,
@@ -1451,37 +1447,6 @@ class AsyncLanceNamespaceDBConnection:
             namespace_path=namespace_path, page_token=page_token, limit=limit
         )
 
-    async def execute_query(
-        self,
-        query: str,
-        *,
-        default_namespace_path: Optional[List[str]] = None,
-    ) -> AsyncRecordBatchReader:
-        """Execute SQL when supported by the underlying connection."""
-        return await self._inner.execute_query(
-            query,
-            default_namespace_path=default_namespace_path,
-        )
-
-    async def execute_query_async(
-        self,
-        query: str,
-        *,
-        default_namespace_path: Optional[List[str]] = None,
-    ) -> AsyncSqlQuery:
-        """Start executing SQL when supported by the underlying connection.
-
-        Namespace-backed local connections do not support SQL.
-        """
-        return await self._inner.execute_query_async(
-            query,
-            default_namespace_path=default_namespace_path,
-        )
-
-    async def describe_query(self, query_id: UUID) -> QueryDescription:
-        """Describe a submitted SQL query when supported."""
-        return await self._inner.describe_query(query_id)
-
     async def namespace_client(self) -> LanceNamespace:
         """Get the namespace client for this connection.
 

python/python/lancedb/query.py

@@ -109,7 +109,6 @@ def _query_is_plain_scan(query: Query) -> bool:
     return (
         query.vector is None
         and query.full_text_query is None
-        and query.take_offsets is None
         and not query.postfilter
         and not query.order_by
     )
@@ -805,10 +804,6 @@ class Query(pydantic.BaseModel):
     # offset to start fetching results from
     offset: Optional[int] = None
 
-    # Dataset offsets whose duplicate occurrences must be restored after lookup.
-    # This is populated when a take query is converted to this serializable form.
-    take_offsets: Optional[List[int]] = None
-
     # if true, will only search the indexed data
     fast_search: Optional[bool] = None
 
@@ -830,7 +825,6 @@ class Query(pydantic.BaseModel):
         query = cls()
         query.limit = req.limit
         query.offset = req.offset
-        query.take_offsets = req.take_offsets
         query.filter = req.filter
         query.full_text_query = req.full_text_search
         query.columns = req.select

python/python/lancedb/remote/db.py

@@ -7,9 +7,8 @@ import json
 import logging
 from concurrent.futures import ThreadPoolExecutor
 import sys
-from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Mapping, Optional, Union
 from urllib.parse import urlparse
-from uuid import UUID
 import warnings
 
 if sys.version_info >= (3, 12):
@@ -26,12 +25,10 @@ from ..common import DATA
 from ..db import DBConnection, LOOP
 from ..functions import FunctionVersion, UdfDefinition
 from ..job import AsyncJob, Job
-from ..sql import Query as SqlQuery
-from ..sql import QueryDescription
 from ..materialized_view import MaterializedView, SelectArg
 
 if TYPE_CHECKING:
-    from .._lancedb import JobInfo
+    from .._lancedb import JobDescription, JobInfo
 from ..embeddings import EmbeddingFunctionConfig
 from lance_namespace import (
     LanceNamespace,
@@ -119,7 +116,6 @@ class RemoteDBConnection(DBConnection):
         read_timeout: Optional[float] = None,
         storage_options: Optional[Dict[str, str]] = None,
         read_consistency_interval: Optional[timedelta] = None,
-        sql_host_override: Optional[str] = None,
     ):
         """Connect to a remote LanceDB database."""
         if isinstance(client_config, dict):
@@ -165,7 +161,6 @@ class RemoteDBConnection(DBConnection):
         self.api_key = api_key
         self.region = region
         self.host_override = host_override
-        self.sql_host_override = sql_host_override
         self.storage_options = storage_options
         self.db_name = parsed.netloc
 
@@ -180,7 +175,6 @@ class RemoteDBConnection(DBConnection):
                 api_key=api_key,
                 region=region,
                 host_override=host_override,
-                sql_host_override=sql_host_override,
                 client_config=client_config,
                 storage_options=storage_options,
                 read_consistency_interval=read_consistency_interval,
@@ -199,7 +193,6 @@ class RemoteDBConnection(DBConnection):
                 "api_key": self.api_key,
                 "region": self.region,
                 "host_override": self.host_override,
-                "sql_host_override": self.sql_host_override,
                 "client_config": _client_config_to_dict(self.client_config),
                 "storage_options": self.storage_options,
             }
@@ -739,33 +732,43 @@ class RemoteDBConnection(DBConnection):
         )
 
     @override
-    def open_job(self, job_id: str) -> Job:
-        """Open a server-side job by id. See
-        [DBConnection.open_job][lancedb.db.DBConnection.open_job].
+    def job(self, job_id: str) -> Job:
+        """A [Job][lancedb.job.Job] handle for a server-side job by id.
+
+        The handle is constructed without a server round trip; an unknown id
+        surfaces when the handle is used. Dropping the handle has no effect
+        on the job itself.
         """
-        return Job(LOOP.run(self._conn.open_job(job_id)))
+        return Job(self._conn.job(job_id))
 
     @override
-    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
-        return Job(LOOP.run(self._conn.create_function_async(definition)))
+    def create_function_async(
+        self,
+        definition: UdfDefinition,
+        *,
+        secrets: Optional[Mapping[str, str]] = None,
+    ) -> Job[FunctionVersion]:
+        return Job(
+            LOOP.run(self._conn.create_function_async(definition, secrets=secrets))
+        )
 
     @override
     def get_function(self, name: str, *, version: str) -> FunctionVersion:
         return LOOP.run(self._conn.get_function(name, version=version))
 
-    @override
-    def list_functions(self) -> List[FunctionVersion]:
-        return LOOP.run(self._conn.list_functions())
-
-    @override
-    def drop_function(self, name: str, *, version: str) -> bool:
-        return LOOP.run(self._conn.drop_function(name, version=version))
-
     @override
     def list_jobs(self) -> List["JobInfo"]:
         """List server-side jobs across the database's tables."""
         return LOOP.run(self._conn.list_jobs())
 
+    @override
+    def get_job(self, job_id: str) -> Optional["JobDescription"]:
+        """Describe a single server-side job by id.
+
+        Returns None when the server has no such job.
+        """
+        return LOOP.run(self._conn.get_job(job_id))
+
     @override
     def cancel_job(self, job_id: str) -> bool:
         """Request cancellation of a server-side job by id.
@@ -777,35 +780,12 @@ class RemoteDBConnection(DBConnection):
         return LOOP.run(self._conn.cancel_job(job_id))
 
     @override
-    def execute_query_async(
-        self,
-        query: str,
-        *,
-        default_namespace_path: Optional[List[str]] = None,
-    ) -> SqlQuery:
-        """Start executing SQL through this remote connection.
+    def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
+        """The lifecycle event history of a server-side job, as Arrow batches.
 
-        Unqualified tables use this connection's database and the
-        ``["public"]`` namespace by default. Fully qualified table names may
-        reference other databases available to the same deployment.
+        Lists history across all jobs when `job_id` is None.
         """
-        return SqlQuery(
-            LOOP.run(
-                self._conn.execute_query_async(
-                    query,
-                    default_namespace_path=default_namespace_path,
-                )
-            )
-        )
-
-    @override
-    def describe_query(self, query_id: UUID) -> QueryDescription:
-        """Describe a submitted SQL query by its connection-scoped id."""
-        return LOOP.run(
-            self._conn.describe_query(
-                query_id,
-            )
-        )
+        return LOOP.run(self._conn.job_history(job_id))
 
     @override
     def namespace_client(self) -> LanceNamespace:

python/python/lancedb/remote/header.py

@@ -177,7 +177,4 @@ class OAuthProvider(HeaderProvider):
         if not self._current_token:
             raise RuntimeError("Failed to obtain OAuth token")
 
-        return {
-            "Authorization": f"Bearer {self._current_token}",
-            "x-lancedb-credential-type": "oidc",
-        }
+        return {"Authorization": f"Bearer {self._current_token}"}

python/python/lancedb/remote/table.py

@@ -67,15 +67,7 @@ from ..query import (
     LanceTakeQueryBuilder,
     LanceVectorQueryBuilder,
 )
-from ..table import (
-    AsyncTable,
-    BlobMode,
-    Branches,
-    IndexStatistics,
-    Query,
-    Table,
-    Tags,
-)
+from ..table import AsyncTable, BlobMode, Branches, IndexStatistics, Query, Table, Tags
 from ..types import BaseTokenizerType
 
 
@@ -548,7 +540,6 @@ class RemoteTable(Table):
         LOOP.run(
             self._table.create_index(
                 column,
-                replace=replace,
                 config=config,
                 wait_timeout=wait_timeout,
                 name=name,

python/python/lancedb/sql.py

@@ -1,88 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-"""Handles to SQL queries running on a remote database."""
-
-from uuid import UUID
-
-import pyarrow as pa
-
-from lancedb.background_loop import LOOP
-
-from . import _lancedb
-from .arrow import AsyncRecordBatchReader
-
-QueryDescription = _lancedb.QueryDescription
-
-
-class AsyncQuery:
-    """A handle to a submitted SQL query on an asynchronous connection."""
-
-    def __init__(self, inner: "_lancedb.SqlQuery"):
-        self._inner = inner
-
-    @property
-    def id(self) -> UUID:
-        """The stable identifier scoped to the connection that submitted it."""
-        return self._inner.id
-
-    async def describe(self) -> QueryDescription:
-        """Get a point-in-time description of the query."""
-        return await self._inner.describe()
-
-    async def reader(self) -> AsyncRecordBatchReader:
-        """Wait for the initial result stream and return its Arrow reader.
-
-        Results are single-consumer. Calling this method more than once on the
-        same query raises an error. Later batches are streamed as they become
-        available without waiting for the full query to finish.
-        """
-        return AsyncRecordBatchReader(await self._inner.reader())
-
-    async def cancel(self) -> None:
-        """Request cancellation of the query."""
-        await self._inner.cancel()
-
-
-class Query:
-    """Synchronous counterpart of :class:`AsyncQuery`."""
-
-    def __init__(self, inner: AsyncQuery):
-        self._inner = inner
-
-    @property
-    def id(self) -> UUID:
-        """The stable identifier scoped to the connection that submitted it."""
-        return self._inner.id
-
-    def describe(self) -> QueryDescription:
-        """Get a point-in-time description of the query."""
-        return LOOP.run(self._inner.describe())
-
-    def reader(self) -> pa.RecordBatchReader:
-        """Wait for the initial result stream and return a blocking reader.
-
-        Results are single-consumer. Calling this method more than once on the
-        same query raises an error. Later batches block only until they become
-        available, without waiting for the full query to finish.
-        """
-        reader = LOOP.run(self._inner.reader())
-
-        def next_batch():
-            try:
-                return LOOP.run(reader.__anext__())
-            except StopAsyncIteration:
-                return None
-
-        def batches():
-            while (batch := next_batch()) is not None:
-                yield batch
-
-        return pa.RecordBatchReader.from_batches(reader.schema, batches())
-
-    def cancel(self) -> None:
-        """Request cancellation of the query."""
-        LOOP.run(self._inner.cancel())
-
-
-__all__ = ["AsyncQuery", "Query", "QueryDescription"]

python/python/lancedb/table.py

@@ -1547,9 +1547,7 @@ class Table(ABC):
         on: Union[str, Iterable[str]]
             A column (or columns) to join on.  This is how records from the
             source table and target table are matched.  Typically this is some
-            kind of key or id column.  Passing several columns matches on the
-            composite key: a source row updates a target row only when it
-            agrees on every one of them.
+            kind of key or id column.
 
         Examples
         --------
@@ -1680,9 +1678,9 @@ class Table(ABC):
         Offsets are mostly useful for sampling as the set of all valid offsets is easily
         known in advance to be [0, len(table)).
 
-        No guarantees are made regarding the order in which results are returned.
-        Repeated offsets produce repeated rows, which makes this method suitable for
-        sampling with replacement.
+        No guarantees are made regarding the order in which results are returned.  If
+        you desire an output order that matches the order of the given offsets, you will
+        need to add the row offset column to the output and align it yourself.
 
         Parameters
         ----------
@@ -1793,9 +1791,6 @@ class Table(ABC):
         The result has the same length and order as ``row_ids``. Null blobs
         produce null slots; valid empty blobs produce ``b""``.
 
-        ``_rowid`` values stay valid after compaction when the table has stable
-        row ids.
-
         Convenience for small payloads. For large values use
         :meth:`fetch_blob_files`.
         """
@@ -1813,9 +1808,6 @@ class Table(ABC):
         The result has the same length and order as ``requests``; null blobs
         produce null slots and empty ranges on non-null blobs produce ``b""``.
 
-        ``_rowid`` values stay valid after compaction when the table has stable
-        row ids.
-
         Row IDs can be obtained from a query with ``with_row_id(True)``. This
         API is currently supported only by local tables.
         """
@@ -1831,9 +1823,6 @@ class Table(ABC):
         ``_rowid`` or a ``_lance_row_id`` field on the blob descriptor. Null
         rows are ``None``. Remote tables require LanceDB Cloud server 0.5.0 or
         newer.
-
-        ``_rowid`` values stay valid after compaction when the table has stable
-        row ids.
         """
 
     @abstractmethod
@@ -2176,11 +2165,9 @@ class Table(ABC):
             Function columns are supported only on LanceDB Cloud and
             Enterprise.
         computed: Dict[str, str], optional
-            A mapping from output column names to SQL expressions derives each
-            output field from its expression. A direct projection of a Blob v2
-            field inherits Blob v2 semantics; other expressions derive their
-            ordinary Arrow type. Mapping order is declaration and dependency
-            order.
+            A map of column name to a SQL expression defining the column. The
+            column's type and inputs are derived from the expression, so no
+            data type is supplied.
 
             Unlike ``transforms``, the expression is stored rather than
             evaluated now: the column is committed with no values, and rows get
@@ -4101,7 +4088,6 @@ class LanceTable(Table):
             )
             and not self._route_pushdown_to_rust
             and self.current_branch() is None
-            and query.take_offsets is None
         ):
             from lancedb.namespace import _execute_server_side_query
 
@@ -5712,9 +5698,7 @@ class AsyncTable:
         on: Union[str, Iterable[str]]
             A column (or columns) to join on.  This is how records from the
             source table and target table are matched.  Typically this is some
-            kind of key or id column.  Passing several columns matches on the
-            composite key: a source row updates a target row only when it
-            agrees on every one of them.
+            kind of key or id column.
 
         Examples
         --------
@@ -5997,23 +5981,7 @@ class AsyncTable:
 
     def _sync_query_to_async(
         self, query: Query
-    ) -> (
-        AsyncHybridQuery
-        | AsyncFTSQuery
-        | AsyncVectorQuery
-        | AsyncQuery
-        | AsyncTakeQuery
-    ):
-        if query.take_offsets is not None:
-            take_query = self.take_offsets(query.take_offsets)
-            if query.columns:
-                take_query = take_query.select(query.columns)
-            if query.use_lsm is not None:
-                take_query = take_query.use_lsm(query.use_lsm)
-            if query.with_row_id:
-                take_query = take_query.with_row_id()
-            return take_query
-
+    ) -> AsyncHybridQuery | AsyncFTSQuery | AsyncVectorQuery | AsyncQuery:
         async_query = self.query()
         if query.limit is not None:
             async_query = async_query.limit(query.limit)
@@ -6078,7 +6046,6 @@ class AsyncTable:
                 self._namespace_client, self._pushdown_operations
             )
             and not self._route_pushdown_to_rust
-            and query.take_offsets is None
         ):
             from lancedb.namespace import _execute_server_side_query
 
@@ -6301,11 +6268,8 @@ class AsyncTable:
             Function columns are supported only on LanceDB Cloud and
             Enterprise.
         computed: Dict[str, str], optional
-            A mapping from output column names to SQL expressions derives each
-            output field from its expression. A direct projection of a Blob v2
-            field inherits Blob v2 semantics; other expressions derive their
-            ordinary Arrow type. Mapping order is declaration and dependency
-            order.
+            A map of column name to a SQL expression defining the column. The
+            column's type and inputs are derived from the expression.
 
             Unlike ``transforms``, the expression is stored rather than
             evaluated now: the column is committed with no values, and rows get
@@ -6576,9 +6540,6 @@ class AsyncTable:
         Offsets are mostly useful for sampling as the set of all valid offsets is easily
         known in advance to be [0, len(table)).
 
-        No guarantees are made regarding the order in which results are returned.
-        Repeated offsets produce repeated rows.
-
         Parameters
         ----------
         offsets: list[int]

python/python/tests/test_blob.py

@@ -66,25 +66,6 @@ def _row_ids_by_id(table):
     return dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
 
 
-def _assert_missing_blob_row_ids(exc_info):
-    message = str(exc_info.value)
-    assert "row ids" in message
-    assert "rowaddr" not in message
-    assert "fragment" not in message
-
-
-def _assert_fetch_apis_reject_missing_row_ids(table, row_ids):
-    with pytest.raises(ValueError) as exc_info:
-        table.fetch_blobs("image", row_ids)
-    _assert_missing_blob_row_ids(exc_info)
-    with pytest.raises(ValueError) as exc_info:
-        table.fetch_blob_files("image", row_ids)
-    _assert_missing_blob_row_ids(exc_info)
-    with pytest.raises(ValueError) as exc_info:
-        table.fetch_blob_ranges("image", [(row_id, 0, 1) for row_id in row_ids])
-    _assert_missing_blob_row_ids(exc_info)
-
-
 def test_blob_factory_declares_v2_field():
     field = lancedb.blob("image")
     assert isinstance(field.type, pa.ExtensionType)
@@ -710,25 +691,6 @@ def test_fetch_blobs_accepts_query_result():
     assert {blobs[i].as_py() for i in range(len(blobs))} == {b"gamma"}
 
 
-def test_fetch_blobs_after_compact_with_stable_row_ids(tmp_path):
-    db = lancedb.connect(
-        tmp_path, storage_options={"new_table_enable_stable_row_ids": "true"}
-    )
-    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
-    table = db.create_table("t", schema=schema)
-    table.add([{"id": 1, "image": b"frag-one"}])
-    table.add([{"id": 2, "image": b"frag-two"}])
-    by_id = _row_ids_by_id(table)
-    ids = [by_id[1], by_id[2]]
-
-    table.optimize()
-
-    blobs = table.fetch_blobs("image", ids)
-    assert blobs.to_pylist() == [b"frag-one", b"frag-two"]
-    ranges = table.fetch_blob_ranges("image", [(ids[0], 5, 3), (ids[1], 5, 3)])
-    assert ranges.to_pylist() == [b"one", b"two"]
-
-
 def test_fetch_blobs_preserves_null_and_empty_values():
     table = _blob_table(
         "nulls",
@@ -777,25 +739,8 @@ def test_fetch_blob_ranges_validates_requests():
     with pytest.raises(ValueError, match="offset \\+ length overflowed"):
         table.fetch_blob_ranges("image", [(row_id, 2**64 - 1, 1)])
 
-    with pytest.raises(ValueError) as exc_info:
+    with pytest.raises(ValueError, match="row IDs"):
         table.fetch_blob_ranges("image", [(2**64 - 1, 0, 1)])
-    _assert_missing_blob_row_ids(exc_info)
-
-
-def test_fetch_blob_apis_reject_missing_fragment_row_addr():
-    table = _blob_table("missing_frag", [{"id": 1, "image": b"x"}])
-    live = _row_ids_by_id(table)[1]
-    _assert_fetch_apis_reject_missing_row_ids(table, [1 << 32, live])
-
-
-def test_fetch_blob_apis_reject_deleted_row_ids():
-    table = _blob_table(
-        "deleted_rows",
-        [{"id": 1, "image": b"one"}, {"id": 2, "image": b"two"}],
-    )
-    by_id = _row_ids_by_id(table)
-    table.delete("id = 2")
-    _assert_fetch_apis_reject_missing_row_ids(table, [by_id[2], by_id[1]])
 
 
 def test_fetch_blob_ranges_empty_requests_returns_empty_array():

python/python/tests/test_first_class_function_slice1.py

@@ -37,6 +37,21 @@ def job_result(name: str) -> dict:
     return json.loads(fixture(name))["result"]
 
 
+def assert_no_secret_values(value):
+    if isinstance(value, dict):
+        for key, child in value.items():
+            assert key not in {
+                "secret_value",
+                "secret_values",
+                "resolved_secret",
+                "resolved_secrets",
+            }
+            assert_no_secret_values(child)
+    elif isinstance(value, list):
+        for child in value:
+            assert_no_secret_values(child)
+
+
 def test_public_function_values_are_in_api_reference():
     docs = Path(__file__).parents[3] / "docs" / "src" / "python" / "python.md"
     rendered = docs.read_text()
@@ -94,6 +109,7 @@ def test_function_version_identity_is_immutable_and_exact():
     version = FunctionVersion.from_json(json.dumps(value))
     assert version.name == "embed"
     assert version.version == "fv_01K3EXACT"
+    assert version.required_secrets == ("HF_TOKEN",)
 
     with pytest.raises((TypeError, ValueError)):
         version.version = "fv_changed"
@@ -276,6 +292,15 @@ def test_refresh_result_rejects_non_u64_values(field):
         RefreshColumnResult.from_json(json.dumps(value))
 
 
+def test_canonical_client_values_contain_secret_names_only():
+    version = FunctionVersion.from_json(
+        json.dumps(job_result("remote_function_job.json"))
+    )
+    canonical = json.loads(version.to_canonical_json())
+    assert canonical["required_secrets"] == ["HF_TOKEN"]
+    assert_no_secret_values(canonical)
+
+
 class _FunctionDeclarationInner:
     def __init__(self):
         self.calls = []

python/python/tests/test_header_provider.py

@@ -54,10 +54,7 @@ class TestOAuthProvider:
         provider = OAuthProvider(fetcher)
         headers = provider.get_headers()
 
-        assert headers == {
-            "Authorization": "Bearer token123",
-            "x-lancedb-credential-type": "oidc",
-        }
+        assert headers == {"Authorization": "Bearer token123"}
         assert provider._current_token == "token123"
         assert provider._token_expires_at is not None
 
@@ -76,20 +73,14 @@ class TestOAuthProvider:
 
         # First call
         headers1 = provider.get_headers()
-        assert headers1 == {
-            "Authorization": "Bearer token1",
-            "x-lancedb-credential-type": "oidc",
-        }
+        assert headers1 == {"Authorization": "Bearer token1"}
 
         # Wait for token to expire
         time.sleep(1.1)
 
         # Second call should refresh
         headers2 = provider.get_headers()
-        assert headers2 == {
-            "Authorization": "Bearer token2",
-            "x-lancedb-credential-type": "oidc",
-        }
+        assert headers2 == {"Authorization": "Bearer token2"}
         assert call_count == 2
 
     def test_no_expiry_info(self):
@@ -101,18 +92,12 @@ class TestOAuthProvider:
         provider = OAuthProvider(fetcher)
         headers = provider.get_headers()
 
-        assert headers == {
-            "Authorization": "Bearer permanent_token",
-            "x-lancedb-credential-type": "oidc",
-        }
+        assert headers == {"Authorization": "Bearer permanent_token"}
         assert provider._token_expires_at is None
 
         # Should not refresh on second call
         headers2 = provider.get_headers()
-        assert headers2 == {
-            "Authorization": "Bearer permanent_token",
-            "x-lancedb-credential-type": "oidc",
-        }
+        assert headers2 == {"Authorization": "Bearer permanent_token"}
 
     def test_missing_access_token(self):
         """Test error handling when access_token is missing."""
@@ -136,10 +121,7 @@ class TestOAuthProvider:
         provider = OAuthProvider(fetcher)
         headers = provider.get_headers()
 
-        assert headers == {
-            "Authorization": "Bearer sync_token",
-            "x-lancedb-credential-type": "oidc",
-        }
+        assert headers == {"Authorization": "Bearer sync_token"}
 
 
 class TestClientConfigIntegration:

python/python/tests/test_materialized_views.py

@@ -266,38 +266,3 @@ async def test_async_namespace_connection_materialized_views(tmp_path):
             handle._route_pushdown_to_rust == through_namespace._route_pushdown_to_rust
         )
         assert handle._namespace_path == through_namespace._namespace_path
-
-
-def test_namespaced_select_kind_is_read_and_unknown_kinds_are_refused():
-    import json
-
-    import pyarrow as pa
-
-    from lancedb.materialized_view import _definition_from_schema
-
-    def schema_with(definition: dict) -> pa.Schema:
-        return pa.schema([pa.field("id", pa.int32())]).with_metadata(
-            {b"mv.definition": json.dumps(definition).encode()}
-        )
-
-    # "namespaced_select" is the namespaced form of "select": same shape,
-    # a separate kind so readers that predate it refuse instead of
-    # resolving the source at the root.
-    definition = _definition_from_schema(
-        schema_with(
-            {
-                "kind": "namespaced_select",
-                "source_table": "people",
-                "source_namespace": ["ns"],
-                "projections": [{"output": "name", "expression": "name"}],
-            }
-        ),
-        "v",
-    )
-    assert definition.source_table == "people"
-    assert definition.source_namespace == ["ns"]
-
-    with pytest.raises(NotImplementedError, match="cannot refresh"):
-        _definition_from_schema(
-            schema_with({"kind": "select_v3", "source_table": "people"}), "v"
-        )

python/python/tests/test_query.py

@@ -1923,21 +1923,6 @@ def test_take_queries(tmp_path):
         17,
     ]
 
-    # Duplicate offsets are occurrences, not set members. Ordering is unspecified.
-    assert sorted(table.take_offsets([5, 2, 5, 17]).to_pandas()["idx"].to_list()) == [
-        2,
-        5,
-        5,
-        17,
-    ]
-
-    # Converting a take builder to its serializable query representation must
-    # retain occurrence metadata and execute with the same multiplicity.
-    query = table.take_offsets([5, 2, 5, 17]).select(["idx"]).to_query_object()
-    assert query.take_offsets == [5, 2, 5, 17]
-    converted = table._execute_query(query).read_all()
-    assert sorted(converted["idx"].to_pylist()) == [2, 5, 5, 17]
-
     # Take by row id
     assert list(
         sorted(table.take_row_ids([5, 2, 17]).to_pandas()["idx"].to_list())

python/python/tests/test_remote_db.py

@@ -479,49 +479,24 @@ def test_remote_permutation_is_picklable():
                 match = re.search(
                     r"_rowoffset\s+in\s+\((.*?)\)", body["filter"], re.IGNORECASE
                 )
-                offsets = list(
-                    dict.fromkeys(int(o.strip()) for o in match.group(1).split(","))
-                )
+                offsets = [int(o.strip()) for o in match.group(1).split(",")]
             else:
                 offsets = list(range(len(rows)))
-            columns = body.get("columns") or ["a"]
-            table = pa.table(
-                {
-                    column: (
-                        [rows[offset] for offset in offsets]
-                        if column == "a"
-                        else offsets
-                    )
-                    for column in columns
-                }
-            )
+            table = pa.table({"a": [rows[offset] for offset in offsets]})
 
             request.send_response(200)
             request.send_header("Content-Type", "application/vnd.apache.arrow.file")
             request.end_headers()
             with pa.ipc.new_file(request.wfile, schema=table.schema) as writer:
-                writer.write_table(table, max_chunksize=2)
+                writer.write_table(table)
         else:
             request.send_response(404)
             request.end_headers()
 
     with mock_lancedb_connection(handler) as db:
-        table = db.open_table("test")
-        assert table.take_offsets([0, 2, 0, 4]).to_list() == [
-            {"a": 0},
-            {"a": 0},
-            {"a": 2},
-            {"a": 4},
-        ]
-
-        permutation = Permutation.identity(table)
+        permutation = Permutation.identity(db.open_table("test"))
         restored = pickle.loads(pickle.dumps(permutation))
-        assert restored.__getitems__([0, 2, 0, 4]) == [
-            {"a": 0},
-            {"a": 2},
-            {"a": 0},
-            {"a": 4},
-        ]
+        assert restored.__getitems__([0, 2, 4]) == [{"a": 0}, {"a": 2}, {"a": 4}]
 
 
 def test_create_table_exist_ok():
@@ -820,13 +795,11 @@ def test_table_create_indices():
         scalar_req = received_requests[0]
         assert "name" in scalar_req
         assert scalar_req["name"] == "custom_scalar_idx"
-        assert scalar_req["replace"] is False
 
         # Check FTS index request has custom name
         fts_req = received_requests[1]
         assert "name" in fts_req
         assert fts_req["name"] == "custom_fts_idx"
-        assert fts_req["replace"] is False
         assert fts_req["block_size"] == 256
         assert fts_req["custom_stop_words"] == ["cloud"]
 
@@ -834,7 +807,6 @@ def test_table_create_indices():
         vector_req = received_requests[2]
         assert "name" in vector_req
         assert vector_req["name"] == "custom_vector_idx"
-        assert "replace" not in vector_req
 
         table.wait_for_index(["custom_scalar_idx"], timedelta(seconds=2))
         table.wait_for_index(
@@ -1107,9 +1079,6 @@ def test_remote_create_index_new_api():
             table.create_index("text", config=FTS(block_size=256))
             # IvfRq via new API
             table.create_index("vector", config=IvfRq(distance_type="l2"))
-            table.create_index(
-                "vector", config=IvfPq(distance_type="l2"), replace=False
-            )
 
         # Legacy index_type="IVF_RQ" routes to IvfRq config under the hood.
         with pytest.warns(DeprecationWarning, match="create_index"):
@@ -1119,17 +1088,15 @@ def test_remote_create_index_new_api():
                 num_partitions=8,
             )
 
-        assert len(received_requests) == 6
+        assert len(received_requests) == 5
         assert [req["column"] for req in received_requests] == [
             "vector",
             "category",
             "text",
             "vector",
             "vector",
-            "vector",
         ]
         assert received_requests[2]["block_size"] == 256
-        assert received_requests[4]["replace"] is False
 
 
 def test_table_wait_for_index_timeout():
@@ -2467,7 +2434,7 @@ def test_remote_blob_byte_apis_not_supported_on_old_server():
 
 
 def test_remote_connection_jobs_surface():
-    from lancedb.exceptions import JobFailedError, JobNotFoundError
+    from lancedb.exceptions import JobFailedError
 
     schema = pa.schema([("state", pa.string())])
     batch = pa.record_batch([pa.array(["created", "done"])], schema=schema)
@@ -2475,7 +2442,6 @@ def test_remote_connection_jobs_surface():
     with pa.ipc.new_stream(sink, schema) as writer:
         writer.write_batch(batch)
     events_body = sink.getvalue().to_pybytes()
-    query_events_payloads = []
 
     def handler(request):
         content_len = int(request.headers.get("Content-Length", 0))
@@ -2513,22 +2479,6 @@ def test_remote_connection_jobs_surface():
             request.end_headers()
             request.wfile.write(json.dumps(rsp).encode())
         elif request.path == "/v1/jobs/describe":
-            if payload["job_id"] == "job-2":
-                request.send_response(200)
-                request.send_header("Content-Type", "application/json")
-                request.end_headers()
-                request.wfile.write(
-                    json.dumps(
-                        dict(
-                            job_id="job-2",
-                            job_type="refresh_column",
-                            job_state="DONE",
-                            creation_ms=2000,
-                            result=dict(rows_assigned=1000000, rows_failed=0),
-                        )
-                    ).encode()
-                )
-                return
             if payload["job_id"] != "job-1":
                 request.send_response(404)
                 request.end_headers()
@@ -2560,7 +2510,7 @@ def test_remote_connection_jobs_surface():
             request.end_headers()
             request.wfile.write(b'{"job_id": "job-1"}')
         elif request.path == "/v1/jobs/query_events":
-            query_events_payloads.append(payload)
+            assert payload["job_id"] == "job-1"
             request.send_response(200)
             request.send_header("Content-Type", "application/vnd.apache.arrow.stream")
             request.end_headers()
@@ -2576,109 +2526,24 @@ def test_remote_connection_jobs_surface():
         assert jobs[0].table == "t1"
         assert jobs[1].state == "finished"
 
+        description = db.get_job("job-1")
+        assert description.job_type == "create_index"
+        assert description.state == "failed"
+        assert json.loads(description.spec_json) == {"column": "vec"}
+        assert description.failure.message == "worker died"
+        assert description.failure.retryable is True
+        assert db.get_job("missing") is None
+
         assert db.cancel_job("job-1") is True
         assert db.cancel_job("missing") is False
 
-        # Opening a job hands back a populated handle; a missing one fails.
-        with pytest.raises(JobNotFoundError, match="missing"):
-            db.open_job("missing")
-        finished = db.open_job("job-2")
-        assert finished.state == "finished"
-        assert finished.result == {"rows_assigned": 1000000, "rows_failed": 0}
+        batches = db.job_history("job-1")
+        assert len(batches) == 1
+        assert batches[0].num_rows == 2
+        assert batches[0].column("state").to_pylist() == ["created", "done"]
 
-        job = db.open_job("job-1")
+        job = db.job("job-1")
         assert job.id == "job-1"
-        # Opening already populated the handle.
-        assert job.state == "failed"
-        assert job.spec == {"column": "vec"}
-        assert job.failure.message == "worker died"
         assert job.status() == "failed"
         with pytest.raises(JobFailedError, match="worker died"):
             job.wait(timeout=timedelta(seconds=5))
-
-
-def test_remote_job_handle_reports_its_own_detail():
-    schema = pa.schema([("state", pa.string())])
-    batch = pa.record_batch([pa.array(["claim_complete"])], schema=schema)
-    sink = pa.BufferOutputStream()
-    with pa.ipc.new_stream(sink, schema) as writer:
-        writer.write_batch(batch)
-    events_body = sink.getvalue().to_pybytes()
-    event_payloads = []
-
-    def handler(request):
-        content_len = int(request.headers.get("Content-Length", 0))
-        body = request.rfile.read(content_len) if content_len > 0 else b""
-        payload = json.loads(body) if body else {}
-        if request.path == "/v1/jobs/describe":
-            request.send_response(200)
-            request.send_header("Content-Type", "application/json")
-            request.end_headers()
-            request.wfile.write(
-                json.dumps(
-                    dict(
-                        job_id="job-1",
-                        job_type="refresh_column",
-                        job_state="DONE",
-                        creation_ms=2000,
-                        spec=dict(column="vec"),
-                        result=dict(rows_assigned=1000000),
-                    )
-                ).encode()
-            )
-        elif request.path == "/v1/jobs/query_events":
-            event_payloads.append(payload)
-            request.send_response(200)
-            request.send_header("Content-Type", "application/vnd.apache.arrow.stream")
-            request.end_headers()
-            request.wfile.write(events_body)
-        else:
-            request.send_response(404)
-            request.end_headers()
-
-    with mock_lancedb_connection(handler) as db:
-        job = db.open_job("job-1")
-
-        # Opening populates the handle in the same round trip.
-        assert job.state == "finished"
-        job.refresh()
-        assert job.job_type == "refresh_column"
-        assert job.creation_ms == 2000
-        assert job.spec == {"column": "vec"}
-        assert job.result == {"rows_assigned": 1000000}
-        assert job.failure is None
-        # The JSON payloads stay reachable, but as internal APIs.
-        assert json.loads(job._spec_json) == {"column": "vec"}
-        assert json.loads(job._result_json) == {"rows_assigned": 1000000}
-
-        # print() shows everything the handle knows and nothing it does not.
-        # print() lays every known field out on its own line, with the JSON
-        # payloads indented rather than crammed onto one line.
-        assert repr(job) == "\n".join(
-            [
-                "Job(",
-                "    id='job-1',",
-                "    state='finished',",
-                "    job_type='refresh_column',",
-                "    creation_ms=2000,",
-                "    spec={",
-                '        "column": "vec"',
-                "    },",
-                "    result={",
-                '        "rows_assigned": 1000000',
-                "    },",
-                ")",
-            ]
-        )
-        # Nothing it does not know shows up.
-        assert "failure" not in repr(job)
-
-        events = job.events(filter="state = 'claim_complete'", limit=500)
-        assert isinstance(events, pa.Table)
-        assert events.column("state").to_pylist() == ["claim_complete"]
-        # The handle supplies job_id; the caller only narrows the query.
-        assert event_payloads[-1] == {
-            "job_id": "job-1",
-            "limit": 500,
-            "filter": "state = 'claim_complete'",
-        }

python/python/tests/test_sql.py

@@ -1,162 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-from uuid import UUID
-
-import pytest
-import pyarrow as pa
-
-import lancedb
-from lancedb import _lancedb
-from lancedb.arrow import AsyncRecordBatchReader
-from lancedb.db import AsyncConnection
-from lancedb.remote.db import RemoteDBConnection
-from lancedb.sql import AsyncQuery, Query
-
-NIL_QUERY_ID = UUID(int=0)
-
-
-class FakeNativeQuery:
-    id = UUID("0198f1b2-c3d4-7e5f-8123-456789abcdef")
-
-    async def reader(self):
-        return pa.table({"value": [1, 2]})
-
-
-class FakeNativeConnection:
-    async def execute_query_async(self, query, *, default_namespace_path=None):
-        return FakeNativeQuery()
-
-
-class FakeAsyncConnection:
-    async def execute_query_async(self, query, *, default_namespace_path=None):
-        return AsyncQuery(FakeNativeQuery())
-
-
-def remote_connection(sql_host_override=None):
-    return lancedb.connect(
-        "db://analytics",
-        api_key="test-key",
-        host_override="http://localhost:10024",
-        sql_host_override=sql_host_override,
-    )
-
-
-def test_sql_is_connection_scoped():
-    assert hasattr(lancedb, "sql")
-    assert not callable(lancedb.sql)
-    assert not hasattr(_lancedb, "sql")
-    assert not hasattr(remote_connection(), "sql")
-    assert hasattr(remote_connection(), "execute_query")
-    assert hasattr(remote_connection(), "execute_query_async")
-    assert hasattr(remote_connection(), "describe_query")
-
-
-def test_query_id_is_uuid():
-    query = AsyncQuery(FakeNativeQuery())
-    assert isinstance(query.id, UUID)
-    assert Query(query).id == query.id
-
-
-def test_connection_serializes_sql_host_override():
-    endpoint = "grpc+tls://sql.example.com:10026"
-    restored = lancedb.deserialize_conn(
-        remote_connection(sql_host_override=endpoint).serialize()
-    )
-    assert restored.sql_host_override == endpoint
-
-
-@pytest.mark.asyncio
-async def test_async_sql_reader_is_record_batch_stream():
-    reader = await AsyncQuery(FakeNativeQuery()).reader()
-    assert isinstance(reader, AsyncRecordBatchReader)
-    assert (await reader.read_all())[0].column(0).to_pylist() == [1, 2]
-
-
-def test_sync_sql_reader_is_record_batch_reader():
-    reader = Query(AsyncQuery(FakeNativeQuery())).reader()
-    assert isinstance(reader, pa.RecordBatchReader)
-    assert reader.read_all().column(0).to_pylist() == [1, 2]
-
-
-def test_execute_query_returns_blocking_reader():
-    connection = RemoteDBConnection.__new__(RemoteDBConnection)
-    connection._conn = FakeAsyncConnection()
-    reader = connection.execute_query("SELECT 1")
-    assert isinstance(reader, pa.RecordBatchReader)
-    assert reader.read_all().column(0).to_pylist() == [1, 2]
-
-
-@pytest.mark.asyncio
-async def test_async_execute_query_returns_async_reader():
-    connection = AsyncConnection(FakeNativeConnection())
-    reader = await connection.execute_query("SELECT 1")
-    assert isinstance(reader, AsyncRecordBatchReader)
-    assert (await reader.read_all())[0].column(0).to_pylist() == [1, 2]
-
-
-def test_local_connection_rejects_sql(tmp_path):
-    connection = lancedb.connect(tmp_path)
-    with pytest.raises(NotImplementedError, match="SQL"):
-        connection.execute_query("SELECT 1")
-    with pytest.raises(NotImplementedError, match="SQL"):
-        connection.execute_query_async("SELECT 1")
-    with pytest.raises(NotImplementedError, match="SQL"):
-        connection.describe_query(NIL_QUERY_ID)
-
-
-@pytest.mark.asyncio
-async def test_local_async_connection_rejects_sql(tmp_path):
-    connection = await lancedb.connect_async(tmp_path)
-    with pytest.raises(NotImplementedError, match="SQL"):
-        await connection.execute_query("SELECT 1")
-    with pytest.raises(NotImplementedError, match="SQL"):
-        await connection.execute_query_async("SELECT 1")
-    with pytest.raises(NotImplementedError, match="SQL"):
-        await connection.describe_query(NIL_QUERY_ID)
-
-
-@pytest.mark.asyncio
-async def test_async_namespace_connection_rejects_sql(tmp_path):
-    connection = lancedb.connect_namespace_async("dir", {"root": str(tmp_path)})
-    with pytest.raises(NotImplementedError, match="SQL"):
-        await connection.execute_query("SELECT 1")
-    with pytest.raises(NotImplementedError, match="SQL"):
-        await connection.execute_query_async("SELECT 1")
-    with pytest.raises(NotImplementedError, match="SQL"):
-        await connection.describe_query(NIL_QUERY_ID)
-
-
-def test_describe_query_requires_uuid():
-    with pytest.raises(TypeError, match="UUID"):
-        remote_connection().describe_query(str(NIL_QUERY_ID))
-
-
-@pytest.mark.parametrize(
-    "default_namespace_path",
-    ["public", ("public",), [1]],
-)
-def test_execute_query_async_requires_namespace_path_list(default_namespace_path):
-    with pytest.raises(ValueError, match="default_namespace_path"):
-        remote_connection().execute_query_async(
-            "SELECT 1", default_namespace_path=default_namespace_path
-        )
-
-
-def test_execute_query_async_rejects_invalid_endpoint():
-    connection = remote_connection(sql_host_override="invalid://localhost")
-    with pytest.raises(ValueError, match="sql_host_override"):
-        connection.execute_query_async("SELECT 1")
-
-
-@pytest.mark.parametrize(
-    "default_namespace_path",
-    [[""], ["café"], ["pub\tlic"], ["events$raw"]],
-)
-def test_execute_query_async_rejects_invalid_namespace_components(
-    default_namespace_path,
-):
-    with pytest.raises(ValueError, match="default_namespace_path"):
-        remote_connection().execute_query_async(
-            "SELECT 1", default_namespace_path=default_namespace_path
-        )

python/python/tests/test_table.py

@@ -2682,43 +2682,6 @@ def test_merge_insert(mem_db: DBConnection):
         )
 
 
-def test_merge_insert_composite_key(mem_db: DBConnection):
-    table = mem_db.create_table(
-        "my_table",
-        data=pa.table(
-            {
-                "shard": ["a", "a", "b"],
-                "id": [1, 2, 1],
-                "val": ["x", "y", "z"],
-            }
-        ),
-    )
-
-    # ("a", 1) matches an existing row and updates it. ("b", 2) agrees with an
-    # existing row on each key column separately but on neither pair, so it is
-    # an insert.
-    new_data = pa.table({"shard": ["a", "b"], "id": [1, 2], "val": ["X", "W"]})
-    res = (
-        table.merge_insert(["shard", "id"])
-        .when_matched_update_all()
-        .when_not_matched_insert_all()
-        .execute(new_data)
-    )
-    assert res.num_updated_rows == 1
-    assert res.num_inserted_rows == 1
-
-    expected = pa.table(
-        {
-            "shard": ["a", "a", "b", "b"],
-            "id": [1, 2, 1, 2],
-            "val": ["X", "y", "z", "W"],
-        }
-    )
-    assert table.to_arrow().sort_by([("shard", "ascending"), ("id", "ascending")]) == (
-        expected
-    )
-
-
 def test_merge_insert_nullable_pandas_into_pydantic_schema(mem_db: DBConnection):
     # Regression test for https://github.com/lancedb/lancedb/issues/2366
     pd = pytest.importorskip("pandas")
@@ -4124,29 +4087,6 @@ def test_computed_column_rejects_transforms_and_computed_together(tmp_path):
         table.add_columns({"a": "x + 1"}, computed={"b": "x * 2"})
 
 
-def test_computed_column_blob_projection_inherits_semantics(tmp_path):
-    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
-    db = lancedb.connect(tmp_path)
-    table = db.create_table("computed_column_blob", schema=schema)
-    table.add(
-        [
-            {"id": 1, "image": b"hello"},
-            {"id": 2, "image": b""},
-            {"id": 3, "image": None},
-        ]
-    )
-
-    table.add_columns(computed={"image_copy": "image", "second_copy": "image_copy"})
-    assert table.refresh_column("image_copy").rows_filled == 2
-    assert table.refresh_column("second_copy").rows_filled == 2
-    assert table.blob_columns() == ["image", "image_copy", "second_copy"]
-
-    hits = table.search().with_row_id(True).limit(10).to_arrow()
-    rows = sorted(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
-    copied = table.fetch_blobs("second_copy", [row_id for _, row_id in rows])
-    assert copied.to_pylist() == [b"hello", b"", None]
-
-
 @pytest.mark.asyncio
 async def test_computed_column_async(tmp_path):
     db = await lancedb.connect_async(tmp_path)

python/src/connection.rs

@@ -13,7 +13,11 @@ use crate::{
     runtime::future_into_py,
     table::Table,
 };
-use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::FromPyArrow};
+use arrow::{
+    datatypes::Schema,
+    ffi_stream::ArrowArrayStreamReader,
+    pyarrow::{FromPyArrow, ToPyArrow},
+};
 use lancedb::{
     connection::Connection as LanceConnection,
     connection::NamespaceClientPushdownOperation,
@@ -24,7 +28,7 @@ use pyo3::{
     Bound, FromPyObject, Py, PyAny, PyRef, PyResult, Python,
     exceptions::{PyRuntimeError, PyValueError},
     pyclass, pyfunction, pymethods,
-    types::{PyAnyMethods, PyDict, PyDictMethods, PyList},
+    types::{PyDict, PyDictMethods, PyList, PyListMethods},
 };
 
 #[pyclass]
@@ -82,24 +86,6 @@ impl Connection {
     }
 }
 
-fn parse_default_namespace_path(path: Option<Bound<'_, PyAny>>) -> PyResult<Vec<String>> {
-    match path {
-        Some(path) => {
-            if !path.is_instance_of::<PyList>() {
-                return Err(PyValueError::new_err(
-                    "Connection.execute_query_async default_namespace_path must be a list",
-                ));
-            }
-            path.extract::<Vec<String>>().map_err(|_| {
-                PyValueError::new_err(
-                    "Connection.execute_query_async default_namespace_path components must be strings",
-                )
-            })
-        }
-        None => Ok(vec!["public".to_string()]),
-    }
-}
-
 #[pymethods]
 impl Connection {
     fn __repr__(&self) -> String {
@@ -122,40 +108,6 @@ impl Connection {
         self.get_inner().map(|inner| inner.uri().to_string())
     }
 
-    #[pyo3(signature = (query, *, default_namespace_path=None))]
-    pub fn execute_query_async<'a>(
-        self_: PyRef<'a, Self>,
-        query: String,
-        default_namespace_path: Option<Bound<'_, PyAny>>,
-    ) -> PyResult<Bound<'a, PyAny>> {
-        let inner = self_.get_inner()?.clone();
-        let default_namespace_path = parse_default_namespace_path(default_namespace_path)?;
-        future_into_py(self_.py(), async move {
-            let operation = inner
-                .execute_query_async(query)
-                .default_namespace_path(default_namespace_path);
-            operation
-                .execute()
-                .await
-                .map(crate::sql::Query::new)
-                .infer_error()
-        })
-    }
-
-    pub fn describe_query<'a>(
-        self_: PyRef<'a, Self>,
-        query_id: uuid::Uuid,
-    ) -> PyResult<Bound<'a, PyAny>> {
-        let inner = self_.get_inner()?.clone();
-        future_into_py(self_.py(), async move {
-            inner
-                .describe_query(query_id)
-                .await
-                .map(crate::sql::QueryDescription::from)
-                .infer_error()
-        })
-    }
-
     #[pyo3(signature = ())]
     pub fn get_read_consistency_interval(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.get_inner()?.clone();
@@ -640,12 +592,9 @@ impl Connection {
         })
     }
 
-    pub fn open_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.get_inner()?.clone();
-        future_into_py(self_.py(), async move {
-            let job = inner.open_job(&job_id).await.infer_error()?;
-            Ok(crate::job::Job::new(job))
-        })
+    pub fn job(&self, job_id: String) -> PyResult<crate::job::Job> {
+        let inner = self.get_inner()?.clone();
+        Ok(crate::job::Job::new(inner.job(job_id).infer_error()?))
     }
 
     pub fn create_function_async(
@@ -680,30 +629,6 @@ impl Connection {
         })
     }
 
-    pub fn list_functions(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.get_inner()?.clone();
-        future_into_py(self_.py(), async move {
-            inner
-                .list_functions()
-                .await
-                .infer_error()?
-                .into_iter()
-                .map(|function| function.to_canonical_json().infer_error())
-                .collect::<PyResult<Vec<_>>>()
-        })
-    }
-
-    pub fn drop_function(
-        self_: PyRef<'_, Self>,
-        name: String,
-        version: String,
-    ) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.get_inner()?.clone();
-        future_into_py(self_.py(), async move {
-            inner.drop_function(name, version).await.infer_error()
-        })
-    }
-
     pub fn list_jobs(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.get_inner()?.clone();
         future_into_py(self_.py(), async move {
@@ -715,16 +640,42 @@ impl Connection {
         })
     }
 
+    pub fn get_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        future_into_py(self_.py(), async move {
+            let description = inner.get_job(&job_id).await.infer_error()?;
+            Ok(description.map(crate::job::JobDescription::from))
+        })
+    }
+
     pub fn cancel_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.get_inner()?.clone();
         future_into_py(self_.py(), async move {
             inner.cancel_job(&job_id).await.infer_error()
         })
     }
+
+    #[pyo3(signature = (job_id=None))]
+    pub fn job_history(
+        self_: PyRef<'_, Self>,
+        job_id: Option<String>,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        future_into_py(self_.py(), async move {
+            let batches = inner.job_history(job_id.as_deref()).await.infer_error()?;
+            Python::attach(|py| {
+                let list = PyList::empty(py);
+                for batch in batches {
+                    list.append(batch.to_pyarrow(py)?)?;
+                }
+                Ok(list.unbind())
+            })
+        })
+    }
 }
 
 #[pyfunction]
-#[pyo3(signature = (uri, api_key=None, region=None, host_override=None, sql_host_override=None, read_consistency_interval=None, client_config=None, storage_options=None, session=None, manifest_enabled=false, namespace_client_properties=None, oauth_config=None))]
+#[pyo3(signature = (uri, api_key=None, region=None, host_override=None, read_consistency_interval=None, client_config=None, storage_options=None, session=None, manifest_enabled=false, namespace_client_properties=None, oauth_config=None))]
 #[allow(clippy::too_many_arguments)]
 pub fn connect(
     py: Python<'_>,
@@ -732,7 +683,6 @@ pub fn connect(
     api_key: Option<String>,
     region: Option<String>,
     host_override: Option<String>,
-    sql_host_override: Option<String>,
     read_consistency_interval: Option<f64>,
     client_config: Option<PyClientConfig>,
     storage_options: Option<HashMap<String, String>>,
@@ -752,12 +702,6 @@ pub fn connect(
         if let Some(host_override) = host_override {
             builder = builder.host_override(&host_override);
         }
-        #[cfg(feature = "remote")]
-        if let Some(sql_host_override) = sql_host_override {
-            builder = builder.sql_host_override(&sql_host_override);
-        }
-        #[cfg(not(feature = "remote"))]
-        let _ = sql_host_override;
         if let Some(read_consistency_interval) = read_consistency_interval {
             let read_consistency_interval = Duration::from_secs_f64(read_consistency_interval);
             builder = builder.read_consistency_interval(read_consistency_interval);

python/src/error.rs

@@ -114,12 +114,6 @@ impl<T> PythonErrorExt<T> for std::result::Result<T, LanceError> {
                         .getattr(intern!(py, "JobCancelledError"))?;
                     Err(PyErr::from_value(cls.call1((err.to_string(),))?))
                 }),
-                LanceError::JobNotFound { .. } => Python::attach(|py| {
-                    let cls = py
-                        .import(intern!(py, "lancedb.exceptions"))?
-                        .getattr(intern!(py, "JobNotFoundError"))?;
-                    Err(PyErr::from_value(cls.call1((err.to_string(),))?))
-                }),
                 _ => self.runtime_error(),
             },
         }

python/src/job.rs

@@ -4,50 +4,11 @@
 use std::sync::Arc;
 
 use crate::runtime::future_into_py;
-use arrow::{
-    datatypes::Schema,
-    pyarrow::{IntoPyArrow, Table as PyArrowTable},
-};
-use lancedb::job::JobEventsRequest;
-use pyo3::{
-    Bound, PyAny, PyRef, PyResult, Python,
-    exceptions::PyValueError,
-    pyclass, pymethods,
-    types::{PyAnyMethods, PyDict, PyDictMethods},
-};
+use pyo3::{Bound, PyAny, PyRef, PyResult, pyclass, pymethods};
 use serde::Serialize;
 
 use crate::error::PythonErrorExt;
 
-const REPR_INDENT: &str = "    ";
-
-/// Parse a stored JSON payload into Python data. The bindings carry these as
-/// strings because that is what crosses the boundary cheaply; the public
-/// Python surface is the parsed form.
-fn parse_json_payload<'py>(
-    py: Python<'py>,
-    raw: Option<&str>,
-) -> PyResult<Option<Bound<'py, PyAny>>> {
-    match raw {
-        None => Ok(None),
-        Some(raw) => Ok(Some(py.import("json")?.call_method1("loads", (raw,))?)),
-    }
-}
-
-/// A payload rendered as indented JSON, aligned under the field that holds it.
-fn pretty_json_payload(py: Python<'_>, raw: Option<&str>) -> PyResult<Option<String>> {
-    let Some(parsed) = parse_json_payload(py, raw)? else {
-        return Ok(None);
-    };
-    let kwargs = PyDict::new(py);
-    kwargs.set_item("indent", 4)?;
-    let rendered: String = py
-        .import("json")?
-        .call_method("dumps", (parsed,), Some(&kwargs))?
-        .extract()?;
-    Ok(Some(rendered.replace('\n', &format!("\n{REPR_INDENT}"))))
-}
-
 #[pyclass]
 pub struct Job {
     inner: Arc<lancedb::Job<std::result::Result<Option<String>, String>>>,
@@ -106,48 +67,6 @@ impl Job {
             Ok(())
         })
     }
-
-    pub fn refresh(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.inner.clone();
-        future_into_py(self_.py(), async move {
-            inner.refresh().await.infer_error()?;
-            Ok(())
-        })
-    }
-
-    /// The last observed lifecycle state, without contacting the backend.
-    #[getter]
-    pub fn _state(&self) -> Option<String> {
-        self.inner.state()
-    }
-
-    /// The last observed server-side record. `None` for an in-process job.
-    #[getter]
-    pub fn _description(&self) -> Option<JobDescription> {
-        self.inner.description().map(JobDescription::from)
-    }
-
-    #[pyo3(signature = (*, limit=None, filter=None))]
-    pub fn events(
-        self_: PyRef<'_, Self>,
-        limit: Option<u32>,
-        filter: Option<String>,
-    ) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.inner.clone();
-        let request = JobEventsRequest { limit, filter };
-        future_into_py(self_.py(), async move {
-            let batches = inner.events(request).await.infer_error()?;
-            Python::attach(|py| {
-                let schema = batches
-                    .first()
-                    .map(|batch| batch.schema())
-                    .unwrap_or_else(|| Arc::new(Schema::empty()));
-                let table = PyArrowTable::try_new(batches, schema)
-                    .map_err(|err| PyValueError::new_err(err.to_string()))?;
-                table.into_pyarrow(py).map(|table| table.unbind())
-            })
-        })
-    }
 }
 
 /// A row from `Connection.list_jobs`: one server-side job.
@@ -202,7 +121,7 @@ impl JobFailureInfo {
     }
 }
 
-/// The server-side record behind a `Job` handle.
+/// A described job from `Connection.get_job`.
 #[pyclass(get_all, skip_from_py_object)]
 #[derive(Clone)]
 pub struct JobDescription {
@@ -210,49 +129,17 @@ pub struct JobDescription {
     job_type: String,
     state: String,
     creation_ms: i64,
-    /// Internal: the wire form behind the `spec` property.
-    _spec_json: Option<String>,
-    /// Internal: the wire form behind the `result` property.
-    _result_json: Option<String>,
+    spec_json: Option<String>,
     failure: Option<JobFailureInfo>,
 }
 
 #[pymethods]
 impl JobDescription {
-    /// The job-type-specific specification it was submitted with.
-    #[getter]
-    fn spec<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyAny>>> {
-        parse_json_payload(py, self._spec_json.as_deref())
-    }
-
-    /// The job-type-specific terminal result. `None` until the job succeeds.
-    #[getter]
-    fn result<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyAny>>> {
-        parse_json_payload(py, self._result_json.as_deref())
-    }
-
-    fn __repr__(&self, py: Python<'_>) -> PyResult<String> {
-        let mut fields = vec![
-            format!("job_id={:?}", self.job_id),
-            format!("job_type={:?}", self.job_type),
-            format!("state={:?}", self.state),
-            format!("creation_ms={}", self.creation_ms),
-        ];
-        // Lay the payloads out as indented JSON, the same way the `Job` repr
-        // does, so the two agree on how the same data looks.
-        for (name, payload) in [("spec", &self._spec_json), ("result", &self._result_json)] {
-            if let Some(rendered) = pretty_json_payload(py, payload.as_deref())? {
-                fields.push(format!("{name}={rendered}"));
-            }
-        }
-        if let Some(failure) = &self.failure {
-            fields.push(format!("failure={}", failure.__repr__()));
-        }
-        let body = fields
-            .iter()
-            .map(|field| format!("\n{REPR_INDENT}{field},"))
-            .collect::<String>();
-        Ok(format!("JobDescription({body}\n)"))
+    fn __repr__(&self) -> String {
+        format!(
+            "JobDescription(job_id={:?}, job_type={:?}, state={:?}, creation_ms={})",
+            self.job_id, self.job_type, self.state, self.creation_ms
+        )
     }
 }
 
@@ -263,11 +150,7 @@ impl From<lancedb::database::JobDescription> for JobDescription {
             job_type: description.job_type,
             state: description.state,
             creation_ms: description.creation_ms,
-            _spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
-            _result_json: description
-                .result
-                .filter(|result| !result.is_null())
-                .map(|result| result.to_string()),
+            spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
             failure: description.failure.map(|failure| JobFailureInfo {
                 phase: failure.phase,
                 message: failure.message,

python/src/lib.rs

@@ -34,7 +34,6 @@ pub mod permutation;
 pub mod query;
 pub mod runtime;
 pub mod session;
-pub mod sql;
 pub mod table;
 pub mod util;
 
@@ -51,8 +50,6 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<crate::job::JobInfo>()?;
     m.add_class::<crate::job::JobDescription>()?;
     m.add_class::<crate::job::JobFailureInfo>()?;
-    m.add_class::<crate::sql::Query>()?;
-    m.add_class::<crate::sql::QueryDescription>()?;
     m.add_class::<PyBlobFile>()?;
     m.add_class::<IndexConfig>()?;
     m.add_class::<Query>()?;

python/src/query.rs

@@ -323,7 +323,6 @@ impl<'py> IntoPyObject<'py> for PyQueryVectors {
 pub struct PyQueryRequest {
     pub limit: Option<usize>,
     pub offset: Option<usize>,
-    pub take_offsets: Option<Vec<u64>>,
     pub filter: Option<PyQueryFilter>,
     pub full_text_search: Option<PyLanceDB<FtsQuery>>,
     pub select: PySelect,
@@ -354,7 +353,6 @@ impl From<AnyQuery> for PyQueryRequest {
             AnyQuery::Query(query_request) => Self {
                 limit: query_request.limit,
                 offset: query_request.offset,
-                take_offsets: query_request.take_offsets,
                 filter: query_request.filter.map(PyQueryFilter),
                 full_text_search: query_request
                     .full_text_search
@@ -383,7 +381,6 @@ impl From<AnyQuery> for PyQueryRequest {
             AnyQuery::VectorQuery(vector_query) => Self {
                 limit: vector_query.base.limit,
                 offset: vector_query.base.offset,
-                take_offsets: vector_query.base.take_offsets,
                 filter: vector_query.base.filter.map(PyQueryFilter),
                 full_text_search: None,
                 select_source_columns: PySelect::source_columns(&vector_query.base.select),

python/src/sql.rs

@@ -1,90 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-use std::sync::Arc;
-
-use chrono::{DateTime, Utc};
-use pyo3::{Bound, PyAny, PyRef, PyResult, pyclass, pymethods};
-use uuid::Uuid;
-
-use crate::arrow::RecordBatchStream;
-use crate::error::PythonErrorExt;
-use crate::runtime::future_into_py;
-
-#[pyclass(name = "SqlQuery")]
-pub struct Query {
-    inner: Arc<lancedb::sql::Query>,
-}
-
-impl Query {
-    pub(crate) fn new(inner: lancedb::sql::Query) -> Self {
-        Self {
-            inner: Arc::new(inner),
-        }
-    }
-}
-
-#[pymethods]
-impl Query {
-    #[getter]
-    pub fn id(&self) -> Uuid {
-        self.inner.id()
-    }
-
-    pub fn describe(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.inner.clone();
-        future_into_py(self_.py(), async move {
-            inner
-                .describe()
-                .await
-                .map(QueryDescription::from)
-                .infer_error()
-        })
-    }
-
-    pub fn reader(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.inner.clone();
-        future_into_py(self_.py(), async move {
-            let stream = inner.reader().await.infer_error()?;
-            Ok(RecordBatchStream::new(stream))
-        })
-    }
-
-    pub fn cancel(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.inner.clone();
-        future_into_py(self_.py(), async move {
-            inner.cancel().await.infer_error()?;
-            Ok(())
-        })
-    }
-}
-
-#[pyclass(get_all, skip_from_py_object)]
-#[derive(Clone)]
-pub struct QueryDescription {
-    id: Uuid,
-    status: String,
-    progress: Option<f64>,
-    expires_at: Option<DateTime<Utc>>,
-}
-
-#[pymethods]
-impl QueryDescription {
-    fn __repr__(&self) -> String {
-        format!(
-            "QueryDescription(id={:?}, status={:?}, progress={:?}, expires_at={:?})",
-            self.id, self.status, self.progress, self.expires_at
-        )
-    }
-}
-
-impl From<lancedb::sql::QueryDescription> for QueryDescription {
-    fn from(description: lancedb::sql::QueryDescription) -> Self {
-        Self {
-            id: description.id,
-            status: description.status.to_string(),
-            progress: description.progress,
-            expires_at: description.expires_at,
-        }
-    }
-}

rust/lancedb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.39.0-beta.5"
+version = "0.38.0-beta.11"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
@@ -21,8 +21,6 @@ arrow-select = { workspace = true }
 arrow-ord = { workspace = true }
 arrow-cast = { workspace = true }
 arrow-ipc.workspace = true
-arrow-flight = { workspace = true, optional = true }
-prost = { version = "0.14", optional = true }
 chrono = { workspace = true }
 datafusion-catalog.workspace = true
 datafusion-common.workspace = true
@@ -79,7 +77,6 @@ reqwest = { version = "0.12.0", default-features = false, features = [
     "rustls-tls-native-roots",
     "stream",
 ], optional = true }
-tonic = { workspace = true, optional = true }
 http = { version = "1", optional = true } # Matching what is in reqwest
 urlencoding = { version = "2", optional = true }
 uuid = { workspace = true, features = ["v5"] }
@@ -148,11 +145,8 @@ huggingface = [
 ]
 dynamodb = ["lance/dynamodb", "aws"]
 remote = [
-    "dep:arrow-flight",
-    "dep:prost",
     "dep:reqwest",
     "dep:http",
-    "dep:tonic",
     "dep:urlencoding",
     "lance-namespace-impls/rest",
     "lance-namespace-impls/rest-adapter",

rust/lancedb/src/blob.rs

@@ -7,9 +7,7 @@
 //! raw `Binary` / `LargeBinary` into the blob struct layout. Queries return
 //! small descriptors, not bytes.
 //!
-//! Blob tables require Lance file format >= 2.2. `_rowid` values stay valid
-//! after compaction when the table has stable row ids. Overwrite is a new
-//! create and does not keep the previous table's stable row id setting.
+//! Blob tables require Lance file format >= 2.2 and stable row ids at create.
 
 use std::ops::Range;
 use std::sync::Arc;
@@ -326,7 +324,6 @@ pub(crate) fn blob_column_names(schema: &Schema) -> Vec<String> {
 }
 
 /// Bumps storage format to at least [`LanceFileVersion::V2_2`] for blob schemas.
-/// Leaves `enable_stable_row_ids` unchanged.
 pub(crate) fn ensure_blob_storage_version(schema: &Schema, params: &mut WriteParams) {
     if !has_blob_columns(schema) {
         return;
@@ -388,30 +385,6 @@ fn ensure_all_row_ids_resolved(column: &str, requested: usize, resolved: usize)
     }
 }
 
-/// Lance take reports a missing physical row address as NotSupported or InvalidInput.
-fn map_blob_take_error(column: &str, requested: usize, err: lance::Error) -> Error {
-    let missing_row_addr = match &err {
-        lance::Error::NotSupported { source, .. } => {
-            source.to_string().contains("must not target deleted rows")
-        }
-        lance::Error::InvalidInput { source, .. } => source
-            .to_string()
-            .contains("belongs to non-existent fragment"),
-        _ => false,
-    };
-
-    if missing_row_addr {
-        Error::InvalidInput {
-            message: format!(
-                "blob read for column '{column}' requested {requested} row ids but some \
-                 do not exist in the table; pass row ids collected from this table"
-            ),
-        }
-    } else {
-        err.into()
-    }
-}
-
 /// Materialize blob-local ranges (same length and order as `requests`, nulls preserved).
 pub(crate) async fn take_blob_ranges_aligned(
     dataset: &Arc<Dataset>,
@@ -432,8 +405,7 @@ pub(crate) async fn take_blob_ranges_aligned(
         .with_row_ids(lance_requests)
         .preserve_order(true)
         .execute()
-        .await
-        .map_err(|err| map_blob_take_error(column, requests.len(), err))?;
+        .await?;
     ensure_all_row_ids_resolved(column, requests.len(), payloads.len())?;
 
     let mut builder = LargeBinaryBuilder::new();
@@ -462,8 +434,7 @@ pub(crate) async fn take_blobs_aligned(
         .with_row_ids(row_ids.to_vec())
         .preserve_order(true)
         .execute()
-        .await
-        .map_err(|err| map_blob_take_error(column, row_ids.len(), err))?;
+        .await?;
     ensure_all_row_ids_resolved(column, row_ids.len(), payloads.len())?;
 
     let mut builder = LargeBinaryBuilder::new();
@@ -487,10 +458,7 @@ pub(crate) async fn take_blob_files_aligned(
         return Ok(Vec::new());
     }
 
-    let handles = dataset
-        .take_blobs(row_ids, column)
-        .await
-        .map_err(|err| map_blob_take_error(column, row_ids.len(), err))?;
+    let handles = dataset.take_blobs(row_ids, column).await?;
     ensure_all_row_ids_resolved(column, row_ids.len(), handles.len())?;
     Ok(handles
         .into_iter()
@@ -536,21 +504,6 @@ mod tests {
             params.data_storage_version.unwrap().resolve(),
             ConcreteFileVersion::V2_2
         );
-        assert!(!params.enable_stable_row_ids);
-    }
-
-    #[test]
-    fn storage_version_leaves_stable_row_ids_enabled() {
-        let mut params = WriteParams {
-            enable_stable_row_ids: true,
-            ..Default::default()
-        };
-        ensure_blob_storage_version(&blob_schema(), &mut params);
-        assert!(params.enable_stable_row_ids);
-        assert_eq!(
-            params.data_storage_version.unwrap().resolve(),
-            ConcreteFileVersion::V2_2
-        );
     }
 
     #[test]
@@ -623,6 +576,5 @@ mod tests {
         let mut params = WriteParams::default();
         ensure_blob_storage_version(&schema, &mut params);
         assert!(params.data_storage_version.is_none());
-        assert!(!params.enable_stable_row_ids);
     }
 }

rust/lancedb/src/connection.rs

@@ -23,18 +23,15 @@ use crate::connection::create_table::CreateTableBuilder;
 use crate::data::scannable::Scannable;
 use crate::database::listing::ListingDatabase;
 use crate::database::{
-    CloneTableRequest, Database, DatabaseOptions, JobInfo, OpenTableRequest, ReadConsistency,
-    TableNamesRequest,
+    CloneTableRequest, Database, DatabaseOptions, JobDescription, JobInfo, OpenTableRequest,
+    ReadConsistency, TableNamesRequest,
 };
 use crate::embeddings::{EmbeddingRegistry, MemoryRegistry};
 use crate::error::{Error, Result};
 #[cfg(feature = "remote")]
 use crate::remote::{
     client::ClientConfig,
-    db::{
-        OPT_REMOTE_API_KEY, OPT_REMOTE_HOST_OVERRIDE, OPT_REMOTE_REGION,
-        OPT_REMOTE_SQL_HOST_OVERRIDE,
-    },
+    db::{OPT_REMOTE_API_KEY, OPT_REMOTE_HOST_OVERRIDE, OPT_REMOTE_REGION},
 };
 use lance::io::ObjectStoreParams;
 pub use lance_file::version::LanceFileVersion;
@@ -325,43 +322,6 @@ pub struct CloneTableBuilder {
     request: CloneTableRequest,
 }
 
-/// Builder for asynchronously executing a SQL statement on a remote database.
-pub struct ExecuteQueryAsyncBuilder {
-    parent: Arc<dyn Database>,
-    query: String,
-    default_namespace_path: Vec<String>,
-}
-
-impl ExecuteQueryAsyncBuilder {
-    fn new(parent: Arc<dyn Database>, query: String) -> Self {
-        Self {
-            parent,
-            query,
-            default_namespace_path: vec!["public".to_string()],
-        }
-    }
-
-    /// Set the namespace used for unqualified table names.
-    ///
-    /// An empty path is treated as `public`, which is the SQL name for the
-    /// root Lance namespace.
-    pub fn default_namespace_path<I, S>(mut self, path: I) -> Self
-    where
-        I: IntoIterator<Item = S>,
-        S: Into<String>,
-    {
-        self.default_namespace_path = path.into_iter().map(Into::into).collect();
-        self
-    }
-
-    /// Start the statement and return its asynchronous query handle.
-    pub async fn execute(self) -> Result<crate::sql::Query> {
-        self.parent
-            .execute_query_async(&self.query, &self.default_namespace_path)
-            .await
-    }
-}
-
 impl CloneTableBuilder {
     fn new(parent: Arc<dyn Database>, target_table_name: String, source_uri: String) -> Self {
         Self {
@@ -445,51 +405,6 @@ impl Connection {
         &self.internal
     }
 
-    /// Start executing SQL on a remote LanceDB database.
-    ///
-    /// The query can reference tables in other databases with SQL dot notation.
-    /// Use [`ExecuteQueryAsyncBuilder::default_namespace_path`] to avoid qualifying
-    /// tables in the default namespace. Local connections return
-    /// [`Error::NotSupported`].
-    ///
-    /// # Example
-    ///
-    /// ```no_run
-    /// # async fn query(db: &lancedb::Connection) -> lancedb::Result<()> {
-    /// use futures::TryStreamExt;
-    ///
-    /// let query = db
-    ///     .execute_query_async("SELECT * FROM events LIMIT 10")
-    ///     .default_namespace_path(["public"])
-    ///     .execute()
-    ///     .await?;
-    /// println!("query id: {}", query.id());
-    /// let mut batches = query.reader().await?;
-    /// while let Some(batch) = batches.try_next().await? {
-    ///     println!("received {} rows", batch.num_rows());
-    /// }
-    /// # Ok(())
-    /// # }
-    /// ```
-    pub fn execute_query_async(&self, query: impl Into<String>) -> ExecuteQueryAsyncBuilder {
-        ExecuteQueryAsyncBuilder::new(self.internal.clone(), query.into())
-    }
-
-    /// Describe a submitted SQL query by its connection-scoped id.
-    ///
-    /// This performs one bounded status poll using state retained by this
-    /// connection. Running state with a live query handle is not evicted;
-    /// abandoned state has bounded retention, and server expiration is
-    /// honored. Terminal state is retained briefly.
-    /// Query ids are not portable to another connection. Local connections
-    /// return [`Error::NotSupported`].
-    pub async fn describe_query(
-        &self,
-        query_id: uuid::Uuid,
-    ) -> Result<crate::sql::QueryDescription> {
-        self.internal.describe_query(query_id).await
-    }
-
     /// Get the names of all tables in the database
     ///
     /// The names will be returned in lexicographical order (ascending)
@@ -608,43 +523,6 @@ impl Connection {
             .await
     }
 
-    /// List every published immutable Function version in the remote catalog.
-    ///
-    /// Results are ordered by Function name then version. The client walks all
-    /// server pages before returning. Local databases return
-    /// [`Error::NotSupported`].
-    ///
-    /// # Example
-    ///
-    /// ```no_run
-    /// # async fn list_functions(
-    /// #     connection: &lancedb::Connection,
-    /// # ) -> Result<(), Box<dyn std::error::Error>> {
-    /// for function in connection.list_functions().await? {
-    ///     println!("{} {}", function.name(), function.version());
-    /// }
-    /// # Ok(())
-    /// # }
-    /// ```
-    pub async fn list_functions(&self) -> Result<Vec<crate::function::FunctionVersion>> {
-        self.internal.list_functions().await
-    }
-
-    /// Drop one exact immutable Function version from the remote catalog.
-    ///
-    /// Returns `true` when the server appended a Dropped transition and
-    /// `false` for an idempotent replay. Local databases return
-    /// [`Error::NotSupported`].
-    pub async fn drop_function(
-        &self,
-        name: impl AsRef<str>,
-        version: impl AsRef<str>,
-    ) -> Result<bool> {
-        self.internal
-            .drop_function(name.as_ref(), version.as_ref())
-            .await
-    }
-
     /// Rename a table in the database.
     ///
     /// This is only supported in LanceDB Cloud.
@@ -670,34 +548,14 @@ impl Connection {
         self.internal.read_consistency().await
     }
 
-    /// Open a server-side job by id, returning a handle with its record
-    /// already populated. Fails with [`crate::Error::JobNotFound`] when the
-    /// server has no such job, the way [`Connection::open_table`] does for a
-    /// missing table.
+    /// A [`crate::job::Job`] handle for a server-side job by id, suitable for
+    /// waiting on or cancelling the job.
     ///
-    /// This is the one way in: the returned [`crate::job::Job`] answers for
-    /// its own state, specification, result, failure and event history, so
-    /// there is no separate connection-level call for any of them.
-    ///
-    /// # Example
-    ///
-    /// ```no_run
-    /// # use lancedb::job::JobEventsRequest;
-    /// # async fn open_job(
-    /// #     connection: &lancedb::Connection,
-    /// #     job_id: &str,
-    /// # ) -> Result<(), Box<dyn std::error::Error>> {
-    /// let job = connection.open_job(job_id).await?;
-    /// println!("{:?} {:?}", job.state(), job.result());
-    /// let done = job
-    ///     .events(JobEventsRequest::default().filter("state = 'claim_complete'"))
-    ///     .await?;
-    /// println!("{} completions", done.iter().map(|b| b.num_rows()).sum::<usize>());
-    /// # Ok(())
-    /// # }
-    /// ```
-    pub async fn open_job(&self, job_id: impl AsRef<str>) -> Result<crate::job::Job> {
-        self.internal.open_job(job_id.as_ref()).await
+    /// The handle is constructed without a server round trip; an unknown id
+    /// surfaces when the handle is used. Only server-backed databases support
+    /// job handles by id.
+    pub fn job(&self, job_id: impl AsRef<str>) -> Result<crate::job::Job> {
+        self.internal.job(job_id.as_ref())
     }
 
     /// List server-side jobs across the database's tables.
@@ -705,12 +563,24 @@ impl Connection {
         self.internal.list_jobs().await
     }
 
+    /// Describe a single server-side job by id. `None` when the server has no
+    /// such job.
+    pub async fn get_job(&self, job_id: impl AsRef<str>) -> Result<Option<JobDescription>> {
+        self.internal.get_job(job_id.as_ref()).await
+    }
+
     /// Request cancellation of a server-side job by id. Returns true if the
     /// server accepted the cancellation, false if no such job exists.
     pub async fn cancel_job(&self, job_id: impl AsRef<str>) -> Result<bool> {
         self.internal.cancel_job(job_id.as_ref()).await
     }
 
+    /// The lifecycle event history of a server-side job (all jobs when
+    /// `job_id` is `None`), as recorded Arrow batches.
+    pub async fn job_history(&self, job_id: Option<&str>) -> Result<Vec<RecordBatch>> {
+        self.internal.job_history(job_id).await
+    }
+
     /// Drop a table in the database.
     ///
     /// # Arguments
@@ -957,19 +827,6 @@ impl ConnectBuilder {
         self
     }
 
-    /// Set the SQL service host override for a remote connection.
-    ///
-    /// The SQL client is initialized lazily when the connection first executes
-    /// SQL and is retained for the connection's lifetime.
-    #[cfg(feature = "remote")]
-    pub fn sql_host_override(mut self, sql_host_override: &str) -> Self {
-        self.request.options.insert(
-            OPT_REMOTE_SQL_HOST_OVERRIDE.to_string(),
-            sql_host_override.to_string(),
-        );
-        self
-    }
-
     /// Set the database specific options
     ///
     /// See [crate::database::listing::ListingDatabaseOptions] for the options available for
@@ -1159,7 +1016,6 @@ impl ConnectBuilder {
 
         let mut merged_options = self.request.options.clone();
         Self::apply_env_defaults(&ENV_VARS_TO_STORAGE_OPTS, &mut merged_options);
-        let sql_host_override = merged_options.get(OPT_REMOTE_SQL_HOST_OVERRIDE).cloned();
         let options = RemoteDatabaseOptions::parse_from_map(&merged_options)?;
 
         let region = options.region.ok_or_else(|| Error::InvalidInput {
@@ -1201,15 +1057,11 @@ impl ConnectBuilder {
         }
 
         let storage_options = StorageOptions(options.storage_options.clone());
-        let host_overrides = crate::remote::db::RemoteHostOverrides {
-            rest: options.host_override,
-            sql: sql_host_override,
-        };
         let internal = Arc::new(crate::remote::db::RemoteDatabase::try_new(
             &self.request.uri,
             &api_key,
             &region,
-            host_overrides,
+            options.host_override,
             client_config,
             storage_options.into(),
             self.request.read_consistency_interval,
@@ -1503,23 +1355,6 @@ mod tests {
         assert_eq!(tc.connection.uri(), tc.uri);
     }
 
-    #[tokio::test]
-    async fn test_local_connection_rejects_sql_queries() {
-        let directory = tempdir().unwrap();
-        let connection = connect(directory.path().to_str().unwrap())
-            .execute()
-            .await
-            .unwrap();
-        assert!(matches!(
-            connection.execute_query_async("SELECT 1").execute().await,
-            Err(Error::NotSupported { .. })
-        ));
-        assert!(matches!(
-            connection.describe_query(uuid::Uuid::nil()).await,
-            Err(Error::NotSupported { .. })
-        ));
-    }
-
     #[cfg(feature = "remote")]
     #[test]
     fn test_apply_env_defaults() {

rust/lancedb/src/database.rs

@@ -18,6 +18,8 @@ use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Duration;
 
+use arrow_array::RecordBatch;
+
 use lance::dataset::ReadParams;
 use lance_namespace::LanceNamespace;
 use lance_namespace::models::{
@@ -204,8 +206,8 @@ pub enum ReadConsistency {
 /// compaction, column refresh, ...).
 #[derive(Debug, Clone)]
 pub struct JobInfo {
-    /// The job id -- what [`Database::open_job`] and
-    /// [`Database::cancel_job`] accept.
+    /// The job id -- what [`Database::get_job`] and [`Database::cancel_job`]
+    /// accept.
     pub job_id: String,
     /// The table the job runs against, without URI or namespace.
     pub table: String,
@@ -216,8 +218,8 @@ pub struct JobInfo {
     pub created_at_millis: i64,
 }
 
-/// The server-side record behind a [`crate::job::Job`] handle: lifecycle
-/// state plus the job-type-specific specification and result.
+/// A described job from [`Database::get_job`]: lifecycle state plus the
+/// job-type-specific specification.
 #[derive(Debug, Clone)]
 pub struct JobDescription {
     pub job_id: String,
@@ -228,10 +230,6 @@ pub struct JobDescription {
     pub creation_ms: i64,
     /// The job-type-specific specification. Null when the server omits it.
     pub spec: serde_json::Value,
-    /// The job-type-specific terminal result, for job types that define one.
-    /// `None` until the job succeeds, so a job that never terminates reports
-    /// its progress through [`crate::job::Job::events`] instead.
-    pub result: Option<serde_json::Value>,
     /// Why the job failed, when the job is failed and the server reports a
     /// reason.
     pub failure: Option<crate::error::JobFailure>,
@@ -309,45 +307,30 @@ pub trait Database:
     ) -> Result<crate::function::FunctionVersion> {
         function_catalog_not_supported()
     }
-    /// List every published immutable Function version in the remote catalog.
-    async fn list_functions(&self) -> Result<Vec<crate::function::FunctionVersion>> {
-        function_catalog_not_supported()
-    }
-    /// Drop one exact immutable Function version from the remote catalog.
-    async fn drop_function(&self, _name: &str, _version: &str) -> Result<bool> {
-        function_catalog_not_supported()
-    }
-    /// Open a job by id, returning a handle with its record already
-    /// populated. Fails with [`crate::Error::JobNotFound`] when the server has
-    /// no such job.
-    async fn open_job(&self, _job_id: &str) -> Result<crate::job::Job> {
-        job_op_not_supported("open_job")
+    /// A [`crate::job::Job`] handle for a server-side job by id, suitable for
+    /// waiting on or cancelling the job. The handle is constructed without a
+    /// server round trip; an unknown id surfaces when the handle is used.
+    fn job(&self, _job_id: &str) -> Result<crate::job::Job> {
+        job_op_not_supported("job")
     }
     /// List server-side jobs across the database's tables.
     async fn list_jobs(&self) -> Result<Vec<JobInfo>> {
         job_op_not_supported("list_jobs")
     }
+    /// Describe a single job by id. `None` when the server has no such job.
+    async fn get_job(&self, _job_id: &str) -> Result<Option<JobDescription>> {
+        job_op_not_supported("get_job")
+    }
     /// Request cancellation of a job by id. Returns true if the server
     /// accepted the cancellation, false if no such job exists. Cancelling an
     /// already-terminal job is a no-op success.
     async fn cancel_job(&self, _job_id: &str) -> Result<bool> {
         job_op_not_supported("cancel_job")
     }
-    /// Start executing a SQL statement on a remote database.
-    async fn execute_query_async(
-        &self,
-        _query: &str,
-        _default_namespace_path: &[String],
-    ) -> Result<crate::sql::Query> {
-        Err(crate::error::Error::NotSupported {
-            message: "SQL is not supported by this database".to_string(),
-        })
-    }
-    /// Describe a submitted SQL query by its connection-scoped id.
-    async fn describe_query(&self, _query_id: uuid::Uuid) -> Result<crate::sql::QueryDescription> {
-        Err(crate::error::Error::NotSupported {
-            message: "SQL is not supported by this database".to_string(),
-        })
+    /// The lifecycle event history of a job (all jobs when `job_id` is
+    /// `None`), as recorded Arrow batches.
+    async fn job_history(&self, _job_id: Option<&str>) -> Result<Vec<RecordBatch>> {
+        job_op_not_supported("job_history")
     }
     /// Open a table in the database
     async fn open_table(&self, request: OpenTableRequest) -> Result<Arc<dyn BaseTable>>;

rust/lancedb/src/database/listing.rs

@@ -18,7 +18,7 @@ use lance_table::io::commit::commit_handler_from_url;
 use object_store::local::LocalFileSystem;
 use snafu::ResultExt;
 
-use crate::blob::ensure_blob_storage_version;
+use crate::blob::{ensure_blob_storage_version, has_blob_columns};
 use crate::connection::ConnectRequest;
 use crate::database::ReadConsistency;
 use crate::database::namespace::LanceNamespaceDatabase;
@@ -827,6 +827,7 @@ impl ListingDatabase {
         if let Some(enable_stable_row_ids) = overrides
             .enable_stable_row_ids
             .or(self.new_table_config.enable_stable_row_ids)
+            .or(has_blob_columns(&data_schema).then_some(true))
         {
             write_params.enable_stable_row_ids = enable_stable_row_ids;
         }

rust/lancedb/src/database/namespace.rs

@@ -3,7 +3,6 @@
 
 //! Namespace-based database implementation that delegates table management to lance-namespace
 
-use lance_datafusion::utils::StreamingWriteSource;
 use std::collections::{HashMap, HashSet};
 use std::sync::{Arc, Mutex};
 
@@ -24,7 +23,7 @@ use lance_namespace_impls::ConnectBuilder;
 use lance_table::io::commit::CommitHandler;
 use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler;
 
-use crate::blob::ensure_blob_storage_version;
+use crate::blob::{ensure_blob_storage_version, has_blob_columns};
 use crate::connection::NamespaceClientPushdownOperation;
 use crate::database::ReadConsistency;
 use crate::database::listing::{NewTableConfig, take_request_creation_overrides};
@@ -218,6 +217,7 @@ impl LanceNamespaceDatabase {
         if let Some(enable_stable_row_ids) = overrides
             .enable_stable_row_ids
             .or(self.new_table_config.enable_stable_row_ids)
+            .or(has_blob_columns(data_schema.as_ref()).then_some(true))
         {
             params.enable_stable_row_ids = enable_stable_row_ids;
         }
@@ -305,10 +305,6 @@ impl Database for LanceNamespaceDatabase {
     }
 
     async fn create_table(&self, request: DbCreateTableRequest) -> Result<Arc<dyn BaseTable>> {
-        // Refuse a bad declaration before the namespace records a table.
-        crate::table::computed_columns::ensure_declarations_are_planned(
-            &request.data.arrow_schema(),
-        )?;
         let mut table_id = request.namespace_path.clone();
         table_id.push(request.name.clone());
         let mut existing_table = None;
@@ -543,7 +539,9 @@ impl Database for LanceNamespaceDatabase {
         self.namespace
             .drop_table(drop_request)
             .await
-            .map_err(|e| map_namespace_lance_error(e, name))?;
+            .map_err(|e| Error::Runtime {
+                message: format!("Failed to drop table: {}", e),
+            })?;
 
         Ok(())
     }
@@ -1497,15 +1495,6 @@ mod tests {
             .expect("Failed to list tables");
         assert!(!table_names_after.contains(&"drop_test".to_string()));
 
-        let error = conn
-            .drop_table("drop_test", &["test_ns".into()])
-            .await
-            .expect_err("dropping a missing table should fail");
-        assert!(
-            matches!(error, Error::TableNotFound { ref name, .. } if name == "drop_test"),
-            "expected TableNotFound, got: {error:?}"
-        );
-
         // Verify: Cannot open dropped table
         let open_result = conn.open_table("drop_test").execute().await;
         assert!(open_result.is_err());

rust/lancedb/src/dataloader/permutation/reader.rs

@@ -31,7 +31,7 @@ use lance::io::RecordBatchStream;
 use lance_arrow::RecordBatchExt;
 use lance_core::ROW_ID;
 use lance_core::error::LanceOptionExt;
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
 use std::sync::Arc;
 
 /// Reads a permutation of a source table based on row IDs stored in a separate table
@@ -234,14 +234,7 @@ impl PermutationReader {
             .expect_ok()?
             .values();
 
-        let mut unique_row_ids = HashSet::with_capacity(num_rows);
-        let in_list: Vec<Expr> = row_ids
-            .iter()
-            .copied()
-            .filter(|row_id| unique_row_ids.insert(*row_id))
-            .map(lit)
-            .collect();
-        let num_unique_row_ids = unique_row_ids.len();
+        let in_list: Vec<Expr> = row_ids.iter().map(|id| lit(*id)).collect();
 
         let base_query = QueryRequest {
             filter: Some(QueryFilter::Datafusion(col(ROW_ID).in_list(in_list, false))),
@@ -254,7 +247,7 @@ impl PermutationReader {
             .query(
                 &AnyQuery::Query(base_query),
                 QueryExecutionOptions {
-                    max_batch_length: num_unique_row_ids as u32,
+                    max_batch_length: num_rows as u32,
                     ..Default::default()
                 },
             )
@@ -269,9 +262,9 @@ impl PermutationReader {
             });
         }
 
-        if batches.iter().map(|b| b.num_rows()).sum::<usize>() != num_unique_row_ids {
+        if batches.iter().map(|b| b.num_rows()).sum::<usize>() != num_rows {
             return Err(Error::InvalidInput {
-                message: "Base table returned a different number of rows than the number of unique row IDs"
+                message: "Base table returned different number of rows than the number of row IDs"
                     .to_string(),
             });
         }
@@ -511,7 +504,6 @@ impl PermutationReader {
             let table = Table::from(self.base_table.clone());
             let batches = table
                 .take_offsets(offsets.to_vec())
-                .preserve_order()
                 .select(selection.clone())
                 .execute()
                 .await?
@@ -811,10 +803,10 @@ mod tests {
         .unwrap();
 
         // Take offsets in reverse order and verify returned rows match that order
-        let offsets = vec![5, 3, 5, 1, 0];
+        let offsets = vec![5, 3, 1, 0];
         let batch = reader.take_offsets(&offsets, Select::All).await.unwrap();
 
-        assert_eq!(batch.num_rows(), 5);
+        assert_eq!(batch.num_rows(), 4);
 
         let idx_values = batch
             .column(0)
@@ -828,52 +820,6 @@ mod tests {
         assert_eq!(idx_values, expected);
     }
 
-    #[tokio::test]
-    async fn test_take_offsets_preserves_repeated_rows_in_permutation() {
-        let base_table = lance_datagen::gen_batch()
-            .col("idx", lance_datagen::array::step::<Int32Type>())
-            .into_mem_table("tbl", RowCount::from(5), BatchCount::from(1))
-            .await;
-        let base_row_ids = collect_column::<UInt64Type>(&base_table, "_rowid").await;
-        let permutation_row_ids = vec![
-            base_row_ids[3],
-            base_row_ids[1],
-            base_row_ids[3],
-            base_row_ids[2],
-        ];
-        let permutation_batch = RecordBatch::try_new(
-            Arc::new(Schema::new(vec![
-                Field::new("row_id", DataType::UInt64, false),
-                Field::new(SPLIT_ID_COLUMN, DataType::UInt64, false),
-            ])),
-            vec![
-                Arc::new(UInt64Array::from(permutation_row_ids)),
-                Arc::new(UInt64Array::from(vec![0; 4])),
-            ],
-        )
-        .unwrap();
-        let permutation_table = virtual_table("row_ids", &permutation_batch).await;
-        let reader = PermutationReader::try_from_tables(
-            base_table.base_table().clone(),
-            permutation_table.base_table().clone(),
-            0,
-        )
-        .await
-        .unwrap();
-
-        let batch = reader
-            .take_offsets(&[0, 1, 2, 3], Select::All)
-            .await
-            .unwrap();
-        let idx_values = batch
-            .column(0)
-            .as_primitive::<Int32Type>()
-            .values()
-            .to_vec();
-
-        assert_eq!(idx_values, vec![3, 1, 3, 2]);
-    }
-
     #[tokio::test]
     async fn test_take_offsets_with_column_selection() {
         let (base_table, row_ids_table, row_ids) = setup_permutation_tables(10).await;
@@ -937,17 +883,17 @@ mod tests {
             .unwrap();
 
         // With no permutation table, take_offsets uses the base table directly
-        let offsets = vec![0, 2, 0, 4, 6];
+        let offsets = vec![0, 2, 4, 6];
         let batch = reader.take_offsets(&offsets, Select::All).await.unwrap();
 
-        assert_eq!(batch.num_rows(), 5);
+        assert_eq!(batch.num_rows(), 4);
 
         let idx_values = batch
             .column(0)
             .as_primitive::<Int32Type>()
             .values()
             .to_vec();
-        assert_eq!(idx_values, vec![0, 2, 0, 4, 6]);
+        assert_eq!(idx_values, vec![0, 2, 4, 6]);
     }
 
     #[tokio::test]

rust/lancedb/src/error.rs

@@ -102,8 +102,6 @@ pub enum Error {
     },
     #[snafu(display("Job{} was cancelled", job_id.as_ref().map(|id| format!(" {id}")).unwrap_or_default()))]
     JobCancelled { job_id: Option<String> },
-    #[snafu(display("Job '{job_id}' was not found"))]
-    JobNotFound { job_id: String },
 
     // 3rd party / external errors
     #[snafu(display("object_store error: {source}"))]

rust/lancedb/src/function.rs

@@ -5,9 +5,9 @@
 //! backend-neutral terminal result of a computed-column refresh.
 //!
 //! This module contains client/wire values only. Catalog persistence,
-//! environment bake, and execution are owned by Sophon.
+//! environment bake, secret resolution, and execution are owned by Sophon.
 
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 
 use serde::de::{self, DeserializeOwned};
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -15,8 +15,15 @@ use serde_json::Value;
 
 use crate::{Error, Result};
 
-/// Semantic Function type for a Blob v2 value.
-pub const FUNCTION_BLOB_V2_TYPE: &str = "blob_v2";
+// Keep these byte limits aligned with Sophon's Function submission validation.
+pub(crate) const MAX_FUNCTION_SECRET_VALUE_BYTES: usize = 64 * 1024;
+const MAX_FUNCTION_SECRET_VALUES_BYTES: usize = 512 * 1024;
+
+fn is_portable_environment_name(name: &str) -> bool {
+    let mut bytes = name.bytes();
+    matches!(bytes.next(), Some(b'A'..=b'Z' | b'a'..=b'z' | b'_'))
+        && bytes.all(|byte| matches!(byte, b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_'))
+}
 
 fn invalid_json(error: impl std::fmt::Display) -> Error {
     Error::InvalidInput {
@@ -201,6 +208,11 @@ pub struct PythonEnvironmentSpec {
 }
 
 /// Reproducible Python runtime definition understood by Sophon.
+///
+/// `env` contains non-secret values. Secret values are submission-only in the
+/// client model and do not become part of this public runtime identity;
+/// [`FunctionVersion::required_secrets`] contains names only. Sophon persists
+/// submitted values separately in the private execution artifact.
 #[derive(Debug, Clone, PartialEq, Eq)]
 #[non_exhaustive]
 pub enum PythonRuntimeSpec {
@@ -210,33 +222,6 @@ pub enum PythonRuntimeSpec {
         environment: PythonEnvironmentSpec,
         env: BTreeMap<String, String>,
     },
-    /// The GPU-enabled Sophon-managed Python runtime.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use std::collections::BTreeMap;
-    /// use lancedb::function::{PythonEnvironmentSpec, PythonRuntimeSpec};
-    ///
-    /// let runtime = PythonRuntimeSpec::PythonV2 {
-    ///     python_version: "3.12".to_string(),
-    ///     environment: PythonEnvironmentSpec {
-    ///         kind: "pip".to_string(),
-    ///         packages: vec!["cupy-cuda12x".to_string()],
-    ///         channels: Vec::new(),
-    ///         path: None,
-    ///         modules: Vec::new(),
-    ///         image: None,
-    ///     },
-    ///     env: BTreeMap::new(),
-    /// };
-    /// assert!(runtime.requires_gpu());
-    /// ```
-    PythonV2 {
-        python_version: String,
-        environment: PythonEnvironmentSpec,
-        env: BTreeMap<String, String>,
-    },
     /// A runtime kind introduced by a newer server.
     ///
     /// Unknown payload fields are intentionally not retained because the
@@ -249,101 +234,61 @@ impl PythonRuntimeSpec {
     pub fn kind(&self) -> &str {
         match self {
             Self::Python { .. } => "python",
-            Self::PythonV2 { .. } => "python_v2",
             Self::Unrecognized { kind } => kind,
         }
     }
 
-    /// The Python version for a known Python runtime, or `None` for an unknown kind.
+    /// The Python version for the V1 runtime, or `None` for an unknown kind.
     pub fn python_version(&self) -> Option<&str> {
         match self {
-            Self::Python { python_version, .. } | Self::PythonV2 { python_version, .. } => {
-                Some(python_version)
-            }
+            Self::Python { python_version, .. } => Some(python_version),
             Self::Unrecognized { .. } => None,
         }
     }
 
-    /// The Python environment for a known Python runtime, or `None` for an unknown kind.
+    /// The Python environment for the V1 runtime, or `None` for an unknown kind.
     pub fn environment(&self) -> Option<&PythonEnvironmentSpec> {
         match self {
-            Self::Python { environment, .. } | Self::PythonV2 { environment, .. } => {
-                Some(environment)
-            }
+            Self::Python { environment, .. } => Some(environment),
             Self::Unrecognized { .. } => None,
         }
     }
 
-    /// Environment variables, or `None` for an unknown kind.
+    /// Non-secret environment variables, or `None` for an unknown kind.
     pub fn env(&self) -> Option<&BTreeMap<String, String>> {
         match self {
-            Self::Python { env, .. } | Self::PythonV2 { env, .. } => Some(env),
+            Self::Python { env, .. } => Some(env),
             Self::Unrecognized { .. } => None,
         }
     }
-
-    /// Whether the runtime requires a GPU selected by the execution platform.
-    pub fn requires_gpu(&self) -> bool {
-        matches!(self, Self::PythonV2 { .. })
-    }
 }
 
 #[derive(Deserialize)]
-struct PythonRuntimeV1Wire {
-    python_version: String,
-    environment: PythonEnvironmentSpec,
+struct PythonRuntimeWire {
+    kind: String,
+    #[serde(default)]
+    python_version: Option<String>,
+    #[serde(default)]
+    environment: Option<PythonEnvironmentSpec>,
     #[serde(default)]
     env: BTreeMap<String, String>,
-    #[serde(default)]
-    gpu: Option<Value>,
-}
-
-#[derive(Deserialize)]
-struct PythonRuntimeV2Wire {
-    python_version: String,
-    environment: PythonEnvironmentSpec,
-    #[serde(default)]
-    env: BTreeMap<String, String>,
-    gpu: bool,
 }
 
 impl<'de> Deserialize<'de> for PythonRuntimeSpec {
     fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
-        let value = Value::deserialize(deserializer)?;
-        let kind = value
-            .get("kind")
-            .ok_or_else(|| de::Error::missing_field("kind"))?
-            .as_str()
-            .ok_or_else(|| de::Error::custom("runtime.kind must be a string"))?
-            .to_string();
-        match kind.as_str() {
-            "python" => {
-                let wire: PythonRuntimeV1Wire =
-                    serde_json::from_value(value).map_err(de::Error::custom)?;
-                if wire.gpu.is_some() {
-                    return Err(de::Error::custom(
-                        "python runtime with gpu requires kind='python_v2'",
-                    ));
-                }
-                Ok(Self::Python {
-                    python_version: wire.python_version,
-                    environment: wire.environment,
-                    env: wire.env,
-                })
-            }
-            "python_v2" => {
-                let wire: PythonRuntimeV2Wire =
-                    serde_json::from_value(value).map_err(de::Error::custom)?;
-                if !wire.gpu {
-                    return Err(de::Error::custom("runtime.gpu must be true"));
-                }
-                Ok(Self::PythonV2 {
-                    python_version: wire.python_version,
-                    environment: wire.environment,
-                    env: wire.env,
-                })
-            }
-            _ => Ok(Self::Unrecognized { kind }),
+        let wire = PythonRuntimeWire::deserialize(deserializer)?;
+        if wire.kind == "python" {
+            Ok(Self::Python {
+                python_version: wire
+                    .python_version
+                    .ok_or_else(|| de::Error::missing_field("python_version"))?,
+                environment: wire
+                    .environment
+                    .ok_or_else(|| de::Error::missing_field("environment"))?,
+                env: wire.env,
+            })
+        } else {
+            Ok(Self::Unrecognized { kind: wire.kind })
         }
     }
 }
@@ -357,8 +302,6 @@ impl Serialize for PythonRuntimeSpec {
             environment: &'a PythonEnvironmentSpec,
             #[serde(skip_serializing_if = "BTreeMap::is_empty")]
             env: &'a BTreeMap<String, String>,
-            #[serde(skip_serializing_if = "Option::is_none")]
-            gpu: Option<bool>,
         }
 
         #[derive(Serialize)]
@@ -376,19 +319,6 @@ impl Serialize for PythonRuntimeSpec {
                 python_version,
                 environment,
                 env,
-                gpu: None,
-            }
-            .serialize(serializer),
-            Self::PythonV2 {
-                python_version,
-                environment,
-                env,
-            } => PythonRuntimeRef {
-                kind: "python_v2",
-                python_version,
-                environment,
-                env,
-                gpu: Some(true),
             }
             .serialize(serializer),
             Self::Unrecognized { kind } => UnrecognizedRuntimeRef { kind }.serialize(serializer),
@@ -398,8 +328,8 @@ impl Serialize for PythonRuntimeSpec {
 
 /// Immutable Function version returned by the Enterprise catalog.
 ///
-/// The GPU execution requirement is part of this identity. CPU and memory sizing,
-/// priority, concurrency, and retry policy belong to the execution platform.
+/// Scheduling resources, priority, concurrency, and retry policy belong to
+/// the submitting Job and are not part of this identity.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct FunctionVersion {
     name: String,
@@ -409,6 +339,8 @@ pub struct FunctionVersion {
     runtime: PythonRuntimeSpec,
     runtime_digest: String,
     environment_digest: String,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    required_secrets: Vec<String>,
     created_at: String,
 }
 
@@ -441,6 +373,12 @@ impl FunctionVersion {
         &self.environment_digest
     }
 
+    /// Required secret names. Resolved values exist only in Sophon's private
+    /// execution artifact and worker launch path.
+    pub fn required_secrets(&self) -> &[String] {
+        &self.required_secrets
+    }
+
     pub fn created_at(&self) -> &str {
         &self.created_at
     }
@@ -482,12 +420,115 @@ pub struct FunctionArtifactRequest {
 }
 
 /// Stable request envelope for remote immutable Function registration.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+///
+/// Secret values are submission-only in the client model. Sophon persists them
+/// in the database-scoped private execution artifact; returned
+/// [`FunctionVersion`] and Job metadata contain only
+/// [`Self::required_secrets`] names. Debug formatting always redacts values.
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct FunctionRegistrationRequest {
     pub name: String,
     pub artifact: FunctionArtifactRequest,
     pub signature: FunctionSignature,
     pub runtime: PythonRuntimeSpec,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub required_secrets: Vec<String>,
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
+    pub secret_values: BTreeMap<String, String>,
+}
+
+impl FunctionRegistrationRequest {
+    pub(crate) fn validate_secret_values(&self) -> Result<()> {
+        let mut required = BTreeSet::new();
+        for name in &self.required_secrets {
+            if !is_portable_environment_name(name) {
+                return Err(Error::InvalidInput {
+                    message: format!(
+                        "Function secret name {name:?} must be a portable environment variable name"
+                    ),
+                });
+            }
+            if !required.insert(name) {
+                return Err(Error::InvalidInput {
+                    message: format!("Function required_secrets contains duplicate name {name:?}"),
+                });
+            }
+        }
+
+        if let PythonRuntimeSpec::Python { env, .. } = &self.runtime
+            && let Some(name) = required.iter().find(|name| env.contains_key(**name))
+        {
+            return Err(Error::InvalidInput {
+                message: format!(
+                    "Function runtime env and secret names must be disjoint: {name:?}"
+                ),
+            });
+        }
+
+        let provided = self.secret_values.keys().collect::<BTreeSet<_>>();
+        if required != provided {
+            return Err(Error::InvalidInput {
+                message: "Function secret_values keys must exactly match required_secrets"
+                    .to_string(),
+            });
+        }
+
+        let mut total_bytes = 0usize;
+        for (name, value) in &self.secret_values {
+            if value.is_empty() {
+                return Err(Error::InvalidInput {
+                    message: format!("Function secret {name:?} value must be non-empty"),
+                });
+            }
+            if value.contains('\0') {
+                return Err(Error::InvalidInput {
+                    message: format!("Function secret {name:?} value must not contain NUL"),
+                });
+            }
+            if value.len() > MAX_FUNCTION_SECRET_VALUE_BYTES {
+                return Err(Error::InvalidInput {
+                    message: format!(
+                        "Function secret {name:?} value exceeds the \
+                         {MAX_FUNCTION_SECRET_VALUE_BYTES}-byte limit"
+                    ),
+                });
+            }
+            total_bytes =
+                total_bytes
+                    .checked_add(value.len())
+                    .ok_or_else(|| Error::InvalidInput {
+                        message: "Function secret values exceed the request byte limit".to_string(),
+                    })?;
+        }
+        if total_bytes > MAX_FUNCTION_SECRET_VALUES_BYTES {
+            return Err(Error::InvalidInput {
+                message: format!(
+                    "Function secret values exceed the \
+                     {MAX_FUNCTION_SECRET_VALUES_BYTES}-byte request limit"
+                ),
+            });
+        }
+        Ok(())
+    }
+}
+
+impl std::fmt::Debug for FunctionRegistrationRequest {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let secret_values = self
+            .secret_values
+            .keys()
+            .map(|name| (name, "[REDACTED]"))
+            .collect::<BTreeMap<_, _>>();
+        formatter
+            .debug_struct("FunctionRegistrationRequest")
+            .field("name", &self.name)
+            .field("artifact", &self.artifact)
+            .field("signature", &self.signature)
+            .field("runtime", &self.runtime)
+            .field("required_secrets", &self.required_secrets)
+            .field("secret_values", &secret_values)
+            .finish()
+    }
 }
 
 impl_json!(FunctionRegistrationRequest);
@@ -582,8 +623,8 @@ pub struct InputBinding {
 
 /// Ordered result-field to table-field mapping for a Function binding.
 ///
-/// `nullable` describes the logical Function result. Physical computed-column
-/// fields remain nullable while unassigned.
+/// Assignment state is not part of the Slice 1 client contract. During the
+/// NULL transition there is no public Lance cell-flag identifier to persist.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct OutputMapping {
     pub result_field: String,
@@ -594,14 +635,6 @@ pub struct OutputMapping {
     pub nullable: bool,
 }
 
-/// Internal physical column preserving the parent validity of a flattened
-/// named-struct result.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct AssignmentMapping {
-    pub output_name: String,
-    pub output_field_id: i32,
-}
-
 /// Immutable Function binding persisted by the Enterprise table service.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct FunctionBinding {
@@ -609,8 +642,6 @@ pub struct FunctionBinding {
     function: FunctionVersionRef,
     inputs: Vec<InputBinding>,
     outputs: Vec<OutputMapping>,
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    assignment: Option<AssignmentMapping>,
     /// Exact Arrow schema presented to the Function, encoded with the Lance
     /// Namespace Arrow JSON representation.
     #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -637,10 +668,6 @@ impl FunctionBinding {
         &self.outputs
     }
 
-    pub fn assignment(&self) -> Option<&AssignmentMapping> {
-        self.assignment.as_ref()
-    }
-
     pub fn input_schema(&self) -> Option<&Value> {
         self.input_schema.as_ref()
     }
@@ -686,9 +713,188 @@ impl RefreshColumnResult {
 
 impl_json!(RefreshColumnResult);
 
+#[cfg(test)]
+mod secret_value_tests {
+    use super::{
+        FunctionRegistrationRequest, MAX_FUNCTION_SECRET_VALUE_BYTES,
+        MAX_FUNCTION_SECRET_VALUES_BYTES, PythonRuntimeSpec,
+    };
+    use crate::Error;
+
+    fn request() -> FunctionRegistrationRequest {
+        FunctionRegistrationRequest::from_json(include_str!(
+            "../tests/fixtures/first_class_functions/v1/remote_function_registration_request.json"
+        ))
+        .unwrap()
+    }
+
+    #[test]
+    fn validates_secret_name_and_value_invariants() {
+        let missing = request();
+        assert!(matches!(
+            missing.validate_secret_values(),
+            Err(Error::InvalidInput { message }) if message.contains("exactly match")
+        ));
+
+        let mut empty = request();
+        empty
+            .secret_values
+            .insert("API_TOKEN".to_string(), String::new());
+        assert!(matches!(
+            empty.validate_secret_values(),
+            Err(Error::InvalidInput { message }) if message.contains("non-empty")
+        ));
+
+        let mut nul = request();
+        nul.secret_values
+            .insert("API_TOKEN".to_string(), "before\0after".to_string());
+        assert!(matches!(
+            nul.validate_secret_values(),
+            Err(Error::InvalidInput { message }) if message.contains("NUL")
+        ));
+
+        let mut unexpected = request();
+        unexpected
+            .secret_values
+            .insert("OTHER".to_string(), "value".to_string());
+        assert!(matches!(
+            unexpected.validate_secret_values(),
+            Err(Error::InvalidInput { message }) if message.contains("exactly match")
+        ));
+    }
+
+    #[test]
+    fn rejects_invalid_duplicate_and_overlapping_secret_declarations() {
+        let mut invalid_name = request();
+        invalid_name.required_secrets = vec!["BAD=NAME".to_string()];
+        invalid_name
+            .secret_values
+            .insert("BAD=NAME".to_string(), "secret".to_string());
+
+        let mut duplicate = request();
+        duplicate.required_secrets = vec!["API_TOKEN".to_string(), "API_TOKEN".to_string()];
+        duplicate
+            .secret_values
+            .insert("API_TOKEN".to_string(), "secret".to_string());
+
+        let mut overlap = request();
+        overlap
+            .secret_values
+            .insert("API_TOKEN".to_string(), "secret".to_string());
+        if let PythonRuntimeSpec::Python { env, .. } = &mut overlap.runtime {
+            env.insert("API_TOKEN".to_string(), "public".to_string());
+        }
+
+        assert!(matches!(
+            invalid_name.validate_secret_values(),
+            Err(Error::InvalidInput { message }) if message.contains("portable environment variable")
+        ));
+        assert!(matches!(
+            duplicate.validate_secret_values(),
+            Err(Error::InvalidInput { message }) if message.contains("duplicate")
+        ));
+        assert!(matches!(
+            overlap.validate_secret_values(),
+            Err(Error::InvalidInput { message }) if message.contains("must be disjoint")
+        ));
+    }
+
+    #[test]
+    fn enforces_portable_secret_name_boundaries() {
+        for name in ["A", "_", "A0_"] {
+            let mut request = request();
+            request.required_secrets = vec![name.to_string()];
+            request
+                .secret_values
+                .insert(name.to_string(), "secret".to_string());
+            request.validate_secret_values().unwrap();
+        }
+
+        for name in ["", "0TOKEN", "BAD-NAME", "TÖKEN"] {
+            let mut request = request();
+            request.required_secrets = vec![name.to_string()];
+            request
+                .secret_values
+                .insert(name.to_string(), "secret".to_string());
+            assert!(matches!(
+                request.validate_secret_values(),
+                Err(Error::InvalidInput { message })
+                    if message.contains("portable environment variable")
+            ));
+        }
+    }
+
+    #[test]
+    fn accepts_exact_secret_value_utf8_byte_limit() {
+        for value in [
+            "x".repeat(MAX_FUNCTION_SECRET_VALUE_BYTES),
+            "é".repeat(MAX_FUNCTION_SECRET_VALUE_BYTES / "é".len()),
+        ] {
+            assert_eq!(value.len(), MAX_FUNCTION_SECRET_VALUE_BYTES);
+            let mut request = request();
+            request.secret_values.insert("API_TOKEN".to_string(), value);
+            request.validate_secret_values().unwrap();
+        }
+    }
+
+    #[test]
+    fn rejects_secret_value_over_utf8_byte_limit() {
+        for value in [
+            "x".repeat(MAX_FUNCTION_SECRET_VALUE_BYTES + 1),
+            "é".repeat(MAX_FUNCTION_SECRET_VALUE_BYTES / "é".len() + 1),
+        ] {
+            assert!(value.len() > MAX_FUNCTION_SECRET_VALUE_BYTES);
+            let mut request = request();
+            request.secret_values.insert("API_TOKEN".to_string(), value);
+            assert!(matches!(
+                request.validate_secret_values(),
+                Err(Error::InvalidInput { message }) if message.contains("65536-byte limit")
+            ));
+        }
+    }
+
+    #[test]
+    fn rejects_aggregate_secret_value_bytes_over_server_limit() {
+        let mut request = request();
+        request.required_secrets = (0..9).map(|index| format!("SECRET_{index}")).collect();
+        request.secret_values = request
+            .required_secrets
+            .iter()
+            .map(|name| (name.clone(), "x".repeat(MAX_FUNCTION_SECRET_VALUE_BYTES)))
+            .collect();
+
+        assert!(matches!(
+            request.validate_secret_values(),
+            Err(Error::InvalidInput { message })
+                if message.contains(&format!("{MAX_FUNCTION_SECRET_VALUES_BYTES}-byte request limit"))
+        ));
+    }
+
+    #[test]
+    fn accepts_exact_aggregate_secret_value_byte_limit() {
+        let mut request = request();
+        request.required_secrets = (0..8).map(|index| format!("SECRET_{index}")).collect();
+        request.secret_values = request
+            .required_secrets
+            .iter()
+            .map(|name| (name.clone(), "x".repeat(MAX_FUNCTION_SECRET_VALUE_BYTES)))
+            .collect();
+
+        assert_eq!(
+            request
+                .secret_values
+                .values()
+                .map(String::len)
+                .sum::<usize>(),
+            MAX_FUNCTION_SECRET_VALUES_BYTES
+        );
+        request.validate_secret_values().unwrap();
+    }
+}
+
 #[cfg(test)]
 mod conda_environment_tests {
-    use super::{PythonEnvironmentSpec, PythonRuntimeSpec};
+    use super::PythonEnvironmentSpec;
 
     #[test]
     fn conda_channels_round_trip_and_pip_stays_bare() {
@@ -707,45 +913,4 @@ mod conda_environment_tests {
             serde_json::from_str(r#"{"kind":"pip","packages":["numpy"]}"#).unwrap();
         assert!(!serde_json::to_string(&pip).unwrap().contains("channels"));
     }
-
-    #[test]
-    fn gpu_python_runtime_marker_round_trips_and_validates() {
-        let runtime: PythonRuntimeSpec = serde_json::from_str(
-            r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":true}"#,
-        )
-        .unwrap();
-        assert_eq!(runtime.kind(), "python_v2");
-        assert!(runtime.requires_gpu());
-        assert_eq!(
-            super::canonical_json(&runtime).unwrap(),
-            r#"{"environment":{"kind":"pip"},"gpu":true,"kind":"python_v2","python_version":"3.12"}"#
-        );
-
-        for invalid in [
-            r#"{"kind":"python","python_version":"3.12","environment":{"kind":"pip"},"gpu":true}"#,
-            r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"}}"#,
-            r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":1}"#,
-            r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":false}"#,
-            r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":"true"}"#,
-            r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":"H100"}"#,
-        ] {
-            assert!(serde_json::from_str::<PythonRuntimeSpec>(invalid).is_err());
-        }
-    }
-
-    #[test]
-    fn unknown_runtime_discards_payload_before_known_field_validation() {
-        for encoded in [
-            r#"{"kind":"python_v3","gpu":{"model":"H100"}}"#,
-            r#"{"kind":"python_v3","resources":[]}"#,
-            r#"{"kind":"python_v3","python_version":3.15,"environment":{"kind":[]}}"#,
-        ] {
-            let runtime: PythonRuntimeSpec = serde_json::from_str(encoded).unwrap();
-            assert_eq!(runtime.kind(), "python_v3");
-            assert_eq!(
-                super::canonical_json(&runtime).unwrap(),
-                r#"{"kind":"python_v3"}"#
-            );
-        }
-    }
 }

rust/lancedb/src/job.rs

@@ -3,53 +3,16 @@
 
 //! Handles to operations a server may run asynchronously.
 
-use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
+use std::sync::Arc;
 
-use arrow_array::RecordBatch;
 use async_trait::async_trait;
 use serde::{Serialize, de::DeserializeOwned};
 use serde_json::Value;
 use tokio::sync::watch;
 use tokio::task::{AbortHandle, JoinHandle};
 
-use crate::database::JobDescription;
 use crate::error::{Error, JobFailure, Result};
 
-/// Which of a job's events [`Job::events`] returns.
-///
-/// The handle already knows which job to ask about, so this narrows the
-/// query rather than naming one.
-#[derive(Debug, Clone, Default)]
-pub struct JobEventsRequest {
-    /// Maximum event rows to return. The server applies its own default
-    /// (1000 rows) and maximum (10,000 rows) when this is `None`, and
-    /// truncates without saying so, which matters for a job with one event
-    /// per fragment.
-    pub limit: Option<u32>,
-    /// SQL-like filter over the event columns `state`, `updated_by`,
-    /// `emitted_from`, `emitted_by`, and `claim_entity`. For example
-    /// `state = 'claim_complete'` selects only per-claim completions.
-    pub filter: Option<String>,
-}
-
-impl JobEventsRequest {
-    pub fn limit(mut self, limit: u32) -> Self {
-        self.limit = Some(limit);
-        self
-    }
-
-    pub fn filter(mut self, filter: impl Into<String>) -> Self {
-        self.filter = Some(filter.into());
-        self
-    }
-}
-
-fn job_detail_not_supported<T>(what: &str) -> Result<T> {
-    Err(Error::NotSupported {
-        message: format!("{what} is only available for server-side jobs"),
-    })
-}
-
 /// Backend-specific tracking for an asynchronous operation.
 #[async_trait]
 pub(crate) trait JobHandle: Send + Sync {
@@ -60,15 +23,6 @@ pub(crate) trait JobHandle: Send + Sync {
     async fn status(&self) -> Result<String>;
     async fn wait(&self) -> Result<TerminalResult>;
     async fn cancel(&self) -> Result<()>;
-    /// The job's full server-side record. Backends that run the operation in
-    /// this process have none and keep the default.
-    async fn describe(&self) -> Result<JobDescription> {
-        job_detail_not_supported("describing a job")
-    }
-    /// The job's recorded lifecycle events.
-    async fn events(&self, _request: JobEventsRequest) -> Result<Vec<RecordBatch>> {
-        job_detail_not_supported("job event history")
-    }
 }
 
 /// A backend-neutral successful terminal result.
@@ -131,34 +85,16 @@ enum JobInner<T> {
     Completed(T),
 }
 
-/// What a handle last learned about its job. `state` is separate because an
-/// in-process job can report one but has no server-side record behind it.
-#[derive(Default)]
-struct JobCache {
-    state: Option<String>,
-    description: Option<JobDescription>,
-}
-
 /// A handle to an operation that may still be running.
 ///
 /// The operation may already be complete when the handle is created. `T` is
 /// the endpoint's successful terminal result; unit-result operations use the
 /// default `Job<()>`.
-///
-/// The detail accessors ([`Job::state`], [`Job::job_type`], ...) read what the
-/// handle last observed. Submitting an operation returns only a job id, so
-/// populating them eagerly would cost an extra round trip on every call:
-///
-/// - [`Job::refresh`] and [`Job::status`] fetch the whole record.
-/// - [`Job::wait`] records the terminal state it establishes, but not the rest
-///   of the record; call [`Job::refresh`] for that.
-/// - Everything is `None` until one of those runs.
 pub struct Job<T = ()>
 where
     T: Clone + Send + Sync + 'static,
 {
     inner: JobInner<T>,
-    cache: RwLock<JobCache>,
 }
 
 impl<T> std::fmt::Debug for Job<T>
@@ -166,40 +102,18 @@ where
     T: Clone + Send + Sync + 'static,
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let cache = self.cache_read();
-        let mut out = f.debug_struct("Job");
-        out.field("id", &self.id())
-            .field("done", &matches!(self.inner, JobInner::Completed(_)));
-        if let Some(state) = &cache.state {
-            out.field("state", state);
-        }
-        if let Some(description) = &cache.description {
-            out.field("job_type", &description.job_type)
-                .field("creation_ms", &description.creation_ms);
-            if !description.spec.is_null() {
-                out.field("spec", &description.spec);
-            }
-            if let Some(result) = &description.result {
-                out.field("result", result);
-            }
-            if let Some(failure) = &description.failure {
-                out.field("failure", failure);
-            }
-        }
-        out.finish()
+        f.debug_struct("Job")
+            .field("id", &self.id())
+            .field("done", &matches!(self.inner, JobInner::Completed(_)))
+            .finish()
     }
 }
 
 impl Job<()> {
-    /// A job whose operation finished before the handle was created. Its
-    /// state is known without asking anyone, so the cache starts populated.
+    /// A job whose operation finished before the handle was created.
     pub(crate) fn new_done() -> Self {
         Self {
             inner: JobInner::Completed(()),
-            cache: RwLock::new(JobCache {
-                state: Some("finished".to_string()),
-                description: None,
-            }),
         }
     }
 
@@ -209,21 +123,8 @@ impl Job<()> {
                 handle,
                 decode: Arc::new(|_| Ok(())),
             },
-            cache: RwLock::default(),
         }
     }
-
-    /// A handle whose record the caller has already fetched, so the detail
-    /// accessors answer without a second round trip.
-    pub(crate) fn opened(handle: Box<dyn JobHandle>, description: JobDescription) -> Self {
-        let job = Self::new(handle);
-        {
-            let mut cache = job.cache_write();
-            cache.state = Some(description.state.clone());
-            cache.description = Some(description);
-        }
-        job
-    }
 }
 
 impl<T> Job<T>
@@ -237,7 +138,6 @@ where
                 handle,
                 decode: Arc::new(TerminalResult::decode::<T>),
             },
-            cache: RwLock::default(),
         }
     }
 }
@@ -269,124 +169,16 @@ where
         }
     }
 
-    fn cache_read(&self) -> RwLockReadGuard<'_, JobCache> {
-        self.cache.read().unwrap_or_else(|err| err.into_inner())
-    }
-
-    fn cache_write(&self) -> RwLockWriteGuard<'_, JobCache> {
-        self.cache.write().unwrap_or_else(|err| err.into_inner())
-    }
-
-    /// Asks the backend for this job's current state, and for a server-side
-    /// job its full record, then caches the answer for the detail accessors.
-    ///
-    /// In-process operations have no server-side record, so only
-    /// [`Job::state`] is populated for them.
-    pub async fn refresh(&self) -> Result<()> {
-        self.refresh_state().await.map(|_| ())
-    }
-
-    /// Refreshes and reports the state, which every backend can answer.
-    async fn refresh_state(&self) -> Result<String> {
-        let JobInner::Handle { handle, .. } = &self.inner else {
-            let state = "finished".to_string();
-            self.cache_write().state = Some(state.clone());
-            return Ok(state);
-        };
-        match handle.describe().await {
-            Ok(description) => {
-                let state = description.state.clone();
-                let mut cache = self.cache_write();
-                cache.state = Some(state.clone());
-                cache.description = Some(description);
-                Ok(state)
-            }
-            // An in-process job knows its own state and nothing more.
-            Err(Error::NotSupported { .. }) => {
-                let state = handle.status().await?;
-                self.cache_write().state = Some(state.clone());
-                Ok(state)
-            }
-            Err(err) => Err(err),
-        }
-    }
-
     /// The operation's current lifecycle state: "running", "finished",
     /// "failed", or "cancelled".
     ///
     /// A point snapshot; unlike [`Job::wait`] it does not block, raise on a
     /// terminal failure state, or retry. States a newer server reports that
-    /// this client version does not know pass through as-is. Also refreshes
-    /// the detail accessors.
+    /// this client version does not know pass through as-is.
     pub async fn status(&self) -> Result<String> {
-        self.refresh_state().await
-    }
-
-    /// The last lifecycle state this handle observed, without contacting the
-    /// backend. `None` until the handle has.
-    pub fn state(&self) -> Option<String> {
-        self.cache_read().state.clone()
-    }
-
-    /// The whole server-side record this handle last observed. The accessors
-    /// below read individual fields out of it. `None` for an in-process job,
-    /// which has no such record.
-    pub fn description(&self) -> Option<JobDescription> {
-        self.cache_read().description.clone()
-    }
-
-    /// The job's type, as the server names it. `None` for an in-process job.
-    pub fn job_type(&self) -> Option<String> {
-        self.with_description(|description| description.job_type.clone())
-    }
-
-    /// When the job was created, in milliseconds since the epoch. `None` for
-    /// an in-process job.
-    pub fn creation_ms(&self) -> Option<i64> {
-        self.with_description(|description| description.creation_ms)
-    }
-
-    /// The job-type-specific specification it was submitted with.
-    pub fn spec(&self) -> Option<Value> {
-        self.with_description(|description| description.spec.clone())
-            .filter(|spec| !spec.is_null())
-    }
-
-    /// The job-type-specific terminal result, as reported data rather than the
-    /// typed model [`Job::wait`] returns. `None` until the job succeeds.
-    pub fn result(&self) -> Option<Value> {
-        self.with_description(|description| description.result.clone())
-            .flatten()
-    }
-
-    /// Why the job failed, when it failed and the server reports a reason.
-    pub fn failure(&self) -> Option<JobFailure> {
-        self.with_description(|description| description.failure.clone())
-            .flatten()
-    }
-
-    fn with_description<R>(&self, read: impl FnOnce(&JobDescription) -> R) -> Option<R> {
-        self.cache_read().description.as_ref().map(read)
-    }
-
-    /// This job's recorded lifecycle events.
-    ///
-    /// Unlike the detail accessors, which report a terminal result only once
-    /// the job reaches one, events are written as the job runs and outlive the
-    /// workers that produced them. A distributed job records a
-    /// `claim`/`claim_complete` pair per unit of work, each carrying
-    /// `rows_processed`, so a job that never finishes still accounts for what
-    /// it did. In-process operations keep no event history.
-    pub async fn events(&self, request: JobEventsRequest) -> Result<Vec<RecordBatch>> {
         match &self.inner {
-            JobInner::Handle { handle, .. } => handle.events(request).await,
-            // The operation finished before the handle existed, so there is no
-            // id to query with even when a server ran it.
-            JobInner::Completed(_) => Err(Error::NotSupported {
-                message: "this operation completed before its handle was created, so it \
-                          carries no job id to query events with"
-                    .to_string(),
-            }),
+            JobInner::Handle { handle, .. } => handle.status().await,
+            JobInner::Completed(_) => Ok("finished".to_string()),
         }
     }
 
@@ -398,19 +190,8 @@ where
     /// [`crate::Error::JobCancelled`] if it was cancelled.
     pub async fn wait(&self) -> Result<T> {
         match &self.inner {
-            JobInner::Handle { handle, decode } => {
-                let settled = handle.wait().await;
-                // Waiting already established a terminal state; record it so
-                // the detail accessors do not need another round trip for it.
-                if let Some(state) = terminal_state(&settled) {
-                    self.cache_write().state = Some(state.to_string());
-                }
-                (decode)(settled?)
-            }
-            JobInner::Completed(result) => {
-                self.cache_write().state = Some("finished".to_string());
-                Ok(result.clone())
-            }
+            JobInner::Handle { handle, decode } => (decode)(handle.wait().await?),
+            JobInner::Completed(result) => Ok(result.clone()),
         }
     }
 
@@ -443,36 +224,20 @@ where
         U: Clone + Send + Sync + 'static,
         F: Fn(T) -> U + Send + Sync + 'static,
     {
-        // The mapped handle tracks the same job, so it inherits what this one
-        // has already learned about it.
-        let Self { inner, cache } = self;
-        match inner {
+        match self.inner {
             JobInner::Handle { handle, decode } => Job {
                 inner: JobInner::Handle {
                     handle,
                     decode: Arc::new(move |result| Ok(map((decode)(result)?))),
                 },
-                cache,
             },
             JobInner::Completed(result) => Job {
                 inner: JobInner::Completed(map(result)),
-                cache,
             },
         }
     }
 }
 
-/// The lifecycle state a settled [`JobHandle::wait`] implies.
-fn terminal_state(settled: &Result<TerminalResult>) -> Option<&'static str> {
-    match settled {
-        Ok(_) => Some("finished"),
-        Err(Error::JobFailed { .. }) => Some("failed"),
-        Err(Error::JobCancelled { .. }) => Some("cancelled"),
-        // Anything else is a transport failure, not a verdict on the job.
-        Err(_) => None,
-    }
-}
-
 /// How an in-process operation ended. Cloneable so every waiter can be given
 /// the outcome; [`Error`] is not, so failures share one behind an [`Arc`].
 #[derive(Clone)]