chore: update lance dependency to v2.0.1-rc.1

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.27.0-beta.3"
+current_version = "0.26.2"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

.github/workflows/build_linux_wheel/action.yml

@@ -29,7 +29,6 @@ runs:
       if: ${{ inputs.arm-build == 'false' }}
       uses: PyO3/maturin-action@v1
       with:
-        maturin-version: "1.12.4"
         command: build
         working-directory: python
         docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
@@ -45,7 +44,6 @@ runs:
       if: ${{ inputs.arm-build == 'true' }}
       uses: PyO3/maturin-action@v1
       with:
-        maturin-version: "1.12.4"
         command: build
         working-directory: python
         docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"

.github/workflows/build_mac_wheel/action.yml

@@ -20,7 +20,6 @@ runs:
       uses: PyO3/maturin-action@v1
       with:
         command: build
-        maturin-version: "1.12.4"
         # TODO: pass through interpreter
         args: ${{ inputs.args }}
         docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"

.github/workflows/build_windows_wheel/action.yml

@@ -25,7 +25,6 @@ runs:
       uses: PyO3/maturin-action@v1
       with:
         command: build
-        maturin-version: "1.12.4"
         args: ${{ inputs.args }}
         docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
         working-directory: python

.github/workflows/codex-fix-ci.yml

@@ -1,173 +0,0 @@
-name: Codex Fix CI
-
-on:
-  workflow_dispatch:
-    inputs:
-      workflow_run_url:
-        description: "Failing CI workflow run URL (e.g., https://github.com/lancedb/lancedb/actions/runs/12345678)"
-        required: true
-        type: string
-      branch:
-        description: "Branch to fix (e.g., main, release/v2.0, or feature-branch)"
-        required: true
-        type: string
-      guidelines:
-        description: "Additional guidelines for the fix (optional)"
-        required: false
-        type: string
-
-permissions:
-  contents: write
-  pull-requests: write
-  actions: read
-
-jobs:
-  fix-ci:
-    runs-on: warp-ubuntu-latest-x64-4x
-    timeout-minutes: 60
-    env:
-      CC: clang
-      CXX: clang++
-    steps:
-      - name: Show inputs
-        run: |
-          echo "workflow_run_url = ${{ inputs.workflow_run_url }}"
-          echo "branch = ${{ inputs.branch }}"
-          echo "guidelines = ${{ inputs.guidelines }}"
-
-      - name: Checkout Repo
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ inputs.branch }}
-          fetch-depth: 0
-          persist-credentials: true
-
-      - name: Set up Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-
-      - name: Install Codex CLI
-        run: npm install -g @openai/codex
-
-      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
-        with:
-          toolchain: stable
-          components: clippy, rustfmt
-
-      - uses: Swatinem/rust-cache@v2
-
-      - name: Install system dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y protobuf-compiler libssl-dev
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.11'
-
-      - name: Install Python dependencies
-        run: |
-          pip install maturin ruff pytest pyarrow pandas polars
-
-      - name: Set up Java
-        uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: '11'
-          cache: maven
-
-      - name: Install Node.js dependencies for TypeScript bindings
-        run: |
-          cd nodejs
-          npm ci
-
-      - name: Configure git user
-        run: |
-          git config user.name "lancedb automation"
-          git config user.email "robot@lancedb.com"
-
-      - name: Run Codex to fix CI failure
-        env:
-          WORKFLOW_RUN_URL: ${{ inputs.workflow_run_url }}
-          BRANCH: ${{ inputs.branch }}
-          GUIDELINES: ${{ inputs.guidelines }}
-          GITHUB_TOKEN: ${{ secrets.ROBOT_TOKEN }}
-          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
-          OPENAI_API_KEY: ${{ secrets.CODEX_TOKEN }}
-        run: |
-          set -euo pipefail
-
-          cat <<EOF >/tmp/codex-prompt.txt
-          You are running inside the lancedb repository on a GitHub Actions runner. Your task is to fix a CI failure.
-
-          Input parameters:
-          - Failing workflow run URL: ${WORKFLOW_RUN_URL}
-          - Branch to fix: ${BRANCH}
-          - Additional guidelines: ${GUIDELINES:-"None provided"}
-
-          Follow these steps exactly:
-
-          1. Extract the run ID from the workflow URL. The URL format is https://github.com/lancedb/lancedb/actions/runs/<run_id>.
-
-          2. Use "gh run view <run_id> --json jobs,conclusion,name" to get information about the failed run.
-
-          3. Identify which jobs failed. For each failed job, use "gh run view <run_id> --job <job_id> --log-failed" to get the failure logs.
-
-          4. Analyze the failure logs to understand what went wrong. Common failures include:
-             - Compilation errors
-             - Test failures
-             - Clippy warnings treated as errors
-             - Formatting issues
-             - Dependency issues
-
-          5. Based on the analysis, fix the issues in the codebase:
-             - For compilation errors: Fix the code that doesn't compile
-             - For test failures: Fix the failing tests or the code they test
-             - For clippy warnings: Apply the suggested fixes
-             - For formatting issues: Run "cargo fmt --all"
-             - For other issues: Apply appropriate fixes
-
-          6. After making fixes, verify them locally:
-             - Run "cargo fmt --all" to ensure formatting is correct
-             - Run "cargo clippy --workspace --tests --all-features -- -D warnings" to check for issues
-             - Run ONLY the specific failing tests to confirm they pass now:
-               - For Rust test failures: Run the specific test with "cargo test -p <crate> <test_name>"
-               - For Python test failures: Build with "cd python && maturin develop" then run "pytest <specific_test_file>::<test_name>"
-               - For Java test failures: Run "cd java && mvn test -Dtest=<TestClass>#<testMethod>"
-               - For TypeScript test failures: Run "cd nodejs && npm run build && npm test -- --testNamePattern='<test_name>'"
-               - Do NOT run the full test suite - only run the tests that were failing
-
-          7. If the additional guidelines are provided, follow them as well.
-
-          8. Inspect "git status --short" and "git diff" to review your changes.
-
-          9. Create a fix branch: "git checkout -b codex/fix-ci-<run_id>".
-
-          10. Stage all changes with "git add -A" and commit with message "fix: resolve CI failures from run <run_id>".
-
-          11. Push the branch: "git push origin codex/fix-ci-<run_id>". If the remote branch exists, delete it first with "gh api -X DELETE repos/lancedb/lancedb/git/refs/heads/codex/fix-ci-<run_id>" then push. Do NOT use "git push --force" or "git push -f".
-
-          12. Create a pull request targeting "${BRANCH}":
-              - Title: "ci: <short summary describing the fix>" (e.g., "ci: fix clippy warnings in lancedb" or "ci: resolve test flakiness in vector search")
-              - First, write the PR body to /tmp/pr-body.md using a heredoc (cat <<'PREOF' > /tmp/pr-body.md). The body should include:
-                - Link to the failing workflow run
-                - Summary of what failed
-                - Description of the fixes applied
-              - Then run "gh pr create --base ${BRANCH} --body-file /tmp/pr-body.md".
-
-          13. Display the new PR URL, "git status --short", and a summary of what was fixed.
-
-          Constraints:
-          - Use bash commands for all operations.
-          - Do not merge the PR.
-          - Do not modify GitHub workflow files unless they are the cause of the failure.
-          - If any command fails, diagnose and attempt to fix the issue instead of aborting immediately.
-          - If you cannot fix the issue automatically, create the PR anyway with a clear explanation of what you tried and what remains to be fixed.
-          - env "GH_TOKEN" is available, use "gh" tools for GitHub-related operations.
-          EOF
-
-          printenv OPENAI_API_KEY | codex login --with-api-key
-          codex --config shell_environment_policy.ignore_default_excludes=true exec --dangerously-bypass-approvals-and-sandbox "$(cat /tmp/codex-prompt.txt)"

.github/workflows/nodejs.yml

@@ -8,7 +8,6 @@ on:
     paths:
       - Cargo.toml
       - nodejs/**
-      - rust/**
       - docs/src/js/**
       - .github/workflows/nodejs.yml
       - docker-compose.yml
@@ -78,11 +77,8 @@ jobs:
         fetch-depth: 0
         lfs: true
     - uses: actions/setup-node@v3
-      name: Setup Node.js 20 for build
       with:
-        # @napi-rs/cli v3 requires Node >= 20.12 (via @inquirer/prompts@8).
-        # Build always on Node 20; tests run on the matrix version below.
-        node-version: 20
+        node-version: ${{ matrix.node-version }}
         cache: 'npm'
         cache-dependency-path: nodejs/package-lock.json
     - uses: Swatinem/rust-cache@v2
@@ -90,16 +86,12 @@ jobs:
       run: |
         sudo apt update
         sudo apt install -y protobuf-compiler libssl-dev
+        npm install -g @napi-rs/cli
     - name: Build
       run: |
         npm ci --include=optional
         npm run build:debug -- --profile ci
-    - uses: actions/setup-node@v3
-      name: Setup Node.js ${{ matrix.node-version }} for test
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: Compile TypeScript
-      run: npm run tsc
+        npm run tsc
     - name: Setup localstack
       working-directory: .
       run: docker compose up --detach --wait
@@ -152,6 +144,7 @@ jobs:
     - name: Install dependencies
       run: |
         brew install protobuf
+        npm install -g @napi-rs/cli
     - name: Build
       run: |
         npm ci --include=optional

.github/workflows/npm-publish.yml

@@ -128,13 +128,16 @@ jobs:
           - target: x86_64-unknown-linux-musl
             # This one seems to need some extra memory
             host: ubuntu-2404-8x-x64
+            # https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile
+            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
             features: fp16kernels
             pre_build: |-
               set -e &&
-              sudo apt-get update &&
-              sudo apt-get install -y protobuf-compiler pkg-config &&
-              rustup target add x86_64-unknown-linux-musl &&
-              export EXTRA_ARGS="-x"
+              apk add protobuf-dev curl &&
+              ln -s /usr/lib/gcc/x86_64-alpine-linux-musl/14.2.0/crtbeginS.o /usr/lib/crtbeginS.o &&
+              ln -s /usr/lib/libgcc_s.so /usr/lib/libgcc.so &&
+              CC=gcc &&
+              CXX=g++
           - target: aarch64-unknown-linux-gnu
             host: ubuntu-2404-8x-x64
             # https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
@@ -150,13 +153,15 @@ jobs:
               rustup target add aarch64-unknown-linux-gnu
           - target: aarch64-unknown-linux-musl
             host: ubuntu-2404-8x-x64
+            # https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile
+            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
             features: ","
             pre_build: |-
               set -e &&
-              sudo apt-get update &&
-              sudo apt-get install -y protobuf-compiler &&
+              apk add protobuf-dev &&
               rustup target add aarch64-unknown-linux-musl &&
-              export EXTRA_ARGS="-x"
+              export CC_aarch64_unknown_linux_musl=aarch64-linux-musl-gcc &&
+              export CXX_aarch64_unknown_linux_musl=aarch64-linux-musl-g++
     name: build - ${{ matrix.settings.target }}
     runs-on: ${{ matrix.settings.host }}
     defaults:
@@ -187,18 +192,12 @@ jobs:
             .cargo-cache
             target/
           key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}
+      - name: Setup toolchain
+        run: ${{ matrix.settings.setup }}
+        if: ${{ matrix.settings.setup }}
+        shell: bash
       - name: Install dependencies
         run: npm ci
-      - name: Install Zig
-        uses: mlugg/setup-zig@v2
-        if: ${{ contains(matrix.settings.target, 'musl') }}
-        with:
-          version: 0.14.1
-      - name: Install cargo-zigbuild
-        uses: taiki-e/install-action@v2
-        if: ${{ contains(matrix.settings.target, 'musl') }}
-        with:
-          tool: cargo-zigbuild
       - name: Build in docker
         uses: addnab/docker-run-action@v3
         if: ${{ matrix.settings.docker }}
@@ -211,24 +210,24 @@ jobs:
           run: |
             set -e
             ${{ matrix.settings.pre_build }}
-            npx napi build --platform --release \
+            npx napi build --platform  --release --no-const-enum \
               --features ${{ matrix.settings.features }} \
               --target ${{ matrix.settings.target }} \
               --dts ../lancedb/native.d.ts \
               --js ../lancedb/native.js \
               --strip \
-              --output-dir dist/
+              dist/
       - name: Build
         run: |
           ${{ matrix.settings.pre_build }}
-          npx napi build --platform --release \
+          npx napi build --platform  --release --no-const-enum \
               --features ${{ matrix.settings.features }} \
               --target ${{ matrix.settings.target }} \
               --dts ../lancedb/native.d.ts \
               --js ../lancedb/native.js \
               --strip \
               $EXTRA_ARGS \
-              --output-dir dist/
+              dist/
         if: ${{ !matrix.settings.docker }}
         shell: bash
       - name: Upload artifact
@@ -356,8 +355,7 @@ jobs:
           if [[ $DRY_RUN == "true" ]]; then
             ARGS="$ARGS --dry-run"
           fi
-          VERSION=$(node -p "require('./package.json').version")
-          if [[ $VERSION == *-* ]]; then
+          if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
             ARGS="$ARGS --tag preview"
           fi
           npm publish $ARGS

.github/workflows/python.yml

@@ -8,12 +8,7 @@ on:
     paths:
       - Cargo.toml
       - python/**
-      - rust/**
       - .github/workflows/python.yml
-      - .github/workflows/build_linux_wheel/**
-      - .github/workflows/build_mac_wheel/**
-      - .github/workflows/build_windows_wheel/**
-      - .github/workflows/run_tests/**
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/rust.yml

@@ -100,9 +100,7 @@ jobs:
           lfs: true
       - uses: Swatinem/rust-cache@v2
       - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
+        run: sudo apt install -y protobuf-compiler libssl-dev
       - uses: rui314/setup-mold@v1
       - name: Make Swap
         run: |
@@ -185,7 +183,7 @@ jobs:
     runs-on: ubuntu-24.04
     strategy:
       matrix:
-        msrv: ["1.91.0"] # This should match up with rust-version in Cargo.toml
+        msrv: ["1.88.0"] # This should match up with rust-version in Cargo.toml
     env:
       # Need up-to-date compilers for kernels
       CC: clang-18

Cargo.toml

@@ -5,30 +5,30 @@ exclude = ["python"]
 resolver = "2"
 
 [workspace.package]
-edition = "2024"
+edition = "2021"
 authors = ["LanceDB Devs <dev@lancedb.com>"]
 license = "Apache-2.0"
 repository = "https://github.com/lancedb/lancedb"
 description = "Serverless, low-latency vector database for AI applications"
 keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
-rust-version = "1.91.0"
+rust-version = "1.88.0"
 
 [workspace.dependencies]
-lance = { "version" = "=3.0.0-rc.2", default-features = false, "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=3.0.0-rc.2", default-features = false, "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=3.0.0-rc.2", default-features = false, "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=2.0.1-rc.1", default-features = false, "tag" = "v2.0.1-rc.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=2.0.1-rc.1", "tag" = "v2.0.1-rc.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=2.0.1-rc.1", "tag" = "v2.0.1-rc.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=2.0.1-rc.1", "tag" = "v2.0.1-rc.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=2.0.1-rc.1", default-features = false, "tag" = "v2.0.1-rc.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=2.0.1-rc.1", "tag" = "v2.0.1-rc.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=2.0.1-rc.1", "tag" = "v2.0.1-rc.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=2.0.1-rc.1", "tag" = "v2.0.1-rc.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=2.0.1-rc.1", default-features = false, "tag" = "v2.0.1-rc.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=2.0.1-rc.1", "tag" = "v2.0.1-rc.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=2.0.1-rc.1", "tag" = "v2.0.1-rc.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=2.0.1-rc.1", "tag" = "v2.0.1-rc.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=2.0.1-rc.1", "tag" = "v2.0.1-rc.1", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=2.0.1-rc.1", "tag" = "v2.0.1-rc.1", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
 arrow = { version = "57.2", optional = false }
@@ -40,15 +40,13 @@ arrow-schema = "57.2"
 arrow-select = "57.2"
 arrow-cast = "57.2"
 async-trait = "0"
-datafusion = { version = "52.1", default-features = false }
-datafusion-catalog = "52.1"
-datafusion-common = { version = "52.1", default-features = false }
-datafusion-execution = "52.1"
-datafusion-expr = "52.1"
-datafusion-functions = "52.1"
-datafusion-physical-plan = "52.1"
-datafusion-physical-expr = "52.1"
-datafusion-sql = "52.1"
+datafusion = { version = "51.0", default-features = false }
+datafusion-catalog = "51.0"
+datafusion-common = { version = "51.0", default-features = false }
+datafusion-execution = "51.0"
+datafusion-expr = "51.0"
+datafusion-physical-plan = "51.0"
+datafusion-physical-expr = "51.0"
 env_logger = "0.11"
 half = { "version" = "2.7.1", default-features = false, features = [
     "num-traits",

docs/mkdocs.yml

@@ -52,21 +52,14 @@ plugins:
           options:
             docstring_style: numpy
             heading_level: 3
+            show_source: true
+            show_symbol_type_in_heading: true
             show_signature_annotations: true
             show_root_heading: true
-            show_docstring_examples: true
-            show_docstring_attributes: false
-            show_docstring_other_parameters: true
-            show_symbol_type_heading: true
-            show_labels: false
-            show_if_no_docstring: true
-            show_source: false
             members_order: source
             docstring_section_style: list
             signature_crossrefs: true
             separate_signature: true
-            filters:
-              - "!^_"
           import:
             # for cross references
             - https://arrow.apache.org/docs/objects.inv
@@ -120,7 +113,7 @@ markdown_extensions:
       emoji_index: !!python/name:material.extensions.emoji.twemoji
       emoji_generator: !!python/name:material.extensions.emoji.to_svg
   - markdown.extensions.toc:
-      toc_depth: 4
+      toc_depth: 3
       permalink: true
       permalink_title: Anchor link to this section
 

docs/src/java/java.md

@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-core</artifactId>
-    <version>0.27.0-beta.3</version>
+    <version>0.26.2</version>
 </dependency>
 ```
 

docs/src/js/interfaces/DeleteResult.md

@@ -8,14 +8,6 @@
 
 ## Properties
 
-### numDeletedRows
-
-```ts
-numDeletedRows: number;
-```
-
-***
-
 ### version
 
 ```ts

java/lancedb-core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
       <groupId>com.lancedb</groupId>
       <artifactId>lancedb-parent</artifactId>
-      <version>0.27.0-beta.3</version>
+      <version>0.26.2-final.0</version>
       <relativePath>../pom.xml</relativePath>
     </parent>
 

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.27.0-beta.3</version>
+    <version>0.26.2-final.0</version>
     <packaging>pom</packaging>
     <name>${project.artifactId}</name>
     <description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>3.1.0-beta.2</lance-core.version>
+        <lance-core.version>2.0.1-rc.1</lance-core.version>
         <spotless.skip>false</spotless.skip>
         <spotless.version>2.30.0</spotless.version>
         <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.27.0-beta.3"
+version = "0.26.2"
 license.workspace = true
 description.workspace = true
 repository.workspace = true
@@ -19,11 +19,11 @@ arrow-schema.workspace = true
 env_logger.workspace = true
 futures.workspace = true
 lancedb = { path = "../rust/lancedb", default-features = false }
-napi = { version = "3.8.3", default-features = false, features = [
+napi = { version = "2.16.8", default-features = false, features = [
     "napi9",
     "async"
 ] }
-napi-derive = "3.5.2"
+napi-derive = "2.16.4"
 # Prevent dynamic linking of lzma, which comes from datafusion
 lzma-sys = { version = "*", features = ["static"] }
 log.workspace = true
@@ -33,7 +33,7 @@ aws-lc-sys = "=0.28.0"
 aws-lc-rs = "=1.13.0"
 
 [build-dependencies]
-napi-build = "2.3.1"
+napi-build = "2.1"
 
 [features]
 default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface"]

nodejs/__test__/table.test.ts

@@ -1697,65 +1697,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       expect(results2[0].text).toBe(data[1].text);
     });
 
-    test("full text search fast search", async () => {
-      const db = await connect(tmpDir.name);
-      const data = [{ text: "hello world", vector: [0.1, 0.2, 0.3], id: 1 }];
-      const table = await db.createTable("test", data);
-      await table.createIndex("text", {
-        config: Index.fts(),
-      });
-
-      // Insert unindexed data after creating the index.
-      await table.add([{ text: "xyz", vector: [0.4, 0.5, 0.6], id: 2 }]);
-
-      const withFlatSearch = await table
-        .search("xyz", "fts")
-        .limit(10)
-        .toArray();
-      expect(withFlatSearch.length).toBeGreaterThan(0);
-
-      const fastSearchResults = await table
-        .search("xyz", "fts")
-        .fastSearch()
-        .limit(10)
-        .toArray();
-      expect(fastSearchResults.length).toBe(0);
-
-      const nearestToTextFastSearch = await table
-        .query()
-        .nearestToText("xyz")
-        .fastSearch()
-        .limit(10)
-        .toArray();
-      expect(nearestToTextFastSearch.length).toBe(0);
-
-      // fastSearch should be chainable with other methods.
-      const chainedFastSearch = await table
-        .search("xyz", "fts")
-        .fastSearch()
-        .select(["text"])
-        .limit(5)
-        .toArray();
-      expect(chainedFastSearch.length).toBe(0);
-
-      await table.optimize();
-
-      const indexedFastSearch = await table
-        .search("xyz", "fts")
-        .fastSearch()
-        .limit(10)
-        .toArray();
-      expect(indexedFastSearch.length).toBeGreaterThan(0);
-
-      const indexedNearestToTextFastSearch = await table
-        .query()
-        .nearestToText("xyz")
-        .fastSearch()
-        .limit(10)
-        .toArray();
-      expect(indexedNearestToTextFastSearch.length).toBeGreaterThan(0);
-    });
-
     test("prewarm full text search index", async () => {
       const db = await connect(tmpDir.name);
       const data = [

nodejs/lancedb/index.ts

@@ -273,9 +273,7 @@ export async function connect(
   let nativeProvider: NativeJsHeaderProvider | undefined;
   if (finalHeaderProvider) {
     if (typeof finalHeaderProvider === "function") {
-      nativeProvider = new NativeJsHeaderProvider(async () =>
-        finalHeaderProvider(),
-      );
+      nativeProvider = new NativeJsHeaderProvider(finalHeaderProvider);
     } else if (
       finalHeaderProvider &&
       typeof finalHeaderProvider.getHeaders === "function"

nodejs/lancedb/query.ts

@@ -684,17 +684,19 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
 
   rerank(reranker: Reranker): VectorQuery {
     super.doCall((inner) =>
-      inner.rerank(async (args) => {
-        const vecResults = await fromBufferToRecordBatch(args.vecResults);
-        const ftsResults = await fromBufferToRecordBatch(args.ftsResults);
-        const result = await reranker.rerankHybrid(
-          args.query,
-          vecResults as RecordBatch,
-          ftsResults as RecordBatch,
-        );
+      inner.rerank({
+        rerankHybrid: async (_, args) => {
+          const vecResults = await fromBufferToRecordBatch(args.vecResults);
+          const ftsResults = await fromBufferToRecordBatch(args.ftsResults);
+          const result = await reranker.rerankHybrid(
+            args.query,
+            vecResults as RecordBatch,
+            ftsResults as RecordBatch,
+          );
 
-        const buffer = fromRecordBatchToBuffer(result);
-        return buffer;
+          const buffer = fromRecordBatchToBuffer(result);
+          return buffer;
+        },
       }),
     );
 

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.27.0-beta.3",
+	"version": "0.26.2",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.27.0-beta.3",
+	"version": "0.26.2",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.27.0-beta.3",
+	"version": "0.26.2",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.27.0-beta.3",
+	"version": "0.26.2",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.27.0-beta.3",
+	"version": "0.26.2",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.27.0-beta.3",
+  "version": "0.26.2",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.27.0-beta.3",
+	"version": "0.26.2",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.27.0-beta.3",
+  "version": "0.26.2",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -21,16 +21,19 @@
   },
   "types": "dist/index.d.ts",
   "napi": {
-    "binaryName": "lancedb",
-    "targets": [
-      "aarch64-apple-darwin",
-      "x86_64-unknown-linux-gnu",
-      "aarch64-unknown-linux-gnu",
-      "x86_64-unknown-linux-musl",
-      "aarch64-unknown-linux-musl",
-      "x86_64-pc-windows-msvc",
-      "aarch64-pc-windows-msvc"
-    ]
+    "name": "lancedb",
+    "triples": {
+      "defaults": false,
+      "additional": [
+        "aarch64-apple-darwin",
+        "x86_64-unknown-linux-gnu",
+        "aarch64-unknown-linux-gnu",
+        "x86_64-unknown-linux-musl",
+        "aarch64-unknown-linux-musl",
+        "x86_64-pc-windows-msvc",
+        "aarch64-pc-windows-msvc"
+      ]
+    }
   },
   "license": "Apache-2.0",
   "repository": {
@@ -43,7 +46,7 @@
     "@aws-sdk/client-s3": "^3.33.0",
     "@biomejs/biome": "^1.7.3",
     "@jest/globals": "^29.7.0",
-    "@napi-rs/cli": "^3.5.1",
+    "@napi-rs/cli": "^2.18.3",
     "@types/axios": "^0.14.0",
     "@types/jest": "^29.1.2",
     "@types/node": "^22.7.4",
@@ -72,9 +75,9 @@
   "os": ["darwin", "linux", "win32"],
   "scripts": {
     "artifacts": "napi artifacts",
-    "build:debug": "napi build --platform --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir lancedb",
+    "build:debug": "napi build --platform --no-const-enum --dts ../lancedb/native.d.ts --js ../lancedb/native.js lancedb",
     "postbuild:debug": "shx mkdir -p dist && shx cp lancedb/*.node dist/",
-    "build:release": "napi build --platform --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir dist",
+    "build:release": "napi build --platform --no-const-enum --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js dist/",
     "postbuild:release": "shx mkdir -p dist && shx cp lancedb/*.node dist/",
     "build": "npm run build:debug && npm run tsc",
     "build-release": "npm run build:release && npm run tsc",
@@ -88,7 +91,7 @@
     "prepublishOnly": "napi prepublish -t npm",
     "test": "jest --verbose",
     "integration": "S3_TEST=1 npm run test",
-    "universal": "napi universalize",
+    "universal": "napi universal",
     "version": "napi version"
   },
   "dependencies": {

nodejs/src/connection.rs

@@ -8,12 +8,11 @@ use lancedb::database::{CreateTableMode, Database};
 use napi::bindgen_prelude::*;
 use napi_derive::*;
 
-use crate::ConnectionOptions;
 use crate::error::NapiErrorExt;
 use crate::header::JsHeaderProvider;
 use crate::table::Table;
+use crate::ConnectionOptions;
 use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection};
-
 use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
 
 #[napi]

nodejs/src/header.rs

@@ -1,19 +1,20 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-use napi::{bindgen_prelude::*, threadsafe_function::ThreadsafeFunction};
+use napi::{
+    bindgen_prelude::*,
+    threadsafe_function::{ErrorStrategy, ThreadsafeFunction},
+};
 use napi_derive::napi;
 use std::collections::HashMap;
 use std::sync::Arc;
 
-type GetHeadersFn = ThreadsafeFunction<(), Promise<HashMap<String, String>>, (), Status, false>;
-
 /// JavaScript HeaderProvider implementation that wraps a JavaScript callback.
 /// This is the only native header provider - all header provider implementations
 /// should provide a JavaScript function that returns headers.
 #[napi]
 pub struct JsHeaderProvider {
-    get_headers_fn: Arc<GetHeadersFn>,
+    get_headers_fn: Arc<ThreadsafeFunction<(), ErrorStrategy::CalleeHandled>>,
 }
 
 impl Clone for JsHeaderProvider {
@@ -28,12 +29,9 @@ impl Clone for JsHeaderProvider {
 impl JsHeaderProvider {
     /// Create a new JsHeaderProvider from a JavaScript callback
     #[napi(constructor)]
-    pub fn new(
-        get_headers_callback: Function<(), Promise<HashMap<String, String>>>,
-    ) -> Result<Self> {
+    pub fn new(get_headers_callback: JsFunction) -> Result<Self> {
         let get_headers_fn = get_headers_callback
-            .build_threadsafe_function()
-            .build()
+            .create_threadsafe_function(0, |ctx| Ok(vec![ctx.value]))
             .map_err(|e| {
                 Error::new(
                     Status::GenericFailure,
@@ -53,7 +51,7 @@ impl lancedb::remote::HeaderProvider for JsHeaderProvider {
     async fn get_headers(&self) -> lancedb::error::Result<HashMap<String, String>> {
         // Call the JavaScript function asynchronously
         let promise: Promise<HashMap<String, String>> =
-            self.get_headers_fn.call_async(()).await.map_err(|e| {
+            self.get_headers_fn.call_async(Ok(())).await.map_err(|e| {
                 lancedb::error::Error::Runtime {
                     message: format!("Failed to call JavaScript get_headers: {}", e),
                 }

nodejs/src/index.rs

@@ -3,12 +3,12 @@
 
 use std::sync::Mutex;
 
-use lancedb::index::Index as LanceDbIndex;
 use lancedb::index::scalar::{BTreeIndexBuilder, FtsIndexBuilder};
 use lancedb::index::vector::{
     IvfFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, IvfPqIndexBuilder,
     IvfRqIndexBuilder,
 };
+use lancedb::index::Index as LanceDbIndex;
 use napi_derive::napi;
 
 use crate::util::parse_distance_type;

nodejs/src/lib.rs

@@ -60,7 +60,7 @@ pub struct OpenTableOptions {
     pub storage_options: Option<HashMap<String, String>>,
 }
 
-#[napi_derive::module_init]
+#[napi::module_init]
 fn init() {
     let env = Env::new()
         .filter_or("LANCEDB_LOG", "warn")

nodejs/src/query.rs

@@ -17,11 +17,11 @@ use lancedb::query::VectorQuery as LanceDbVectorQuery;
 use napi::bindgen_prelude::*;
 use napi_derive::napi;
 
-use crate::error::NapiErrorExt;
 use crate::error::convert_error;
+use crate::error::NapiErrorExt;
 use crate::iterator::RecordBatchIterator;
-use crate::rerankers::RerankHybridCallbackArgs;
 use crate::rerankers::Reranker;
+use crate::rerankers::RerankerCallbacks;
 use crate::util::{parse_distance_type, schema_to_buffer};
 
 #[napi]
@@ -42,7 +42,7 @@ impl Query {
     }
 
     #[napi]
-    pub fn full_text_search(&mut self, query: Object) -> napi::Result<()> {
+    pub fn full_text_search(&mut self, query: napi::JsObject) -> napi::Result<()> {
         let query = parse_fts_query(query)?;
         self.inner = self.inner.clone().full_text_search(query);
         Ok(())
@@ -235,7 +235,7 @@ impl VectorQuery {
     }
 
     #[napi]
-    pub fn full_text_search(&mut self, query: Object) -> napi::Result<()> {
+    pub fn full_text_search(&mut self, query: napi::JsObject) -> napi::Result<()> {
         let query = parse_fts_query(query)?;
         self.inner = self.inner.clone().full_text_search(query);
         Ok(())
@@ -272,13 +272,11 @@ impl VectorQuery {
     }
 
     #[napi]
-    pub fn rerank(
-        &mut self,
-        rerank_hybrid: Function<RerankHybridCallbackArgs, Promise<Buffer>>,
-    ) -> napi::Result<()> {
-        let reranker = Reranker::new(rerank_hybrid)?;
-        self.inner = self.inner.clone().rerank(Arc::new(reranker));
-        Ok(())
+    pub fn rerank(&mut self, callbacks: RerankerCallbacks) {
+        self.inner = self
+            .inner
+            .clone()
+            .rerank(Arc::new(Reranker::new(callbacks)));
     }
 
     #[napi(catch_unwind)]
@@ -525,12 +523,12 @@ impl JsFullTextQuery {
     }
 }
 
-fn parse_fts_query(query: Object) -> napi::Result<FullTextSearchQuery> {
-    if let Ok(Some(query)) = query.get::<&JsFullTextQuery>("query") {
+fn parse_fts_query(query: napi::JsObject) -> napi::Result<FullTextSearchQuery> {
+    if let Ok(Some(query)) = query.get::<_, &JsFullTextQuery>("query") {
         Ok(FullTextSearchQuery::new_query(query.inner.clone()))
-    } else if let Ok(Some(query_text)) = query.get::<String>("query") {
+    } else if let Ok(Some(query_text)) = query.get::<_, String>("query") {
         let mut query_text = query_text;
-        let columns = query.get::<Option<Vec<String>>>("columns")?.flatten();
+        let columns = query.get::<_, Option<Vec<String>>>("columns")?.flatten();
 
         let is_phrase =
             query_text.len() >= 2 && query_text.starts_with('"') && query_text.ends_with('"');
@@ -551,12 +549,15 @@ fn parse_fts_query(query: Object) -> napi::Result<FullTextSearchQuery> {
             }
         };
         let mut query = FullTextSearchQuery::new_query(query);
-        if let Some(cols) = columns
-            && !cols.is_empty()
-        {
-            query = query.with_columns(&cols).map_err(|e| {
-                napi::Error::from_reason(format!("Failed to set full text search columns: {}", e))
-            })?;
+        if let Some(cols) = columns {
+            if !cols.is_empty() {
+                query = query.with_columns(&cols).map_err(|e| {
+                    napi::Error::from_reason(format!(
+                        "Failed to set full text search columns: {}",
+                        e
+                    ))
+                })?;
+            }
         }
         Ok(query)
     } else {

nodejs/src/rerankers.rs

@@ -3,7 +3,10 @@
 
 use arrow_array::RecordBatch;
 use async_trait::async_trait;
-use napi::{bindgen_prelude::*, threadsafe_function::ThreadsafeFunction};
+use napi::{
+    bindgen_prelude::*,
+    threadsafe_function::{ErrorStrategy, ThreadsafeFunction},
+};
 use napi_derive::napi;
 
 use lancedb::ipc::batches_to_ipc_file;
@@ -12,28 +15,27 @@ use lancedb::{error::Error, ipc::ipc_file_to_batches};
 
 use crate::error::NapiErrorExt;
 
-type RerankHybridFn = ThreadsafeFunction<
-    RerankHybridCallbackArgs,
-    Promise<Buffer>,
-    RerankHybridCallbackArgs,
-    Status,
-    false,
->;
-
 /// Reranker implementation that "wraps" a NodeJS Reranker implementation.
 /// This contains references to the callbacks that can be used to invoke the
 /// reranking methods on the NodeJS implementation and handles serializing the
 /// record batches to Arrow IPC buffers.
+#[napi]
 pub struct Reranker {
-    rerank_hybrid: RerankHybridFn,
+    /// callback to the Javascript which will call the rerankHybrid method of
+    /// some Reranker implementation
+    rerank_hybrid: ThreadsafeFunction<RerankHybridCallbackArgs, ErrorStrategy::CalleeHandled>,
 }
 
+#[napi]
 impl Reranker {
-    pub fn new(
-        rerank_hybrid: Function<RerankHybridCallbackArgs, Promise<Buffer>>,
-    ) -> napi::Result<Self> {
-        let rerank_hybrid = rerank_hybrid.build_threadsafe_function().build()?;
-        Ok(Self { rerank_hybrid })
+    #[napi]
+    pub fn new(callbacks: RerankerCallbacks) -> Self {
+        let rerank_hybrid = callbacks
+            .rerank_hybrid
+            .create_threadsafe_function(0, move |ctx| Ok(vec![ctx.value]))
+            .unwrap();
+
+        Self { rerank_hybrid }
     }
 }
 
@@ -47,16 +49,16 @@ impl lancedb::rerankers::Reranker for Reranker {
     ) -> lancedb::error::Result<RecordBatch> {
         let callback_args = RerankHybridCallbackArgs {
             query: query.to_string(),
-            vec_results: Buffer::from(batches_to_ipc_file(&[vector_results])?.as_ref()),
-            fts_results: Buffer::from(batches_to_ipc_file(&[fts_results])?.as_ref()),
+            vec_results: batches_to_ipc_file(&[vector_results])?,
+            fts_results: batches_to_ipc_file(&[fts_results])?,
         };
         let promised_buffer: Promise<Buffer> = self
             .rerank_hybrid
-            .call_async(callback_args)
+            .call_async(Ok(callback_args))
             .await
             .map_err(|e| Error::Runtime {
-            message: format!("napi error status={}, reason={}", e.status, e.reason),
-        })?;
+                message: format!("napi error status={}, reason={}", e.status, e.reason),
+            })?;
         let buffer = promised_buffer.await.map_err(|e| Error::Runtime {
             message: format!("napi error status={}, reason={}", e.status, e.reason),
         })?;
@@ -75,11 +77,16 @@ impl std::fmt::Debug for Reranker {
     }
 }
 
+#[napi(object)]
+pub struct RerankerCallbacks {
+    pub rerank_hybrid: JsFunction,
+}
+
 #[napi(object)]
 pub struct RerankHybridCallbackArgs {
     pub query: String,
-    pub vec_results: Buffer,
-    pub fts_results: Buffer,
+    pub vec_results: Vec<u8>,
+    pub fts_results: Vec<u8>,
 }
 
 fn buffer_to_record_batch(buffer: Buffer) -> Result<RecordBatch> {

nodejs/src/session.rs

@@ -95,7 +95,8 @@ impl napi::bindgen_prelude::FromNapiValue for Session {
         napi_val: napi::sys::napi_value,
     ) -> napi::Result<Self> {
         let object: napi::bindgen_prelude::ClassInstance<Self> =
-            unsafe { napi::bindgen_prelude::ClassInstance::from_napi_value(env, napi_val)? };
-        Ok((*object).clone())
+            napi::bindgen_prelude::ClassInstance::from_napi_value(env, napi_val)?;
+        let copy = object.clone();
+        Ok(copy)
     }
 }

nodejs/src/table.rs

@@ -71,17 +71,6 @@ impl Table {
     pub async fn add(&self, buf: Buffer, mode: String) -> napi::Result<AddResult> {
         let batches = ipc_file_to_batches(buf.to_vec())
             .map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
-        let batches = batches
-            .into_iter()
-            .map(|batch| {
-                batch.map_err(|e| {
-                    napi::Error::from_reason(format!(
-                        "Failed to read record batch from IPC file: {}",
-                        e
-                    ))
-                })
-            })
-            .collect::<Result<Vec<_>>>()?;
         let mut op = self.inner_ref()?.add(batches);
 
         op = if mode == "append" {
@@ -753,14 +742,12 @@ impl From<lancedb::table::AddResult> for AddResult {
 
 #[napi(object)]
 pub struct DeleteResult {
-    pub num_deleted_rows: i64,
     pub version: i64,
 }
 
 impl From<lancedb::table::DeleteResult> for DeleteResult {
     fn from(value: lancedb::table::DeleteResult) -> Self {
         Self {
-            num_deleted_rows: value.num_deleted_rows as i64,
             version: value.version as i64,
         }
     }

python/.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.30.0-beta.4"
+current_version = "0.29.2"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

python/Cargo.toml

@@ -1,13 +1,13 @@
 [package]
 name = "lancedb-python"
-version = "0.30.0-beta.4"
+version = "0.29.2"
 edition.workspace = true
 description = "Python bindings for LanceDB"
 license.workspace = true
 repository.workspace = true
 keywords.workspace = true
 categories.workspace = true
-rust-version = "1.91.0"
+rust-version = "1.88.0"
 
 [lib]
 name = "_lancedb"
@@ -16,11 +16,9 @@ crate-type = ["cdylib"]
 [dependencies]
 arrow = { version = "57.2", features = ["pyarrow"] }
 async-trait = "0.1"
-bytes = "1"
 lancedb = { path = "../rust/lancedb", default-features = false }
 lance-core.workspace = true
 lance-namespace.workspace = true
-lance-namespace-impls.workspace = true
 lance-io.workspace = true
 env_logger.workspace = true
 pyo3 = { version = "0.26", features = ["extension-module", "abi3-py39"] }
@@ -30,8 +28,6 @@ pyo3-async-runtimes = { version = "0.26", features = [
 ] }
 pin-project = "1.1.5"
 futures.workspace = true
-serde = "1"
-serde_json = "1"
 snafu.workspace = true
 tokio = { version = "1.40", features = ["sync"] }
 

python/pyproject.toml

@@ -45,7 +45,7 @@ repository = "https://github.com/lancedb/lancedb"
 
 [project.optional-dependencies]
 pylance = [
-    "pylance>=4.0.0b7",
+    "pylance>=1.0.0b14",
 ]
 tests = [
     "aiohttp",
@@ -59,9 +59,9 @@ tests = [
     "polars>=0.19, <=1.3.0",
     "tantivy",
     "pyarrow-stubs",
-    "pylance>=4.0.0b7",
+    "pylance>=1.0.0b14",
     "requests",
-    "datafusion>=52,<53",
+    "datafusion",
 ]
 dev = [
     "ruff",

python/python/lancedb/arrow.py

@@ -1,10 +1,8 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-from functools import singledispatch
 from typing import List, Optional, Tuple, Union
 
-from lancedb.pydantic import LanceModel, model_to_dict
 import pyarrow as pa
 
 from ._lancedb import RecordBatchStream
@@ -82,32 +80,3 @@ def peek_reader(
         yield from reader
 
     return batch, pa.RecordBatchReader.from_batches(batch.schema, all_batches())
-
-
-@singledispatch
-def to_arrow(data) -> pa.Table:
-    """Convert a single data object to a pa.Table."""
-    raise NotImplementedError(f"to_arrow not implemented for type {type(data)}")
-
-
-@to_arrow.register(pa.RecordBatch)
-def _arrow_from_batch(data: pa.RecordBatch) -> pa.Table:
-    return pa.Table.from_batches([data])
-
-
-@to_arrow.register(pa.Table)
-def _arrow_from_table(data: pa.Table) -> pa.Table:
-    return data
-
-
-@to_arrow.register(list)
-def _arrow_from_list(data: list) -> pa.Table:
-    if not data:
-        raise ValueError("Cannot create table from empty list without a schema")
-
-    if isinstance(data[0], LanceModel):
-        schema = data[0].__class__.to_arrow_schema()
-        dicts = [model_to_dict(d) for d in data]
-        return pa.Table.from_pylist(dicts, schema=schema)
-
-    return pa.Table.from_pylist(data)

python/python/lancedb/db.py

@@ -8,7 +8,7 @@ from abc import abstractmethod
 from datetime import timedelta
 from pathlib import Path
 import sys
-from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Literal, Optional, Union
+from typing import TYPE_CHECKING, Dict, Iterable, List, Literal, Optional, Union
 
 if sys.version_info >= (3, 12):
     from typing import override
@@ -1541,8 +1541,6 @@ class AsyncConnection(object):
         storage_options_provider: Optional["StorageOptionsProvider"] = None,
         index_cache_size: Optional[int] = None,
         location: Optional[str] = None,
-        namespace_client: Optional[Any] = None,
-        managed_versioning: Optional[bool] = None,
     ) -> AsyncTable:
         """Open a Lance Table in the database.
 
@@ -1575,9 +1573,6 @@ class AsyncConnection(object):
             The explicit location (URI) of the table. If provided, the table will be
             opened from this location instead of deriving it from the database URI
             and table name.
-        managed_versioning: bool, optional
-            Whether managed versioning is enabled for this table. If provided,
-            avoids a redundant describe_table call when namespace_client is set.
 
         Returns
         -------
@@ -1592,8 +1587,6 @@ class AsyncConnection(object):
             storage_options_provider=storage_options_provider,
             index_cache_size=index_cache_size,
             location=location,
-            namespace_client=namespace_client,
-            managed_versioning=managed_versioning,
         )
         return AsyncTable(table)
 

python/python/lancedb/embeddings/gte.py

@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 
-import warnings
 from typing import List, Union
 
 import numpy as np
@@ -16,8 +15,6 @@ from .utils import weak_lru
 @register("gte-text")
 class GteEmbeddings(TextEmbeddingFunction):
     """
-    Deprecated: GTE embeddings should be used through sentence-transformers.
-
     An embedding function that uses GTE-LARGE MLX format(for Apple silicon devices only)
     as well as the standard cpu/gpu version from: https://huggingface.co/thenlper/gte-large.
 
@@ -64,13 +61,6 @@ class GteEmbeddings(TextEmbeddingFunction):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        warnings.warn(
-            "GTE embeddings as a standalone embedding function are deprecated. "
-            "Use the 'sentence-transformers' embedding function with a GTE model "
-            "instead.",
-            DeprecationWarning,
-            stacklevel=3,
-        )
         self._ndims = None
         if kwargs:
             self.mlx = kwargs.get("mlx", False)

python/python/lancedb/embeddings/openai.py

@@ -110,9 +110,6 @@ class OpenAIEmbeddings(TextEmbeddingFunction):
             valid_embeddings = {
                 idx: v.embedding for v, idx in zip(rs.data, valid_indices)
             }
-        except openai.AuthenticationError:
-            logging.error("Authentication failed: Invalid API key provided")
-            raise
         except openai.BadRequestError:
             logging.exception("Bad request: %s", texts)
             return [None] * len(texts)

python/python/lancedb/embeddings/siglip.py

@@ -6,7 +6,6 @@ import io
 import os
 from typing import TYPE_CHECKING, List, Union
 import urllib.parse as urlparse
-import warnings
 
 import numpy as np
 import pyarrow as pa
@@ -25,7 +24,6 @@ if TYPE_CHECKING:
 
 @register("siglip")
 class SigLipEmbeddings(EmbeddingFunction):
-    # Deprecated: prefer CLIP embeddings via `open-clip`.
     model_name: str = "google/siglip-base-patch16-224"
     device: str = "cpu"
     batch_size: int = 64
@@ -38,12 +36,6 @@ class SigLipEmbeddings(EmbeddingFunction):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        warnings.warn(
-            "SigLip embeddings are deprecated. Use CLIP embeddings via the "
-            "'open-clip' embedding function instead.",
-            DeprecationWarning,
-            stacklevel=3,
-        )
         transformers = attempt_import_or_raise("transformers")
         self._torch = attempt_import_or_raise("torch")
 

python/python/lancedb/embeddings/utils.py

@@ -269,11 +269,6 @@ def retry_with_exponential_backoff(
             # and say that it is assumed that if this portion errors out, it's due
             # to rate limit but the user should check the error message to be sure.
             except Exception as e:  # noqa: PERF203
-                # Don't retry on authentication errors (e.g., OpenAI 401)
-                # These are permanent failures that won't be fixed by retrying
-                if _is_non_retryable_error(e):
-                    raise
-
                 num_retries += 1
 
                 if num_retries > max_retries:
@@ -294,29 +289,6 @@ def retry_with_exponential_backoff(
     return wrapper
 
 
-def _is_non_retryable_error(error: Exception) -> bool:
-    """Check if an error should not be retried.
-
-    Args:
-        error: The exception to check
-
-    Returns:
-        True if the error should not be retried, False otherwise
-    """
-    # Check for OpenAI authentication errors
-    error_type = type(error).__name__
-    if error_type == "AuthenticationError":
-        return True
-
-    # Check for other common non-retryable HTTP status codes
-    # 401 Unauthorized, 403 Forbidden
-    if hasattr(error, "status_code"):
-        if error.status_code in (401, 403):
-            return True
-
-    return False
-
-
 def url_retrieve(url: str):
     """
     Parameters

python/python/lancedb/namespace.py

@@ -12,7 +12,7 @@ from __future__ import annotations
 
 import asyncio
 import sys
-from typing import Any, Dict, Iterable, List, Optional, Union
+from typing import Dict, Iterable, List, Optional, Union
 
 if sys.version_info >= (3, 12):
     from typing import override
@@ -44,7 +44,7 @@ from lance_namespace import (
     ListNamespacesRequest,
     CreateNamespaceRequest,
     DropNamespaceRequest,
-    DeclareTableRequest,
+    CreateEmptyTableRequest,
 )
 from lancedb.table import AsyncTable, LanceTable, Table
 from lancedb.util import validate_table_name
@@ -240,7 +240,7 @@ class LanceNamespaceDBConnection(DBConnection):
         session : Optional[Session]
             A session to use for this connection
         """
-        self._namespace_client = namespace
+        self._ns = namespace
         self.read_consistency_interval = read_consistency_interval
         self.storage_options = storage_options or {}
         self.session = session
@@ -269,7 +269,7 @@ class LanceNamespaceDBConnection(DBConnection):
         if namespace is None:
             namespace = []
         request = ListTablesRequest(id=namespace, page_token=page_token, limit=limit)
-        response = self._namespace_client.list_tables(request)
+        response = self._ns.list_tables(request)
         return response.tables if response.tables else []
 
     @override
@@ -309,9 +309,7 @@ class LanceNamespaceDBConnection(DBConnection):
             # Try to describe the table first to see if it exists
             try:
                 describe_request = DescribeTableRequest(id=table_id)
-                describe_response = self._namespace_client.describe_table(
-                    describe_request
-                )
+                describe_response = self._ns.describe_table(describe_request)
                 location = describe_response.location
                 namespace_storage_options = describe_response.storage_options
             except Exception:
@@ -320,20 +318,20 @@ class LanceNamespaceDBConnection(DBConnection):
 
         if location is None:
             # Table doesn't exist or mode is "create", reserve a new location
-            declare_request = DeclareTableRequest(
+            create_empty_request = CreateEmptyTableRequest(
                 id=table_id,
                 location=None,
                 properties=self.storage_options if self.storage_options else None,
             )
-            declare_response = self._namespace_client.declare_table(declare_request)
+            create_empty_response = self._ns.create_empty_table(create_empty_request)
 
-            if not declare_response.location:
+            if not create_empty_response.location:
                 raise ValueError(
-                    "Table location is missing from declare_table response"
+                    "Table location is missing from create_empty_table response"
                 )
 
-            location = declare_response.location
-            namespace_storage_options = declare_response.storage_options
+            location = create_empty_response.location
+            namespace_storage_options = create_empty_response.storage_options
 
         # Merge storage options: self.storage_options < user options < namespace options
         merged_storage_options = dict(self.storage_options)
@@ -355,7 +353,7 @@ class LanceNamespaceDBConnection(DBConnection):
         # Only create if namespace returned storage_options (not None)
         if storage_options_provider is None and namespace_storage_options is not None:
             storage_options_provider = LanceNamespaceStorageOptionsProvider(
-                namespace=self._namespace_client,
+                namespace=self._ns,
                 table_id=table_id,
             )
 
@@ -373,7 +371,6 @@ class LanceNamespaceDBConnection(DBConnection):
             storage_options=merged_storage_options,
             storage_options_provider=storage_options_provider,
             location=location,
-            namespace_client=self._namespace_client,
         )
 
         return tbl
@@ -392,7 +389,7 @@ class LanceNamespaceDBConnection(DBConnection):
             namespace = []
         table_id = namespace + [name]
         request = DescribeTableRequest(id=table_id)
-        response = self._namespace_client.describe_table(request)
+        response = self._ns.describe_table(request)
 
         # Merge storage options: self.storage_options < user options < namespace options
         merged_storage_options = dict(self.storage_options)
@@ -405,14 +402,10 @@ class LanceNamespaceDBConnection(DBConnection):
         # Only create if namespace returned storage_options (not None)
         if storage_options_provider is None and response.storage_options is not None:
             storage_options_provider = LanceNamespaceStorageOptionsProvider(
-                namespace=self._namespace_client,
+                namespace=self._ns,
                 table_id=table_id,
             )
 
-        # Pass managed_versioning to avoid redundant describe_table call in Rust.
-        # Convert None to False since we already have the answer from describe_table.
-        managed_versioning = response.managed_versioning is True
-
         return self._lance_table_from_uri(
             name,
             response.location,
@@ -420,8 +413,6 @@ class LanceNamespaceDBConnection(DBConnection):
             storage_options=merged_storage_options,
             storage_options_provider=storage_options_provider,
             index_cache_size=index_cache_size,
-            namespace_client=self._namespace_client,
-            managed_versioning=managed_versioning,
         )
 
     @override
@@ -431,7 +422,7 @@ class LanceNamespaceDBConnection(DBConnection):
             namespace = []
         table_id = namespace + [name]
         request = DropTableRequest(id=table_id)
-        self._namespace_client.drop_table(request)
+        self._ns.drop_table(request)
 
     @override
     def rename_table(
@@ -493,7 +484,7 @@ class LanceNamespaceDBConnection(DBConnection):
         request = ListNamespacesRequest(
             id=namespace, page_token=page_token, limit=limit
         )
-        response = self._namespace_client.list_namespaces(request)
+        response = self._ns.list_namespaces(request)
         return ListNamespacesResponse(
             namespaces=response.namespaces if response.namespaces else [],
             page_token=response.page_token,
@@ -529,7 +520,7 @@ class LanceNamespaceDBConnection(DBConnection):
             mode=_normalize_create_namespace_mode(mode),
             properties=properties,
         )
-        response = self._namespace_client.create_namespace(request)
+        response = self._ns.create_namespace(request)
         return CreateNamespaceResponse(
             properties=response.properties if hasattr(response, "properties") else None
         )
@@ -564,7 +555,7 @@ class LanceNamespaceDBConnection(DBConnection):
             mode=_normalize_drop_namespace_mode(mode),
             behavior=_normalize_drop_namespace_behavior(behavior),
         )
-        response = self._namespace_client.drop_namespace(request)
+        response = self._ns.drop_namespace(request)
         return DropNamespaceResponse(
             properties=(
                 response.properties if hasattr(response, "properties") else None
@@ -590,7 +581,7 @@ class LanceNamespaceDBConnection(DBConnection):
             Response containing the namespace properties.
         """
         request = DescribeNamespaceRequest(id=namespace)
-        response = self._namespace_client.describe_namespace(request)
+        response = self._ns.describe_namespace(request)
         return DescribeNamespaceResponse(
             properties=response.properties if hasattr(response, "properties") else None
         )
@@ -624,7 +615,7 @@ class LanceNamespaceDBConnection(DBConnection):
         if namespace is None:
             namespace = []
         request = ListTablesRequest(id=namespace, page_token=page_token, limit=limit)
-        response = self._namespace_client.list_tables(request)
+        response = self._ns.list_tables(request)
         return ListTablesResponse(
             tables=response.tables if response.tables else [],
             page_token=response.page_token,
@@ -639,8 +630,6 @@ class LanceNamespaceDBConnection(DBConnection):
         storage_options: Optional[Dict[str, str]] = None,
         storage_options_provider: Optional[StorageOptionsProvider] = None,
         index_cache_size: Optional[int] = None,
-        namespace_client: Optional[Any] = None,
-        managed_versioning: Optional[bool] = None,
     ) -> LanceTable:
         # Open a table directly from a URI using the location parameter
         # Note: storage_options should already be merged by the caller
@@ -654,8 +643,6 @@ class LanceNamespaceDBConnection(DBConnection):
         )
 
         # Open the table using the temporary connection with the location parameter
-        # Pass namespace_client to enable managed versioning support
-        # Pass managed_versioning to avoid redundant describe_table call
         return LanceTable.open(
             temp_conn,
             name,
@@ -664,8 +651,6 @@ class LanceNamespaceDBConnection(DBConnection):
             storage_options_provider=storage_options_provider,
             index_cache_size=index_cache_size,
             location=table_uri,
-            namespace_client=namespace_client,
-            managed_versioning=managed_versioning,
         )
 
 
@@ -700,7 +685,7 @@ class AsyncLanceNamespaceDBConnection:
         session : Optional[Session]
             A session to use for this connection
         """
-        self._namespace_client = namespace
+        self._ns = namespace
         self.read_consistency_interval = read_consistency_interval
         self.storage_options = storage_options or {}
         self.session = session
@@ -728,7 +713,7 @@ class AsyncLanceNamespaceDBConnection:
         if namespace is None:
             namespace = []
         request = ListTablesRequest(id=namespace, page_token=page_token, limit=limit)
-        response = self._namespace_client.list_tables(request)
+        response = self._ns.list_tables(request)
         return response.tables if response.tables else []
 
     async def create_table(
@@ -765,9 +750,7 @@ class AsyncLanceNamespaceDBConnection:
             # Try to describe the table first to see if it exists
             try:
                 describe_request = DescribeTableRequest(id=table_id)
-                describe_response = self._namespace_client.describe_table(
-                    describe_request
-                )
+                describe_response = self._ns.describe_table(describe_request)
                 location = describe_response.location
                 namespace_storage_options = describe_response.storage_options
             except Exception:
@@ -776,20 +759,20 @@ class AsyncLanceNamespaceDBConnection:
 
         if location is None:
             # Table doesn't exist or mode is "create", reserve a new location
-            declare_request = DeclareTableRequest(
+            create_empty_request = CreateEmptyTableRequest(
                 id=table_id,
                 location=None,
                 properties=self.storage_options if self.storage_options else None,
             )
-            declare_response = self._namespace_client.declare_table(declare_request)
+            create_empty_response = self._ns.create_empty_table(create_empty_request)
 
-            if not declare_response.location:
+            if not create_empty_response.location:
                 raise ValueError(
-                    "Table location is missing from declare_table response"
+                    "Table location is missing from create_empty_table response"
                 )
 
-            location = declare_response.location
-            namespace_storage_options = declare_response.storage_options
+            location = create_empty_response.location
+            namespace_storage_options = create_empty_response.storage_options
 
         # Merge storage options: self.storage_options < user options < namespace options
         merged_storage_options = dict(self.storage_options)
@@ -814,7 +797,7 @@ class AsyncLanceNamespaceDBConnection:
                 and namespace_storage_options is not None
             ):
                 provider = LanceNamespaceStorageOptionsProvider(
-                    namespace=self._namespace_client,
+                    namespace=self._ns,
                     table_id=table_id,
                 )
             else:
@@ -834,7 +817,6 @@ class AsyncLanceNamespaceDBConnection:
                 storage_options=merged_storage_options,
                 storage_options_provider=provider,
                 location=location,
-                namespace_client=self._namespace_client,
             )
 
         lance_table = await asyncio.to_thread(_create_table)
@@ -855,7 +837,7 @@ class AsyncLanceNamespaceDBConnection:
             namespace = []
         table_id = namespace + [name]
         request = DescribeTableRequest(id=table_id)
-        response = self._namespace_client.describe_table(request)
+        response = self._ns.describe_table(request)
 
         # Merge storage options: self.storage_options < user options < namespace options
         merged_storage_options = dict(self.storage_options)
@@ -867,14 +849,10 @@ class AsyncLanceNamespaceDBConnection:
         # Create a storage options provider if not provided by user
         if storage_options_provider is None and response.storage_options is not None:
             storage_options_provider = LanceNamespaceStorageOptionsProvider(
-                namespace=self._namespace_client,
+                namespace=self._ns,
                 table_id=table_id,
             )
 
-        # Capture managed_versioning from describe response.
-        # Convert None to False since we already have the answer from describe_table.
-        managed_versioning = response.managed_versioning is True
-
         # Open table in a thread
         def _open_table():
             temp_conn = LanceDBConnection(
@@ -892,8 +870,6 @@ class AsyncLanceNamespaceDBConnection:
                 storage_options_provider=storage_options_provider,
                 index_cache_size=index_cache_size,
                 location=response.location,
-                namespace_client=self._namespace_client,
-                managed_versioning=managed_versioning,
             )
 
         lance_table = await asyncio.to_thread(_open_table)
@@ -905,7 +881,7 @@ class AsyncLanceNamespaceDBConnection:
             namespace = []
         table_id = namespace + [name]
         request = DropTableRequest(id=table_id)
-        self._namespace_client.drop_table(request)
+        self._ns.drop_table(request)
 
     async def rename_table(
         self,
@@ -967,7 +943,7 @@ class AsyncLanceNamespaceDBConnection:
         request = ListNamespacesRequest(
             id=namespace, page_token=page_token, limit=limit
         )
-        response = self._namespace_client.list_namespaces(request)
+        response = self._ns.list_namespaces(request)
         return ListNamespacesResponse(
             namespaces=response.namespaces if response.namespaces else [],
             page_token=response.page_token,
@@ -1002,7 +978,7 @@ class AsyncLanceNamespaceDBConnection:
             mode=_normalize_create_namespace_mode(mode),
             properties=properties,
         )
-        response = self._namespace_client.create_namespace(request)
+        response = self._ns.create_namespace(request)
         return CreateNamespaceResponse(
             properties=response.properties if hasattr(response, "properties") else None
         )
@@ -1036,7 +1012,7 @@ class AsyncLanceNamespaceDBConnection:
             mode=_normalize_drop_namespace_mode(mode),
             behavior=_normalize_drop_namespace_behavior(behavior),
         )
-        response = self._namespace_client.drop_namespace(request)
+        response = self._ns.drop_namespace(request)
         return DropNamespaceResponse(
             properties=(
                 response.properties if hasattr(response, "properties") else None
@@ -1063,7 +1039,7 @@ class AsyncLanceNamespaceDBConnection:
             Response containing the namespace properties.
         """
         request = DescribeNamespaceRequest(id=namespace)
-        response = self._namespace_client.describe_namespace(request)
+        response = self._ns.describe_namespace(request)
         return DescribeNamespaceResponse(
             properties=response.properties if hasattr(response, "properties") else None
         )
@@ -1096,7 +1072,7 @@ class AsyncLanceNamespaceDBConnection:
         if namespace is None:
             namespace = []
         request = ListTablesRequest(id=namespace, page_token=page_token, limit=limit)
-        response = self._namespace_client.list_tables(request)
+        response = self._ns.list_tables(request)
         return ListTablesResponse(
             tables=response.tables if response.tables else [],
             page_token=response.page_token,

python/python/lancedb/permutation.py

@@ -9,7 +9,7 @@ import json
 from ._lancedb import async_permutation_builder, PermutationReader
 from .table import LanceTable
 from .background_loop import LOOP
-from .util import batch_to_tensor, batch_to_tensor_rows
+from .util import batch_to_tensor
 from typing import Any, Callable, Iterator, Literal, Optional, TYPE_CHECKING, Union
 
 if TYPE_CHECKING:
@@ -333,11 +333,7 @@ class Transforms:
     """
 
     @staticmethod
-    def arrow2python(batch: pa.RecordBatch) -> list[dict[str, Any]]:
-        return batch.to_pylist()
-
-    @staticmethod
-    def arrow2pythoncol(batch: pa.RecordBatch) -> dict[str, list[Any]]:
+    def arrow2python(batch: pa.RecordBatch) -> dict[str, list[Any]]:
         return batch.to_pydict()
 
     @staticmethod
@@ -691,17 +687,7 @@ class Permutation:
             return
 
     def with_format(
-        self,
-        format: Literal[
-            "numpy",
-            "python",
-            "python_col",
-            "pandas",
-            "arrow",
-            "torch",
-            "torch_col",
-            "polars",
-        ],
+        self, format: Literal["numpy", "python", "pandas", "arrow", "torch", "polars"]
     ) -> "Permutation":
         """
         Set the format for batches
@@ -710,18 +696,16 @@ class Permutation:
 
         The format can be one of:
         - "numpy" - the batch will be a dict of numpy arrays (one per column)
-        - "python" - the batch will be a list of dicts (one per row)
-        - "python_col" - the batch will be a dict of lists (one entry per column)
+        - "python" - the batch will be a dict of lists (one per column)
         - "pandas" - the batch will be a pandas DataFrame
         - "arrow" - the batch will be a pyarrow RecordBatch
-        - "torch" - the batch will be a list of tensors, one per row
-        - "torch_col" - the batch will be a 2D torch tensor (first dim indexes columns)
+        - "torch" - the batch will be a two dimensional torch tensor
         - "polars" - the batch will be a polars DataFrame
 
         Conversion may or may not involve a data copy.  Lance uses Arrow internally
-        and so it is able to zero-copy to the arrow and polars formats.
+        and so it is able to zero-copy to the arrow and polars.
 
-        Conversion to torch_col will be zero-copy but will only support a subset of data
+        Conversion to torch will be zero-copy but will only support a subset of data
         types (numeric types).
 
         Conversion to numpy and/or pandas will typically be zero-copy for numeric
@@ -734,8 +718,6 @@ class Permutation:
         assert format is not None, "format is required"
         if format == "python":
             return self.with_transform(Transforms.arrow2python)
-        if format == "python_col":
-            return self.with_transform(Transforms.arrow2pythoncol)
         elif format == "numpy":
             return self.with_transform(Transforms.arrow2numpy)
         elif format == "pandas":
@@ -743,8 +725,6 @@ class Permutation:
         elif format == "arrow":
             return self.with_transform(Transforms.arrow2arrow)
         elif format == "torch":
-            return self.with_transform(batch_to_tensor_rows)
-        elif format == "torch_col":
             return self.with_transform(batch_to_tensor)
         elif format == "polars":
             return self.with_transform(Transforms.arrow2polars())

python/python/lancedb/query.py

@@ -606,7 +606,6 @@ class LanceQueryBuilder(ABC):
                 query,
                 ordering_field_name=ordering_field_name,
                 fts_columns=fts_columns,
-                fast_search=fast_search,
             )
 
         if isinstance(query, list):
@@ -1457,14 +1456,12 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
         query: str | FullTextQuery,
         ordering_field_name: Optional[str] = None,
         fts_columns: Optional[Union[str, List[str]]] = None,
-        fast_search: bool = None,
     ):
         super().__init__(table)
         self._query = query
         self._phrase_query = False
         self.ordering_field_name = ordering_field_name
         self._reranker = None
-        self._fast_search = fast_search
         if isinstance(fts_columns, str):
             fts_columns = [fts_columns]
         self._fts_columns = fts_columns
@@ -1486,19 +1483,6 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
         self._phrase_query = phrase_query
         return self
 
-    def fast_search(self) -> LanceFtsQueryBuilder:
-        """
-        Skip a flat search of unindexed data. This will improve
-        search performance but search results will not include unindexed data.
-
-        Returns
-        -------
-        LanceFtsQueryBuilder
-            The LanceFtsQueryBuilder object.
-        """
-        self._fast_search = True
-        return self
-
     def to_query_object(self) -> Query:
         return Query(
             columns=self._columns,
@@ -1510,7 +1494,6 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
                 query=self._query, columns=self._fts_columns
             ),
             offset=self._offset,
-            fast_search=self._fast_search,
         )
 
     def output_schema(self) -> pa.Schema:
@@ -1799,26 +1782,6 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
             vector_results = LanceHybridQueryBuilder._rank(vector_results, "_distance")
             fts_results = LanceHybridQueryBuilder._rank(fts_results, "_score")
 
-        # If both result sets are empty (e.g. after hard filtering),
-        # return early to avoid errors in reranking or score restoration.
-        if vector_results.num_rows == 0 and fts_results.num_rows == 0:
-            # Build a minimal empty table with the _relevance_score column
-            combined_schema = pa.unify_schemas(
-                [vector_results.schema, fts_results.schema],
-            )
-            empty = pa.table(
-                {
-                    col: pa.array([], type=combined_schema.field(col).type)
-                    for col in combined_schema.names
-                }
-            )
-            empty = empty.append_column(
-                "_relevance_score", pa.array([], type=pa.float32())
-            )
-            if not with_row_ids and "_rowid" in empty.column_names:
-                empty = empty.drop(["_rowid"])
-            return empty
-
         original_distances = None
         original_scores = None
         original_distance_row_ids = None

python/python/lancedb/remote/table.py

@@ -218,6 +218,8 @@ class RemoteTable(Table):
         train: bool = True,
     ):
         """Create an index on the table.
+        Currently, the only parameters that matter are
+        the metric and the vector column name.
 
         Parameters
         ----------
@@ -248,6 +250,11 @@ class RemoteTable(Table):
         >>> table.create_index("l2", "vector") # doctest: +SKIP
         """
 
+        if num_sub_vectors is not None:
+            logging.warning(
+                "num_sub_vectors is not supported on LanceDB cloud."
+                "This parameter will be tuned automatically."
+            )
         if accelerator is not None:
             logging.warning(
                 "GPU accelerator is not yet supported on LanceDB cloud."

python/python/lancedb/scannable.py

@@ -1,214 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-from dataclasses import dataclass
-from functools import singledispatch
-import sys
-from typing import Callable, Iterator, Optional
-from lancedb.arrow import to_arrow
-import pyarrow as pa
-import pyarrow.dataset as ds
-
-from .pydantic import LanceModel
-
-
-@dataclass
-class Scannable:
-    schema: pa.Schema
-    num_rows: Optional[int]
-    # Factory function to create a new reader each time (supports re-scanning)
-    reader: Callable[[], pa.RecordBatchReader]
-    # Whether reader can be called more than once. For example, an iterator can
-    # only be consumed once, while a DataFrame can be converted to a new reader
-    # each time.
-    rescannable: bool = True
-
-
-@singledispatch
-def to_scannable(data) -> Scannable:
-    # Fallback: try iterable protocol
-    if hasattr(data, "__iter__"):
-        return _from_iterable(iter(data))
-    raise NotImplementedError(f"to_scannable not implemented for type {type(data)}")
-
-
-@to_scannable.register(pa.RecordBatchReader)
-def _from_reader(data: pa.RecordBatchReader) -> Scannable:
-    # RecordBatchReader can only be consumed once - not rescannable
-    return Scannable(
-        schema=data.schema, num_rows=None, reader=lambda: data, rescannable=False
-    )
-
-
-@to_scannable.register(pa.RecordBatch)
-def _from_batch(data: pa.RecordBatch) -> Scannable:
-    return Scannable(
-        schema=data.schema,
-        num_rows=data.num_rows,
-        reader=lambda: pa.RecordBatchReader.from_batches(data.schema, [data]),
-    )
-
-
-@to_scannable.register(pa.Table)
-def _from_table(data: pa.Table) -> Scannable:
-    return Scannable(schema=data.schema, num_rows=data.num_rows, reader=data.to_reader)
-
-
-@to_scannable.register(ds.Dataset)
-def _from_dataset(data: ds.Dataset) -> Scannable:
-    return Scannable(
-        schema=data.schema,
-        num_rows=data.count_rows(),
-        reader=lambda: data.scanner().to_reader(),
-    )
-
-
-@to_scannable.register(ds.Scanner)
-def _from_scanner(data: ds.Scanner) -> Scannable:
-    # Scanner can only be consumed once - not rescannable
-    return Scannable(
-        schema=data.projected_schema,
-        num_rows=None,
-        reader=data.to_reader,
-        rescannable=False,
-    )
-
-
-@to_scannable.register(list)
-def _from_list(data: list) -> Scannable:
-    if not data:
-        raise ValueError("Cannot create table from empty list without a schema")
-    table = to_arrow(data)
-    return Scannable(
-        schema=table.schema, num_rows=table.num_rows, reader=table.to_reader
-    )
-
-
-@to_scannable.register(dict)
-def _from_dict(data: dict) -> Scannable:
-    raise ValueError("Cannot add a single dictionary to a table. Use a list.")
-
-
-@to_scannable.register(LanceModel)
-def _from_lance_model(data: LanceModel) -> Scannable:
-    raise ValueError("Cannot add a single LanceModel to a table. Use a list.")
-
-
-def _from_iterable(data: Iterator) -> Scannable:
-    first_item = next(data, None)
-    if first_item is None:
-        raise ValueError("Cannot create table from empty iterator")
-    first = to_arrow(first_item)
-    schema = first.schema
-
-    def iter():
-        yield from first.to_batches()
-        for item in data:
-            batch = to_arrow(item)
-            if batch.schema != schema:
-                try:
-                    batch = batch.cast(schema)
-                except pa.lib.ArrowInvalid:
-                    raise ValueError(
-                        f"Input iterator yielded a batch with schema that "
-                        f"does not match the schema of other batches.\n"
-                        f"Expected:\n{schema}\nGot:\n{batch.schema}"
-                    )
-            yield from batch.to_batches()
-
-    reader = pa.RecordBatchReader.from_batches(schema, iter())
-    return to_scannable(reader)
-
-
-_registered_modules: set[str] = set()
-
-
-def _register_optional_converters():
-    """Register converters for optional dependencies that are already imported."""
-
-    if "pandas" in sys.modules and "pandas" not in _registered_modules:
-        _registered_modules.add("pandas")
-        import pandas as pd
-
-        @to_arrow.register(pd.DataFrame)
-        def _arrow_from_pandas(data: pd.DataFrame) -> pa.Table:
-            table = pa.Table.from_pandas(data, preserve_index=False)
-            return table.replace_schema_metadata(None)
-
-        @to_scannable.register(pd.DataFrame)
-        def _from_pandas(data: pd.DataFrame) -> Scannable:
-            return to_scannable(_arrow_from_pandas(data))
-
-    if "polars" in sys.modules and "polars" not in _registered_modules:
-        _registered_modules.add("polars")
-        import polars as pl
-
-        @to_arrow.register(pl.DataFrame)
-        def _arrow_from_polars(data: pl.DataFrame) -> pa.Table:
-            return data.to_arrow()
-
-        @to_scannable.register(pl.DataFrame)
-        def _from_polars(data: pl.DataFrame) -> Scannable:
-            arrow = data.to_arrow()
-            return Scannable(
-                schema=arrow.schema, num_rows=len(data), reader=arrow.to_reader
-            )
-
-        @to_scannable.register(pl.LazyFrame)
-        def _from_polars_lazy(data: pl.LazyFrame) -> Scannable:
-            arrow = data.collect().to_arrow()
-            return Scannable(
-                schema=arrow.schema, num_rows=arrow.num_rows, reader=arrow.to_reader
-            )
-
-    if "datasets" in sys.modules and "datasets" not in _registered_modules:
-        _registered_modules.add("datasets")
-        from datasets import Dataset as HFDataset
-        from datasets import DatasetDict as HFDatasetDict
-
-        @to_scannable.register(HFDataset)
-        def _from_hf_dataset(data: HFDataset) -> Scannable:
-            table = data.data.table  # Access underlying Arrow table
-            return Scannable(
-                schema=table.schema, num_rows=len(data), reader=table.to_reader
-            )
-
-        @to_scannable.register(HFDatasetDict)
-        def _from_hf_dataset_dict(data: HFDatasetDict) -> Scannable:
-            # HuggingFace DatasetDict: combine all splits with a 'split' column
-            schema = data[list(data.keys())[0]].features.arrow_schema
-            if "split" not in schema.names:
-                schema = schema.append(pa.field("split", pa.string()))
-
-            def gen():
-                for split_name, dataset in data.items():
-                    for batch in dataset.data.to_batches():
-                        split_arr = pa.array(
-                            [split_name] * len(batch), type=pa.string()
-                        )
-                        yield pa.RecordBatch.from_arrays(
-                            list(batch.columns) + [split_arr], schema=schema
-                        )
-
-            total_rows = sum(len(dataset) for dataset in data.values())
-            return Scannable(
-                schema=schema,
-                num_rows=total_rows,
-                reader=lambda: pa.RecordBatchReader.from_batches(schema, gen()),
-            )
-
-    if "lance" in sys.modules and "lance" not in _registered_modules:
-        _registered_modules.add("lance")
-        import lance
-
-        @to_scannable.register(lance.LanceDataset)
-        def _from_lance(data: lance.LanceDataset) -> Scannable:
-            return Scannable(
-                schema=data.schema,
-                num_rows=data.count_rows(),
-                reader=lambda: data.scanner().to_reader(),
-            )
-
-
-# Register on module load
-_register_optional_converters()

python/python/lancedb/table.py

@@ -25,8 +25,6 @@ from typing import (
 )
 from urllib.parse import urlparse
 
-from lancedb.scannable import _register_optional_converters, to_scannable
-
 from . import __version__
 from lancedb.arrow import peek_reader
 from lancedb.background_loop import LOOP
@@ -1331,7 +1329,7 @@ class Table(ABC):
         1  2  [3.0, 4.0]
         2  3  [5.0, 6.0]
         >>> table.delete("x = 2")
-        DeleteResult(num_deleted_rows=1, version=2)
+        DeleteResult(version=2)
         >>> table.to_pandas()
            x      vector
         0  1  [1.0, 2.0]
@@ -1345,7 +1343,7 @@ class Table(ABC):
         >>> to_remove
         '1, 5'
         >>> table.delete(f"x IN ({to_remove})")
-        DeleteResult(num_deleted_rows=1, version=3)
+        DeleteResult(version=3)
         >>> table.to_pandas()
            x      vector
         0  3  [5.0, 6.0]
@@ -1746,8 +1744,6 @@ class LanceTable(Table):
         storage_options_provider: Optional["StorageOptionsProvider"] = None,
         index_cache_size: Optional[int] = None,
         location: Optional[str] = None,
-        namespace_client: Optional[Any] = None,
-        managed_versioning: Optional[bool] = None,
         _async: AsyncTable = None,
     ):
         if namespace is None:
@@ -1755,7 +1751,6 @@ class LanceTable(Table):
         self._conn = connection
         self._namespace = namespace
         self._location = location  # Store location for use in _dataset_path
-        self._namespace_client = namespace_client
         if _async is not None:
             self._table = _async
         else:
@@ -1767,8 +1762,6 @@ class LanceTable(Table):
                     storage_options_provider=storage_options_provider,
                     index_cache_size=index_cache_size,
                     location=location,
-                    namespace_client=namespace_client,
-                    managed_versioning=managed_versioning,
                 )
             )
 
@@ -1811,8 +1804,6 @@ class LanceTable(Table):
         storage_options_provider: Optional["StorageOptionsProvider"] = None,
         index_cache_size: Optional[int] = None,
         location: Optional[str] = None,
-        namespace_client: Optional[Any] = None,
-        managed_versioning: Optional[bool] = None,
     ):
         if namespace is None:
             namespace = []
@@ -1824,8 +1815,6 @@ class LanceTable(Table):
             storage_options_provider=storage_options_provider,
             index_cache_size=index_cache_size,
             location=location,
-            namespace_client=namespace_client,
-            managed_versioning=managed_versioning,
         )
 
         # check the dataset exists
@@ -1857,16 +1846,6 @@ class LanceTable(Table):
                 "Please install with `pip install pylance`."
             )
 
-        if self._namespace_client is not None:
-            table_id = self._namespace + [self.name]
-            return lance.dataset(
-                version=self.version,
-                storage_options=self._conn.storage_options,
-                namespace=self._namespace_client,
-                table_id=table_id,
-                **kwargs,
-            )
-
         return lance.dataset(
             self._dataset_path,
             version=self.version,
@@ -2732,7 +2711,6 @@ class LanceTable(Table):
         data_storage_version: Optional[str] = None,
         enable_v2_manifest_paths: Optional[bool] = None,
         location: Optional[str] = None,
-        namespace_client: Optional[Any] = None,
     ):
         """
         Create a new table.
@@ -2793,7 +2771,6 @@ class LanceTable(Table):
         self._conn = db
         self._namespace = namespace
         self._location = location
-        self._namespace_client = namespace_client
 
         if data_storage_version is not None:
             warnings.warn(
@@ -3750,31 +3727,18 @@ class AsyncTable:
             on_bad_vectors = "error"
         if fill_value is None:
             fill_value = 0.0
+        data = _sanitize_data(
+            data,
+            schema,
+            metadata=schema.metadata,
+            on_bad_vectors=on_bad_vectors,
+            fill_value=fill_value,
+            allow_subschema=True,
+        )
+        if isinstance(data, pa.Table):
+            data = data.to_reader()
 
-        # _santitize_data is an old code path, but we will use it until the
-        # new code path is ready.
-        if on_bad_vectors != "error" or (
-            schema.metadata is not None and b"embedding_functions" in schema.metadata
-        ):
-            data = _sanitize_data(
-                data,
-                schema,
-                metadata=schema.metadata,
-                on_bad_vectors=on_bad_vectors,
-                fill_value=fill_value,
-                allow_subschema=True,
-            )
-        _register_optional_converters()
-        data = to_scannable(data)
-        try:
-            return await self._inner.add(data, mode or "append")
-        except RuntimeError as e:
-            if "Cast error" in str(e):
-                raise ValueError(e)
-            elif "Vector column contains NaN" in str(e):
-                raise ValueError(e)
-            else:
-                raise
+        return await self._inner.add(data, mode or "append")
 
     def merge_insert(self, on: Union[str, Iterable[str]]) -> LanceMergeInsertBuilder:
         """
@@ -4236,7 +4200,7 @@ class AsyncTable:
         1  2  [3.0, 4.0]
         2  3  [5.0, 6.0]
         >>> table.delete("x = 2")
-        DeleteResult(num_deleted_rows=1, version=2)
+        DeleteResult(version=2)
         >>> table.to_pandas()
            x      vector
         0  1  [1.0, 2.0]
@@ -4250,7 +4214,7 @@ class AsyncTable:
         >>> to_remove
         '1, 5'
         >>> table.delete(f"x IN ({to_remove})")
-        DeleteResult(num_deleted_rows=1, version=3)
+        DeleteResult(version=3)
         >>> table.to_pandas()
            x      vector
         0  3  [5.0, 6.0]

python/python/lancedb/util.py

@@ -324,16 +324,6 @@ def _(value: list):
     return "[" + ", ".join(map(value_to_sql, value)) + "]"
 
 
-@value_to_sql.register(dict)
-def _(value: dict):
-    # https://datafusion.apache.org/user-guide/sql/scalar_functions.html#named-struct
-    return (
-        "named_struct("
-        + ", ".join(f"'{k}', {value_to_sql(v)}" for k, v in value.items())
-        + ")"
-    )
-
-
 @value_to_sql.register(np.ndarray)
 def _(value: np.ndarray):
     return value_to_sql(value.tolist())
@@ -429,22 +419,3 @@ def batch_to_tensor(batch: pa.RecordBatch):
     """
     torch = attempt_import_or_raise("torch", "torch")
     return torch.stack([torch.from_dlpack(col) for col in batch.columns])
-
-
-def batch_to_tensor_rows(batch: pa.RecordBatch):
-    """
-    Convert a PyArrow RecordBatch to a list of PyTorch Tensor, one per row
-
-    Each column is converted to a tensor (using zero-copy via DLPack)
-    and the columns are then stacked into a single tensor.  The 2D tensor
-    is then converted to a list of tensors, one per row
-
-    Fails if torch or numpy is not installed.
-    Fails if a column's data type is not supported by PyTorch.
-    """
-    torch = attempt_import_or_raise("torch", "torch")
-    numpy = attempt_import_or_raise("numpy", "numpy")
-    columns = [col.to_numpy(zero_copy_only=False) for col in batch.columns]
-    stacked = torch.tensor(numpy.column_stack(columns))
-    rows = list(stacked.unbind(dim=0))
-    return rows

python/python/tests/test_embeddings.py

@@ -515,34 +515,3 @@ def test_openai_propagates_api_key(monkeypatch):
     query = "greetings"
     actual = table.search(query).limit(1).to_pydantic(Words)[0]
     assert len(actual.text) > 0
-
-
-@patch("time.sleep")
-def test_openai_no_retry_on_401(mock_sleep):
-    """
-    Test that OpenAI embedding function does not retry on 401 authentication
-    errors.
-    """
-    from lancedb.embeddings.utils import retry_with_exponential_backoff
-
-    # Create a mock that raises an AuthenticationError
-    class MockAuthenticationError(Exception):
-        """Mock OpenAI AuthenticationError"""
-
-        pass
-
-    MockAuthenticationError.__name__ = "AuthenticationError"
-
-    mock_func = MagicMock(side_effect=MockAuthenticationError("Invalid API key"))
-
-    # Wrap the function with retry logic
-    wrapped_func = retry_with_exponential_backoff(mock_func, max_retries=3)
-
-    # Should raise without retrying
-    with pytest.raises(MockAuthenticationError):
-        wrapped_func()
-
-    # Verify that the function was only called once (no retries)
-    assert mock_func.call_count == 1
-    # Verify that sleep was never called (no retries)
-    assert mock_sleep.call_count == 0

python/python/tests/test_fts.py

@@ -27,7 +27,6 @@ from lancedb.query import (
     PhraseQuery,
     BooleanQuery,
     Occur,
-    LanceFtsQueryBuilder,
 )
 import numpy as np
 import pyarrow as pa
@@ -883,109 +882,3 @@ def test_fts_query_to_json():
         '"must_not":[]}}'
     )
     assert json_str == expected
-
-
-def test_fts_fast_search(table):
-    table.create_fts_index("text", use_tantivy=False)
-
-    # Insert some unindexed data
-    table.add(
-        [
-            {
-                "text": "xyz",
-                "vector": [0 for _ in range(128)],
-                "id": 101,
-                "text2": "xyz",
-                "nested": {"text": "xyz"},
-                "count": 10,
-            }
-        ]
-    )
-
-    # Without fast_search, the query object should not have fast_search set
-    builder = table.search("xyz", query_type="fts").limit(10)
-    query = builder.to_query_object()
-    assert query.fast_search is None
-
-    # With fast_search, the query object should have fast_search=True
-    builder = table.search("xyz", query_type="fts").fast_search().limit(10)
-    query = builder.to_query_object()
-    assert query.fast_search is True
-
-    # fast_search should be chainable with other methods
-    builder = (
-        table.search("xyz", query_type="fts").fast_search().select(["text"]).limit(5)
-    )
-    query = builder.to_query_object()
-    assert query.fast_search is True
-    assert query.limit == 5
-    assert query.columns == ["text"]
-
-    # fast_search should be enabled by keyword argument too
-    query = LanceFtsQueryBuilder(table, "xyz", fast_search=True).to_query_object()
-    assert query.fast_search is True
-
-    # Verify it executes without error and skips unindexed data
-    results = table.search("xyz", query_type="fts").fast_search().limit(5).to_list()
-    assert len(results) == 0
-
-    # Update index and verify it returns results
-    table.optimize()
-    results = table.search("xyz", query_type="fts").fast_search().limit(5).to_list()
-    assert len(results) > 0
-
-
-@pytest.mark.asyncio
-async def test_fts_fast_search_async(async_table):
-    await async_table.create_index("text", config=FTS())
-
-    # Insert some unindexed data
-    await async_table.add(
-        [
-            {
-                "text": "xyz",
-                "vector": [0 for _ in range(128)],
-                "id": 101,
-                "text2": "xyz",
-                "nested": {"text": "xyz"},
-                "count": 10,
-            }
-        ]
-    )
-
-    # Without fast_search, should return results
-    results = await async_table.query().nearest_to_text("xyz").limit(5).to_list()
-    assert len(results) > 0
-
-    # With fast_search, should return no results data unindexed
-    fast_results = (
-        await async_table.query()
-        .nearest_to_text("xyz")
-        .fast_search()
-        .limit(5)
-        .to_list()
-    )
-    assert len(fast_results) == 0
-
-    # Update index and verify it returns results
-    await async_table.optimize()
-
-    fast_results = (
-        await async_table.query()
-        .nearest_to_text("xyz")
-        .fast_search()
-        .limit(5)
-        .to_list()
-    )
-    assert len(fast_results) > 0
-
-    # fast_search should be chainable with other methods
-    results = (
-        await async_table.query()
-        .nearest_to_text("xyz")
-        .fast_search()
-        .select(["text"])
-        .limit(5)
-        .to_list()
-    )
-    assert len(results) > 0

python/python/tests/test_permutation.py

@@ -664,20 +664,23 @@ def test_iter_basic(some_permutation: Permutation):
     expected_batches = (950 + batch_size - 1) // batch_size  # ceiling division
     assert len(batches) == expected_batches
 
-    # Check that all batches are lists of dicts (default python format)
-    assert all(isinstance(batch, list) for batch in batches)
+    # Check that all batches are dicts (default python format)
+    assert all(isinstance(batch, dict) for batch in batches)
 
     # Check that batches have the correct structure
     for batch in batches:
-        assert "id" in batch[0]
-        assert "value" in batch[0]
+        assert "id" in batch
+        assert "value" in batch
+        assert isinstance(batch["id"], list)
+        assert isinstance(batch["value"], list)
 
     # Check that all batches except the last have the correct size
     for batch in batches[:-1]:
-        assert len(batch) == batch_size
+        assert len(batch["id"]) == batch_size
+        assert len(batch["value"]) == batch_size
 
     # Last batch might be smaller
-    assert len(batches[-1]) <= batch_size
+    assert len(batches[-1]["id"]) <= batch_size
 
 
 def test_iter_skip_last_batch(some_permutation: Permutation):
@@ -696,11 +699,11 @@ def test_iter_skip_last_batch(some_permutation: Permutation):
     if 950 % batch_size != 0:
         assert len(batches_without_skip) == num_full_batches + 1
         # Last batch should be smaller
-        assert len(batches_without_skip[-1]) == 950 % batch_size
+        assert len(batches_without_skip[-1]["id"]) == 950 % batch_size
 
     # All batches with skip_last_batch should be full size
     for batch in batches_with_skip:
-        assert len(batch) == batch_size
+        assert len(batch["id"]) == batch_size
 
 
 def test_iter_different_batch_sizes(some_permutation: Permutation):
@@ -717,12 +720,12 @@ def test_iter_different_batch_sizes(some_permutation: Permutation):
     # Test with batch size equal to total rows
     single_batch = list(some_permutation.iter(950, skip_last_batch=False))
     assert len(single_batch) == 1
-    assert len(single_batch[0]) == 950
+    assert len(single_batch[0]["id"]) == 950
 
     # Test with batch size larger than total rows
     oversized_batch = list(some_permutation.iter(10000, skip_last_batch=False))
     assert len(oversized_batch) == 1
-    assert len(oversized_batch[0]) == 950
+    assert len(oversized_batch[0]["id"]) == 950
 
 
 def test_dunder_iter(some_permutation: Permutation):
@@ -735,13 +738,15 @@ def test_dunder_iter(some_permutation: Permutation):
 
     # All batches should be full size
     for batch in batches:
-        assert len(batch) == 100
+        assert len(batch["id"]) == 100
+        assert len(batch["value"]) == 100
 
     some_permutation = some_permutation.with_batch_size(400)
     batches = list(some_permutation)
     assert len(batches) == 2  # floor(950 / 400) since skip_last_batch=True
     for batch in batches:
-        assert len(batch) == 400
+        assert len(batch["id"]) == 400
+        assert len(batch["value"]) == 400
 
 
 def test_iter_with_different_formats(some_permutation: Permutation):
@@ -756,7 +761,7 @@ def test_iter_with_different_formats(some_permutation: Permutation):
     # Test with python format (default)
     python_perm = some_permutation.with_format("python")
     python_batches = list(python_perm.iter(batch_size, skip_last_batch=False))
-    assert all(isinstance(batch, list) for batch in python_batches)
+    assert all(isinstance(batch, dict) for batch in python_batches)
 
     # Test with pandas format
     pandas_perm = some_permutation.with_format("pandas")
@@ -775,8 +780,8 @@ def test_iter_with_column_selection(some_permutation: Permutation):
 
     # Check that batches only contain the id column
     for batch in batches:
-        assert "id" in batch[0]
-        assert "value" not in batch[0]
+        assert "id" in batch
+        assert "value" not in batch
 
 
 def test_iter_with_column_rename(some_permutation: Permutation):
@@ -786,9 +791,9 @@ def test_iter_with_column_rename(some_permutation: Permutation):
 
     # Check that batches have the renamed column
     for batch in batches:
-        assert "id" in batch[0]
-        assert "data" in batch[0]
-        assert "value" not in batch[0]
+        assert "id" in batch
+        assert "data" in batch
+        assert "value" not in batch
 
 
 def test_iter_with_limit_offset(some_permutation: Permutation):
@@ -807,14 +812,14 @@ def test_iter_with_limit_offset(some_permutation: Permutation):
     assert len(limit_batches) == 5
 
     no_skip = some_permutation.iter(101, skip_last_batch=False)
-    row_100 = next(no_skip)[100]["id"]
+    row_100 = next(no_skip)["id"][100]
 
     # Test with both limit and offset
     limited_perm = some_permutation.with_skip(100).with_take(300)
     limited_batches = list(limited_perm.iter(100, skip_last_batch=False))
     # Should have 3 batches (300 / 100)
     assert len(limited_batches) == 3
-    assert limited_batches[0][0]["id"] == row_100
+    assert limited_batches[0]["id"][0] == row_100
 
 
 def test_iter_empty_permutation(mem_db):
@@ -837,7 +842,7 @@ def test_iter_single_row(mem_db):
     # With skip_last_batch=False, should get one batch
     batches = list(perm.iter(10, skip_last_batch=False))
     assert len(batches) == 1
-    assert len(batches[0]) == 1
+    assert len(batches[0]["id"]) == 1
 
     # With skip_last_batch=True, should skip the single row (since it's < batch_size)
     batches_skip = list(perm.iter(10, skip_last_batch=True))
@@ -855,7 +860,8 @@ def test_identity_permutation(mem_db):
 
     batches = list(permutation.iter(10, skip_last_batch=False))
     assert len(batches) == 1
-    assert len(batches[0]) == 10
+    assert len(batches[0]["id"]) == 10
+    assert len(batches[0]["value"]) == 10
 
     permutation = permutation.remove_columns(["value"])
     assert permutation.num_columns == 1
@@ -898,10 +904,10 @@ def test_transform_fn(mem_db):
     py_result = list(permutation.with_format("python").iter(10, skip_last_batch=False))[
         0
     ]
-    assert len(py_result) == 10
-    assert "id" in py_result[0]
-    assert "value" in py_result[0]
-    assert isinstance(py_result, list)
+    assert len(py_result) == 2
+    assert len(py_result["id"]) == 10
+    assert len(py_result["value"]) == 10
+    assert isinstance(py_result, dict)
 
     try:
         import torch
@@ -909,11 +915,9 @@ def test_transform_fn(mem_db):
         torch_result = list(
             permutation.with_format("torch").iter(10, skip_last_batch=False)
         )[0]
-        assert isinstance(torch_result, list)
-        assert len(torch_result) == 10
-        assert isinstance(torch_result[0], torch.Tensor)
-        assert torch_result[0].shape == (2,)
-        assert torch_result[0].dtype == torch.int64
+        assert torch_result.shape == (2, 10)
+        assert torch_result.dtype == torch.int64
+        assert isinstance(torch_result, torch.Tensor)
     except ImportError:
         # Skip check if torch is not installed
         pass
@@ -946,16 +950,17 @@ def test_custom_transform(mem_db):
 def test_getitems_basic(some_permutation: Permutation):
     """Test __getitems__ returns correct rows by offset."""
     result = some_permutation.__getitems__([0, 1, 2])
-    assert isinstance(result, list)
-    assert "id" in result[0]
-    assert "value" in result[0]
-    assert len(result) == 3
+    assert isinstance(result, dict)
+    assert "id" in result
+    assert "value" in result
+    assert len(result["id"]) == 3
 
 
 def test_getitems_single_index(some_permutation: Permutation):
     """Test __getitems__ with a single index."""
     result = some_permutation.__getitems__([0])
-    assert len(result) == 1
+    assert len(result["id"]) == 1
+    assert len(result["value"]) == 1
 
 
 def test_getitems_preserves_order(some_permutation: Permutation):
@@ -965,40 +970,38 @@ def test_getitems_preserves_order(some_permutation: Permutation):
     # Get the same rows in reverse order
     reverse = some_permutation.__getitems__([4, 3, 2, 1, 0])
 
-    assert [r["id"] for r in forward] == list(reversed([r["id"] for r in reverse]))
-    assert [r["value"] for r in forward] == list(
-        reversed([r["value"] for r in reverse])
-    )
+    assert forward["id"] == list(reversed(reverse["id"]))
+    assert forward["value"] == list(reversed(reverse["value"]))
 
 
 def test_getitems_non_contiguous(some_permutation: Permutation):
     """Test __getitems__ with non-contiguous indices."""
     result = some_permutation.__getitems__([0, 10, 50, 100, 500])
-    assert len(result) == 5
+    assert len(result["id"]) == 5
 
     # Each id/value pair should match what we'd get individually
     for i, offset in enumerate([0, 10, 50, 100, 500]):
         single = some_permutation.__getitems__([offset])
-        assert result[i]["id"] == single[0]["id"]
-        assert result[i]["value"] == single[0]["value"]
+        assert result["id"][i] == single["id"][0]
+        assert result["value"][i] == single["value"][0]
 
 
 def test_getitems_with_column_selection(some_permutation: Permutation):
     """Test __getitems__ respects column selection."""
     id_only = some_permutation.select_columns(["id"])
     result = id_only.__getitems__([0, 1, 2])
-    assert "id" in result[0]
-    assert "value" not in result[0]
-    assert len(result) == 3
+    assert "id" in result
+    assert "value" not in result
+    assert len(result["id"]) == 3
 
 
 def test_getitems_with_column_rename(some_permutation: Permutation):
     """Test __getitems__ respects column renames."""
     renamed = some_permutation.rename_column("value", "data")
     result = renamed.__getitems__([0, 1])
-    assert "data" in result[0]
-    assert "value" not in result[0]
-    assert len(result) == 2
+    assert "data" in result
+    assert "value" not in result
+    assert len(result["data"]) == 2
 
 
 def test_getitems_with_format(some_permutation: Permutation):
@@ -1029,8 +1032,8 @@ def test_getitems_identity_permutation(mem_db):
     perm = Permutation.identity(tbl)
 
     result = perm.__getitems__([0, 5, 9])
-    assert [r["id"] for r in result] == [0, 5, 9]
-    assert [r["value"] for r in result] == [0, 5, 9]
+    assert result["id"] == [0, 5, 9]
+    assert result["value"] == [0, 5, 9]
 
 
 def test_getitems_with_limit_offset(some_permutation: Permutation):
@@ -1039,12 +1042,12 @@ def test_getitems_with_limit_offset(some_permutation: Permutation):
 
     # Should be able to access offsets within the limited range
     result = limited.__getitems__([0, 1, 199])
-    assert len(result) == 3
+    assert len(result["id"]) == 3
 
     # The first item of the limited permutation should match offset 100 of original
     full_result = some_permutation.__getitems__([100])
     limited_result = limited.__getitems__([0])
-    assert limited_result[0]["id"] == full_result[0]["id"]
+    assert limited_result["id"][0] == full_result["id"][0]
 
 
 def test_getitems_invalid_offset(some_permutation: Permutation):

python/python/tests/test_rerankers.py

@@ -531,78 +531,6 @@ def test_empty_result_reranker():
         )
 
 
-def test_empty_hybrid_result_reranker():
-    """Test that hybrid search with empty results after filtering doesn't crash.
-
-    Regression test for https://github.com/lancedb/lancedb/issues/2425
-    """
-    from lancedb.query import LanceHybridQueryBuilder
-
-    # Simulate empty vector and FTS results with the expected schema
-    vector_schema = pa.schema(
-        [
-            ("text", pa.string()),
-            ("vector", pa.list_(pa.float32(), 4)),
-            ("_rowid", pa.uint64()),
-            ("_distance", pa.float32()),
-        ]
-    )
-    fts_schema = pa.schema(
-        [
-            ("text", pa.string()),
-            ("vector", pa.list_(pa.float32(), 4)),
-            ("_rowid", pa.uint64()),
-            ("_score", pa.float32()),
-        ]
-    )
-    empty_vector = pa.table(
-        {
-            "text": pa.array([], type=pa.string()),
-            "vector": pa.array([], type=pa.list_(pa.float32(), 4)),
-            "_rowid": pa.array([], type=pa.uint64()),
-            "_distance": pa.array([], type=pa.float32()),
-        },
-        schema=vector_schema,
-    )
-    empty_fts = pa.table(
-        {
-            "text": pa.array([], type=pa.string()),
-            "vector": pa.array([], type=pa.list_(pa.float32(), 4)),
-            "_rowid": pa.array([], type=pa.uint64()),
-            "_score": pa.array([], type=pa.float32()),
-        },
-        schema=fts_schema,
-    )
-
-    for reranker in [LinearCombinationReranker(), RRFReranker()]:
-        result = LanceHybridQueryBuilder._combine_hybrid_results(
-            fts_results=empty_fts,
-            vector_results=empty_vector,
-            norm="score",
-            fts_query="nonexistent query",
-            reranker=reranker,
-            limit=10,
-            with_row_ids=False,
-        )
-        assert len(result) == 0
-        assert "_relevance_score" in result.column_names
-        assert "_rowid" not in result.column_names
-
-    # Also test with with_row_ids=True
-    result = LanceHybridQueryBuilder._combine_hybrid_results(
-        fts_results=empty_fts,
-        vector_results=empty_vector,
-        norm="score",
-        fts_query="nonexistent query",
-        reranker=LinearCombinationReranker(),
-        limit=10,
-        with_row_ids=True,
-    )
-    assert len(result) == 0
-    assert "_relevance_score" in result.column_names
-    assert "_rowid" in result.column_names
-
-
 @pytest.mark.parametrize("use_tantivy", [True, False])
 def test_cross_encoder_reranker_return_all(tmp_path, use_tantivy):
     pytest.importorskip("sentence_transformers")

python/python/tests/test_table.py

@@ -326,24 +326,6 @@ def test_add_struct(mem_db: DBConnection):
     table = mem_db.create_table("test2", schema=schema)
     table.add(data)
 
-    struct_type = pa.struct(
-        [
-            ("b", pa.int64()),
-            ("a", pa.int64()),
-        ]
-    )
-    expected = pa.table(
-        {
-            "s_list": [
-                [
-                    pa.scalar({"b": 1, "a": 2}, type=struct_type),
-                    pa.scalar({"b": 4, "a": None}, type=struct_type),
-                ]
-            ],
-        }
-    )
-    assert table.to_arrow() == expected
-
 
 def test_add_subschema(mem_db: DBConnection):
     schema = pa.schema(
@@ -828,7 +810,7 @@ def test_create_index_name_and_train_parameters(
     )
 
 
-def test_create_with_nans(mem_db: DBConnection):
+def test_add_with_nans(mem_db: DBConnection):
     # by default we raise an error on bad input vectors
     bad_data = [
         {"vector": [np.nan], "item": "bar", "price": 20.0},
@@ -872,57 +854,6 @@ def test_create_with_nans(mem_db: DBConnection):
     assert np.allclose(v, np.array([0.0, 0.0]))
 
 
-def test_add_with_nans(mem_db: DBConnection):
-    schema = pa.schema(
-        [
-            pa.field("vector", pa.list_(pa.float32(), 2), nullable=True),
-            pa.field("item", pa.string(), nullable=True),
-            pa.field("price", pa.float64(), nullable=False),
-        ],
-    )
-    table = mem_db.create_table("test", schema=schema)
-    # by default we raise an error on bad input vectors
-    bad_data = [
-        {"vector": [np.nan], "item": "bar", "price": 20.0},
-        {"vector": [5], "item": "bar", "price": 20.0},
-        {"vector": [np.nan, np.nan], "item": "bar", "price": 20.0},
-        {"vector": [np.nan, 5.0], "item": "bar", "price": 20.0},
-    ]
-    for row in bad_data:
-        with pytest.raises(ValueError):
-            table.add(
-                data=[row],
-            )
-
-    table.add(
-        [
-            {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-            {"vector": [2.1, 4.1], "item": "foo", "price": 9.0},
-            {"vector": [np.nan], "item": "bar", "price": 20.0},
-            {"vector": [5], "item": "bar", "price": 20.0},
-            {"vector": [np.nan, np.nan], "item": "bar", "price": 20.0},
-        ],
-        on_bad_vectors="drop",
-    )
-    assert len(table) == 2
-    table.delete("true")
-
-    # We can fill bad input with some value
-    table.add(
-        data=[
-            {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-            {"vector": [np.nan], "item": "bar", "price": 20.0},
-            {"vector": [np.nan, np.nan], "item": "bar", "price": 20.0},
-        ],
-        on_bad_vectors="fill",
-        fill_value=0.0,
-    )
-    assert len(table) == 3
-    arrow_tbl = table.search().where("item == 'bar'").to_arrow()
-    v = arrow_tbl["vector"].to_pylist()[0]
-    assert np.allclose(v, np.array([0.0, 0.0]))
-
-
 def test_restore(mem_db: DBConnection):
     table = mem_db.create_table(
         "my_table",

python/python/tests/test_torch.py

@@ -4,7 +4,6 @@
 import pyarrow as pa
 import pytest
 from lancedb.util import tbl_to_tensor
-from lancedb.permutation import Permutation
 
 torch = pytest.importorskip("torch")
 
@@ -17,26 +16,3 @@ def test_table_dataloader(mem_db):
     for batch in dataloader:
         assert batch.size(0) == 1
         assert batch.size(1) == 10
-
-
-def test_permutation_dataloader(mem_db):
-    table = mem_db.create_table("test_table", pa.table({"a": range(1000)}))
-
-    permutation = Permutation.identity(table)
-    dataloader = torch.utils.data.DataLoader(permutation, batch_size=10, shuffle=True)
-    for batch in dataloader:
-        assert batch["a"].size(0) == 10
-
-    permutation = permutation.with_format("torch")
-    dataloader = torch.utils.data.DataLoader(permutation, batch_size=10, shuffle=True)
-    for batch in dataloader:
-        assert batch.size(0) == 10
-        assert batch.size(1) == 1
-
-    permutation = permutation.with_format("torch_col")
-    dataloader = torch.utils.data.DataLoader(
-        permutation, collate_fn=lambda x: x, batch_size=10, shuffle=True
-    )
-    for batch in dataloader:
-        assert batch.size(0) == 1
-        assert batch.size(1) == 10

python/python/tests/test_util.py

@@ -121,32 +121,6 @@ def test_value_to_sql_string(tmp_path):
         assert table.to_pandas().query("search == @value")["replace"].item() == value
 
 
-def test_value_to_sql_dict():
-    # Simple flat struct
-    assert value_to_sql({"a": 1, "b": "hello"}) == "named_struct('a', 1, 'b', 'hello')"
-
-    # Nested struct
-    assert (
-        value_to_sql({"outer": {"inner": 1}})
-        == "named_struct('outer', named_struct('inner', 1))"
-    )
-
-    # List inside struct
-    assert value_to_sql({"a": [1, 2]}) == "named_struct('a', [1, 2])"
-
-    # Mixed types
-    assert (
-        value_to_sql({"name": "test", "count": 42, "rate": 3.14, "active": True})
-        == "named_struct('name', 'test', 'count', 42, 'rate', 3.14, 'active', TRUE)"
-    )
-
-    # Null value inside struct
-    assert value_to_sql({"a": None}) == "named_struct('a', NULL)"
-
-    # Empty dict
-    assert value_to_sql({}) == "named_struct()"
-
-
 def test_append_vector_columns():
     registry = EmbeddingFunctionRegistry.get_instance()
     registry.register("test")(MockTextEmbeddingFunction)
@@ -318,14 +292,18 @@ class TestModel(lancedb.pydantic.LanceModel):
         lambda: pa.table({"a": [1], "b": [2]}),
         lambda: pa.table({"a": [1], "b": [2]}).to_reader(),
         lambda: iter(pa.table({"a": [1], "b": [2]}).to_batches()),
-        lambda: lance.write_dataset(
-            pa.table({"a": [1], "b": [2]}),
-            "memory://test",
+        lambda: (
+            lance.write_dataset(
+                pa.table({"a": [1], "b": [2]}),
+                "memory://test",
+            )
+        ),
+        lambda: (
+            lance.write_dataset(
+                pa.table({"a": [1], "b": [2]}),
+                "memory://test",
+            ).scanner()
         ),
-        lambda: lance.write_dataset(
-            pa.table({"a": [1], "b": [2]}),
-            "memory://test",
-        ).scanner(),
         lambda: pd.DataFrame({"a": [1], "b": [2]}),
         lambda: pl.DataFrame({"a": [1], "b": [2]}),
         lambda: pl.LazyFrame({"a": [1], "b": [2]}),

python/src/arrow.rs

@@ -10,7 +10,7 @@ use arrow::{
 use futures::stream::StreamExt;
 use lancedb::arrow::SendableRecordBatchStream;
 use pyo3::{
-    Bound, Py, PyAny, PyRef, PyResult, Python, exceptions::PyStopAsyncIteration, pyclass, pymethods,
+    exceptions::PyStopAsyncIteration, pyclass, pymethods, Bound, Py, PyAny, PyRef, PyResult, Python,
 };
 use pyo3_async_runtimes::tokio::future_into_py;
 

python/src/connection.rs

@@ -9,16 +9,15 @@ use lancedb::{
     database::{CreateTableMode, Database, ReadConsistency},
 };
 use pyo3::{
-    Bound, FromPyObject, Py, PyAny, PyRef, PyResult, Python,
     exceptions::{PyRuntimeError, PyValueError},
     pyclass, pyfunction, pymethods,
     types::{PyDict, PyDictMethods},
+    Bound, FromPyObject, Py, PyAny, PyRef, PyResult, Python,
 };
 use pyo3_async_runtimes::tokio::future_into_py;
 
 use crate::{
-    error::PythonErrorExt, namespace::extract_namespace_arc,
-    storage_options::py_object_to_storage_options_provider, table::Table,
+    error::PythonErrorExt, storage_options::py_object_to_storage_options_provider, table::Table,
 };
 
 #[pyclass]
@@ -122,8 +121,7 @@ impl Connection {
 
         let mode = Self::parse_create_mode_str(mode)?;
 
-        let batches: Box<dyn arrow::array::RecordBatchReader + Send> =
-            Box::new(ArrowArrayStreamReader::from_pyarrow_bound(&data)?);
+        let batches = ArrowArrayStreamReader::from_pyarrow_bound(&data)?;
 
         let mut builder = inner.create_table(name, batches).mode(mode);
 
@@ -183,8 +181,7 @@ impl Connection {
         })
     }
 
-    #[allow(clippy::too_many_arguments)]
-    #[pyo3(signature = (name, namespace=vec![], storage_options = None, storage_options_provider=None, index_cache_size = None, location=None, namespace_client=None, managed_versioning=None))]
+    #[pyo3(signature = (name, namespace=vec![], storage_options = None, storage_options_provider=None, index_cache_size = None, location=None))]
     pub fn open_table(
         self_: PyRef<'_, Self>,
         name: String,
@@ -193,13 +190,11 @@ impl Connection {
         storage_options_provider: Option<Py<PyAny>>,
         index_cache_size: Option<u32>,
         location: Option<String>,
-        namespace_client: Option<Py<PyAny>>,
-        managed_versioning: Option<bool>,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.get_inner()?.clone();
 
         let mut builder = inner.open_table(name);
-        builder = builder.namespace(namespace.clone());
+        builder = builder.namespace(namespace);
         if let Some(storage_options) = storage_options {
             builder = builder.storage_options(storage_options);
         }
@@ -213,20 +208,6 @@ impl Connection {
         if let Some(location) = location {
             builder = builder.location(location);
         }
-        // Extract namespace client from Python object if provided
-        let ns_client = if let Some(ns_obj) = namespace_client {
-            let py = self_.py();
-            Some(extract_namespace_arc(py, ns_obj)?)
-        } else {
-            None
-        };
-        if let Some(ns_client) = ns_client {
-            builder = builder.namespace_client(ns_client);
-        }
-        // Pass managed_versioning if provided to avoid redundant describe_table call
-        if let Some(enabled) = managed_versioning {
-            builder = builder.managed_versioning(enabled);
-        }
 
         future_into_py(self_.py(), async move {
             let table = builder.execute().await.infer_error()?;

python/src/error.rs

@@ -2,10 +2,10 @@
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 use pyo3::{
-    PyErr, PyResult, Python,
     exceptions::{PyIOError, PyNotImplementedError, PyOSError, PyRuntimeError, PyValueError},
     intern,
     types::{PyAnyMethods, PyNone},
+    PyErr, PyResult, Python,
 };
 
 use lancedb::error::Error as LanceError;

python/src/index.rs

@@ -3,17 +3,17 @@
 
 use lancedb::index::vector::{IvfFlatIndexBuilder, IvfRqIndexBuilder, IvfSqIndexBuilder};
 use lancedb::index::{
-    Index as LanceDbIndex,
     scalar::{BTreeIndexBuilder, FtsIndexBuilder},
     vector::{IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, IvfPqIndexBuilder},
+    Index as LanceDbIndex,
 };
-use pyo3::IntoPyObject;
 use pyo3::types::PyStringMethods;
+use pyo3::IntoPyObject;
 use pyo3::{
-    Bound, FromPyObject, PyAny, PyResult, Python,
     exceptions::{PyKeyError, PyValueError},
     intern, pyclass, pymethods,
     types::PyAnyMethods,
+    Bound, FromPyObject, PyAny, PyResult, Python,
 };
 
 use crate::util::parse_distance_type;
@@ -41,12 +41,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                 let inner_opts = FtsIndexBuilder::default()
                     .base_tokenizer(params.base_tokenizer)
                     .language(&params.language)
-                    .map_err(|_| {
-                        PyValueError::new_err(format!(
-                            "LanceDB does not support the requested language: '{}'",
-                            params.language
-                        ))
-                    })?
+                    .map_err(|_| PyValueError::new_err(format!("LanceDB does not support the requested language: '{}'", params.language)))?
                     .with_position(params.with_position)
                     .lower_case(params.lower_case)
                     .max_token_length(params.max_token_length)
@@ -57,7 +52,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                     .ngram_max_length(params.ngram_max_length)
                     .ngram_prefix_only(params.prefix_only);
                 Ok(LanceDbIndex::FTS(inner_opts))
-            }
+            },
             "IvfFlat" => {
                 let params = source.extract::<IvfFlatParams>()?;
                 let distance_type = parse_distance_type(params.distance_type)?;
@@ -69,11 +64,10 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                     ivf_flat_builder = ivf_flat_builder.num_partitions(num_partitions);
                 }
                 if let Some(target_partition_size) = params.target_partition_size {
-                    ivf_flat_builder =
-                        ivf_flat_builder.target_partition_size(target_partition_size);
+                    ivf_flat_builder = ivf_flat_builder.target_partition_size(target_partition_size);
                 }
                 Ok(LanceDbIndex::IvfFlat(ivf_flat_builder))
-            }
+            },
             "IvfPq" => {
                 let params = source.extract::<IvfPqParams>()?;
                 let distance_type = parse_distance_type(params.distance_type)?;
@@ -92,7 +86,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                     ivf_pq_builder = ivf_pq_builder.num_sub_vectors(num_sub_vectors);
                 }
                 Ok(LanceDbIndex::IvfPq(ivf_pq_builder))
-            }
+            },
             "IvfSq" => {
                 let params = source.extract::<IvfSqParams>()?;
                 let distance_type = parse_distance_type(params.distance_type)?;
@@ -107,7 +101,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                     ivf_sq_builder = ivf_sq_builder.target_partition_size(target_partition_size);
                 }
                 Ok(LanceDbIndex::IvfSq(ivf_sq_builder))
-            }
+            },
             "IvfRq" => {
                 let params = source.extract::<IvfRqParams>()?;
                 let distance_type = parse_distance_type(params.distance_type)?;
@@ -123,7 +117,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                     ivf_rq_builder = ivf_rq_builder.target_partition_size(target_partition_size);
                 }
                 Ok(LanceDbIndex::IvfRq(ivf_rq_builder))
-            }
+            },
             "HnswPq" => {
                 let params = source.extract::<IvfHnswPqParams>()?;
                 let distance_type = parse_distance_type(params.distance_type)?;
@@ -144,7 +138,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                     hnsw_pq_builder = hnsw_pq_builder.num_sub_vectors(num_sub_vectors);
                 }
                 Ok(LanceDbIndex::IvfHnswPq(hnsw_pq_builder))
-            }
+            },
             "HnswSq" => {
                 let params = source.extract::<IvfHnswSqParams>()?;
                 let distance_type = parse_distance_type(params.distance_type)?;
@@ -161,7 +155,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                     hnsw_sq_builder = hnsw_sq_builder.target_partition_size(target_partition_size);
                 }
                 Ok(LanceDbIndex::IvfHnswSq(hnsw_sq_builder))
-            }
+            },
             not_supported => Err(PyValueError::new_err(format!(
                 "Invalid index type '{}'.  Must be one of BTree, Bitmap, LabelList, FTS, IvfPq, IvfSq, IvfHnswPq, or IvfHnswSq",
                 not_supported

python/src/lib.rs

@@ -2,14 +2,14 @@
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 use arrow::RecordBatchStream;
-use connection::{Connection, connect};
+use connection::{connect, Connection};
 use env_logger::Env;
 use index::IndexConfig;
 use permutation::{PyAsyncPermutationBuilder, PyPermutationReader};
 use pyo3::{
-    Bound, PyResult, Python, pymodule,
+    pymodule,
     types::{PyModule, PyModuleMethods},
-    wrap_pyfunction,
+    wrap_pyfunction, Bound, PyResult, Python,
 };
 use query::{FTSQuery, HybridQuery, Query, VectorQuery};
 use session::Session;
@@ -23,7 +23,6 @@ pub mod connection;
 pub mod error;
 pub mod header;
 pub mod index;
-pub mod namespace;
 pub mod permutation;
 pub mod query;
 pub mod session;

python/src/namespace.rs

@@ -1,746 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-//! Namespace utilities for Python bindings
-
-use std::collections::HashMap;
-use std::sync::Arc;
-
-use async_trait::async_trait;
-use bytes::Bytes;
-use lance_namespace::LanceNamespace as LanceNamespaceTrait;
-use lance_namespace::models::*;
-use pyo3::prelude::*;
-use pyo3::types::PyDict;
-
-/// Wrapper that allows any Python object implementing LanceNamespace protocol
-/// to be used as a Rust LanceNamespace.
-///
-/// This is similar to PyLanceNamespace in lance's Python bindings - it wraps a Python
-/// object and calls back into Python when namespace methods are invoked.
-pub struct PyLanceNamespace {
-    py_namespace: Arc<Py<PyAny>>,
-    namespace_id: String,
-}
-
-impl PyLanceNamespace {
-    /// Create a new PyLanceNamespace wrapper around a Python namespace object.
-    pub fn new(_py: Python<'_>, py_namespace: &Bound<'_, PyAny>) -> PyResult<Self> {
-        let namespace_id = py_namespace
-            .call_method0("namespace_id")?
-            .extract::<String>()?;
-
-        Ok(Self {
-            py_namespace: Arc::new(py_namespace.clone().unbind()),
-            namespace_id,
-        })
-    }
-
-    /// Create an Arc<dyn LanceNamespace> from a Python namespace object.
-    pub fn create_arc(
-        py: Python<'_>,
-        py_namespace: &Bound<'_, PyAny>,
-    ) -> PyResult<Arc<dyn LanceNamespaceTrait>> {
-        let wrapper = Self::new(py, py_namespace)?;
-        Ok(Arc::new(wrapper))
-    }
-}
-
-impl std::fmt::Debug for PyLanceNamespace {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "PyLanceNamespace {{ id: {} }}", self.namespace_id)
-    }
-}
-
-/// Get or create the DictWithModelDump class in Python.
-/// This class acts like a dict but also has model_dump() method.
-/// This allows it to work with both:
-/// - depythonize (which expects a dict/Mapping)
-/// - Python code that calls .model_dump() (like DirectoryNamespace wrapper)
-fn get_dict_with_model_dump_class(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
-    // Use a module-level cache via __builtins__
-    let builtins = py.import("builtins")?;
-    if builtins.hasattr("_DictWithModelDump")? {
-        return builtins.getattr("_DictWithModelDump");
-    }
-
-    // Create the class using exec
-    let locals = PyDict::new(py);
-    py.run(
-        c"class DictWithModelDump(dict):
-    def model_dump(self):
-        return dict(self)",
-        None,
-        Some(&locals),
-    )?;
-    let class = locals.get_item("DictWithModelDump")?.ok_or_else(|| {
-        pyo3::exceptions::PyRuntimeError::new_err("Failed to create DictWithModelDump class")
-    })?;
-
-    // Cache it
-    builtins.setattr("_DictWithModelDump", &class)?;
-    Ok(class)
-}
-
-/// Helper to call a Python namespace method with JSON serialization.
-/// For methods that take a request and return a response.
-/// Uses DictWithModelDump to pass a dict that also has model_dump() method,
-/// making it compatible with both depythonize and Python wrappers.
-async fn call_py_method<Req, Resp>(
-    py_namespace: Arc<Py<PyAny>>,
-    method_name: &'static str,
-    request: Req,
-) -> lance_core::Result<Resp>
-where
-    Req: serde::Serialize + Send + 'static,
-    Resp: serde::de::DeserializeOwned + Send + 'static,
-{
-    let request_json = serde_json::to_string(&request).map_err(|e| {
-        lance_core::Error::io(
-            format!("Failed to serialize request for {}: {}", method_name, e),
-            Default::default(),
-        )
-    })?;
-
-    let response_json = tokio::task::spawn_blocking(move || {
-        Python::attach(|py| {
-            let json_module = py.import("json")?;
-            let request_dict = json_module.call_method1("loads", (&request_json,))?;
-
-            // Wrap dict in DictWithModelDump so it works with both depythonize and .model_dump()
-            let dict_class = get_dict_with_model_dump_class(py)?;
-            let request_arg = dict_class.call1((request_dict,))?;
-
-            // Call the Python method
-            let result = py_namespace.call_method1(py, method_name, (request_arg,))?;
-
-            // Convert response to dict, then to JSON
-            // Pydantic models have model_dump() method
-            let result_dict = if result.bind(py).hasattr("model_dump")? {
-                result.call_method0(py, "model_dump")?
-            } else {
-                result
-            };
-            let response_json: String = json_module
-                .call_method1("dumps", (result_dict,))?
-                .extract()?;
-            Ok::<_, PyErr>(response_json)
-        })
-    })
-    .await
-    .map_err(|e| {
-        lance_core::Error::io(
-            format!("Task join error for {}: {}", method_name, e),
-            Default::default(),
-        )
-    })?
-    .map_err(|e: PyErr| {
-        lance_core::Error::io(
-            format!("Python error in {}: {}", method_name, e),
-            Default::default(),
-        )
-    })?;
-
-    serde_json::from_str(&response_json).map_err(|e| {
-        lance_core::Error::io(
-            format!("Failed to deserialize response from {}: {}", method_name, e),
-            Default::default(),
-        )
-    })
-}
-
-/// Helper for methods that return () on success
-async fn call_py_method_unit<Req>(
-    py_namespace: Arc<Py<PyAny>>,
-    method_name: &'static str,
-    request: Req,
-) -> lance_core::Result<()>
-where
-    Req: serde::Serialize + Send + 'static,
-{
-    let request_json = serde_json::to_string(&request).map_err(|e| {
-        lance_core::Error::io(
-            format!("Failed to serialize request for {}: {}", method_name, e),
-            Default::default(),
-        )
-    })?;
-
-    tokio::task::spawn_blocking(move || {
-        Python::attach(|py| {
-            let json_module = py.import("json")?;
-            let request_dict = json_module.call_method1("loads", (&request_json,))?;
-
-            // Wrap dict in DictWithModelDump
-            let dict_class = get_dict_with_model_dump_class(py)?;
-            let request_arg = dict_class.call1((request_dict,))?;
-
-            // Call the Python method
-            py_namespace.call_method1(py, method_name, (request_arg,))?;
-            Ok::<_, PyErr>(())
-        })
-    })
-    .await
-    .map_err(|e| {
-        lance_core::Error::io(
-            format!("Task join error for {}: {}", method_name, e),
-            Default::default(),
-        )
-    })?
-    .map_err(|e: PyErr| {
-        lance_core::Error::io(
-            format!("Python error in {}: {}", method_name, e),
-            Default::default(),
-        )
-    })
-}
-
-/// Helper for methods that return a primitive type
-async fn call_py_method_primitive<Req, Resp>(
-    py_namespace: Arc<Py<PyAny>>,
-    method_name: &'static str,
-    request: Req,
-) -> lance_core::Result<Resp>
-where
-    Req: serde::Serialize + Send + 'static,
-    Resp: for<'py> pyo3::FromPyObject<'py> + Send + 'static,
-{
-    let request_json = serde_json::to_string(&request).map_err(|e| {
-        lance_core::Error::io(
-            format!("Failed to serialize request for {}: {}", method_name, e),
-            Default::default(),
-        )
-    })?;
-
-    tokio::task::spawn_blocking(move || {
-        Python::attach(|py| {
-            let json_module = py.import("json")?;
-            let request_dict = json_module.call_method1("loads", (&request_json,))?;
-
-            // Wrap dict in DictWithModelDump
-            let dict_class = get_dict_with_model_dump_class(py)?;
-            let request_arg = dict_class.call1((request_dict,))?;
-
-            // Call the Python method
-            let result = py_namespace.call_method1(py, method_name, (request_arg,))?;
-            let value: Resp = result.extract(py)?;
-            Ok::<_, PyErr>(value)
-        })
-    })
-    .await
-    .map_err(|e| {
-        lance_core::Error::io(
-            format!("Task join error for {}: {}", method_name, e),
-            Default::default(),
-        )
-    })?
-    .map_err(|e: PyErr| {
-        lance_core::Error::io(
-            format!("Python error in {}: {}", method_name, e),
-            Default::default(),
-        )
-    })
-}
-
-/// Helper for methods that return Bytes
-async fn call_py_method_bytes<Req>(
-    py_namespace: Arc<Py<PyAny>>,
-    method_name: &'static str,
-    request: Req,
-) -> lance_core::Result<Bytes>
-where
-    Req: serde::Serialize + Send + 'static,
-{
-    let request_json = serde_json::to_string(&request).map_err(|e| {
-        lance_core::Error::io(
-            format!("Failed to serialize request for {}: {}", method_name, e),
-            Default::default(),
-        )
-    })?;
-
-    tokio::task::spawn_blocking(move || {
-        Python::attach(|py| {
-            let json_module = py.import("json")?;
-            let request_dict = json_module.call_method1("loads", (&request_json,))?;
-
-            // Wrap dict in DictWithModelDump
-            let dict_class = get_dict_with_model_dump_class(py)?;
-            let request_arg = dict_class.call1((request_dict,))?;
-
-            // Call the Python method
-            let result = py_namespace.call_method1(py, method_name, (request_arg,))?;
-            let bytes_data: Vec<u8> = result.extract(py)?;
-            Ok::<_, PyErr>(Bytes::from(bytes_data))
-        })
-    })
-    .await
-    .map_err(|e| {
-        lance_core::Error::io(
-            format!("Task join error for {}: {}", method_name, e),
-            Default::default(),
-        )
-    })?
-    .map_err(|e: PyErr| {
-        lance_core::Error::io(
-            format!("Python error in {}: {}", method_name, e),
-            Default::default(),
-        )
-    })
-}
-
-/// Helper for methods that take request + data and return a response
-async fn call_py_method_with_data<Req, Resp>(
-    py_namespace: Arc<Py<PyAny>>,
-    method_name: &'static str,
-    request: Req,
-    data: Bytes,
-) -> lance_core::Result<Resp>
-where
-    Req: serde::Serialize + Send + 'static,
-    Resp: serde::de::DeserializeOwned + Send + 'static,
-{
-    let request_json = serde_json::to_string(&request).map_err(|e| {
-        lance_core::Error::io(
-            format!("Failed to serialize request for {}: {}", method_name, e),
-            Default::default(),
-        )
-    })?;
-
-    let response_json = tokio::task::spawn_blocking(move || {
-        Python::attach(|py| {
-            let json_module = py.import("json")?;
-            let request_dict = json_module.call_method1("loads", (&request_json,))?;
-
-            // Wrap dict in DictWithModelDump
-            let dict_class = get_dict_with_model_dump_class(py)?;
-            let request_arg = dict_class.call1((request_dict,))?;
-
-            // Pass request and bytes to Python method
-            let py_bytes = pyo3::types::PyBytes::new(py, &data);
-            let result = py_namespace.call_method1(py, method_name, (request_arg, py_bytes))?;
-
-            // Convert response dict to JSON
-            let response_json: String = json_module.call_method1("dumps", (result,))?.extract()?;
-            Ok::<_, PyErr>(response_json)
-        })
-    })
-    .await
-    .map_err(|e| {
-        lance_core::Error::io(
-            format!("Task join error for {}: {}", method_name, e),
-            Default::default(),
-        )
-    })?
-    .map_err(|e: PyErr| {
-        lance_core::Error::io(
-            format!("Python error in {}: {}", method_name, e),
-            Default::default(),
-        )
-    })?;
-
-    serde_json::from_str(&response_json).map_err(|e| {
-        lance_core::Error::io(
-            format!("Failed to deserialize response from {}: {}", method_name, e),
-            Default::default(),
-        )
-    })
-}
-
-#[async_trait]
-impl LanceNamespaceTrait for PyLanceNamespace {
-    fn namespace_id(&self) -> String {
-        self.namespace_id.clone()
-    }
-
-    async fn list_namespaces(
-        &self,
-        request: ListNamespacesRequest,
-    ) -> lance_core::Result<ListNamespacesResponse> {
-        call_py_method(self.py_namespace.clone(), "list_namespaces", request).await
-    }
-
-    async fn describe_namespace(
-        &self,
-        request: DescribeNamespaceRequest,
-    ) -> lance_core::Result<DescribeNamespaceResponse> {
-        call_py_method(self.py_namespace.clone(), "describe_namespace", request).await
-    }
-
-    async fn create_namespace(
-        &self,
-        request: CreateNamespaceRequest,
-    ) -> lance_core::Result<CreateNamespaceResponse> {
-        call_py_method(self.py_namespace.clone(), "create_namespace", request).await
-    }
-
-    async fn drop_namespace(
-        &self,
-        request: DropNamespaceRequest,
-    ) -> lance_core::Result<DropNamespaceResponse> {
-        call_py_method(self.py_namespace.clone(), "drop_namespace", request).await
-    }
-
-    async fn namespace_exists(&self, request: NamespaceExistsRequest) -> lance_core::Result<()> {
-        call_py_method_unit(self.py_namespace.clone(), "namespace_exists", request).await
-    }
-
-    async fn list_tables(
-        &self,
-        request: ListTablesRequest,
-    ) -> lance_core::Result<ListTablesResponse> {
-        call_py_method(self.py_namespace.clone(), "list_tables", request).await
-    }
-
-    async fn describe_table(
-        &self,
-        request: DescribeTableRequest,
-    ) -> lance_core::Result<DescribeTableResponse> {
-        call_py_method(self.py_namespace.clone(), "describe_table", request).await
-    }
-
-    async fn register_table(
-        &self,
-        request: RegisterTableRequest,
-    ) -> lance_core::Result<RegisterTableResponse> {
-        call_py_method(self.py_namespace.clone(), "register_table", request).await
-    }
-
-    async fn table_exists(&self, request: TableExistsRequest) -> lance_core::Result<()> {
-        call_py_method_unit(self.py_namespace.clone(), "table_exists", request).await
-    }
-
-    async fn drop_table(&self, request: DropTableRequest) -> lance_core::Result<DropTableResponse> {
-        call_py_method(self.py_namespace.clone(), "drop_table", request).await
-    }
-
-    async fn deregister_table(
-        &self,
-        request: DeregisterTableRequest,
-    ) -> lance_core::Result<DeregisterTableResponse> {
-        call_py_method(self.py_namespace.clone(), "deregister_table", request).await
-    }
-
-    async fn count_table_rows(&self, request: CountTableRowsRequest) -> lance_core::Result<i64> {
-        call_py_method_primitive(self.py_namespace.clone(), "count_table_rows", request).await
-    }
-
-    async fn create_table(
-        &self,
-        request: CreateTableRequest,
-        request_data: Bytes,
-    ) -> lance_core::Result<CreateTableResponse> {
-        call_py_method_with_data(
-            self.py_namespace.clone(),
-            "create_table",
-            request,
-            request_data,
-        )
-        .await
-    }
-
-    async fn declare_table(
-        &self,
-        request: DeclareTableRequest,
-    ) -> lance_core::Result<DeclareTableResponse> {
-        call_py_method(self.py_namespace.clone(), "declare_table", request).await
-    }
-
-    async fn insert_into_table(
-        &self,
-        request: InsertIntoTableRequest,
-        request_data: Bytes,
-    ) -> lance_core::Result<InsertIntoTableResponse> {
-        call_py_method_with_data(
-            self.py_namespace.clone(),
-            "insert_into_table",
-            request,
-            request_data,
-        )
-        .await
-    }
-
-    async fn merge_insert_into_table(
-        &self,
-        request: MergeInsertIntoTableRequest,
-        request_data: Bytes,
-    ) -> lance_core::Result<MergeInsertIntoTableResponse> {
-        call_py_method_with_data(
-            self.py_namespace.clone(),
-            "merge_insert_into_table",
-            request,
-            request_data,
-        )
-        .await
-    }
-
-    async fn update_table(
-        &self,
-        request: UpdateTableRequest,
-    ) -> lance_core::Result<UpdateTableResponse> {
-        call_py_method(self.py_namespace.clone(), "update_table", request).await
-    }
-
-    async fn delete_from_table(
-        &self,
-        request: DeleteFromTableRequest,
-    ) -> lance_core::Result<DeleteFromTableResponse> {
-        call_py_method(self.py_namespace.clone(), "delete_from_table", request).await
-    }
-
-    async fn query_table(&self, request: QueryTableRequest) -> lance_core::Result<Bytes> {
-        call_py_method_bytes(self.py_namespace.clone(), "query_table", request).await
-    }
-
-    async fn create_table_index(
-        &self,
-        request: CreateTableIndexRequest,
-    ) -> lance_core::Result<CreateTableIndexResponse> {
-        call_py_method(self.py_namespace.clone(), "create_table_index", request).await
-    }
-
-    async fn list_table_indices(
-        &self,
-        request: ListTableIndicesRequest,
-    ) -> lance_core::Result<ListTableIndicesResponse> {
-        call_py_method(self.py_namespace.clone(), "list_table_indices", request).await
-    }
-
-    async fn describe_table_index_stats(
-        &self,
-        request: DescribeTableIndexStatsRequest,
-    ) -> lance_core::Result<DescribeTableIndexStatsResponse> {
-        call_py_method(
-            self.py_namespace.clone(),
-            "describe_table_index_stats",
-            request,
-        )
-        .await
-    }
-
-    async fn describe_transaction(
-        &self,
-        request: DescribeTransactionRequest,
-    ) -> lance_core::Result<DescribeTransactionResponse> {
-        call_py_method(self.py_namespace.clone(), "describe_transaction", request).await
-    }
-
-    async fn alter_transaction(
-        &self,
-        request: AlterTransactionRequest,
-    ) -> lance_core::Result<AlterTransactionResponse> {
-        call_py_method(self.py_namespace.clone(), "alter_transaction", request).await
-    }
-
-    async fn create_table_scalar_index(
-        &self,
-        request: CreateTableIndexRequest,
-    ) -> lance_core::Result<CreateTableScalarIndexResponse> {
-        call_py_method(
-            self.py_namespace.clone(),
-            "create_table_scalar_index",
-            request,
-        )
-        .await
-    }
-
-    async fn drop_table_index(
-        &self,
-        request: DropTableIndexRequest,
-    ) -> lance_core::Result<DropTableIndexResponse> {
-        call_py_method(self.py_namespace.clone(), "drop_table_index", request).await
-    }
-
-    async fn list_all_tables(
-        &self,
-        request: ListTablesRequest,
-    ) -> lance_core::Result<ListTablesResponse> {
-        call_py_method(self.py_namespace.clone(), "list_all_tables", request).await
-    }
-
-    async fn restore_table(
-        &self,
-        request: RestoreTableRequest,
-    ) -> lance_core::Result<RestoreTableResponse> {
-        call_py_method(self.py_namespace.clone(), "restore_table", request).await
-    }
-
-    async fn rename_table(
-        &self,
-        request: RenameTableRequest,
-    ) -> lance_core::Result<RenameTableResponse> {
-        call_py_method(self.py_namespace.clone(), "rename_table", request).await
-    }
-
-    async fn list_table_versions(
-        &self,
-        request: ListTableVersionsRequest,
-    ) -> lance_core::Result<ListTableVersionsResponse> {
-        call_py_method(self.py_namespace.clone(), "list_table_versions", request).await
-    }
-
-    async fn create_table_version(
-        &self,
-        request: CreateTableVersionRequest,
-    ) -> lance_core::Result<CreateTableVersionResponse> {
-        call_py_method(self.py_namespace.clone(), "create_table_version", request).await
-    }
-
-    async fn describe_table_version(
-        &self,
-        request: DescribeTableVersionRequest,
-    ) -> lance_core::Result<DescribeTableVersionResponse> {
-        call_py_method(self.py_namespace.clone(), "describe_table_version", request).await
-    }
-
-    async fn batch_delete_table_versions(
-        &self,
-        request: BatchDeleteTableVersionsRequest,
-    ) -> lance_core::Result<BatchDeleteTableVersionsResponse> {
-        call_py_method(
-            self.py_namespace.clone(),
-            "batch_delete_table_versions",
-            request,
-        )
-        .await
-    }
-
-    async fn update_table_schema_metadata(
-        &self,
-        request: UpdateTableSchemaMetadataRequest,
-    ) -> lance_core::Result<UpdateTableSchemaMetadataResponse> {
-        call_py_method(
-            self.py_namespace.clone(),
-            "update_table_schema_metadata",
-            request,
-        )
-        .await
-    }
-
-    async fn get_table_stats(
-        &self,
-        request: GetTableStatsRequest,
-    ) -> lance_core::Result<GetTableStatsResponse> {
-        call_py_method(self.py_namespace.clone(), "get_table_stats", request).await
-    }
-
-    async fn explain_table_query_plan(
-        &self,
-        request: ExplainTableQueryPlanRequest,
-    ) -> lance_core::Result<String> {
-        call_py_method_primitive(
-            self.py_namespace.clone(),
-            "explain_table_query_plan",
-            request,
-        )
-        .await
-    }
-
-    async fn analyze_table_query_plan(
-        &self,
-        request: AnalyzeTableQueryPlanRequest,
-    ) -> lance_core::Result<String> {
-        call_py_method_primitive(
-            self.py_namespace.clone(),
-            "analyze_table_query_plan",
-            request,
-        )
-        .await
-    }
-
-    async fn alter_table_add_columns(
-        &self,
-        request: AlterTableAddColumnsRequest,
-    ) -> lance_core::Result<AlterTableAddColumnsResponse> {
-        call_py_method(
-            self.py_namespace.clone(),
-            "alter_table_add_columns",
-            request,
-        )
-        .await
-    }
-
-    async fn alter_table_alter_columns(
-        &self,
-        request: AlterTableAlterColumnsRequest,
-    ) -> lance_core::Result<AlterTableAlterColumnsResponse> {
-        call_py_method(
-            self.py_namespace.clone(),
-            "alter_table_alter_columns",
-            request,
-        )
-        .await
-    }
-
-    async fn alter_table_drop_columns(
-        &self,
-        request: AlterTableDropColumnsRequest,
-    ) -> lance_core::Result<AlterTableDropColumnsResponse> {
-        call_py_method(
-            self.py_namespace.clone(),
-            "alter_table_drop_columns",
-            request,
-        )
-        .await
-    }
-
-    async fn list_table_tags(
-        &self,
-        request: ListTableTagsRequest,
-    ) -> lance_core::Result<ListTableTagsResponse> {
-        call_py_method(self.py_namespace.clone(), "list_table_tags", request).await
-    }
-
-    async fn create_table_tag(
-        &self,
-        request: CreateTableTagRequest,
-    ) -> lance_core::Result<CreateTableTagResponse> {
-        call_py_method(self.py_namespace.clone(), "create_table_tag", request).await
-    }
-
-    async fn delete_table_tag(
-        &self,
-        request: DeleteTableTagRequest,
-    ) -> lance_core::Result<DeleteTableTagResponse> {
-        call_py_method(self.py_namespace.clone(), "delete_table_tag", request).await
-    }
-
-    async fn update_table_tag(
-        &self,
-        request: UpdateTableTagRequest,
-    ) -> lance_core::Result<UpdateTableTagResponse> {
-        call_py_method(self.py_namespace.clone(), "update_table_tag", request).await
-    }
-
-    async fn get_table_tag_version(
-        &self,
-        request: GetTableTagVersionRequest,
-    ) -> lance_core::Result<GetTableTagVersionResponse> {
-        call_py_method(self.py_namespace.clone(), "get_table_tag_version", request).await
-    }
-}
-
-/// Convert Python dict to HashMap<String, String>
-#[allow(dead_code)]
-fn dict_to_hashmap(dict: &Bound<'_, PyDict>) -> PyResult<HashMap<String, String>> {
-    let mut map = HashMap::new();
-    for (key, value) in dict.iter() {
-        let key_str: String = key.extract()?;
-        let value_str: String = value.extract()?;
-        map.insert(key_str, value_str);
-    }
-    Ok(map)
-}
-
-/// Extract an Arc<dyn LanceNamespace> from a Python namespace object.
-///
-/// This function wraps any Python namespace object with PyLanceNamespace.
-/// The PyLanceNamespace wrapper uses DictWithModelDump to pass requests,
-/// which works with both:
-/// - Native namespaces (DirectoryNamespace, RestNamespace) that use depythonize (expects dict)
-/// - Custom Python implementations that call .model_dump() on the request
-pub fn extract_namespace_arc(
-    py: Python<'_>,
-    ns: Py<PyAny>,
-) -> PyResult<Arc<dyn LanceNamespaceTrait>> {
-    let ns_ref = ns.bind(py);
-    PyLanceNamespace::create_arc(py, ns_ref)
-}

python/src/permutation.rs

@@ -16,32 +16,17 @@ use lancedb::{
     query::Select,
 };
 use pyo3::{
-    Bound, PyAny, PyRef, PyRefMut, PyResult, Python,
     exceptions::PyRuntimeError,
     pyclass, pymethods,
     types::{PyAnyMethods, PyDict, PyDictMethods, PyType},
+    Bound, PyAny, PyRef, PyRefMut, PyResult, Python,
 };
 use pyo3_async_runtimes::tokio::future_into_py;
 
-fn table_from_py<'a>(table: Bound<'a, PyAny>) -> PyResult<Bound<'a, Table>> {
-    if table.hasattr("_inner")? {
-        Ok(table.getattr("_inner")?.downcast_into::<Table>()?)
-    } else if table.hasattr("_table")? {
-        Ok(table
-            .getattr("_table")?
-            .getattr("_inner")?
-            .downcast_into::<Table>()?)
-    } else {
-        Err(PyRuntimeError::new_err(
-            "Provided table does not appear to be a Table or RemoteTable instance",
-        ))
-    }
-}
-
 /// Create a permutation builder for the given table
 #[pyo3::pyfunction]
 pub fn async_permutation_builder(table: Bound<'_, PyAny>) -> PyResult<PyAsyncPermutationBuilder> {
-    let table = table_from_py(table)?;
+    let table = table.getattr("_inner")?.downcast_into::<Table>()?;
     let inner_table = table.borrow().inner_ref()?.clone();
     let inner_builder = LancePermutationBuilder::new(inner_table);
 
@@ -265,8 +250,10 @@ impl PyPermutationReader {
         permutation_table: Option<Bound<'py, PyAny>>,
         split: u64,
     ) -> PyResult<Bound<'py, PyAny>> {
-        let base_table = table_from_py(base_table)?;
-        let permutation_table = permutation_table.map(table_from_py).transpose()?;
+        let base_table = base_table.getattr("_inner")?.downcast_into::<Table>()?;
+        let permutation_table = permutation_table
+            .map(|p| PyResult::Ok(p.getattr("_inner")?.downcast_into::<Table>()?))
+            .transpose()?;
 
         let base_table = base_table.borrow().inner_ref()?.base_table().clone();
         let permutation_table = permutation_table

python/src/query.rs

@@ -4,9 +4,9 @@
 use std::sync::Arc;
 use std::time::Duration;
 
+use arrow::array::make_array;
 use arrow::array::Array;
 use arrow::array::ArrayData;
-use arrow::array::make_array;
 use arrow::pyarrow::FromPyArrow;
 use arrow::pyarrow::IntoPyArrow;
 use arrow::pyarrow::ToPyArrow;
@@ -22,23 +22,23 @@ use lancedb::query::{
     VectorQuery as LanceDbVectorQuery,
 };
 use lancedb::table::AnyQuery;
+use pyo3::prelude::{PyAnyMethods, PyDictMethods};
+use pyo3::pyfunction;
+use pyo3::pymethods;
+use pyo3::types::PyList;
+use pyo3::types::{PyDict, PyString};
 use pyo3::Bound;
 use pyo3::IntoPyObject;
 use pyo3::PyAny;
 use pyo3::PyRef;
 use pyo3::PyResult;
 use pyo3::Python;
-use pyo3::prelude::{PyAnyMethods, PyDictMethods};
-use pyo3::pyfunction;
-use pyo3::pymethods;
-use pyo3::types::PyList;
-use pyo3::types::{PyDict, PyString};
-use pyo3::{FromPyObject, exceptions::PyRuntimeError};
-use pyo3::{PyErr, pyclass};
+use pyo3::{exceptions::PyRuntimeError, FromPyObject};
 use pyo3::{
     exceptions::{PyNotImplementedError, PyValueError},
     intern,
 };
+use pyo3::{pyclass, PyErr};
 use pyo3_async_runtimes::tokio::future_into_py;
 
 use crate::util::parse_distance_type;

python/src/session.rs

@@ -4,7 +4,7 @@
 use std::sync::Arc;
 
 use lancedb::{ObjectStoreRegistry, Session as LanceSession};
-use pyo3::{PyResult, pyclass, pymethods};
+use pyo3::{pyclass, pymethods, PyResult};
 
 /// A session for managing caches and object stores across LanceDB operations.
 ///

python/src/table.rs

@@ -5,9 +5,8 @@ use std::{collections::HashMap, sync::Arc};
 use crate::{
     connection::Connection,
     error::PythonErrorExt,
-    index::{IndexConfig, extract_index_params},
+    index::{extract_index_params, IndexConfig},
     query::{Query, TakeQuery},
-    table::scannable::PyScannable,
 };
 use arrow::{
     datatypes::{DataType, Schema},
@@ -19,15 +18,13 @@ use lancedb::table::{
     Table as LanceDbTable,
 };
 use pyo3::{
-    Bound, FromPyObject, PyAny, PyRef, PyResult, Python,
     exceptions::{PyKeyError, PyRuntimeError, PyValueError},
     pyclass, pymethods,
     types::{IntoPyDict, PyAnyMethods, PyDict, PyDictMethods},
+    Bound, FromPyObject, PyAny, PyRef, PyResult, Python,
 };
 use pyo3_async_runtimes::tokio::future_into_py;
 
-mod scannable;
-
 /// Statistics about a compaction operation.
 #[pyclass(get_all)]
 #[derive(Clone, Debug)]
@@ -112,24 +109,19 @@ impl From<lancedb::table::AddResult> for AddResult {
 #[pyclass(get_all)]
 #[derive(Clone, Debug)]
 pub struct DeleteResult {
-    pub num_deleted_rows: u64,
     pub version: u64,
 }
 
 #[pymethods]
 impl DeleteResult {
     pub fn __repr__(&self) -> String {
-        format!(
-            "DeleteResult(num_deleted_rows={}, version={})",
-            self.num_deleted_rows, self.version
-        )
+        format!("DeleteResult(version={})", self.version)
     }
 }
 
 impl From<lancedb::table::DeleteResult> for DeleteResult {
     fn from(result: lancedb::table::DeleteResult) -> Self {
         Self {
-            num_deleted_rows: result.num_deleted_rows,
             version: result.version,
         }
     }
@@ -301,10 +293,11 @@ impl Table {
 
     pub fn add<'a>(
         self_: PyRef<'a, Self>,
-        data: PyScannable,
+        data: Bound<'_, PyAny>,
         mode: String,
     ) -> PyResult<Bound<'a, PyAny>> {
-        let mut op = self_.inner_ref()?.add(data);
+        let batches = ArrowArrayStreamReader::from_pyarrow_bound(&data)?;
+        let mut op = self_.inner_ref()?.add(batches);
         if mode == "append" {
             op = op.mode(AddDataMode::Append);
         } else if mode == "overwrite" {
@@ -542,7 +535,7 @@ impl Table {
         let inner = self_.inner_ref()?.clone();
         future_into_py(self_.py(), async move {
             let versions = inner.list_versions().await.infer_error()?;
-            Python::attach(|py| {
+            let versions_as_dict = Python::attach(|py| {
                 versions
                     .iter()
                     .map(|v| {
@@ -559,7 +552,9 @@ impl Table {
                         Ok(dict.unbind())
                     })
                     .collect::<PyResult<Vec<_>>>()
-            })
+            });
+
+            versions_as_dict
         })
     }
 

python/src/table/scannable.rs

@@ -1,145 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-use std::sync::Arc;
-
-use arrow::{
-    datatypes::{Schema, SchemaRef},
-    ffi_stream::ArrowArrayStreamReader,
-    pyarrow::{FromPyArrow, PyArrowType},
-};
-use futures::StreamExt;
-use lancedb::{
-    Error,
-    arrow::{SendableRecordBatchStream, SimpleRecordBatchStream},
-    data::scannable::Scannable,
-};
-use pyo3::{FromPyObject, Py, PyAny, Python, types::PyAnyMethods};
-
-/// Adapter that implements Scannable for a Python reader factory callable.
-///
-/// This holds a Python callable that returns a RecordBatchReader when called.
-/// For rescannable sources, the callable can be invoked multiple times to
-/// get fresh readers.
-pub struct PyScannable {
-    /// Python callable that returns a RecordBatchReader
-    reader_factory: Py<PyAny>,
-    schema: SchemaRef,
-    num_rows: Option<usize>,
-    rescannable: bool,
-}
-
-impl std::fmt::Debug for PyScannable {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("PyScannable")
-            .field("schema", &self.schema)
-            .field("num_rows", &self.num_rows)
-            .field("rescannable", &self.rescannable)
-            .finish()
-    }
-}
-
-impl Scannable for PyScannable {
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-
-    fn scan_as_stream(&mut self) -> SendableRecordBatchStream {
-        let reader: Result<ArrowArrayStreamReader, Error> = {
-            Python::attach(|py| {
-                let result =
-                    self.reader_factory
-                        .call0(py)
-                        .map_err(|e| lancedb::Error::Runtime {
-                            message: format!("Python reader factory failed: {}", e),
-                        })?;
-                ArrowArrayStreamReader::from_pyarrow_bound(result.bind(py)).map_err(|e| {
-                    lancedb::Error::Runtime {
-                        message: format!("Failed to create Arrow reader from Python: {}", e),
-                    }
-                })
-            })
-        };
-
-        // Reader is blocking but stream is non-blocking, so we need to spawn a task to pull.
-        let (tx, rx) = tokio::sync::mpsc::channel(1);
-
-        let join_handle = tokio::task::spawn_blocking(move || {
-            let reader = match reader {
-                Ok(reader) => reader,
-                Err(e) => {
-                    let _ = tx.blocking_send(Err(e));
-                    return;
-                }
-            };
-            for batch in reader {
-                match batch {
-                    Ok(batch) => {
-                        if tx.blocking_send(Ok(batch)).is_err() {
-                            // Receiver dropped, stop processing
-                            break;
-                        }
-                    }
-                    Err(source) => {
-                        let _ = tx.blocking_send(Err(Error::Arrow { source }));
-                        break;
-                    }
-                }
-            }
-        });
-
-        let schema = self.schema.clone();
-        let stream = futures::stream::unfold(
-            (rx, Some(join_handle)),
-            |(mut rx, join_handle)| async move {
-                match rx.recv().await {
-                    Some(Ok(batch)) => Some((Ok(batch), (rx, join_handle))),
-                    Some(Err(e)) => Some((Err(e), (rx, join_handle))),
-                    None => {
-                        // Channel closed. Check if the task panicked — a panic
-                        // drops the sender without sending an error, so without
-                        // this check we'd silently return a truncated stream.
-                        if let Some(handle) = join_handle
-                            && let Err(join_err) = handle.await
-                        {
-                            return Some((
-                                Err(Error::Runtime {
-                                    message: format!("Reader task panicked: {}", join_err),
-                                }),
-                                (rx, None),
-                            ));
-                        }
-                        None
-                    }
-                }
-            },
-        );
-        Box::pin(SimpleRecordBatchStream::new(stream.fuse(), schema))
-    }
-
-    fn num_rows(&self) -> Option<usize> {
-        self.num_rows
-    }
-
-    fn rescannable(&self) -> bool {
-        self.rescannable
-    }
-}
-
-impl<'py> FromPyObject<'py> for PyScannable {
-    fn extract_bound(ob: &pyo3::Bound<'py, PyAny>) -> pyo3::PyResult<Self> {
-        // Convert from Scannable dataclass.
-        let schema: PyArrowType<Schema> = ob.getattr("schema")?.extract()?;
-        let schema = Arc::new(schema.0);
-        let num_rows: Option<usize> = ob.getattr("num_rows")?.extract()?;
-        let rescannable: bool = ob.getattr("rescannable")?.extract()?;
-        let reader_factory: Py<PyAny> = ob.getattr("reader")?.unbind();
-
-        Ok(Self {
-            schema,
-            reader_factory,
-            num_rows,
-            rescannable,
-        })
-    }
-}

python/src/util.rs

@@ -5,9 +5,8 @@ use std::sync::Mutex;
 
 use lancedb::DistanceType;
 use pyo3::{
-    PyResult,
     exceptions::{PyRuntimeError, PyValueError},
-    pyfunction,
+    pyfunction, PyResult,
 };
 
 /// A wrapper around a rust builder

python/uv.lock

@@ -2006,7 +2006,7 @@ requires-dist = [
     { name = "botocore", marker = "extra == 'embeddings'", specifier = ">=1.31.57" },
     { name = "cohere", marker = "extra == 'embeddings'" },
     { name = "colpali-engine", marker = "extra == 'embeddings'", specifier = ">=0.3.10" },
-    { name = "datafusion", marker = "extra == 'tests'", specifier = "<52" },
+    { name = "datafusion", marker = "extra == 'tests'" },
     { name = "deprecation" },
     { name = "duckdb", marker = "extra == 'tests'" },
     { name = "google-generativeai", marker = "extra == 'embeddings'" },
@@ -2035,7 +2035,7 @@ requires-dist = [
     { name = "pyarrow-stubs", marker = "extra == 'tests'" },
     { name = "pydantic", specifier = ">=1.10" },
     { name = "pylance", marker = "extra == 'pylance'", specifier = ">=1.0.0b14" },
-    { name = "pylance", marker = "extra == 'tests'", specifier = ">=1.0.0b14,<3.0.0" },
+    { name = "pylance", marker = "extra == 'tests'", specifier = ">=1.0.0b14" },
     { name = "pyright", marker = "extra == 'dev'" },
     { name = "pytest", marker = "extra == 'tests'" },
     { name = "pytest-asyncio", marker = "extra == 'tests'" },

rust-toolchain.toml

@@ -1,2 +1,2 @@
 [toolchain]
-channel = "1.91.0"
+channel = "1.90.0"

rust/lancedb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.27.0-beta.3"
+version = "0.26.2"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
@@ -25,9 +25,7 @@ datafusion-catalog.workspace = true
 datafusion-common.workspace = true
 datafusion-execution.workspace = true
 datafusion-expr.workspace = true
-datafusion-functions.workspace = true
 datafusion-physical-expr.workspace = true
-datafusion-sql.workspace = true
 datafusion-physical-plan.workspace = true
 datafusion.workspace = true
 object_store = { workspace = true }

rust/lancedb/examples/bedrock.rs

@@ -3,15 +3,17 @@
 
 use std::{iter::once, sync::Arc};
 
-use arrow_array::{Float64Array, Int32Array, RecordBatch, StringArray};
+use arrow_array::{Float64Array, Int32Array, RecordBatch, RecordBatchIterator, StringArray};
 use arrow_schema::{DataType, Field, Schema};
 use aws_config::Region;
 use aws_sdk_bedrockruntime::Client;
 use futures::StreamExt;
 use lancedb::{
-    Result, connect,
-    embeddings::{EmbeddingDefinition, EmbeddingFunction, bedrock::BedrockEmbeddingFunction},
+    arrow::IntoArrow,
+    connect,
+    embeddings::{bedrock::BedrockEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
     query::{ExecutableQuery, QueryBase},
+    Result,
 };
 
 #[tokio::main]
@@ -65,7 +67,7 @@ async fn main() -> Result<()> {
     Ok(())
 }
 
-fn make_data() -> RecordBatch {
+fn make_data() -> impl IntoArrow {
     let schema = Schema::new(vec![
         Field::new("id", DataType::Int32, true),
         Field::new("text", DataType::Utf8, false),
@@ -81,9 +83,10 @@ fn make_data() -> RecordBatch {
     ]);
     let price = Float64Array::from(vec![10.0, 50.0, 100.0, 30.0]);
     let schema = Arc::new(schema);
-    RecordBatch::try_new(
+    let rb = RecordBatch::try_new(
         schema.clone(),
         vec![Arc::new(id), Arc::new(text), Arc::new(price)],
     )
-    .unwrap()
+    .unwrap();
+    Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema))
 }

rust/lancedb/examples/full_text_search.rs

@@ -3,17 +3,16 @@
 
 use std::sync::Arc;
 
-use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator, StringArray};
+use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};
 use arrow_schema::{DataType, Field, Schema};
 
 use futures::TryStreamExt;
 use lance_index::scalar::FullTextSearchQuery;
 use lancedb::connection::Connection;
-
-use lancedb::index::Index;
 use lancedb::index::scalar::FtsIndexBuilder;
+use lancedb::index::Index;
 use lancedb::query::{ExecutableQuery, QueryBase};
-use lancedb::{Result, Table, connect};
+use lancedb::{connect, Result, Table};
 use rand::random;
 
 #[tokio::main]
@@ -30,7 +29,7 @@ async fn main() -> Result<()> {
     Ok(())
 }
 
-fn create_some_records() -> Result<Box<dyn arrow_array::RecordBatchReader + Send>> {
+fn create_some_records() -> Result<Box<dyn RecordBatchReader + Send>> {
     const TOTAL: usize = 1000;
 
     let schema = Arc::new(Schema::new(vec![
@@ -46,21 +45,19 @@ fn create_some_records() -> Result<Box<dyn arrow_array::RecordBatchReader + Send
         .collect::<Vec<_>>();
     let n_terms = 3;
     let batches = RecordBatchIterator::new(
-        vec![
-            RecordBatch::try_new(
-                schema.clone(),
-                vec![
-                    Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
-                    Arc::new(StringArray::from_iter_values((0..TOTAL).map(|_| {
-                        (0..n_terms)
-                            .map(|_| words[random::<u32>() as usize % words.len()])
-                            .collect::<Vec<_>>()
-                            .join(" ")
-                    }))),
-                ],
-            )
-            .unwrap(),
-        ]
+        vec![RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
+                Arc::new(StringArray::from_iter_values((0..TOTAL).map(|_| {
+                    (0..n_terms)
+                        .map(|_| words[random::<u32>() as usize % words.len()])
+                        .collect::<Vec<_>>()
+                        .join(" ")
+                }))),
+            ],
+        )
+        .unwrap()]
         .into_iter()
         .map(Ok),
         schema.clone(),
@@ -69,7 +66,7 @@ fn create_some_records() -> Result<Box<dyn arrow_array::RecordBatchReader + Send
 }
 
 async fn create_table(db: &Connection) -> Result<Table> {
-    let initial_data = create_some_records()?;
+    let initial_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
     let tbl = db.create_table("my_table", initial_data).execute().await?;
     Ok(tbl)
 }

rust/lancedb/examples/hybrid_search.rs

@@ -1,19 +1,21 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-use arrow_array::{RecordBatch, StringArray};
+use arrow_array::{RecordBatch, RecordBatchIterator, StringArray};
 use arrow_schema::{DataType, Field, Schema};
 use futures::TryStreamExt;
 use lance_index::scalar::FullTextSearchQuery;
-use lancedb::index::Index;
 use lancedb::index::scalar::FtsIndexBuilder;
+use lancedb::index::Index;
 use lancedb::{
-    Result, Table, connect,
+    arrow::IntoArrow,
+    connect,
     embeddings::{
-        EmbeddingDefinition, EmbeddingFunction,
-        sentence_transformers::SentenceTransformersEmbeddings,
+        sentence_transformers::SentenceTransformersEmbeddings, EmbeddingDefinition,
+        EmbeddingFunction,
     },
     query::{QueryBase, QueryExecutionOptions},
+    Result, Table,
 };
 use std::{iter::once, sync::Arc};
 
@@ -68,7 +70,7 @@ async fn main() -> Result<()> {
     Ok(())
 }
 
-fn make_data() -> RecordBatch {
+fn make_data() -> impl IntoArrow {
     let schema = Schema::new(vec![Field::new("facts", DataType::Utf8, false)]);
 
     let facts = StringArray::from_iter_values(vec![
@@ -99,7 +101,8 @@ fn make_data() -> RecordBatch {
         "The first chatbot was ELIZA, created in the 1960s.",
     ]);
     let schema = Arc::new(schema);
-    RecordBatch::try_new(schema.clone(), vec![Arc::new(facts)]).unwrap()
+    let rb = RecordBatch::try_new(schema.clone(), vec![Arc::new(facts)]).unwrap();
+    Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema))
 }
 
 async fn create_index(table: &Table) -> Result<()> {

rust/lancedb/examples/ivf_pq.rs

@@ -8,16 +8,17 @@
 use std::sync::Arc;
 
 use arrow_array::types::Float32Type;
-use arrow_array::{FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator};
+use arrow_array::{
+    FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader,
+};
 use arrow_schema::{DataType, Field, Schema};
 
 use futures::TryStreamExt;
 use lancedb::connection::Connection;
-
-use lancedb::index::Index;
 use lancedb::index::vector::IvfPqIndexBuilder;
+use lancedb::index::Index;
 use lancedb::query::{ExecutableQuery, QueryBase};
-use lancedb::{DistanceType, Result, Table, connect};
+use lancedb::{connect, DistanceType, Result, Table};
 
 #[tokio::main]
 async fn main() -> Result<()> {
@@ -33,7 +34,7 @@ async fn main() -> Result<()> {
     Ok(())
 }
 
-fn create_some_records() -> Result<Box<dyn arrow_array::RecordBatchReader + Send>> {
+fn create_some_records() -> Result<Box<dyn RecordBatchReader + Send>> {
     const TOTAL: usize = 1000;
     const DIM: usize = 128;
 
@@ -51,21 +52,19 @@ fn create_some_records() -> Result<Box<dyn arrow_array::RecordBatchReader + Send
 
     // Create a RecordBatch stream.
     let batches = RecordBatchIterator::new(
-        vec![
-            RecordBatch::try_new(
-                schema.clone(),
-                vec![
-                    Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
-                    Arc::new(
-                        FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
-                            (0..TOTAL).map(|_| Some(vec![Some(1.0); DIM])),
-                            DIM as i32,
-                        ),
+        vec![RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
+                Arc::new(
+                    FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
+                        (0..TOTAL).map(|_| Some(vec![Some(1.0); DIM])),
+                        DIM as i32,
                     ),
-                ],
-            )
-            .unwrap(),
-        ]
+                ),
+            ],
+        )
+        .unwrap()]
         .into_iter()
         .map(Ok),
         schema.clone(),
@@ -74,9 +73,9 @@ fn create_some_records() -> Result<Box<dyn arrow_array::RecordBatchReader + Send
 }
 
 async fn create_table(db: &Connection) -> Result<Table> {
-    let initial_data = create_some_records()?;
+    let initial_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
     let tbl = db
-        .create_table("my_table", initial_data)
+        .create_table("my_table", Box::new(initial_data))
         .execute()
         .await
         .unwrap();

rust/lancedb/examples/openai.rs

@@ -5,12 +5,15 @@
 
 use std::{iter::once, sync::Arc};
 
-use arrow_array::{RecordBatch, StringArray};
+use arrow_array::{Float64Array, Int32Array, RecordBatch, RecordBatchIterator, StringArray};
+use arrow_schema::{DataType, Field, Schema};
 use futures::StreamExt;
 use lancedb::{
-    Result, connect,
-    embeddings::{EmbeddingDefinition, EmbeddingFunction, openai::OpenAIEmbeddingFunction},
+    arrow::IntoArrow,
+    connect,
+    embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
     query::{ExecutableQuery, QueryBase},
+    Result,
 };
 
 // --8<-- [end:imports]
@@ -61,20 +64,26 @@ async fn main() -> Result<()> {
 }
 // --8<-- [end:openai_embeddings]
 
-fn make_data() -> RecordBatch {
-    arrow_array::record_batch!(
-        ("id", Int32, [1, 2, 3, 4]),
-        (
-            "text",
-            Utf8,
-            [
-                "Black T-Shirt",
-                "Leather Jacket",
-                "Winter Parka",
-                "Hooded Sweatshirt"
-            ]
-        ),
-        ("price", Float64, [10.0, 50.0, 100.0, 30.0])
+fn make_data() -> impl IntoArrow {
+    let schema = Schema::new(vec![
+        Field::new("id", DataType::Int32, true),
+        Field::new("text", DataType::Utf8, false),
+        Field::new("price", DataType::Float64, false),
+    ]);
+
+    let id = Int32Array::from(vec![1, 2, 3, 4]);
+    let text = StringArray::from_iter_values(vec![
+        "Black T-Shirt",
+        "Leather Jacket",
+        "Winter Parka",
+        "Hooded Sweatshirt",
+    ]);
+    let price = Float64Array::from(vec![10.0, 50.0, 100.0, 30.0]);
+    let schema = Arc::new(schema);
+    let rb = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(id), Arc::new(text), Arc::new(price)],
     )
-    .unwrap()
+    .unwrap();
+    Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema))
 }

rust/lancedb/examples/sentence_transformers.rs

@@ -3,16 +3,18 @@
 
 use std::{iter::once, sync::Arc};
 
-use arrow_array::{RecordBatch, StringArray};
+use arrow_array::{RecordBatch, RecordBatchIterator, StringArray};
 use arrow_schema::{DataType, Field, Schema};
 use futures::StreamExt;
 use lancedb::{
-    Result, connect,
+    arrow::IntoArrow,
+    connect,
     embeddings::{
-        EmbeddingDefinition, EmbeddingFunction,
-        sentence_transformers::SentenceTransformersEmbeddings,
+        sentence_transformers::SentenceTransformersEmbeddings, EmbeddingDefinition,
+        EmbeddingFunction,
     },
     query::{ExecutableQuery, QueryBase},
+    Result,
 };
 
 #[tokio::main]
@@ -57,7 +59,7 @@ async fn main() -> Result<()> {
     Ok(())
 }
 
-fn make_data() -> RecordBatch {
+fn make_data() -> impl IntoArrow {
     let schema = Schema::new(vec![Field::new("facts", DataType::Utf8, false)]);
 
     let facts = StringArray::from_iter_values(vec![
@@ -88,5 +90,6 @@ fn make_data() -> RecordBatch {
         "The first chatbot was ELIZA, created in the 1960s.",
     ]);
     let schema = Arc::new(schema);
-    RecordBatch::try_new(schema.clone(), vec![Arc::new(facts)]).unwrap()
+    let rb = RecordBatch::try_new(schema.clone(), vec![Arc::new(facts)]).unwrap();
+    Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema))
 }

rust/lancedb/examples/simple.rs

@@ -8,13 +8,15 @@
 use std::sync::Arc;
 
 use arrow_array::types::Float32Type;
-use arrow_array::{FixedSizeListArray, Int32Array, RecordBatch};
+use arrow_array::{FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator};
 use arrow_schema::{DataType, Field, Schema};
 use futures::TryStreamExt;
+
+use lancedb::arrow::IntoArrow;
 use lancedb::connection::Connection;
 use lancedb::index::Index;
 use lancedb::query::{ExecutableQuery, QueryBase};
-use lancedb::{Result, Table as LanceDbTable, connect};
+use lancedb::{connect, Result, Table as LanceDbTable};
 
 #[tokio::main]
 async fn main() -> Result<()> {
@@ -57,7 +59,7 @@ async fn open_with_existing_tbl() -> Result<()> {
     Ok(())
 }
 
-fn create_some_records() -> Result<RecordBatch> {
+fn create_some_records() -> Result<impl IntoArrow> {
     const TOTAL: usize = 1000;
     const DIM: usize = 128;
 
@@ -74,18 +76,25 @@ fn create_some_records() -> Result<RecordBatch> {
     ]));
 
     // Create a RecordBatch stream.
-    Ok(RecordBatch::try_new(
-        schema.clone(),
-        vec![
-            Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
-            Arc::new(
-                FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
-                    (0..TOTAL).map(|_| Some(vec![Some(1.0); DIM])),
-                    DIM as i32,
+    let batches = RecordBatchIterator::new(
+        vec![RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
+                Arc::new(
+                    FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
+                        (0..TOTAL).map(|_| Some(vec![Some(1.0); DIM])),
+                        DIM as i32,
+                    ),
                 ),
-            ),
-        ],
-    )?)
+            ],
+        )
+        .unwrap()]
+        .into_iter()
+        .map(Ok),
+        schema.clone(),
+    );
+    Ok(Box::new(batches))
 }
 
 async fn create_table(db: &Connection) -> Result<LanceDbTable> {

rust/lancedb/src/arrow.rs

@@ -12,7 +12,7 @@ use lance_datagen::{BatchCount, BatchGeneratorBuilder, RowCount};
 #[cfg(feature = "polars")]
 use {crate::polars_arrow_convertors, polars::frame::ArrowChunk, polars::prelude::DataFrame};
 
-use crate::{Error, error::Result};
+use crate::{error::Result, Error};
 
 /// An iterator of batches that also has a schema
 pub trait RecordBatchReader: Iterator<Item = Result<arrow_array::RecordBatch>> {
@@ -155,7 +155,9 @@ impl IntoArrowStream for SendableRecordBatchStream {
 impl IntoArrowStream for datafusion_physical_plan::SendableRecordBatchStream {
     fn into_arrow(self) -> Result<SendableRecordBatchStream> {
         let schema = self.schema();
-        let stream = self.map_err(|df_err| df_err.into());
+        let stream = self.map_err(|df_err| Error::Runtime {
+            message: df_err.to_string(),
+        });
         Ok(Box::pin(SimpleRecordBatchStream::new(stream, schema)))
     }
 }

rust/lancedb/src/connection.rs

@@ -6,8 +6,8 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use arrow_array::RecordBatch;
-use arrow_schema::SchemaRef;
+use arrow_array::RecordBatchReader;
+use arrow_schema::{Field, SchemaRef};
 use lance::dataset::ReadParams;
 use lance_namespace::models::{
     CreateNamespaceRequest, CreateNamespaceResponse, DescribeNamespaceRequest,
@@ -17,29 +17,31 @@ use lance_namespace::models::{
 #[cfg(feature = "aws")]
 use object_store::aws::AwsCredential;
 
-use crate::Table;
-use crate::connection::create_table::CreateTableBuilder;
-use crate::data::scannable::Scannable;
-use crate::database::listing::ListingDatabase;
-use crate::database::{
-    CloneTableRequest, Database, DatabaseOptions, OpenTableRequest, ReadConsistency,
-    TableNamesRequest,
+use crate::arrow::{IntoArrow, IntoArrowStream, SendableRecordBatchStream};
+use crate::database::listing::{
+    ListingDatabase, OPT_NEW_TABLE_STORAGE_VERSION, OPT_NEW_TABLE_V2_MANIFEST_PATHS,
+};
+use crate::database::{
+    CloneTableRequest, CreateTableData, CreateTableMode, CreateTableRequest, Database,
+    DatabaseOptions, OpenTableRequest, ReadConsistency, TableNamesRequest,
+};
+use crate::embeddings::{
+    EmbeddingDefinition, EmbeddingFunction, EmbeddingRegistry, MemoryRegistry, WithEmbeddings,
 };
-use crate::embeddings::{EmbeddingRegistry, MemoryRegistry};
 use crate::error::{Error, Result};
 #[cfg(feature = "remote")]
 use crate::remote::{
     client::ClientConfig,
     db::{OPT_REMOTE_API_KEY, OPT_REMOTE_HOST_OVERRIDE, OPT_REMOTE_REGION},
 };
+use crate::table::{TableDefinition, WriteOptions};
+use crate::Table;
 use lance::io::ObjectStoreParams;
 pub use lance_encoding::version::LanceFileVersion;
 #[cfg(feature = "remote")]
 use lance_io::object_store::StorageOptions;
 use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
 
-mod create_table;
-
 fn merge_storage_options(
     store_params: &mut ObjectStoreParams,
     pairs: impl IntoIterator<Item = (String, String)>,
@@ -114,6 +116,337 @@ impl TableNamesBuilder {
     }
 }
 
+pub struct NoData {}
+
+impl IntoArrow for NoData {
+    fn into_arrow(self) -> Result<Box<dyn arrow_array::RecordBatchReader + Send>> {
+        unreachable!("NoData should never be converted to Arrow")
+    }
+}
+
+// Stores the value given from the initial CreateTableBuilder::new call
+// and defers errors until `execute` is called
+enum CreateTableBuilderInitialData {
+    None,
+    Iterator(Result<Box<dyn RecordBatchReader + Send>>),
+    Stream(Result<SendableRecordBatchStream>),
+}
+
+/// A builder for configuring a [`Connection::create_table`] operation
+pub struct CreateTableBuilder<const HAS_DATA: bool> {
+    parent: Arc<dyn Database>,
+    embeddings: Vec<(EmbeddingDefinition, Arc<dyn EmbeddingFunction>)>,
+    embedding_registry: Arc<dyn EmbeddingRegistry>,
+    request: CreateTableRequest,
+    // This is a bit clumsy but we defer errors until `execute` is called
+    // to maintain backwards compatibility
+    data: CreateTableBuilderInitialData,
+}
+
+// Builder methods that only apply when we have initial data
+impl CreateTableBuilder<true> {
+    fn new<T: IntoArrow>(
+        parent: Arc<dyn Database>,
+        name: String,
+        data: T,
+        embedding_registry: Arc<dyn EmbeddingRegistry>,
+    ) -> Self {
+        let dummy_schema = Arc::new(arrow_schema::Schema::new(Vec::<Field>::default()));
+        Self {
+            parent,
+            request: CreateTableRequest::new(
+                name,
+                CreateTableData::Empty(TableDefinition::new_from_schema(dummy_schema)),
+            ),
+            embeddings: Vec::new(),
+            embedding_registry,
+            data: CreateTableBuilderInitialData::Iterator(data.into_arrow()),
+        }
+    }
+
+    fn new_streaming<T: IntoArrowStream>(
+        parent: Arc<dyn Database>,
+        name: String,
+        data: T,
+        embedding_registry: Arc<dyn EmbeddingRegistry>,
+    ) -> Self {
+        let dummy_schema = Arc::new(arrow_schema::Schema::new(Vec::<Field>::default()));
+        Self {
+            parent,
+            request: CreateTableRequest::new(
+                name,
+                CreateTableData::Empty(TableDefinition::new_from_schema(dummy_schema)),
+            ),
+            embeddings: Vec::new(),
+            embedding_registry,
+            data: CreateTableBuilderInitialData::Stream(data.into_arrow()),
+        }
+    }
+
+    /// Execute the create table operation
+    pub async fn execute(self) -> Result<Table> {
+        let embedding_registry = self.embedding_registry.clone();
+        let parent = self.parent.clone();
+        let request = self.into_request()?;
+        Ok(Table::new_with_embedding_registry(
+            parent.create_table(request).await?,
+            parent,
+            embedding_registry,
+        ))
+    }
+
+    fn into_request(self) -> Result<CreateTableRequest> {
+        if self.embeddings.is_empty() {
+            match self.data {
+                CreateTableBuilderInitialData::Iterator(maybe_iter) => {
+                    let data = maybe_iter?;
+                    Ok(CreateTableRequest {
+                        data: CreateTableData::Data(data),
+                        ..self.request
+                    })
+                }
+                CreateTableBuilderInitialData::None => {
+                    unreachable!("No data provided for CreateTableBuilder<true>")
+                }
+                CreateTableBuilderInitialData::Stream(maybe_stream) => {
+                    let data = maybe_stream?;
+                    Ok(CreateTableRequest {
+                        data: CreateTableData::StreamingData(data),
+                        ..self.request
+                    })
+                }
+            }
+        } else {
+            let CreateTableBuilderInitialData::Iterator(maybe_iter) = self.data else {
+                return Err(Error::NotSupported { message: "Creating a table with embeddings is currently not support when the input is streaming".to_string() });
+            };
+            let data = maybe_iter?;
+            let data = Box::new(WithEmbeddings::new(data, self.embeddings));
+            Ok(CreateTableRequest {
+                data: CreateTableData::Data(data),
+                ..self.request
+            })
+        }
+    }
+}
+
+// Builder methods that only apply when we do not have initial data
+impl CreateTableBuilder<false> {
+    fn new(
+        parent: Arc<dyn Database>,
+        name: String,
+        schema: SchemaRef,
+        embedding_registry: Arc<dyn EmbeddingRegistry>,
+    ) -> Self {
+        let table_definition = TableDefinition::new_from_schema(schema);
+        Self {
+            parent,
+            request: CreateTableRequest::new(name, CreateTableData::Empty(table_definition)),
+            data: CreateTableBuilderInitialData::None,
+            embeddings: Vec::default(),
+            embedding_registry,
+        }
+    }
+
+    /// Execute the create table operation
+    pub async fn execute(self) -> Result<Table> {
+        let parent = self.parent.clone();
+        let embedding_registry = self.embedding_registry.clone();
+        let request = self.into_request()?;
+        Ok(Table::new_with_embedding_registry(
+            parent.create_table(request).await?,
+            parent,
+            embedding_registry,
+        ))
+    }
+
+    fn into_request(self) -> Result<CreateTableRequest> {
+        if self.embeddings.is_empty() {
+            return Ok(self.request);
+        }
+
+        let CreateTableData::Empty(table_def) = self.request.data else {
+            unreachable!("CreateTableBuilder<false> should always have Empty data")
+        };
+
+        let schema = table_def.schema.clone();
+        let empty_batch = arrow_array::RecordBatch::new_empty(schema.clone());
+
+        let reader = Box::new(std::iter::once(Ok(empty_batch)).collect::<Vec<_>>());
+        let reader = arrow_array::RecordBatchIterator::new(reader.into_iter(), schema);
+        let with_embeddings = WithEmbeddings::new(reader, self.embeddings);
+        let table_definition = with_embeddings.table_definition()?;
+
+        Ok(CreateTableRequest {
+            data: CreateTableData::Empty(table_definition),
+            ..self.request
+        })
+    }
+}
+
+impl<const HAS_DATA: bool> CreateTableBuilder<HAS_DATA> {
+    /// Set the mode for creating the table
+    ///
+    /// This controls what happens if a table with the given name already exists
+    pub fn mode(mut self, mode: CreateTableMode) -> Self {
+        self.request.mode = mode;
+        self
+    }
+
+    /// Apply the given write options when writing the initial data
+    pub fn write_options(mut self, write_options: WriteOptions) -> Self {
+        self.request.write_options = write_options;
+        self
+    }
+
+    /// Set an option for the storage layer.
+    ///
+    /// Options already set on the connection will be inherited by the table,
+    /// but can be overridden here.
+    ///
+    /// See available options at <https://lancedb.com/docs/storage/>
+    pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
+        let store_params = self
+            .request
+            .write_options
+            .lance_write_params
+            .get_or_insert(Default::default())
+            .store_params
+            .get_or_insert(Default::default());
+        merge_storage_options(store_params, [(key.into(), value.into())]);
+        self
+    }
+
+    /// Set multiple options for the storage layer.
+    ///
+    /// Options already set on the connection will be inherited by the table,
+    /// but can be overridden here.
+    ///
+    /// See available options at <https://lancedb.com/docs/storage/>
+    pub fn storage_options(
+        mut self,
+        pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
+    ) -> Self {
+        let store_params = self
+            .request
+            .write_options
+            .lance_write_params
+            .get_or_insert(Default::default())
+            .store_params
+            .get_or_insert(Default::default());
+        let updates = pairs
+            .into_iter()
+            .map(|(key, value)| (key.into(), value.into()));
+        merge_storage_options(store_params, updates);
+        self
+    }
+
+    /// Add an embedding definition to the table.
+    ///
+    /// The `embedding_name` must match the name of an embedding function that
+    /// was previously registered with the connection's [`EmbeddingRegistry`].
+    pub fn add_embedding(mut self, definition: EmbeddingDefinition) -> Result<Self> {
+        // Early verification of the embedding name
+        let embedding_func = self
+            .embedding_registry
+            .get(&definition.embedding_name)
+            .ok_or_else(|| Error::EmbeddingFunctionNotFound {
+                name: definition.embedding_name.clone(),
+                reason: "No embedding function found in the connection's embedding_registry"
+                    .to_string(),
+            })?;
+
+        self.embeddings.push((definition, embedding_func));
+        Ok(self)
+    }
+
+    /// Set whether to use V2 manifest paths for the table. (default: false)
+    ///
+    /// These paths provide more efficient opening of tables with many
+    /// versions on object stores.
+    ///
+    /// <div class="warning">Turning this on will make the dataset unreadable
+    /// for older versions of LanceDB (prior to 0.10.0).</div>
+    ///
+    /// To migrate an existing dataset, instead use the
+    /// [[NativeTable::migrate_manifest_paths_v2]].
+    ///
+    /// This has no effect in LanceDB Cloud.
+    #[deprecated(since = "0.15.1", note = "Use `database_options` instead")]
+    pub fn enable_v2_manifest_paths(mut self, use_v2_manifest_paths: bool) -> Self {
+        let store_params = self
+            .request
+            .write_options
+            .lance_write_params
+            .get_or_insert_with(Default::default)
+            .store_params
+            .get_or_insert_with(Default::default);
+        let value = if use_v2_manifest_paths {
+            "true".to_string()
+        } else {
+            "false".to_string()
+        };
+        merge_storage_options(
+            store_params,
+            [(OPT_NEW_TABLE_V2_MANIFEST_PATHS.to_string(), value)],
+        );
+        self
+    }
+
+    /// Set the data storage version.
+    ///
+    /// The default is `LanceFileVersion::Stable`.
+    #[deprecated(since = "0.15.1", note = "Use `database_options` instead")]
+    pub fn data_storage_version(mut self, data_storage_version: LanceFileVersion) -> Self {
+        let store_params = self
+            .request
+            .write_options
+            .lance_write_params
+            .get_or_insert_with(Default::default)
+            .store_params
+            .get_or_insert_with(Default::default);
+        merge_storage_options(
+            store_params,
+            [(
+                OPT_NEW_TABLE_STORAGE_VERSION.to_string(),
+                data_storage_version.to_string(),
+            )],
+        );
+        self
+    }
+
+    /// Set the namespace for the table
+    pub fn namespace(mut self, namespace: Vec<String>) -> Self {
+        self.request.namespace = namespace;
+        self
+    }
+
+    /// Set a custom location for the table.
+    ///
+    /// If not set, the database will derive a location from its URI and the table name.
+    /// This is useful when integrating with namespace systems that manage table locations.
+    pub fn location(mut self, location: impl Into<String>) -> Self {
+        self.request.location = Some(location.into());
+        self
+    }
+
+    /// Set a storage options provider for automatic credential refresh.
+    ///
+    /// This allows tables to automatically refresh cloud storage credentials
+    /// when they expire, enabling long-running operations on remote storage.
+    pub fn storage_options_provider(mut self, provider: Arc<dyn StorageOptionsProvider>) -> Self {
+        let store_params = self
+            .request
+            .write_options
+            .lance_write_params
+            .get_or_insert(Default::default())
+            .store_params
+            .get_or_insert(Default::default());
+        set_storage_options_provider(store_params, provider);
+        self
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct OpenTableBuilder {
     parent: Arc<dyn Database>,
@@ -136,7 +469,6 @@ impl OpenTableBuilder {
                 lance_read_params: None,
                 location: None,
                 namespace_client: None,
-                managed_versioning: None,
             },
             embedding_registry,
         }
@@ -236,29 +568,6 @@ impl OpenTableBuilder {
         self
     }
 
-    /// Set a namespace client for managed versioning support.
-    ///
-    /// When a namespace client is provided and the table has `managed_versioning` enabled,
-    /// the table will use the namespace's commit handler to notify the namespace of
-    /// version changes. This enables features like event emission for table modifications.
-    pub fn namespace_client(mut self, client: Arc<dyn lance_namespace::LanceNamespace>) -> Self {
-        self.request.namespace_client = Some(client);
-        self
-    }
-
-    /// Set whether managed versioning is enabled for this table.
-    ///
-    /// When set to `Some(true)`, the table will use namespace-managed commits.
-    /// When set to `Some(false)`, the table will use local commits even if namespace_client is set.
-    /// When set to `None` (default), the value will be fetched from the namespace if namespace_client is set.
-    ///
-    /// This is typically set when the caller has already queried the namespace and knows the
-    /// managed_versioning status, avoiding a redundant describe_table call.
-    pub fn managed_versioning(mut self, enabled: bool) -> Self {
-        self.request.managed_versioning = Some(enabled);
-        self
-    }
-
     /// Open the table
     pub async fn execute(self) -> Result<Table> {
         let table = self.parent.open_table(self.request).await?;
@@ -318,12 +627,6 @@ impl CloneTableBuilder {
         self
     }
 
-    /// Set a namespace client for managed versioning support.
-    pub fn namespace_client(mut self, client: Arc<dyn lance_namespace::LanceNamespace>) -> Self {
-        self.request.namespace_client = Some(client);
-        self
-    }
-
     /// Execute the clone operation
     pub async fn execute(self) -> Result<Table> {
         let parent = self.parent.clone();
@@ -381,17 +684,35 @@ impl Connection {
     ///
     /// * `name` - The name of the table
     /// * `initial_data` - The initial data to write to the table
-    pub fn create_table<T: Scannable + 'static>(
+    pub fn create_table<T: IntoArrow>(
         &self,
         name: impl Into<String>,
         initial_data: T,
-    ) -> CreateTableBuilder {
-        let initial_data = Box::new(initial_data);
-        CreateTableBuilder::new(
+    ) -> CreateTableBuilder<true> {
+        CreateTableBuilder::<true>::new(
             self.internal.clone(),
-            self.embedding_registry.clone(),
             name.into(),
             initial_data,
+            self.embedding_registry.clone(),
+        )
+    }
+
+    /// Create a new table from a stream of data
+    ///
+    /// # Parameters
+    ///
+    /// * `name` - The name of the table
+    /// * `initial_data` - The initial data to write to the table
+    pub fn create_table_streaming<T: IntoArrowStream>(
+        &self,
+        name: impl Into<String>,
+        initial_data: T,
+    ) -> CreateTableBuilder<true> {
+        CreateTableBuilder::<true>::new_streaming(
+            self.internal.clone(),
+            name.into(),
+            initial_data,
+            self.embedding_registry.clone(),
         )
     }
 
@@ -405,9 +726,13 @@ impl Connection {
         &self,
         name: impl Into<String>,
         schema: SchemaRef,
-    ) -> CreateTableBuilder {
-        let empty_batch = RecordBatch::new_empty(schema);
-        self.create_table(name, empty_batch)
+    ) -> CreateTableBuilder<false> {
+        CreateTableBuilder::<false>::new(
+            self.internal.clone(),
+            name.into(),
+            schema,
+            self.embedding_registry.clone(),
+        )
     }
 
     /// Open an existing table in the database
@@ -596,11 +921,8 @@ pub struct ConnectBuilder {
 }
 
 #[cfg(feature = "remote")]
-const ENV_VARS_TO_STORAGE_OPTS: [(&str, &str); 3] = [
-    ("AZURE_STORAGE_ACCOUNT_NAME", "azure_storage_account_name"),
-    ("AZURE_CLIENT_ID", "azure_client_id"),
-    ("AZURE_TENANT_ID", "azure_tenant_id"),
-];
+const ENV_VARS_TO_STORAGE_OPTS: [(&str, &str); 1] =
+    [("AZURE_STORAGE_ACCOUNT_NAME", "azure_storage_account_name")];
 
 impl ConnectBuilder {
     /// Create a new [`ConnectOptions`] with the given database URI.
@@ -791,10 +1113,10 @@ impl ConnectBuilder {
         options: &mut HashMap<String, String>,
     ) {
         for (env_key, opt_key) in env_var_to_remote_storage_option {
-            if let Ok(env_value) = std::env::var(env_key)
-                && !options.contains_key(*opt_key)
-            {
-                options.insert((*opt_key).to_string(), env_value);
+            if let Ok(env_value) = std::env::var(env_key) {
+                if !options.contains_key(*opt_key) {
+                    options.insert((*opt_key).to_string(), env_value);
+                }
             }
         }
     }
@@ -1027,11 +1349,20 @@ mod test_utils {
 
 #[cfg(test)]
 mod tests {
+    use crate::database::listing::{ListingDatabaseOptions, NewTableConfig};
+    use crate::query::QueryBase;
+    use crate::query::{ExecutableQuery, QueryExecutionOptions};
+    use crate::test_utils::connection::new_test_connection;
+    use arrow::compute::concat_batches;
+    use arrow_array::RecordBatchReader;
     use arrow_schema::{DataType, Field, Schema};
+    use datafusion_physical_plan::stream::RecordBatchStreamAdapter;
+    use futures::{stream, TryStreamExt};
+    use lance_core::error::{ArrowResult, DataFusionResult};
     use lance_testing::datagen::{BatchGenerator, IncrementingInt32};
     use tempfile::tempdir;
 
-    use crate::test_utils::connection::new_test_connection;
+    use crate::arrow::SimpleRecordBatchStream;
 
     use super::*;
 
@@ -1044,13 +1375,14 @@ mod tests {
     #[cfg(feature = "remote")]
     #[test]
     fn test_apply_env_defaults() {
-        let env_key = "PATH";
-        let env_val = std::env::var(env_key).expect("PATH should be set in test environment");
+        let env_key = "TEST_APPLY_ENV_DEFAULTS_ENVIRONMENT_VARIABLE_ENV_KEY";
+        let env_val = "TEST_APPLY_ENV_DEFAULTS_ENVIRONMENT_VARIABLE_ENV_VAL";
         let opts_key = "test_apply_env_defaults_environment_variable_opts_key";
+        std::env::set_var(env_key, env_val);
 
         let mut options = HashMap::new();
         ConnectBuilder::apply_env_defaults(&[(env_key, opts_key)], &mut options);
-        assert_eq!(Some(&env_val), options.get(opts_key));
+        assert_eq!(Some(&env_val.to_string()), options.get(opts_key));
 
         options.insert(opts_key.to_string(), "EXPLICIT-VALUE".to_string());
         ConnectBuilder::apply_env_defaults(&[(env_key, opts_key)], &mut options);
@@ -1146,6 +1478,139 @@ mod tests {
         assert_eq!(tables, vec!["table1".to_owned()]);
     }
 
+    fn make_data() -> Box<dyn RecordBatchReader + Send + 'static> {
+        let id = Box::new(IncrementingInt32::new().named("id".to_string()));
+        Box::new(BatchGenerator::new().col(id).batches(10, 2000))
+    }
+
+    #[tokio::test]
+    async fn test_create_table_v2() {
+        let tmp_dir = tempdir().unwrap();
+        let uri = tmp_dir.path().to_str().unwrap();
+        let db = connect(uri)
+            .database_options(&ListingDatabaseOptions {
+                new_table_config: NewTableConfig {
+                    data_storage_version: Some(LanceFileVersion::Legacy),
+                    ..Default::default()
+                },
+                ..Default::default()
+            })
+            .execute()
+            .await
+            .unwrap();
+
+        let tbl = db
+            .create_table("v1_test", make_data())
+            .execute()
+            .await
+            .unwrap();
+
+        // In v1 the row group size will trump max_batch_length
+        let batches = tbl
+            .query()
+            .limit(20000)
+            .execute_with_options(QueryExecutionOptions {
+                max_batch_length: 50000,
+                ..Default::default()
+            })
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(batches.len(), 20);
+
+        let db = connect(uri)
+            .database_options(&ListingDatabaseOptions {
+                new_table_config: NewTableConfig {
+                    data_storage_version: Some(LanceFileVersion::Stable),
+                    ..Default::default()
+                },
+                ..Default::default()
+            })
+            .execute()
+            .await
+            .unwrap();
+
+        let tbl = db
+            .create_table("v2_test", make_data())
+            .execute()
+            .await
+            .unwrap();
+
+        // In v2 the page size is much bigger than 50k so we should get a single batch
+        let batches = tbl
+            .query()
+            .execute_with_options(QueryExecutionOptions {
+                max_batch_length: 50000,
+                ..Default::default()
+            })
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+
+        assert_eq!(batches.len(), 1);
+    }
+
+    #[tokio::test]
+    async fn test_create_table_streaming() {
+        let tmp_dir = tempdir().unwrap();
+
+        let uri = tmp_dir.path().to_str().unwrap();
+        let db = connect(uri).execute().await.unwrap();
+
+        let batches = make_data().collect::<ArrowResult<Vec<_>>>().unwrap();
+
+        let schema = batches.first().unwrap().schema();
+        let one_batch = concat_batches(&schema, batches.iter()).unwrap();
+
+        let ldb_stream = stream::iter(batches.clone().into_iter().map(Result::Ok));
+        let ldb_stream: SendableRecordBatchStream =
+            Box::pin(SimpleRecordBatchStream::new(ldb_stream, schema.clone()));
+
+        let tbl1 = db
+            .create_table_streaming("one", ldb_stream)
+            .execute()
+            .await
+            .unwrap();
+
+        let df_stream = stream::iter(batches.into_iter().map(DataFusionResult::Ok));
+        let df_stream: datafusion_physical_plan::SendableRecordBatchStream =
+            Box::pin(RecordBatchStreamAdapter::new(schema.clone(), df_stream));
+
+        let tbl2 = db
+            .create_table_streaming("two", df_stream)
+            .execute()
+            .await
+            .unwrap();
+
+        let tbl1_data = tbl1
+            .query()
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+
+        let tbl1_data = concat_batches(&schema, tbl1_data.iter()).unwrap();
+        assert_eq!(tbl1_data, one_batch);
+
+        let tbl2_data = tbl2
+            .query()
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+
+        let tbl2_data = concat_batches(&schema, tbl2_data.iter()).unwrap();
+        assert_eq!(tbl2_data, one_batch);
+    }
+
     #[tokio::test]
     async fn drop_table() {
         let tc = new_test_connection().await.unwrap();
@@ -1175,6 +1640,41 @@ mod tests {
         assert_eq!(tables.len(), 0);
     }
 
+    #[tokio::test]
+    async fn test_create_table_already_exists() {
+        let tmp_dir = tempdir().unwrap();
+        let uri = tmp_dir.path().to_str().unwrap();
+        let db = connect(uri).execute().await.unwrap();
+        let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)]));
+        db.create_empty_table("test", schema.clone())
+            .execute()
+            .await
+            .unwrap();
+        // TODO: None of the open table options are "inspectable" right now but once one is we
+        // should assert we are passing these options in correctly
+        db.create_empty_table("test", schema)
+            .mode(CreateTableMode::exist_ok(|mut req| {
+                req.index_cache_size = Some(16);
+                req
+            }))
+            .execute()
+            .await
+            .unwrap();
+        let other_schema = Arc::new(Schema::new(vec![Field::new("y", DataType::Int32, false)]));
+        assert!(db
+            .create_empty_table("test", other_schema.clone())
+            .execute()
+            .await
+            .is_err());
+        let overwritten = db
+            .create_empty_table("test", other_schema.clone())
+            .mode(CreateTableMode::Overwrite)
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(other_schema, overwritten.schema().await.unwrap());
+    }
+
     #[tokio::test]
     async fn test_clone_table() {
         let tmp_dir = tempdir().unwrap();
@@ -1185,8 +1685,7 @@ mod tests {
         let mut batch_gen = BatchGenerator::new()
             .col(Box::new(IncrementingInt32::new().named("id")))
             .col(Box::new(IncrementingInt32::new().named("value")));
-        let reader: Box<dyn arrow_array::RecordBatchReader + Send> =
-            Box::new(batch_gen.batches(5, 100));
+        let reader = batch_gen.batches(5, 100);
 
         let source_table = db
             .create_table("source_table", reader)
@@ -1221,4 +1720,128 @@ mod tests {
         let cloned_count = cloned_table.count_rows(None).await.unwrap();
         assert_eq!(source_count, cloned_count);
     }
+
+    #[tokio::test]
+    async fn test_create_empty_table_with_embeddings() {
+        use crate::embeddings::{EmbeddingDefinition, EmbeddingFunction};
+        use arrow_array::{
+            Array, FixedSizeListArray, Float32Array, RecordBatch, RecordBatchIterator, StringArray,
+        };
+        use std::borrow::Cow;
+
+        #[derive(Debug, Clone)]
+        struct MockEmbedding {
+            dim: usize,
+        }
+
+        impl EmbeddingFunction for MockEmbedding {
+            fn name(&self) -> &str {
+                "test_embedding"
+            }
+
+            fn source_type(&self) -> Result<Cow<'_, DataType>> {
+                Ok(Cow::Owned(DataType::Utf8))
+            }
+
+            fn dest_type(&self) -> Result<Cow<'_, DataType>> {
+                Ok(Cow::Owned(DataType::new_fixed_size_list(
+                    DataType::Float32,
+                    self.dim as i32,
+                    true,
+                )))
+            }
+
+            fn compute_source_embeddings(&self, source: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
+                let len = source.len();
+                let values = vec![1.0f32; len * self.dim];
+                let values = Arc::new(Float32Array::from(values));
+                let field = Arc::new(Field::new("item", DataType::Float32, true));
+                Ok(Arc::new(FixedSizeListArray::new(
+                    field,
+                    self.dim as i32,
+                    values,
+                    None,
+                )))
+            }
+
+            fn compute_query_embeddings(&self, _input: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
+                unimplemented!()
+            }
+        }
+
+        let tmp_dir = tempdir().unwrap();
+        let uri = tmp_dir.path().to_str().unwrap();
+        let db = connect(uri).execute().await.unwrap();
+
+        let embed_func = Arc::new(MockEmbedding { dim: 128 });
+        db.embedding_registry()
+            .register("test_embedding", embed_func.clone())
+            .unwrap();
+
+        let schema = Arc::new(Schema::new(vec![Field::new("name", DataType::Utf8, true)]));
+        let ed = EmbeddingDefinition {
+            source_column: "name".to_owned(),
+            dest_column: Some("name_embedding".to_owned()),
+            embedding_name: "test_embedding".to_owned(),
+        };
+
+        let table = db
+            .create_empty_table("test", schema)
+            .mode(CreateTableMode::Overwrite)
+            .add_embedding(ed)
+            .unwrap()
+            .execute()
+            .await
+            .unwrap();
+
+        let table_schema = table.schema().await.unwrap();
+        assert!(table_schema.column_with_name("name").is_some());
+        assert!(table_schema.column_with_name("name_embedding").is_some());
+
+        let embedding_field = table_schema.field_with_name("name_embedding").unwrap();
+        assert_eq!(
+            embedding_field.data_type(),
+            &DataType::new_fixed_size_list(DataType::Float32, 128, true)
+        );
+
+        let input_schema = Arc::new(Schema::new(vec![Field::new("name", DataType::Utf8, true)]));
+        let input_batch = RecordBatch::try_new(
+            input_schema.clone(),
+            vec![Arc::new(StringArray::from(vec![
+                Some("Alice"),
+                Some("Bob"),
+                Some("Charlie"),
+            ]))],
+        )
+        .unwrap();
+
+        let input_reader = Box::new(RecordBatchIterator::new(
+            vec![Ok(input_batch)].into_iter(),
+            input_schema,
+        ));
+
+        table.add(input_reader).execute().await.unwrap();
+
+        let results = table
+            .query()
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+
+        assert_eq!(results.len(), 1);
+        let batch = &results[0];
+        assert_eq!(batch.num_rows(), 3);
+        assert!(batch.column_by_name("name_embedding").is_some());
+
+        let embedding_col = batch
+            .column_by_name("name_embedding")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<FixedSizeListArray>()
+            .unwrap();
+        assert_eq!(embedding_col.len(), 3);
+    }
 }

rust/lancedb/src/connection/create_table.rs

@@ -1,613 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-use std::sync::Arc;
-
-use lance_io::object_store::StorageOptionsProvider;
-
-use crate::{
-    Error, Result, Table,
-    connection::{merge_storage_options, set_storage_options_provider},
-    data::scannable::{Scannable, WithEmbeddingsScannable},
-    database::{CreateTableMode, CreateTableRequest, Database},
-    embeddings::{EmbeddingDefinition, EmbeddingFunction, EmbeddingRegistry},
-    table::WriteOptions,
-};
-
-pub struct CreateTableBuilder {
-    parent: Arc<dyn Database>,
-    embeddings: Vec<(EmbeddingDefinition, Arc<dyn EmbeddingFunction>)>,
-    embedding_registry: Arc<dyn EmbeddingRegistry>,
-    request: CreateTableRequest,
-}
-
-impl CreateTableBuilder {
-    pub(super) fn new(
-        parent: Arc<dyn Database>,
-        embedding_registry: Arc<dyn EmbeddingRegistry>,
-        name: String,
-        data: Box<dyn Scannable>,
-    ) -> Self {
-        Self {
-            parent,
-            embeddings: Vec::new(),
-            embedding_registry,
-            request: CreateTableRequest::new(name, data),
-        }
-    }
-
-    /// Set the mode for creating the table
-    ///
-    /// This controls what happens if a table with the given name already exists
-    pub fn mode(mut self, mode: CreateTableMode) -> Self {
-        self.request.mode = mode;
-        self
-    }
-
-    /// Apply the given write options when writing the initial data
-    pub fn write_options(mut self, write_options: WriteOptions) -> Self {
-        self.request.write_options = write_options;
-        self
-    }
-
-    /// Set an option for the storage layer.
-    ///
-    /// Options already set on the connection will be inherited by the table,
-    /// but can be overridden here.
-    ///
-    /// See available options at <https://lancedb.com/docs/storage/>
-    pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
-        let store_params = self
-            .request
-            .write_options
-            .lance_write_params
-            .get_or_insert(Default::default())
-            .store_params
-            .get_or_insert(Default::default());
-        merge_storage_options(store_params, [(key.into(), value.into())]);
-        self
-    }
-
-    /// Set multiple options for the storage layer.
-    ///
-    /// Options already set on the connection will be inherited by the table,
-    /// but can be overridden here.
-    ///
-    /// See available options at <https://lancedb.com/docs/storage/>
-    pub fn storage_options(
-        mut self,
-        pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
-    ) -> Self {
-        let store_params = self
-            .request
-            .write_options
-            .lance_write_params
-            .get_or_insert(Default::default())
-            .store_params
-            .get_or_insert(Default::default());
-        let updates = pairs
-            .into_iter()
-            .map(|(key, value)| (key.into(), value.into()));
-        merge_storage_options(store_params, updates);
-        self
-    }
-
-    /// Add an embedding definition to the table.
-    ///
-    /// The `embedding_name` must match the name of an embedding function that
-    /// was previously registered with the connection's [`EmbeddingRegistry`].
-    pub fn add_embedding(mut self, definition: EmbeddingDefinition) -> Result<Self> {
-        // Early verification of the embedding name
-        let embedding_func = self
-            .embedding_registry
-            .get(&definition.embedding_name)
-            .ok_or_else(|| Error::EmbeddingFunctionNotFound {
-                name: definition.embedding_name.clone(),
-                reason: "No embedding function found in the connection's embedding_registry"
-                    .to_string(),
-            })?;
-
-        self.embeddings.push((definition, embedding_func));
-        Ok(self)
-    }
-
-    /// Set the namespace for the table
-    pub fn namespace(mut self, namespace: Vec<String>) -> Self {
-        self.request.namespace = namespace;
-        self
-    }
-
-    /// Set a custom location for the table.
-    ///
-    /// If not set, the database will derive a location from its URI and the table name.
-    /// This is useful when integrating with namespace systems that manage table locations.
-    pub fn location(mut self, location: impl Into<String>) -> Self {
-        self.request.location = Some(location.into());
-        self
-    }
-
-    /// Set a storage options provider for automatic credential refresh.
-    ///
-    /// This allows tables to automatically refresh cloud storage credentials
-    /// when they expire, enabling long-running operations on remote storage.
-    pub fn storage_options_provider(mut self, provider: Arc<dyn StorageOptionsProvider>) -> Self {
-        let store_params = self
-            .request
-            .write_options
-            .lance_write_params
-            .get_or_insert(Default::default())
-            .store_params
-            .get_or_insert(Default::default());
-        set_storage_options_provider(store_params, provider);
-        self
-    }
-
-    /// Execute the create table operation
-    pub async fn execute(mut self) -> Result<Table> {
-        let embedding_registry = self.embedding_registry.clone();
-        let parent = self.parent.clone();
-
-        // If embeddings were configured via add_embedding(), wrap the data
-        if !self.embeddings.is_empty() {
-            let wrapped_data: Box<dyn Scannable> = Box::new(WithEmbeddingsScannable::try_new(
-                self.request.data,
-                self.embeddings,
-            )?);
-            self.request.data = wrapped_data;
-        }
-
-        Ok(Table::new_with_embedding_registry(
-            parent.create_table(self.request).await?,
-            parent,
-            embedding_registry,
-        ))
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use arrow_array::{
-        Array, FixedSizeListArray, Float32Array, RecordBatch, RecordBatchIterator, record_batch,
-    };
-    use arrow_schema::{ArrowError, DataType, Field, Schema};
-    use futures::TryStreamExt;
-    use lance_file::version::LanceFileVersion;
-    use tempfile::tempdir;
-
-    use crate::{
-        arrow::{SendableRecordBatchStream, SimpleRecordBatchStream},
-        connect,
-        database::listing::{ListingDatabaseOptions, NewTableConfig},
-        embeddings::{EmbeddingDefinition, EmbeddingFunction, MemoryRegistry},
-        query::{ExecutableQuery, QueryBase, Select},
-        test_utils::embeddings::MockEmbed,
-    };
-    use std::borrow::Cow;
-
-    use super::*;
-
-    #[tokio::test]
-    async fn create_empty_table() {
-        let db = connect("memory://").execute().await.unwrap();
-        let schema = Arc::new(Schema::new(vec![
-            Field::new("id", DataType::Int64, false),
-            Field::new("value", DataType::Float64, false),
-        ]));
-        db.create_empty_table("name", schema.clone())
-            .execute()
-            .await
-            .unwrap();
-        let table = db.open_table("name").execute().await.unwrap();
-        assert_eq!(table.schema().await.unwrap(), schema);
-        assert_eq!(table.count_rows(None).await.unwrap(), 0);
-    }
-
-    async fn test_create_table_with_data<T>(data: T)
-    where
-        T: Scannable + 'static,
-    {
-        let db = connect("memory://").execute().await.unwrap();
-        let schema = data.schema();
-        db.create_table("data_table", data).execute().await.unwrap();
-        let table = db.open_table("data_table").execute().await.unwrap();
-        assert_eq!(table.count_rows(None).await.unwrap(), 3);
-        assert_eq!(table.schema().await.unwrap(), schema);
-    }
-
-    #[tokio::test]
-    async fn create_table_with_batch() {
-        let batch = record_batch!(("id", Int64, [1, 2, 3])).unwrap();
-        test_create_table_with_data(batch).await;
-    }
-
-    #[tokio::test]
-    async fn test_create_table_with_vec_batch() {
-        let data = vec![
-            record_batch!(("id", Int64, [1, 2])).unwrap(),
-            record_batch!(("id", Int64, [3])).unwrap(),
-        ];
-        test_create_table_with_data(data).await;
-    }
-
-    #[tokio::test]
-    async fn test_create_table_with_record_batch_reader() {
-        let data = vec![
-            record_batch!(("id", Int64, [1, 2])).unwrap(),
-            record_batch!(("id", Int64, [3])).unwrap(),
-        ];
-        let schema = data[0].schema();
-        let reader: Box<dyn arrow_array::RecordBatchReader + Send> = Box::new(
-            RecordBatchIterator::new(data.into_iter().map(Ok), schema.clone()),
-        );
-        test_create_table_with_data(reader).await;
-    }
-
-    #[tokio::test]
-    async fn test_create_table_with_stream() {
-        let data = vec![
-            record_batch!(("id", Int64, [1, 2])).unwrap(),
-            record_batch!(("id", Int64, [3])).unwrap(),
-        ];
-        let schema = data[0].schema();
-        let inner = futures::stream::iter(data.into_iter().map(Ok));
-        let stream: SendableRecordBatchStream = Box::pin(SimpleRecordBatchStream {
-            schema,
-            stream: inner,
-        });
-        test_create_table_with_data(stream).await;
-    }
-
-    #[derive(Debug)]
-    struct MyError;
-
-    impl std::fmt::Display for MyError {
-        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-            write!(f, "MyError occurred")
-        }
-    }
-
-    impl std::error::Error for MyError {}
-
-    #[tokio::test]
-    async fn test_create_preserves_reader_error() {
-        let first_batch = record_batch!(("id", Int64, [1, 2])).unwrap();
-        let schema = first_batch.schema();
-        let iterator = vec![
-            Ok(first_batch),
-            Err(ArrowError::ExternalError(Box::new(MyError))),
-        ];
-        let reader: Box<dyn arrow_array::RecordBatchReader + Send> = Box::new(
-            RecordBatchIterator::new(iterator.into_iter(), schema.clone()),
-        );
-
-        let db = connect("memory://").execute().await.unwrap();
-        let result = db.create_table("failing_table", reader).execute().await;
-
-        assert!(result.is_err());
-        // TODO: when we upgrade to Lance 2.0.0, this should pass
-        // assert!(matches!(result, Err(Error::External { source})
-        //     if source.downcast_ref::<MyError>().is_some()
-        // ));
-    }
-
-    #[tokio::test]
-    async fn test_create_preserves_stream_error() {
-        let first_batch = record_batch!(("id", Int64, [1, 2])).unwrap();
-        let schema = first_batch.schema();
-        let iterator = vec![
-            Ok(first_batch),
-            Err(Error::External {
-                source: Box::new(MyError),
-            }),
-        ];
-        let stream = futures::stream::iter(iterator);
-        let stream: SendableRecordBatchStream = Box::pin(SimpleRecordBatchStream {
-            schema: schema.clone(),
-            stream,
-        });
-
-        let db = connect("memory://").execute().await.unwrap();
-        let result = db
-            .create_table("failing_stream_table", stream)
-            .execute()
-            .await;
-
-        assert!(result.is_err());
-        // TODO: when we upgrade to Lance 2.0.0, this should pass
-        // assert!(matches!(result, Err(Error::External { source})
-        //     if source.downcast_ref::<MyError>().is_some()
-        // ));
-    }
-
-    #[tokio::test]
-    #[allow(deprecated)]
-    async fn test_create_table_with_storage_options() {
-        let batch = record_batch!(("id", Int64, [1, 2, 3])).unwrap();
-        let db = connect("memory://").execute().await.unwrap();
-
-        let table = db
-            .create_table("options_table", batch)
-            .storage_option("timeout", "30s")
-            .storage_options([("retry_count", "3")])
-            .execute()
-            .await
-            .unwrap();
-
-        let final_options = table.storage_options().await.unwrap();
-        assert_eq!(final_options.get("timeout"), Some(&"30s".to_string()));
-        assert_eq!(final_options.get("retry_count"), Some(&"3".to_string()));
-    }
-
-    #[tokio::test]
-    async fn test_create_table_unregistered_embedding() {
-        let db = connect("memory://").execute().await.unwrap();
-        let batch = record_batch!(("text", Utf8, ["hello", "world"])).unwrap();
-
-        // Try to add an embedding that doesn't exist in the registry
-        let result = db
-            .create_table("embed_table", batch)
-            .add_embedding(EmbeddingDefinition::new(
-                "text",
-                "nonexistent_embedding_function",
-                None::<&str>,
-            ));
-
-        match result {
-            Err(Error::EmbeddingFunctionNotFound { name, .. }) => {
-                assert_eq!(name, "nonexistent_embedding_function");
-            }
-            Err(other) => panic!("Expected EmbeddingFunctionNotFound error, got: {:?}", other),
-            Ok(_) => panic!("Expected error, but got Ok"),
-        }
-    }
-
-    #[tokio::test]
-    async fn test_create_table_already_exists() {
-        let tmp_dir = tempdir().unwrap();
-        let uri = tmp_dir.path().to_str().unwrap();
-        let db = connect(uri).execute().await.unwrap();
-        let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)]));
-        db.create_empty_table("test", schema.clone())
-            .execute()
-            .await
-            .unwrap();
-        db.create_empty_table("test", schema)
-            .mode(CreateTableMode::exist_ok(|mut req| {
-                req.index_cache_size = Some(16);
-                req
-            }))
-            .execute()
-            .await
-            .unwrap();
-        let other_schema = Arc::new(Schema::new(vec![Field::new("y", DataType::Int32, false)]));
-        assert!(
-            db.create_empty_table("test", other_schema.clone())
-                .execute()
-                .await
-                .is_err()
-        ); // TODO: assert what this error is
-        let overwritten = db
-            .create_empty_table("test", other_schema.clone())
-            .mode(CreateTableMode::Overwrite)
-            .execute()
-            .await
-            .unwrap();
-        assert_eq!(other_schema, overwritten.schema().await.unwrap());
-    }
-
-    #[tokio::test]
-    #[rstest::rstest]
-    #[case(LanceFileVersion::Legacy)]
-    #[case(LanceFileVersion::Stable)]
-    async fn test_create_table_with_storage_version(
-        #[case] data_storage_version: LanceFileVersion,
-    ) {
-        let db = connect("memory://")
-            .database_options(&ListingDatabaseOptions {
-                new_table_config: NewTableConfig {
-                    data_storage_version: Some(data_storage_version),
-                    ..Default::default()
-                },
-                ..Default::default()
-            })
-            .execute()
-            .await
-            .unwrap();
-
-        let batch = record_batch!(("id", Int64, [1, 2, 3])).unwrap();
-        let table = db
-            .create_table("legacy_table", batch)
-            .execute()
-            .await
-            .unwrap();
-
-        let native_table = table.as_native().unwrap();
-        let storage_format = native_table
-            .manifest()
-            .await
-            .unwrap()
-            .data_storage_format
-            .lance_file_version()
-            .unwrap();
-        // Compare resolved versions since Stable/Next are aliases that resolve at storage time
-        assert_eq!(storage_format.resolve(), data_storage_version.resolve());
-    }
-
-    #[tokio::test]
-    async fn test_create_table_with_embedding() {
-        // Register the mock embedding function
-        let registry = Arc::new(MemoryRegistry::new());
-        let mock_embedding: Arc<dyn EmbeddingFunction> = Arc::new(MockEmbed::new("mock", 4));
-        registry.register("mock", mock_embedding).unwrap();
-
-        // Connect with the custom registry
-        let conn = connect("memory://")
-            .embedding_registry(registry)
-            .execute()
-            .await
-            .unwrap();
-
-        // Create data without the embedding column
-        let batch = record_batch!(("text", Utf8, ["hello", "world", "test"])).unwrap();
-
-        // Create table with add_embedding - embeddings should be computed automatically
-        let table = conn
-            .create_table("embed_test", batch)
-            .add_embedding(EmbeddingDefinition::new(
-                "text",
-                "mock",
-                Some("text_embedding"),
-            ))
-            .unwrap()
-            .execute()
-            .await
-            .unwrap();
-
-        // Verify row count
-        assert_eq!(table.count_rows(None).await.unwrap(), 3);
-
-        // Verify the schema includes the embedding column
-        let result_schema = table.schema().await.unwrap();
-        assert_eq!(result_schema.fields().len(), 2);
-        assert_eq!(result_schema.field(0).name(), "text");
-        assert_eq!(result_schema.field(1).name(), "text_embedding");
-
-        // Verify the embedding column has the correct type
-        assert!(matches!(
-            result_schema.field(1).data_type(),
-            DataType::FixedSizeList(_, 4)
-        ));
-
-        // Query to verify the embeddings were computed
-        let results: Vec<RecordBatch> = table
-            .query()
-            .select(Select::columns(&["text", "text_embedding"]))
-            .execute()
-            .await
-            .unwrap()
-            .try_collect()
-            .await
-            .unwrap();
-
-        let total_rows: usize = results.iter().map(|b| b.num_rows()).sum();
-        assert_eq!(total_rows, 3);
-
-        // Check that all rows have embedding values (not null)
-        for batch in &results {
-            let embedding_col = batch.column(1);
-            assert_eq!(embedding_col.null_count(), 0);
-            assert_eq!(embedding_col.len(), batch.num_rows());
-        }
-
-        // Verify the schema metadata contains the column definitions
-        assert!(
-            result_schema
-                .metadata
-                .contains_key("lancedb::column_definitions"),
-            "Schema metadata should contain column definitions"
-        );
-    }
-
-    #[tokio::test]
-    async fn test_create_empty_table_with_embeddings() {
-        #[derive(Debug, Clone)]
-        struct MockEmbedding {
-            dim: usize,
-        }
-
-        impl EmbeddingFunction for MockEmbedding {
-            fn name(&self) -> &str {
-                "test_embedding"
-            }
-
-            fn source_type(&self) -> Result<Cow<'_, DataType>> {
-                Ok(Cow::Owned(DataType::Utf8))
-            }
-
-            fn dest_type(&self) -> Result<Cow<'_, DataType>> {
-                Ok(Cow::Owned(DataType::new_fixed_size_list(
-                    DataType::Float32,
-                    self.dim as i32,
-                    true,
-                )))
-            }
-
-            fn compute_source_embeddings(&self, source: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
-                let len = source.len();
-                let values = vec![1.0f32; len * self.dim];
-                let values = Arc::new(Float32Array::from(values));
-                let field = Arc::new(Field::new("item", DataType::Float32, true));
-                Ok(Arc::new(FixedSizeListArray::new(
-                    field,
-                    self.dim as i32,
-                    values,
-                    None,
-                )))
-            }
-
-            fn compute_query_embeddings(&self, _input: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
-                unimplemented!()
-            }
-        }
-
-        let tmp_dir = tempdir().unwrap();
-        let uri = tmp_dir.path().to_str().unwrap();
-        let db = connect(uri).execute().await.unwrap();
-
-        let embed_func = Arc::new(MockEmbedding { dim: 128 });
-        db.embedding_registry()
-            .register("test_embedding", embed_func.clone())
-            .unwrap();
-
-        let schema = Arc::new(Schema::new(vec![Field::new("name", DataType::Utf8, true)]));
-        let ed = EmbeddingDefinition {
-            source_column: "name".to_owned(),
-            dest_column: Some("name_embedding".to_owned()),
-            embedding_name: "test_embedding".to_owned(),
-        };
-
-        let table = db
-            .create_empty_table("test", schema)
-            .mode(CreateTableMode::Overwrite)
-            .add_embedding(ed)
-            .unwrap()
-            .execute()
-            .await
-            .unwrap();
-
-        let table_schema = table.schema().await.unwrap();
-        assert!(table_schema.column_with_name("name").is_some());
-        assert!(table_schema.column_with_name("name_embedding").is_some());
-
-        let embedding_field = table_schema.field_with_name("name_embedding").unwrap();
-        assert_eq!(
-            embedding_field.data_type(),
-            &DataType::new_fixed_size_list(DataType::Float32, 128, true)
-        );
-
-        let input_batch = record_batch!(("name", Utf8, ["Alice", "Bob", "Charlie"])).unwrap();
-        table.add(input_batch).execute().await.unwrap();
-
-        let results = table
-            .query()
-            .execute()
-            .await
-            .unwrap()
-            .try_collect::<Vec<_>>()
-            .await
-            .unwrap();
-
-        assert_eq!(results.len(), 1);
-        let batch = &results[0];
-        assert_eq!(batch.num_rows(), 3);
-        assert!(batch.column_by_name("name_embedding").is_some());
-
-        let embedding_col = batch
-            .column_by_name("name_embedding")
-            .unwrap()
-            .as_any()
-            .downcast_ref::<FixedSizeListArray>()
-            .unwrap();
-        assert_eq!(embedding_col.len(), 3);
-    }
-}

rust/lancedb/src/data.rs

@@ -5,4 +5,3 @@
 
 pub mod inspect;
 pub mod sanitize;
-pub mod scannable;

rust/lancedb/src/data/inspect.rs

@@ -5,9 +5,9 @@ use std::collections::HashMap;
 
 use arrow::compute::kernels::{aggregate::bool_and, length::length};
 use arrow_array::{
-    Array, GenericListArray, OffsetSizeTrait, PrimitiveArray, RecordBatchReader,
     cast::AsArray,
     types::{ArrowPrimitiveType, Int32Type, Int64Type},
+    Array, GenericListArray, OffsetSizeTrait, PrimitiveArray, RecordBatchReader,
 };
 use arrow_ord::cmp::eq;
 use arrow_schema::DataType;
@@ -78,7 +78,7 @@ pub fn infer_vector_columns(
                 _ => {
                     return Err(Error::Schema {
                         message: format!("Column {} is not a list", col_name),
-                    });
+                    })
                 }
             } {
                 if let Some(Some(prev_dim)) = columns_to_infer.get(&col_name) {
@@ -102,8 +102,8 @@ mod tests {
     use super::*;
 
     use arrow_array::{
-        FixedSizeListArray, Float32Array, ListArray, RecordBatch, RecordBatchIterator, StringArray,
         types::{Float32Type, Float64Type},
+        FixedSizeListArray, Float32Array, ListArray, RecordBatch, RecordBatchIterator, StringArray,
     };
     use arrow_schema::{DataType, Field, Schema};
     use std::{sync::Arc, vec};

rust/lancedb/src/data/sanitize.rs

@@ -4,10 +4,10 @@
 use std::{iter::repeat_with, sync::Arc};
 
 use arrow_array::{
-    Array, ArrowNumericType, FixedSizeListArray, PrimitiveArray, RecordBatch, RecordBatchIterator,
-    RecordBatchReader,
     cast::AsArray,
     types::{Float16Type, Float32Type, Float64Type, Int32Type, Int64Type},
+    Array, ArrowNumericType, FixedSizeListArray, PrimitiveArray, RecordBatch, RecordBatchIterator,
+    RecordBatchReader,
 };
 use arrow_cast::{can_cast_types, cast};
 use arrow_schema::{ArrowError, DataType, Field, Schema};
@@ -184,7 +184,7 @@ mod tests {
     use std::sync::Arc;
 
     use arrow_array::{
-        FixedSizeListArray, Float16Array, Float32Array, Float64Array, Int8Array, Int32Array,
+        FixedSizeListArray, Float16Array, Float32Array, Float64Array, Int32Array, Int8Array,
         RecordBatch, RecordBatchIterator, StringArray,
     };
     use arrow_schema::Field;

rust/lancedb/src/data/scannable.rs

@@ -1,968 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-//! Data source abstraction for LanceDB.
-//!
-//! This module provides a [`Scannable`] trait that allows input data sources to express
-//! capabilities (row count, rescannability) so the insert pipeline can make
-//! better decisions about write parallelism and retry strategies.
-
-use std::sync::Arc;
-
-use crate::arrow::{
-    SendableRecordBatchStream, SendableRecordBatchStreamExt, SimpleRecordBatchStream,
-};
-use crate::embeddings::{
-    EmbeddingDefinition, EmbeddingFunction, EmbeddingRegistry, compute_embeddings_for_batch,
-    compute_output_schema,
-};
-use crate::table::{ColumnDefinition, ColumnKind, TableDefinition};
-use crate::{Error, Result};
-use arrow_array::{ArrayRef, RecordBatch, RecordBatchIterator, RecordBatchReader};
-use arrow_schema::{ArrowError, SchemaRef};
-use async_trait::async_trait;
-use futures::StreamExt;
-use futures::stream::once;
-use lance_datafusion::utils::StreamingWriteSource;
-
-pub trait Scannable: Send {
-    /// Returns the schema of the data.
-    fn schema(&self) -> SchemaRef;
-
-    /// Read data as a stream of record batches.
-    ///
-    /// For rescannable sources (in-memory data like RecordBatch, Vec<RecordBatch>),
-    /// this can be called multiple times and returns cloned data each time.
-    ///
-    /// For non-rescannable sources (streams, readers), this can only be called once.
-    /// Calling it a second time returns a stream whose first item is an error.
-    fn scan_as_stream(&mut self) -> SendableRecordBatchStream;
-
-    /// Optional hint about the number of rows.
-    ///
-    /// When available, this allows the pipeline to estimate total data size
-    /// and choose appropriate partitioning.
-    fn num_rows(&self) -> Option<usize> {
-        None
-    }
-
-    /// Whether the source can be re-read from the beginning.
-    ///
-    /// `true` for in-memory data (Tables, DataFrames) and disk-based sources (Datasets).
-    /// `false` for streaming sources (DuckDB results, network streams).
-    ///
-    /// When true, the pipeline can retry failed writes by rescanning.
-    fn rescannable(&self) -> bool {
-        false
-    }
-}
-
-impl std::fmt::Debug for dyn Scannable {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("Scannable")
-            .field("schema", &self.schema())
-            .field("num_rows", &self.num_rows())
-            .field("rescannable", &self.rescannable())
-            .finish()
-    }
-}
-
-impl Scannable for RecordBatch {
-    fn schema(&self) -> SchemaRef {
-        Self::schema(self)
-    }
-
-    fn scan_as_stream(&mut self) -> SendableRecordBatchStream {
-        let batch = self.clone();
-        let schema = batch.schema();
-        Box::pin(SimpleRecordBatchStream {
-            schema,
-            stream: once(async move { Ok(batch) }),
-        })
-    }
-
-    fn num_rows(&self) -> Option<usize> {
-        Some(Self::num_rows(self))
-    }
-
-    fn rescannable(&self) -> bool {
-        true
-    }
-}
-
-impl Scannable for Vec<RecordBatch> {
-    fn schema(&self) -> SchemaRef {
-        if self.is_empty() {
-            Arc::new(arrow_schema::Schema::empty())
-        } else {
-            self[0].schema()
-        }
-    }
-
-    fn scan_as_stream(&mut self) -> SendableRecordBatchStream {
-        if self.is_empty() {
-            let schema = Scannable::schema(self);
-            return Box::pin(SimpleRecordBatchStream {
-                schema,
-                stream: once(async {
-                    Err(Error::InvalidInput {
-                        message: "Cannot scan an empty Vec<RecordBatch>".to_string(),
-                    })
-                }),
-            });
-        }
-        let schema = Scannable::schema(self);
-        let batches = self.clone();
-        let stream = futures::stream::iter(batches.into_iter().map(Ok));
-        Box::pin(SimpleRecordBatchStream { schema, stream })
-    }
-
-    fn num_rows(&self) -> Option<usize> {
-        Some(self.iter().map(|b| b.num_rows()).sum())
-    }
-
-    fn rescannable(&self) -> bool {
-        true
-    }
-}
-
-impl Scannable for Box<dyn RecordBatchReader + Send> {
-    fn schema(&self) -> SchemaRef {
-        RecordBatchReader::schema(self.as_ref())
-    }
-
-    fn scan_as_stream(&mut self) -> SendableRecordBatchStream {
-        let schema = Scannable::schema(self);
-
-        // Swap self with a reader that errors on iteration, so a second call
-        // produces a clear error instead of silently returning empty data.
-        let err_reader: Box<dyn RecordBatchReader + Send> = Box::new(RecordBatchIterator::new(
-            vec![Err(ArrowError::InvalidArgumentError(
-                "Reader has already been consumed".into(),
-            ))],
-            schema.clone(),
-        ));
-        let reader = std::mem::replace(self, err_reader);
-
-        // Bridge the blocking RecordBatchReader to an async stream via a channel.
-        let (tx, rx) = tokio::sync::mpsc::channel::<crate::Result<RecordBatch>>(2);
-        tokio::task::spawn_blocking(move || {
-            for batch_result in reader {
-                let result = batch_result.map_err(Into::into);
-                if tx.blocking_send(result).is_err() {
-                    break;
-                }
-            }
-        });
-
-        let stream = futures::stream::unfold(rx, |mut rx| async move {
-            rx.recv().await.map(|batch| (batch, rx))
-        })
-        .fuse();
-
-        Box::pin(SimpleRecordBatchStream { schema, stream })
-    }
-}
-
-impl Scannable for SendableRecordBatchStream {
-    fn schema(&self) -> SchemaRef {
-        self.as_ref().schema()
-    }
-
-    fn scan_as_stream(&mut self) -> SendableRecordBatchStream {
-        let schema = Scannable::schema(self);
-
-        // Swap self with an error stream so a second call produces a clear error.
-        let error_stream = Box::pin(SimpleRecordBatchStream {
-            schema: schema.clone(),
-            stream: once(async {
-                Err(Error::InvalidInput {
-                    message: "Stream has already been consumed".to_string(),
-                })
-            }),
-        });
-        std::mem::replace(self, error_stream)
-    }
-}
-
-#[async_trait]
-impl StreamingWriteSource for Box<dyn Scannable> {
-    fn arrow_schema(&self) -> SchemaRef {
-        self.schema()
-    }
-
-    fn into_stream(mut self) -> datafusion_physical_plan::SendableRecordBatchStream {
-        self.scan_as_stream().into_df_stream()
-    }
-}
-
-/// A scannable that applies embeddings to the stream.
-pub struct WithEmbeddingsScannable {
-    inner: Box<dyn Scannable>,
-    embeddings: Vec<(EmbeddingDefinition, Arc<dyn EmbeddingFunction>)>,
-    output_schema: SchemaRef,
-}
-
-impl WithEmbeddingsScannable {
-    /// Create a new WithEmbeddingsScannable.
-    ///
-    /// The embeddings are applied to the inner scannable's data as new columns.
-    pub fn try_new(
-        inner: Box<dyn Scannable>,
-        embeddings: Vec<(EmbeddingDefinition, Arc<dyn EmbeddingFunction>)>,
-    ) -> Result<Self> {
-        let output_schema = compute_output_schema(&inner.schema(), &embeddings)?;
-
-        // Build column definitions: Physical for base columns, Embedding for new ones
-        let base_col_count = inner.schema().fields().len();
-        let column_definitions: Vec<ColumnDefinition> = (0..base_col_count)
-            .map(|_| ColumnDefinition {
-                kind: ColumnKind::Physical,
-            })
-            .chain(embeddings.iter().map(|(ed, _)| ColumnDefinition {
-                kind: ColumnKind::Embedding(ed.clone()),
-            }))
-            .collect();
-
-        let table_definition = TableDefinition::new(output_schema, column_definitions);
-        let output_schema = table_definition.into_rich_schema();
-
-        Self::with_schema(inner, embeddings, output_schema)
-    }
-
-    /// Create a WithEmbeddingsScannable with a specific output schema.
-    ///
-    /// Use this when the table schema is already known (e.g. during add) to
-    /// avoid nullability mismatches between the embedding function's declared
-    /// type and the table's stored type.
-    pub fn with_schema(
-        inner: Box<dyn Scannable>,
-        embeddings: Vec<(EmbeddingDefinition, Arc<dyn EmbeddingFunction>)>,
-        output_schema: SchemaRef,
-    ) -> Result<Self> {
-        Ok(Self {
-            inner,
-            embeddings,
-            output_schema,
-        })
-    }
-}
-
-impl Scannable for WithEmbeddingsScannable {
-    fn schema(&self) -> SchemaRef {
-        self.output_schema.clone()
-    }
-
-    fn scan_as_stream(&mut self) -> SendableRecordBatchStream {
-        let inner_stream = self.inner.scan_as_stream();
-        let embeddings = self.embeddings.clone();
-        let output_schema = self.output_schema.clone();
-        let stream_schema = output_schema.clone();
-
-        let mapped_stream = inner_stream.then(move |batch_result| {
-            let embeddings = embeddings.clone();
-            let output_schema = output_schema.clone();
-            async move {
-                let batch = batch_result?;
-                let result = tokio::task::spawn_blocking(move || {
-                    compute_embeddings_for_batch(batch, &embeddings)
-                })
-                .await
-                .map_err(|e| Error::Runtime {
-                    message: format!("Task panicked during embedding computation: {}", e),
-                })??;
-                // Cast columns to match the declared output schema. The data is
-                // identical but field metadata (e.g. nested nullability) may
-                // differ between the embedding function output and the table.
-                let columns: Vec<ArrayRef> = result
-                    .columns()
-                    .iter()
-                    .enumerate()
-                    .map(|(i, col)| {
-                        let target_type = output_schema.field(i).data_type();
-                        if col.data_type() == target_type {
-                            Ok(col.clone())
-                        } else {
-                            arrow_cast::cast(col, target_type).map_err(Error::from)
-                        }
-                    })
-                    .collect::<Result<_>>()?;
-                let result = RecordBatch::try_new(output_schema, columns)?;
-                Ok(result)
-            }
-        });
-
-        Box::pin(SimpleRecordBatchStream {
-            schema: stream_schema,
-            stream: mapped_stream,
-        })
-    }
-
-    fn num_rows(&self) -> Option<usize> {
-        self.inner.num_rows()
-    }
-
-    fn rescannable(&self) -> bool {
-        self.inner.rescannable()
-    }
-}
-
-pub fn scannable_with_embeddings(
-    inner: Box<dyn Scannable>,
-    table_definition: &TableDefinition,
-    registry: Option<&Arc<dyn EmbeddingRegistry>>,
-) -> Result<Box<dyn Scannable>> {
-    if let Some(registry) = registry {
-        let mut embeddings = Vec::with_capacity(table_definition.column_definitions.len());
-        for cd in table_definition.column_definitions.iter() {
-            if let ColumnKind::Embedding(embedding_def) = &cd.kind {
-                match registry.get(&embedding_def.embedding_name) {
-                    Some(func) => {
-                        embeddings.push((embedding_def.clone(), func));
-                    }
-                    None => {
-                        return Err(Error::EmbeddingFunctionNotFound {
-                            name: embedding_def.embedding_name.clone(),
-                            reason: format!(
-                                "Table was defined with an embedding column `{}` but no embedding function was found with that name within the registry.",
-                                embedding_def.embedding_name
-                            ),
-                        });
-                    }
-                }
-            }
-        }
-
-        if !embeddings.is_empty() {
-            // Use the table's schema so embedding column types (including nested
-            // nullability) match what's stored, avoiding mismatches with the
-            // embedding function's declared dest_type.
-            return Ok(Box::new(WithEmbeddingsScannable::with_schema(
-                inner,
-                embeddings,
-                table_definition.schema.clone(),
-            )?));
-        }
-    }
-
-    Ok(inner)
-}
-
-/// A wrapper that buffers the first RecordBatch from a Scannable so we can
-/// inspect it (e.g. to estimate data size) without losing it.
-pub(crate) struct PeekedScannable {
-    inner: Box<dyn Scannable>,
-    peeked: Option<RecordBatch>,
-    /// The first item from the stream, if it was an error. Stored so we can
-    /// re-emit it from `scan_as_stream` instead of silently dropping it.
-    first_error: Option<crate::Error>,
-    stream: Option<SendableRecordBatchStream>,
-}
-
-impl PeekedScannable {
-    pub fn new(inner: Box<dyn Scannable>) -> Self {
-        Self {
-            inner,
-            peeked: None,
-            first_error: None,
-            stream: None,
-        }
-    }
-
-    /// Reads and buffers the first batch from the inner scannable.
-    /// Returns a clone of it. Subsequent calls return the same batch.
-    ///
-    /// Returns `None` if the stream is empty or the first item is an error.
-    /// Errors are preserved and re-emitted by `scan_as_stream`.
-    pub async fn peek(&mut self) -> Option<RecordBatch> {
-        if self.peeked.is_some() {
-            return self.peeked.clone();
-        }
-        // Already peeked and got an error or empty stream.
-        if self.stream.is_some() || self.first_error.is_some() {
-            return None;
-        }
-        let mut stream = self.inner.scan_as_stream();
-        match stream.next().await {
-            Some(Ok(batch)) => {
-                self.peeked = Some(batch.clone());
-                self.stream = Some(stream);
-                Some(batch)
-            }
-            Some(Err(e)) => {
-                self.first_error = Some(e);
-                self.stream = Some(stream);
-                None
-            }
-            None => {
-                self.stream = Some(stream);
-                None
-            }
-        }
-    }
-}
-
-impl Scannable for PeekedScannable {
-    fn schema(&self) -> SchemaRef {
-        self.inner.schema()
-    }
-
-    fn num_rows(&self) -> Option<usize> {
-        self.inner.num_rows()
-    }
-
-    fn rescannable(&self) -> bool {
-        self.inner.rescannable()
-    }
-
-    fn scan_as_stream(&mut self) -> SendableRecordBatchStream {
-        let schema = self.inner.schema();
-
-        // If peek() hit an error, prepend it so downstream sees the error.
-        let error_item = self.first_error.take().map(Err);
-
-        match (self.peeked.take(), self.stream.take()) {
-            (Some(batch), Some(rest)) => {
-                let prepend = futures::stream::once(std::future::ready(Ok(batch)));
-                Box::pin(SimpleRecordBatchStream {
-                    schema,
-                    stream: prepend.chain(rest),
-                })
-            }
-            (Some(batch), None) => Box::pin(SimpleRecordBatchStream {
-                schema,
-                stream: futures::stream::once(std::future::ready(Ok(batch))),
-            }),
-            (None, Some(rest)) => {
-                if let Some(err) = error_item {
-                    let stream = futures::stream::once(std::future::ready(err));
-                    Box::pin(SimpleRecordBatchStream { schema, stream })
-                } else {
-                    rest
-                }
-            }
-            (None, None) => {
-                // peek() was never called — just delegate
-                self.inner.scan_as_stream()
-            }
-        }
-    }
-}
-
-/// Compute the number of write partitions based on data size estimates.
-///
-/// `sample_bytes` and `sample_rows` come from a representative batch and are
-/// used to estimate per-row size. `total_rows_hint` is the total row count
-/// when known; otherwise `sample_rows` row count is used as a lower bound
-/// estimate.
-///
-/// Targets roughly 1 million rows or 2 GB per partition, capped at
-/// `max_partitions` (typically the number of available CPU cores).
-pub(crate) fn estimate_write_partitions(
-    sample_bytes: usize,
-    sample_rows: usize,
-    total_rows_hint: Option<usize>,
-    max_partitions: usize,
-) -> usize {
-    if sample_rows == 0 {
-        return 1;
-    }
-    let bytes_per_row = sample_bytes / sample_rows;
-    let total_rows = total_rows_hint.unwrap_or(sample_rows);
-    let total_bytes = total_rows * bytes_per_row;
-    let by_rows = total_rows.div_ceil(1_000_000);
-    let by_bytes = total_bytes.div_ceil(2 * 1024 * 1024 * 1024);
-    by_rows.max(by_bytes).max(1).min(max_partitions)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use arrow_array::record_batch;
-    use futures::TryStreamExt;
-
-    #[tokio::test]
-    async fn test_record_batch_rescannable() {
-        let mut batch = record_batch!(("id", Int64, [0, 1, 2])).unwrap();
-
-        let stream1 = batch.scan_as_stream();
-        let batches1: Vec<RecordBatch> = stream1.try_collect().await.unwrap();
-        assert_eq!(batches1.len(), 1);
-        assert_eq!(batches1[0], batch);
-
-        assert!(batch.rescannable());
-        let stream2 = batch.scan_as_stream();
-        let batches2: Vec<RecordBatch> = stream2.try_collect().await.unwrap();
-        assert_eq!(batches2.len(), 1);
-        assert_eq!(batches2[0], batch);
-    }
-
-    #[tokio::test]
-    async fn test_vec_batch_rescannable() {
-        let mut batches = vec![
-            record_batch!(("id", Int64, [0, 1])).unwrap(),
-            record_batch!(("id", Int64, [2, 3, 4])).unwrap(),
-        ];
-
-        let stream1 = batches.scan_as_stream();
-        let result1: Vec<RecordBatch> = stream1.try_collect().await.unwrap();
-        assert_eq!(result1.len(), 2);
-        assert_eq!(result1[0], batches[0]);
-        assert_eq!(result1[1], batches[1]);
-
-        assert!(batches.rescannable());
-        let stream2 = batches.scan_as_stream();
-        let result2: Vec<RecordBatch> = stream2.try_collect().await.unwrap();
-        assert_eq!(result2.len(), 2);
-        assert_eq!(result2[0], batches[0]);
-        assert_eq!(result2[1], batches[1]);
-    }
-
-    #[tokio::test]
-    async fn test_vec_batch_empty_errors() {
-        let mut empty: Vec<RecordBatch> = vec![];
-        let mut stream = empty.scan_as_stream();
-        let result = stream.next().await;
-        assert!(result.is_some());
-        assert!(result.unwrap().is_err());
-    }
-
-    #[tokio::test]
-    async fn test_reader_not_rescannable() {
-        let batch = record_batch!(("id", Int64, [0, 1, 2])).unwrap();
-        let schema = batch.schema();
-        let mut reader: Box<dyn arrow_array::RecordBatchReader + Send> = Box::new(
-            RecordBatchIterator::new(vec![Ok(batch.clone())], schema.clone()),
-        );
-
-        let stream1 = reader.scan_as_stream();
-        let result1: Vec<RecordBatch> = stream1.try_collect().await.unwrap();
-        assert_eq!(result1.len(), 1);
-        assert_eq!(result1[0], batch);
-
-        assert!(!reader.rescannable());
-        // Second call returns a stream whose first item is an error
-        let mut stream2 = reader.scan_as_stream();
-        let result2 = stream2.next().await;
-        assert!(result2.is_some());
-        assert!(result2.unwrap().is_err());
-    }
-
-    #[tokio::test]
-    async fn test_stream_not_rescannable() {
-        let batch = record_batch!(("id", Int64, [0, 1, 2])).unwrap();
-        let schema = batch.schema();
-        let inner_stream = futures::stream::iter(vec![Ok(batch.clone())]);
-        let mut stream: SendableRecordBatchStream = Box::pin(SimpleRecordBatchStream {
-            schema: schema.clone(),
-            stream: inner_stream,
-        });
-
-        let stream1 = stream.scan_as_stream();
-        let result1: Vec<RecordBatch> = stream1.try_collect().await.unwrap();
-        assert_eq!(result1.len(), 1);
-        assert_eq!(result1[0], batch);
-
-        assert!(!stream.rescannable());
-        // Second call returns a stream whose first item is an error
-        let mut stream2 = stream.scan_as_stream();
-        let result2 = stream2.next().await;
-        assert!(result2.is_some());
-        assert!(result2.unwrap().is_err());
-    }
-
-    mod peeked_scannable_tests {
-        use crate::test_utils::TestCustomError;
-
-        use super::*;
-
-        #[tokio::test]
-        async fn test_peek_returns_first_batch() {
-            let batch = record_batch!(("id", Int64, [1, 2, 3])).unwrap();
-            let mut peeked = PeekedScannable::new(Box::new(batch.clone()));
-
-            let first = peeked.peek().await.unwrap();
-            assert_eq!(first, batch);
-        }
-
-        #[tokio::test]
-        async fn test_peek_is_idempotent() {
-            let batch = record_batch!(("id", Int64, [1, 2, 3])).unwrap();
-            let mut peeked = PeekedScannable::new(Box::new(batch.clone()));
-
-            let first = peeked.peek().await.unwrap();
-            let second = peeked.peek().await.unwrap();
-            assert_eq!(first, second);
-        }
-
-        #[tokio::test]
-        async fn test_scan_after_peek_returns_all_data() {
-            let batches = vec![
-                record_batch!(("id", Int64, [1, 2])).unwrap(),
-                record_batch!(("id", Int64, [3, 4, 5])).unwrap(),
-            ];
-            let mut peeked = PeekedScannable::new(Box::new(batches.clone()));
-
-            let first = peeked.peek().await.unwrap();
-            assert_eq!(first, batches[0]);
-
-            let result: Vec<RecordBatch> = peeked.scan_as_stream().try_collect().await.unwrap();
-            assert_eq!(result.len(), 2);
-            assert_eq!(result[0], batches[0]);
-            assert_eq!(result[1], batches[1]);
-        }
-
-        #[tokio::test]
-        async fn test_scan_without_peek_passes_through() {
-            let batch = record_batch!(("id", Int64, [1, 2, 3])).unwrap();
-            let mut peeked = PeekedScannable::new(Box::new(batch.clone()));
-
-            let result: Vec<RecordBatch> = peeked.scan_as_stream().try_collect().await.unwrap();
-            assert_eq!(result.len(), 1);
-            assert_eq!(result[0], batch);
-        }
-
-        #[tokio::test]
-        async fn test_delegates_num_rows() {
-            let batches = vec![
-                record_batch!(("id", Int64, [1, 2])).unwrap(),
-                record_batch!(("id", Int64, [3])).unwrap(),
-            ];
-            let peeked = PeekedScannable::new(Box::new(batches));
-            assert_eq!(peeked.num_rows(), Some(3));
-        }
-
-        #[tokio::test]
-        async fn test_non_rescannable_stream_data_preserved() {
-            let batches = vec![
-                record_batch!(("id", Int64, [1, 2])).unwrap(),
-                record_batch!(("id", Int64, [3])).unwrap(),
-            ];
-            let schema = batches[0].schema();
-            let inner = futures::stream::iter(batches.clone().into_iter().map(Ok));
-            let stream: SendableRecordBatchStream = Box::pin(SimpleRecordBatchStream {
-                schema,
-                stream: inner,
-            });
-
-            let mut peeked = PeekedScannable::new(Box::new(stream));
-            assert!(!peeked.rescannable());
-            assert_eq!(peeked.num_rows(), None);
-
-            let first = peeked.peek().await.unwrap();
-            assert_eq!(first, batches[0]);
-
-            // All data is still available via scan_as_stream
-            let result: Vec<RecordBatch> = peeked.scan_as_stream().try_collect().await.unwrap();
-            assert_eq!(result.len(), 2);
-            assert_eq!(result[0], batches[0]);
-            assert_eq!(result[1], batches[1]);
-        }
-
-        #[tokio::test]
-        async fn test_error_in_first_batch_propagates() {
-            let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
-                "id",
-                arrow_schema::DataType::Int64,
-                false,
-            )]));
-            let inner = futures::stream::iter(vec![Err(Error::External {
-                source: Box::new(TestCustomError),
-            })]);
-            let stream: SendableRecordBatchStream = Box::pin(SimpleRecordBatchStream {
-                schema,
-                stream: inner,
-            });
-
-            let mut peeked = PeekedScannable::new(Box::new(stream));
-
-            // peek returns None for errors
-            assert!(peeked.peek().await.is_none());
-
-            // But the error should come through when scanning
-            let mut stream = peeked.scan_as_stream();
-            let first = stream.next().await.unwrap();
-            assert!(first.is_err());
-            let err = first.unwrap_err();
-            assert!(
-                matches!(&err, Error::External { source } if source.downcast_ref::<TestCustomError>().is_some()),
-                "Expected TestCustomError to be preserved, got: {err}"
-            );
-        }
-
-        #[tokio::test]
-        async fn test_error_in_later_batch_propagates() {
-            let good_batch = record_batch!(("id", Int64, [1, 2])).unwrap();
-            let schema = good_batch.schema();
-            let inner = futures::stream::iter(vec![
-                Ok(good_batch.clone()),
-                Err(Error::External {
-                    source: Box::new(TestCustomError),
-                }),
-            ]);
-            let stream: SendableRecordBatchStream = Box::pin(SimpleRecordBatchStream {
-                schema,
-                stream: inner,
-            });
-
-            let mut peeked = PeekedScannable::new(Box::new(stream));
-
-            // peek succeeds with the first batch
-            let first = peeked.peek().await.unwrap();
-            assert_eq!(first, good_batch);
-
-            // scan_as_stream should yield the first batch, then the error
-            let mut stream = peeked.scan_as_stream();
-            let batch1 = stream.next().await.unwrap().unwrap();
-            assert_eq!(batch1, good_batch);
-
-            let batch2 = stream.next().await.unwrap();
-            assert!(batch2.is_err());
-            let err = batch2.unwrap_err();
-            assert!(
-                matches!(&err, Error::External { source } if source.downcast_ref::<TestCustomError>().is_some()),
-                "Expected TestCustomError to be preserved, got: {err}"
-            );
-        }
-
-        #[tokio::test]
-        async fn test_empty_stream_returns_none() {
-            let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
-                "id",
-                arrow_schema::DataType::Int64,
-                false,
-            )]));
-            let inner = futures::stream::empty();
-            let stream: SendableRecordBatchStream = Box::pin(SimpleRecordBatchStream {
-                schema,
-                stream: inner,
-            });
-
-            let mut peeked = PeekedScannable::new(Box::new(stream));
-            assert!(peeked.peek().await.is_none());
-
-            // Scanning an empty (post-peek) stream should yield nothing
-            let result: Vec<RecordBatch> = peeked.scan_as_stream().try_collect().await.unwrap();
-            assert!(result.is_empty());
-        }
-    }
-
-    mod estimate_write_partitions_tests {
-        use super::*;
-
-        #[test]
-        fn test_small_data_single_partition() {
-            // 100 rows * 24 bytes/row = 2400 bytes — well under both thresholds
-            assert_eq!(estimate_write_partitions(2400, 100, Some(100), 8), 1);
-        }
-
-        #[test]
-        fn test_scales_by_row_count() {
-            // 2.5M rows at 24 bytes/row — row threshold dominates
-            // ceil(2_500_000 / 1_000_000) = 3
-            assert_eq!(estimate_write_partitions(72, 3, Some(2_500_000), 8), 3);
-        }
-
-        #[test]
-        fn test_scales_by_byte_size() {
-            // 100k rows at 40KB/row = ~4GB total → ceil(4GB / 2GB) = 2
-            let sample_bytes = 40_000 * 10;
-            assert_eq!(
-                estimate_write_partitions(sample_bytes, 10, Some(100_000), 8),
-                2
-            );
-        }
-
-        #[test]
-        fn test_capped_at_max_partitions() {
-            // 10M rows would want 10 partitions, but capped at 4
-            assert_eq!(estimate_write_partitions(72, 3, Some(10_000_000), 4), 4);
-        }
-
-        #[test]
-        fn test_zero_sample_rows_returns_one() {
-            assert_eq!(estimate_write_partitions(0, 0, Some(1_000_000), 8), 1);
-        }
-
-        #[test]
-        fn test_no_row_hint_uses_sample_size() {
-            // Without a hint, uses sample_rows (3), which is small
-            assert_eq!(estimate_write_partitions(72, 3, None, 8), 1);
-        }
-
-        #[test]
-        fn test_always_at_least_one() {
-            assert_eq!(estimate_write_partitions(24, 1, Some(1), 8), 1);
-        }
-    }
-
-    mod embedding_tests {
-        use super::*;
-        use crate::embeddings::MemoryRegistry;
-        use crate::table::{ColumnDefinition, ColumnKind};
-        use crate::test_utils::embeddings::MockEmbed;
-        use arrow_array::Array as _;
-        use arrow_array::{ArrayRef, StringArray};
-        use arrow_schema::{DataType, Field, Schema};
-
-        #[tokio::test]
-        async fn test_with_embeddings_scannable() {
-            let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)]));
-            let text_array = StringArray::from(vec!["hello", "world", "test"]);
-            let batch =
-                RecordBatch::try_new(schema.clone(), vec![Arc::new(text_array) as ArrayRef])
-                    .unwrap();
-
-            let mock_embedding: Arc<dyn EmbeddingFunction> = Arc::new(MockEmbed::new("mock", 4));
-            let embedding_def = EmbeddingDefinition::new("text", "mock", Some("text_embedding"));
-
-            let mut scannable = WithEmbeddingsScannable::try_new(
-                Box::new(batch.clone()),
-                vec![(embedding_def, mock_embedding)],
-            )
-            .unwrap();
-
-            // Check that schema has the embedding column
-            let output_schema = scannable.schema();
-            assert_eq!(output_schema.fields().len(), 2);
-            assert_eq!(output_schema.field(0).name(), "text");
-            assert_eq!(output_schema.field(1).name(), "text_embedding");
-
-            // Check num_rows and rescannable are preserved
-            assert_eq!(scannable.num_rows(), Some(3));
-            assert!(scannable.rescannable());
-
-            // Read the data
-            let stream = scannable.scan_as_stream();
-            let results: Vec<RecordBatch> = stream.try_collect().await.unwrap();
-            assert_eq!(results.len(), 1);
-
-            let result_batch = &results[0];
-            assert_eq!(result_batch.num_rows(), 3);
-            assert_eq!(result_batch.num_columns(), 2);
-
-            // Verify the embedding column is present and has the right shape
-            let embedding_col = result_batch.column(1);
-            assert_eq!(embedding_col.len(), 3);
-        }
-
-        #[tokio::test]
-        async fn test_maybe_embedded_scannable_no_embeddings() {
-            let batch = record_batch!(("id", Int64, [1, 2, 3])).unwrap();
-
-            // Create a table definition with no embedding columns
-            let table_def = TableDefinition::new_from_schema(batch.schema());
-
-            // Even with a registry, if there are no embedding columns, it's a passthrough
-            let registry: Arc<dyn EmbeddingRegistry> = Arc::new(MemoryRegistry::new());
-            let mut scannable =
-                scannable_with_embeddings(Box::new(batch.clone()), &table_def, Some(&registry))
-                    .unwrap();
-
-            // Check that data passes through unchanged
-            let stream = scannable.scan_as_stream();
-            let results: Vec<RecordBatch> = stream.try_collect().await.unwrap();
-            assert_eq!(results.len(), 1);
-            assert_eq!(results[0], batch);
-        }
-
-        #[tokio::test]
-        async fn test_maybe_embedded_scannable_with_embeddings() {
-            let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)]));
-            let text_array = StringArray::from(vec!["hello", "world"]);
-            let batch =
-                RecordBatch::try_new(schema.clone(), vec![Arc::new(text_array) as ArrayRef])
-                    .unwrap();
-
-            // Create a table definition with an embedding column
-            let embedding_def = EmbeddingDefinition::new("text", "mock", Some("text_embedding"));
-            let embedding_schema = Arc::new(Schema::new(vec![
-                Field::new("text", DataType::Utf8, false),
-                Field::new(
-                    "text_embedding",
-                    DataType::FixedSizeList(
-                        Arc::new(Field::new("item", DataType::Float32, true)),
-                        4,
-                    ),
-                    false,
-                ),
-            ]));
-            let table_def = TableDefinition::new(
-                embedding_schema,
-                vec![
-                    ColumnDefinition {
-                        kind: ColumnKind::Physical,
-                    },
-                    ColumnDefinition {
-                        kind: ColumnKind::Embedding(embedding_def.clone()),
-                    },
-                ],
-            );
-
-            // Register the mock embedding function
-            let registry: Arc<dyn EmbeddingRegistry> = Arc::new(MemoryRegistry::new());
-            let mock_embedding: Arc<dyn EmbeddingFunction> = Arc::new(MockEmbed::new("mock", 4));
-            registry.register("mock", mock_embedding).unwrap();
-
-            let mut scannable =
-                scannable_with_embeddings(Box::new(batch), &table_def, Some(&registry)).unwrap();
-
-            // Read and verify the data has embeddings
-            let stream = scannable.scan_as_stream();
-            let results: Vec<RecordBatch> = stream.try_collect().await.unwrap();
-            assert_eq!(results.len(), 1);
-
-            let result_batch = &results[0];
-            assert_eq!(result_batch.num_columns(), 2);
-            assert_eq!(result_batch.schema().field(1).name(), "text_embedding");
-        }
-
-        #[tokio::test]
-        async fn test_maybe_embedded_scannable_missing_function() {
-            let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)]));
-            let text_array = StringArray::from(vec!["hello"]);
-            let batch =
-                RecordBatch::try_new(schema.clone(), vec![Arc::new(text_array) as ArrayRef])
-                    .unwrap();
-
-            // Create a table definition with an embedding column
-            let embedding_def =
-                EmbeddingDefinition::new("text", "nonexistent", Some("text_embedding"));
-            let embedding_schema = Arc::new(Schema::new(vec![
-                Field::new("text", DataType::Utf8, false),
-                Field::new(
-                    "text_embedding",
-                    DataType::FixedSizeList(
-                        Arc::new(Field::new("item", DataType::Float32, true)),
-                        4,
-                    ),
-                    false,
-                ),
-            ]));
-            let table_def = TableDefinition::new(
-                embedding_schema,
-                vec![
-                    ColumnDefinition {
-                        kind: ColumnKind::Physical,
-                    },
-                    ColumnDefinition {
-                        kind: ColumnKind::Embedding(embedding_def),
-                    },
-                ],
-            );
-
-            // Registry has no embedding functions registered
-            let registry: Arc<dyn EmbeddingRegistry> = Arc::new(MemoryRegistry::new());
-
-            let result = scannable_with_embeddings(Box::new(batch), &table_def, Some(&registry));
-
-            // Should fail because the embedding function is not found
-            assert!(result.is_err());
-            let err = result.err().unwrap();
-            assert!(
-                matches!(err, Error::EmbeddingFunctionNotFound { .. }),
-                "Expected EmbeddingFunctionNotFound"
-            );
-        }
-    }
-}

rust/lancedb/src/database.rs

@@ -18,17 +18,22 @@ use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Duration;
 
+use arrow_array::RecordBatchReader;
+use async_trait::async_trait;
+use datafusion_physical_plan::stream::RecordBatchStreamAdapter;
+use futures::stream;
 use lance::dataset::ReadParams;
-use lance_namespace::LanceNamespace;
+use lance_datafusion::utils::StreamingWriteSource;
 use lance_namespace::models::{
     CreateNamespaceRequest, CreateNamespaceResponse, DescribeNamespaceRequest,
     DescribeNamespaceResponse, DropNamespaceRequest, DropNamespaceResponse, ListNamespacesRequest,
     ListNamespacesResponse, ListTablesRequest, ListTablesResponse,
 };
+use lance_namespace::LanceNamespace;
 
-use crate::data::scannable::Scannable;
+use crate::arrow::{SendableRecordBatchStream, SendableRecordBatchStreamExt};
 use crate::error::Result;
-use crate::table::{BaseTable, WriteOptions};
+use crate::table::{BaseTable, TableDefinition, WriteOptions};
 
 pub mod listing;
 pub mod namespace;
@@ -66,10 +71,6 @@ pub struct OpenTableRequest {
     /// Optional namespace client for server-side query execution.
     /// When set, queries will be executed on the namespace server instead of locally.
     pub namespace_client: Option<Arc<dyn LanceNamespace>>,
-    /// Whether managed versioning is enabled for this table.
-    /// When Some(true), the table will use namespace-managed commits instead of local commits.
-    /// When None and namespace_client is provided, the value will be fetched from the namespace.
-    pub managed_versioning: Option<bool>,
 }
 
 impl std::fmt::Debug for OpenTableRequest {
@@ -81,7 +82,6 @@ impl std::fmt::Debug for OpenTableRequest {
             .field("lance_read_params", &self.lance_read_params)
             .field("location", &self.location)
             .field("namespace_client", &self.namespace_client)
-            .field("managed_versioning", &self.managed_versioning)
             .finish()
     }
 }
@@ -90,10 +90,8 @@ pub type TableBuilderCallback = Box<dyn FnOnce(OpenTableRequest) -> OpenTableReq
 
 /// Describes what happens when creating a table and a table with
 /// the same name already exists
-#[derive(Default)]
 pub enum CreateTableMode {
     /// If the table already exists, an error is returned
-    #[default]
     Create,
     /// If the table already exists, it is opened.  Any provided data is
     /// ignored.  The function will be passed an OpenTableBuilder to customize
@@ -111,14 +109,57 @@ impl CreateTableMode {
     }
 }
 
+impl Default for CreateTableMode {
+    fn default() -> Self {
+        Self::Create
+    }
+}
+
+/// The data to start a table or a schema to create an empty table
+pub enum CreateTableData {
+    /// Creates a table using an iterator of data, the schema will be obtained from the data
+    Data(Box<dyn RecordBatchReader + Send>),
+    /// Creates a table using a stream of data, the schema will be obtained from the data
+    StreamingData(SendableRecordBatchStream),
+    /// Creates an empty table, the definition / schema must be provided separately
+    Empty(TableDefinition),
+}
+
+impl CreateTableData {
+    pub fn schema(&self) -> Arc<arrow_schema::Schema> {
+        match self {
+            Self::Data(reader) => reader.schema(),
+            Self::StreamingData(stream) => stream.schema(),
+            Self::Empty(definition) => definition.schema.clone(),
+        }
+    }
+}
+
+#[async_trait]
+impl StreamingWriteSource for CreateTableData {
+    fn arrow_schema(&self) -> Arc<arrow_schema::Schema> {
+        self.schema()
+    }
+    fn into_stream(self) -> datafusion_physical_plan::SendableRecordBatchStream {
+        match self {
+            Self::Data(reader) => reader.into_stream(),
+            Self::StreamingData(stream) => stream.into_df_stream(),
+            Self::Empty(table_definition) => {
+                let schema = table_definition.schema.clone();
+                Box::pin(RecordBatchStreamAdapter::new(schema, stream::empty()))
+            }
+        }
+    }
+}
+
 /// A request to create a table
 pub struct CreateTableRequest {
     /// The name of the new table
     pub name: String,
     /// The namespace to create the table in. Empty list represents root namespace.
     pub namespace: Vec<String>,
-    /// Initial data to write to the table, can be empty.
-    pub data: Box<dyn Scannable>,
+    /// Initial data to write to the table, can be None to create an empty table
+    pub data: CreateTableData,
     /// The mode to use when creating the table
     pub mode: CreateTableMode,
     /// Options to use when writing data (only used if `data` is not None)
@@ -132,7 +173,7 @@ pub struct CreateTableRequest {
 }
 
 impl CreateTableRequest {
-    pub fn new(name: String, data: Box<dyn Scannable>) -> Self {
+    pub fn new(name: String, data: CreateTableData) -> Self {
         Self {
             name,
             namespace: vec![],
@@ -166,9 +207,6 @@ pub struct CloneTableRequest {
     /// Whether to perform a shallow clone (true) or deep clone (false). Defaults to true.
     /// Currently only shallow clone is supported.
     pub is_shallow: bool,
-    /// Optional namespace client for managed versioning support.
-    /// When set, enables the commit handler to track table versions through the namespace.
-    pub namespace_client: Option<Arc<dyn LanceNamespace>>,
 }
 
 impl CloneTableRequest {
@@ -180,7 +218,6 @@ impl CloneTableRequest {
             source_version: None,
             source_tag: None,
             is_shallow: true,
-            namespace_client: None,
         }
     }
 }

rust/lancedb/src/database/listing.rs

@@ -8,7 +8,7 @@ use std::path::Path;
 use std::{collections::HashMap, sync::Arc};
 
 use lance::dataset::refs::Ref;
-use lance::dataset::{ReadParams, WriteMode, builder::DatasetBuilder};
+use lance::dataset::{builder::DatasetBuilder, ReadParams, WriteMode};
 use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore};
 use lance_datafusion::utils::StreamingWriteSource;
 use lance_encoding::version::LanceFileVersion;
@@ -669,7 +669,6 @@ impl ListingDatabase {
                     lance_read_params: None,
                     location: None,
                     namespace_client: None,
-                    managed_versioning: None,
                 };
                 let req = (callback)(req);
                 let table = self.open_table(req).await?;
@@ -870,7 +869,6 @@ impl Database for ListingDatabase {
             Some(write_params),
             self.read_consistency_interval,
             request.namespace_client,
-            false, // server_side_query_enabled - listing database doesn't support server-side queries
         )
         .await
         {
@@ -924,7 +922,7 @@ impl Database for ListingDatabase {
             .with_read_params(read_params.clone())
             .load()
             .await
-            .map_err(|e| -> Error { e.into() })?;
+            .map_err(|e| Error::Lance { source: e })?;
 
         let version_ref = match (request.source_version, request.source_tag) {
             (Some(v), None) => Ok(Ref::Version(None, Some(v))),
@@ -939,7 +937,7 @@ impl Database for ListingDatabase {
         source_dataset
             .shallow_clone(&target_uri, version_ref, Some(storage_params))
             .await
-            .map_err(|e| -> Error { e.into() })?;
+            .map_err(|e| Error::Lance { source: e })?;
 
         let cloned_table = NativeTable::open_with_params(
             &target_uri,
@@ -948,9 +946,7 @@ impl Database for ListingDatabase {
             self.store_wrapper.clone(),
             None,
             self.read_consistency_interval,
-            request.namespace_client,
-            false, // server_side_query_enabled - listing database doesn't support server-side queries
-            None,  // managed_versioning - will be queried if namespace_client is provided
+            None,
         )
         .await?;
 
@@ -1026,8 +1022,6 @@ impl Database for ListingDatabase {
                 Some(read_params),
                 self.read_consistency_interval,
                 request.namespace_client,
-                false, // server_side_query_enabled - listing database doesn't support server-side queries
-                request.managed_versioning, // Pass through managed_versioning from request
             )
             .await?,
         );
@@ -1103,11 +1097,9 @@ impl Database for ListingDatabase {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::Table;
     use crate::connection::ConnectRequest;
-    use crate::data::scannable::Scannable;
-    use crate::database::{CreateTableMode, CreateTableRequest};
-    use crate::table::WriteOptions;
+    use crate::database::{CreateTableData, CreateTableMode, CreateTableRequest, WriteOptions};
+    use crate::table::{Table, TableDefinition};
     use arrow_array::{Int32Array, RecordBatch, StringArray};
     use arrow_schema::{DataType, Field, Schema};
     use std::path::PathBuf;
@@ -1147,7 +1139,7 @@ mod tests {
             .create_table(CreateTableRequest {
                 name: "source_table".to_string(),
                 namespace: vec![],
-                data: Box::new(RecordBatch::new_empty(schema.clone())) as Box<dyn Scannable>,
+                data: CreateTableData::Empty(TableDefinition::new_from_schema(schema.clone())),
                 mode: CreateTableMode::Create,
                 write_options: Default::default(),
                 location: None,
@@ -1168,7 +1160,6 @@ mod tests {
                 source_version: None,
                 source_tag: None,
                 is_shallow: true,
-                namespace_client: None,
             })
             .await
             .unwrap();
@@ -1205,11 +1196,16 @@ mod tests {
         )
         .unwrap();
 
+        let reader = Box::new(arrow_array::RecordBatchIterator::new(
+            vec![Ok(batch)],
+            schema.clone(),
+        ));
+
         let source_table = db
             .create_table(CreateTableRequest {
                 name: "source_with_data".to_string(),
                 namespace: vec![],
-                data: Box::new(batch) as Box<dyn Scannable>,
+                data: CreateTableData::Data(reader),
                 mode: CreateTableMode::Create,
                 write_options: Default::default(),
                 location: None,
@@ -1229,7 +1225,6 @@ mod tests {
                 source_version: None,
                 source_tag: None,
                 is_shallow: true,
-                namespace_client: None,
             })
             .await
             .unwrap();
@@ -1269,7 +1264,7 @@ mod tests {
         db.create_table(CreateTableRequest {
             name: "source".to_string(),
             namespace: vec![],
-            data: Box::new(RecordBatch::new_empty(schema)) as Box<dyn Scannable>,
+            data: CreateTableData::Empty(TableDefinition::new_from_schema(schema)),
             mode: CreateTableMode::Create,
             write_options: Default::default(),
             location: None,
@@ -1289,7 +1284,6 @@ mod tests {
                 source_version: None,
                 source_tag: None,
                 is_shallow: true,
-                namespace_client: None,
             })
             .await;
 
@@ -1306,7 +1300,7 @@ mod tests {
         db.create_table(CreateTableRequest {
             name: "source".to_string(),
             namespace: vec![],
-            data: Box::new(RecordBatch::new_empty(schema)) as Box<dyn Scannable>,
+            data: CreateTableData::Empty(TableDefinition::new_from_schema(schema)),
             mode: CreateTableMode::Create,
             write_options: Default::default(),
             location: None,
@@ -1326,7 +1320,6 @@ mod tests {
                 source_version: None,
                 source_tag: None,
                 is_shallow: false, // Request deep clone
-                namespace_client: None,
             })
             .await;
 
@@ -1347,7 +1340,7 @@ mod tests {
         db.create_table(CreateTableRequest {
             name: "source".to_string(),
             namespace: vec![],
-            data: Box::new(RecordBatch::new_empty(schema)) as Box<dyn Scannable>,
+            data: CreateTableData::Empty(TableDefinition::new_from_schema(schema)),
             mode: CreateTableMode::Create,
             write_options: Default::default(),
             location: None,
@@ -1367,7 +1360,6 @@ mod tests {
                 source_version: None,
                 source_tag: None,
                 is_shallow: true,
-                namespace_client: None,
             })
             .await;
 
@@ -1388,7 +1380,7 @@ mod tests {
         db.create_table(CreateTableRequest {
             name: "source".to_string(),
             namespace: vec![],
-            data: Box::new(RecordBatch::new_empty(schema)) as Box<dyn Scannable>,
+            data: CreateTableData::Empty(TableDefinition::new_from_schema(schema)),
             mode: CreateTableMode::Create,
             write_options: Default::default(),
             location: None,
@@ -1408,7 +1400,6 @@ mod tests {
                 source_version: None,
                 source_tag: None,
                 is_shallow: true,
-                namespace_client: None,
             })
             .await;
 
@@ -1428,7 +1419,6 @@ mod tests {
                 source_version: None,
                 source_tag: None,
                 is_shallow: true,
-                namespace_client: None,
             })
             .await;
 
@@ -1445,7 +1435,7 @@ mod tests {
         db.create_table(CreateTableRequest {
             name: "source".to_string(),
             namespace: vec![],
-            data: Box::new(RecordBatch::new_empty(schema)) as Box<dyn Scannable>,
+            data: CreateTableData::Empty(TableDefinition::new_from_schema(schema)),
             mode: CreateTableMode::Create,
             write_options: Default::default(),
             location: None,
@@ -1465,7 +1455,6 @@ mod tests {
                 source_version: Some(1),
                 source_tag: Some("v1.0".to_string()),
                 is_shallow: true,
-                namespace_client: None,
             })
             .await;
 
@@ -1495,11 +1484,16 @@ mod tests {
         )
         .unwrap();
 
+        let reader = Box::new(arrow_array::RecordBatchIterator::new(
+            vec![Ok(batch1)],
+            schema.clone(),
+        ));
+
         let source_table = db
             .create_table(CreateTableRequest {
                 name: "versioned_source".to_string(),
                 namespace: vec![],
-                data: Box::new(batch1) as Box<dyn Scannable>,
+                data: CreateTableData::Data(reader),
                 mode: CreateTableMode::Create,
                 write_options: Default::default(),
                 location: None,
@@ -1523,7 +1517,14 @@ mod tests {
 
         let db = Arc::new(db);
         let source_table_obj = Table::new(source_table.clone(), db.clone());
-        source_table_obj.add(batch2).execute().await.unwrap();
+        source_table_obj
+            .add(Box::new(arrow_array::RecordBatchIterator::new(
+                vec![Ok(batch2)],
+                schema.clone(),
+            )))
+            .execute()
+            .await
+            .unwrap();
 
         // Verify source table now has 4 rows
         assert_eq!(source_table.count_rows(None).await.unwrap(), 4);
@@ -1539,7 +1540,6 @@ mod tests {
                 source_version: Some(initial_version),
                 source_tag: None,
                 is_shallow: true,
-                namespace_client: None,
             })
             .await
             .unwrap();
@@ -1570,11 +1570,16 @@ mod tests {
         )
         .unwrap();
 
+        let reader = Box::new(arrow_array::RecordBatchIterator::new(
+            vec![Ok(batch1)],
+            schema.clone(),
+        ));
+
         let source_table = db
             .create_table(CreateTableRequest {
                 name: "tagged_source".to_string(),
                 namespace: vec![],
-                data: Box::new(batch1),
+                data: CreateTableData::Data(reader),
                 mode: CreateTableMode::Create,
                 write_options: Default::default(),
                 location: None,
@@ -1602,7 +1607,14 @@ mod tests {
         .unwrap();
 
         let source_table_obj = Table::new(source_table.clone(), db.clone());
-        source_table_obj.add(batch2).execute().await.unwrap();
+        source_table_obj
+            .add(Box::new(arrow_array::RecordBatchIterator::new(
+                vec![Ok(batch2)],
+                schema.clone(),
+            )))
+            .execute()
+            .await
+            .unwrap();
 
         // Source table should have 4 rows
         assert_eq!(source_table.count_rows(None).await.unwrap(), 4);
@@ -1618,7 +1630,6 @@ mod tests {
                 source_version: None,
                 source_tag: Some("v1.0".to_string()),
                 is_shallow: true,
-                namespace_client: None,
             })
             .await
             .unwrap();
@@ -1646,11 +1657,16 @@ mod tests {
         )
         .unwrap();
 
+        let reader = Box::new(arrow_array::RecordBatchIterator::new(
+            vec![Ok(batch1)],
+            schema.clone(),
+        ));
+
         let source_table = db
             .create_table(CreateTableRequest {
                 name: "independent_source".to_string(),
                 namespace: vec![],
-                data: Box::new(batch1),
+                data: CreateTableData::Data(reader),
                 mode: CreateTableMode::Create,
                 write_options: Default::default(),
                 location: None,
@@ -1670,7 +1686,6 @@ mod tests {
                 source_version: None,
                 source_tag: None,
                 is_shallow: true,
-                namespace_client: None,
             })
             .await
             .unwrap();
@@ -1691,7 +1706,14 @@ mod tests {
 
         let db = Arc::new(db);
         let cloned_table_obj = Table::new(cloned_table.clone(), db.clone());
-        cloned_table_obj.add(batch_clone).execute().await.unwrap();
+        cloned_table_obj
+            .add(Box::new(arrow_array::RecordBatchIterator::new(
+                vec![Ok(batch_clone)],
+                schema.clone(),
+            )))
+            .execute()
+            .await
+            .unwrap();
 
         // Add different data to the source table
         let batch_source = RecordBatch::try_new(
@@ -1704,7 +1726,14 @@ mod tests {
         .unwrap();
 
         let source_table_obj = Table::new(source_table.clone(), db);
-        source_table_obj.add(batch_source).execute().await.unwrap();
+        source_table_obj
+            .add(Box::new(arrow_array::RecordBatchIterator::new(
+                vec![Ok(batch_source)],
+                schema.clone(),
+            )))
+            .execute()
+            .await
+            .unwrap();
 
         // Verify they have evolved independently
         assert_eq!(source_table.count_rows(None).await.unwrap(), 4); // 2 + 2
@@ -1722,11 +1751,16 @@ mod tests {
             RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1, 2]))])
                 .unwrap();
 
+        let reader = Box::new(arrow_array::RecordBatchIterator::new(
+            vec![Ok(batch1)],
+            schema.clone(),
+        ));
+
         let source_table = db
             .create_table(CreateTableRequest {
                 name: "latest_version_source".to_string(),
                 namespace: vec![],
-                data: Box::new(batch1),
+                data: CreateTableData::Data(reader),
                 mode: CreateTableMode::Create,
                 write_options: Default::default(),
                 location: None,
@@ -1745,7 +1779,14 @@ mod tests {
             .unwrap();
 
             let source_table_obj = Table::new(source_table.clone(), db.clone());
-            source_table_obj.add(batch).execute().await.unwrap();
+            source_table_obj
+                .add(Box::new(arrow_array::RecordBatchIterator::new(
+                    vec![Ok(batch)],
+                    schema.clone(),
+                )))
+                .execute()
+                .await
+                .unwrap();
         }
 
         // Source should have 8 rows total (2 + 2 + 2 + 2)
@@ -1763,7 +1804,6 @@ mod tests {
                 source_version: None,
                 source_tag: None,
                 is_shallow: true,
-                namespace_client: None,
             })
             .await
             .unwrap();
@@ -1809,11 +1849,16 @@ mod tests {
         )
         .unwrap();
 
+        let reader = Box::new(arrow_array::RecordBatchIterator::new(
+            vec![Ok(batch)],
+            schema.clone(),
+        ));
+
         let table = db
             .create_table(CreateTableRequest {
                 name: "test_stable".to_string(),
                 namespace: vec![],
-                data: Box::new(batch),
+                data: CreateTableData::Data(reader),
                 mode: CreateTableMode::Create,
                 write_options: Default::default(),
                 location: None,
@@ -1842,6 +1887,11 @@ mod tests {
         )
         .unwrap();
 
+        let reader = Box::new(arrow_array::RecordBatchIterator::new(
+            vec![Ok(batch)],
+            schema.clone(),
+        ));
+
         let mut storage_options = HashMap::new();
         storage_options.insert(
             OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
@@ -1864,7 +1914,7 @@ mod tests {
             .create_table(CreateTableRequest {
                 name: "test_stable_table_level".to_string(),
                 namespace: vec![],
-                data: Box::new(batch),
+                data: CreateTableData::Data(reader),
                 mode: CreateTableMode::Create,
                 write_options,
                 location: None,
@@ -1913,6 +1963,11 @@ mod tests {
         )
         .unwrap();
 
+        let reader = Box::new(arrow_array::RecordBatchIterator::new(
+            vec![Ok(batch)],
+            schema.clone(),
+        ));
+
         let mut storage_options = HashMap::new();
         storage_options.insert(
             OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
@@ -1935,7 +1990,7 @@ mod tests {
             .create_table(CreateTableRequest {
                 name: "test_override".to_string(),
                 namespace: vec![],
-                data: Box::new(batch),
+                data: CreateTableData::Data(reader),
                 mode: CreateTableMode::Create,
                 write_options,
                 location: None,
@@ -2053,7 +2108,7 @@ mod tests {
         db.create_table(CreateTableRequest {
             name: "table1".to_string(),
             namespace: vec![],
-            data: Box::new(RecordBatch::new_empty(schema.clone())) as Box<dyn Scannable>,
+            data: CreateTableData::Empty(TableDefinition::new_from_schema(schema.clone())),
             mode: CreateTableMode::Create,
             write_options: Default::default(),
             location: None,
@@ -2065,7 +2120,7 @@ mod tests {
         db.create_table(CreateTableRequest {
             name: "table2".to_string(),
             namespace: vec![],
-            data: Box::new(RecordBatch::new_empty(schema)) as Box<dyn Scannable>,
+            data: CreateTableData::Empty(TableDefinition::new_from_schema(schema)),
             mode: CreateTableMode::Create,
             write_options: Default::default(),
             location: None,

rust/lancedb/src/database/namespace.rs

@@ -7,20 +7,17 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use async_trait::async_trait;
-use lance::io::commit::namespace_manifest::LanceNamespaceExternalManifestStore;
-use lance_io::object_store::{ObjectStoreParams, StorageOptionsAccessor};
 use lance_namespace::{
-    LanceNamespace,
     models::{
-        CreateNamespaceRequest, CreateNamespaceResponse, DeclareTableRequest,
-        DescribeNamespaceRequest, DescribeNamespaceResponse, DescribeTableRequest,
-        DropNamespaceRequest, DropNamespaceResponse, DropTableRequest, ListNamespacesRequest,
-        ListNamespacesResponse, ListTablesRequest, ListTablesResponse,
+        CreateEmptyTableRequest, CreateNamespaceRequest, CreateNamespaceResponse,
+        DeclareTableRequest, DescribeNamespaceRequest, DescribeNamespaceResponse,
+        DescribeTableRequest, DropNamespaceRequest, DropNamespaceResponse, DropTableRequest,
+        ListNamespacesRequest, ListNamespacesResponse, ListTablesRequest, ListTablesResponse,
     },
+    LanceNamespace,
 };
 use lance_namespace_impls::ConnectBuilder;
-use lance_table::io::commit::CommitHandler;
-use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler;
+use log::warn;
 
 use crate::database::ReadConsistency;
 use crate::error::{Error, Result};
@@ -208,55 +205,53 @@ impl Database for LanceNamespaceDatabase {
         let mut table_id = request.namespace.clone();
         table_id.push(request.name.clone());
 
+        // Try declare_table first, falling back to create_empty_table for backwards
+        // compatibility with older namespace clients that don't support declare_table
         let declare_request = DeclareTableRequest {
             id: Some(table_id.clone()),
             ..Default::default()
         };
 
-        let response = self
-            .namespace
-            .declare_table(declare_request)
-            .await
-            .map_err(|e| Error::Runtime {
-                message: format!("Failed to declare table: {}", e),
-            })?;
+        let location = match self.namespace.declare_table(declare_request).await {
+            Ok(response) => response.location.ok_or_else(|| Error::Runtime {
+                message: "Table location is missing from declare_table response".to_string(),
+            })?,
+            Err(e) => {
+                // Check if the error is "not supported" and try create_empty_table as fallback
+                let err_str = e.to_string().to_lowercase();
+                if err_str.contains("not supported") || err_str.contains("not implemented") {
+                    warn!(
+                        "declare_table is not supported by the namespace client, \
+                        falling back to deprecated create_empty_table. \
+                        create_empty_table is deprecated and will be removed in Lance 3.0.0. \
+                        Please upgrade your namespace client to support declare_table."
+                    );
+                    #[allow(deprecated)]
+                    let create_empty_request = CreateEmptyTableRequest {
+                        id: Some(table_id.clone()),
+                        ..Default::default()
+                    };
 
-        let location = response.location.ok_or_else(|| Error::Runtime {
-            message: "Table location is missing from declare_table response".to_string(),
-        })?;
+                    #[allow(deprecated)]
+                    let create_response = self
+                        .namespace
+                        .create_empty_table(create_empty_request)
+                        .await
+                        .map_err(|e| Error::Runtime {
+                            message: format!("Failed to create empty table: {}", e),
+                        })?;
 
-        // Use storage options from response, fall back to self.storage_options
-        let initial_storage_options = response
-            .storage_options
-            .or_else(|| Some(self.storage_options.clone()))
-            .filter(|o| !o.is_empty());
-
-        let managed_versioning = response.managed_versioning;
-
-        // Build write params with storage options and commit handler
-        let mut params = request.write_options.lance_write_params.unwrap_or_default();
-
-        // Set up storage options if provided
-        if let Some(storage_opts) = initial_storage_options {
-            let store_params = params
-                .store_params
-                .get_or_insert_with(ObjectStoreParams::default);
-            store_params.storage_options_accessor = Some(Arc::new(
-                StorageOptionsAccessor::with_static_options(storage_opts),
-            ));
-        }
-
-        // Set up commit handler when managed_versioning is enabled
-        if managed_versioning == Some(true) {
-            let external_store =
-                LanceNamespaceExternalManifestStore::new(self.namespace.clone(), table_id.clone());
-            let commit_handler: Arc<dyn CommitHandler> = Arc::new(ExternalManifestCommitHandler {
-                external_manifest_store: Arc::new(external_store),
-            });
-            params.commit_handler = Some(commit_handler);
-        }
-
-        let write_params = Some(params);
+                    create_response.location.ok_or_else(|| Error::Runtime {
+                        message: "Table location is missing from create_empty_table response"
+                            .to_string(),
+                    })?
+                } else {
+                    return Err(Error::Runtime {
+                        message: format!("Failed to declare table: {}", e),
+                    });
+                }
+            }
+        };
 
         let native_table = NativeTable::create_from_namespace(
             self.namespace.clone(),
@@ -265,7 +260,7 @@ impl Database for LanceNamespaceDatabase {
             request.namespace.clone(),
             request.data,
             None, // write_store_wrapper not used for namespace connections
-            write_params,
+            request.write_options.lance_write_params,
             self.read_consistency_interval,
             self.server_side_query_enabled,
             self.session.clone(),
@@ -359,13 +354,15 @@ mod tests {
     use super::*;
     use crate::connect_namespace;
     use crate::query::ExecutableQuery;
-    use arrow_array::{Int32Array, RecordBatch, StringArray};
+    use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator, StringArray};
     use arrow_schema::{DataType, Field, Schema};
     use futures::TryStreamExt;
     use tempfile::tempdir;
 
     /// Helper function to create test data
-    fn create_test_data() -> RecordBatch {
+    fn create_test_data() -> RecordBatchIterator<
+        std::vec::IntoIter<std::result::Result<RecordBatch, arrow_schema::ArrowError>>,
+    > {
         let schema = Arc::new(Schema::new(vec![
             Field::new("id", DataType::Int32, false),
             Field::new("name", DataType::Utf8, false),
@@ -374,7 +371,12 @@ mod tests {
         let id_array = Int32Array::from(vec![1, 2, 3, 4, 5]);
         let name_array = StringArray::from(vec!["Alice", "Bob", "Charlie", "David", "Eve"]);
 
-        RecordBatch::try_new(schema, vec![Arc::new(id_array), Arc::new(name_array)]).unwrap()
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(id_array), Arc::new(name_array)],
+        )
+        .unwrap();
+        RecordBatchIterator::new(vec![std::result::Result::Ok(batch)].into_iter(), schema)
     }
 
     #[tokio::test]
@@ -616,7 +618,13 @@ mod tests {
 
         // Test: Overwrite the table
         let table2 = conn
-            .create_table("overwrite_test", test_data2)
+            .create_table(
+                "overwrite_test",
+                RecordBatchIterator::new(
+                    vec![std::result::Result::Ok(test_data2)].into_iter(),
+                    schema,
+                ),
+            )
             .namespace(vec!["test_ns".into()])
             .mode(CreateTableMode::Overwrite)
             .execute()

rust/lancedb/src/dataloader/permutation/builder.rs

@@ -11,16 +11,16 @@ use lance_core::ROW_ID;
 use lance_datafusion::exec::SessionContextExt;
 
 use crate::{
-    Error, Result, Table,
     arrow::{SendableRecordBatchStream, SendableRecordBatchStreamExt, SimpleRecordBatchStream},
     connect,
-    database::{CreateTableRequest, Database},
+    database::{CreateTableData, CreateTableRequest, Database},
     dataloader::permutation::{
         shuffle::{Shuffler, ShufflerConfig},
-        split::{SPLIT_ID_COLUMN, SplitStrategy, Splitter},
-        util::{TemporaryDirectory, rename_column},
+        split::{SplitStrategy, Splitter, SPLIT_ID_COLUMN},
+        util::{rename_column, TemporaryDirectory},
     },
     query::{ExecutableQuery, QueryBase, Select},
+    Error, Result, Table,
 };
 
 pub const SRC_ROW_ID_COL: &str = "row_id";
@@ -57,7 +57,7 @@ pub struct PermutationConfig {
 }
 
 /// Strategy for shuffling the data.
-#[derive(Debug, Clone, Default)]
+#[derive(Debug, Clone)]
 pub enum ShuffleStrategy {
     /// The data is randomly shuffled
     ///
@@ -78,10 +78,15 @@ pub enum ShuffleStrategy {
     /// The data is not shuffled
     ///
     /// This is useful for debugging and testing.
-    #[default]
     None,
 }
 
+impl Default for ShuffleStrategy {
+    fn default() -> Self {
+        Self::None
+    }
+}
+
 /// Builder for creating a permutation table.
 ///
 /// A permutation table is a table that stores split assignments and a shuffled order of rows.  This
@@ -308,8 +313,10 @@ impl PermutationBuilder {
             }
         };
 
-        let create_table_request =
-            CreateTableRequest::new(name.to_string(), Box::new(streaming_data));
+        let create_table_request = CreateTableRequest::new(
+            name.to_string(),
+            CreateTableData::StreamingData(streaming_data),
+        );
 
         let table = database.create_table(create_table_request).await?;
 
@@ -340,7 +347,7 @@ mod tests {
             .col("col_b", lance_datagen::array::step::<Int32Type>())
             .into_ldb_stream(RowCount::from(100), BatchCount::from(10));
         let data_table = db
-            .create_table("base_tbl", initial_data)
+            .create_table_streaming("base_tbl", initial_data)
             .execute()
             .await
             .unwrap();
@@ -380,7 +387,7 @@ mod tests {
             .col("some_value", lance_datagen::array::step::<Int32Type>())
             .into_ldb_stream(RowCount::from(100), BatchCount::from(10));
         let data_table = db
-            .create_table("mytbl", initial_data)
+            .create_table_streaming("mytbl", initial_data)
             .execute()
             .await
             .unwrap();

rust/lancedb/src/dataloader/permutation/reader.rs

@@ -25,8 +25,8 @@ use futures::{StreamExt, TryStreamExt};
 use lance::dataset::scanner::DatasetRecordBatchStream;
 use lance::io::RecordBatchStream;
 use lance_arrow::RecordBatchExt;
-use lance_core::ROW_ID;
 use lance_core::error::LanceOptionExt;
+use lance_core::ROW_ID;
 use std::collections::HashMap;
 use std::sync::Arc;
 
@@ -426,7 +426,6 @@ impl PermutationReader {
             row_ids_query = row_ids_query.limit(limit as usize);
         }
         let mut row_ids = row_ids_query.execute().await?;
-        let mut idx_offset = 0;
         while let Some(batch) = row_ids.try_next().await? {
             let row_ids = batch
                 .column(0)
@@ -434,9 +433,8 @@ impl PermutationReader {
                 .values()
                 .to_vec();
             for (i, row_id) in row_ids.iter().enumerate() {
-                offset_map.insert(i as u64 + idx_offset, *row_id);
+                offset_map.insert(i as u64, *row_id);
             }
-            idx_offset += batch.num_rows() as u64;
         }
         let offset_map = Arc::new(offset_map);
         *offset_map_ref = Some(offset_map.clone());
@@ -500,10 +498,10 @@ mod tests {
     use rand::seq::SliceRandom;
 
     use crate::{
-        Table,
         arrow::SendableRecordBatchStream,
         query::{ExecutableQuery, QueryBase},
-        test_utils::datagen::{LanceDbDatagenExt, virtual_table},
+        test_utils::datagen::{virtual_table, LanceDbDatagenExt},
+        Table,
     };
 
     use super::*;
@@ -847,106 +845,4 @@ mod tests {
             .to_vec();
         assert_eq!(idx_values, vec![row_ids[2] as i32]);
     }
-
-    #[tokio::test]
-    async fn test_filtered_permutation_full_iteration() {
-        use crate::dataloader::permutation::builder::PermutationBuilder;
-
-        // Create a base table with 10000 rows where idx goes 0..10000.
-        // Filter to even values only, giving 5000 rows in the permutation.
-        let base_table = lance_datagen::gen_batch()
-            .col("idx", lance_datagen::array::step::<Int32Type>())
-            .into_mem_table("tbl", RowCount::from(10000), BatchCount::from(1))
-            .await;
-
-        let permutation_table = PermutationBuilder::new(base_table.clone())
-            .with_filter("idx % 2 = 0".to_string())
-            .build()
-            .await
-            .unwrap();
-
-        assert_eq!(permutation_table.count_rows(None).await.unwrap(), 5000);
-
-        let reader = PermutationReader::try_from_tables(
-            base_table.base_table().clone(),
-            permutation_table.base_table().clone(),
-            0,
-        )
-        .await
-        .unwrap();
-
-        assert_eq!(reader.count_rows(), 5000);
-
-        // Iterate through all batches using a batch size that doesn't evenly divide
-        // the row count (5000 / 128 = 39 full batches + 1 batch of 8 rows).
-        let batch_size = 128;
-        let mut stream = reader
-            .read(
-                Select::All,
-                QueryExecutionOptions {
-                    max_batch_length: batch_size,
-                    ..Default::default()
-                },
-            )
-            .await
-            .unwrap();
-
-        let mut total_rows = 0u64;
-        let mut all_idx_values = Vec::new();
-        while let Some(batch) = stream.try_next().await.unwrap() {
-            assert!(batch.num_rows() <= batch_size as usize);
-            total_rows += batch.num_rows() as u64;
-            let idx_col = batch.column(0).as_primitive::<Int32Type>().values();
-            all_idx_values.extend(idx_col.iter().copied());
-        }
-
-        assert_eq!(total_rows, 5000);
-        assert_eq!(all_idx_values.len(), 5000);
-
-        // Every value should be even (from the filter)
-        assert!(all_idx_values.iter().all(|v| v % 2 == 0));
-
-        // Should have 5000 unique values
-        let unique: std::collections::HashSet<i32> = all_idx_values.iter().copied().collect();
-        assert_eq!(unique.len(), 5000);
-
-        // Use take_offsets to fetch rows from the beginning, middle, and end
-        // of the permutation. The values should match what we saw during iteration.
-
-        // Beginning
-        let batch = reader.take_offsets(&[0, 1, 2], Select::All).await.unwrap();
-        assert_eq!(batch.num_rows(), 3);
-        let idx_values = batch
-            .column(0)
-            .as_primitive::<Int32Type>()
-            .values()
-            .to_vec();
-        assert_eq!(idx_values, &all_idx_values[0..3]);
-
-        // Middle
-        let batch = reader
-            .take_offsets(&[2499, 2500, 2501], Select::All)
-            .await
-            .unwrap();
-        assert_eq!(batch.num_rows(), 3);
-        let idx_values = batch
-            .column(0)
-            .as_primitive::<Int32Type>()
-            .values()
-            .to_vec();
-        assert_eq!(idx_values, &all_idx_values[2499..2502]);
-
-        // End (last 3 rows)
-        let batch = reader
-            .take_offsets(&[4997, 4998, 4999], Select::All)
-            .await
-            .unwrap();
-        assert_eq!(batch.num_rows(), 3);
-        let idx_values = batch
-            .column(0)
-            .as_primitive::<Int32Type>()
-            .values()
-            .to_vec();
-        assert_eq!(idx_values, &all_idx_values[4997..5000]);
-    }
 }

rust/lancedb/src/dataloader/permutation/shuffle.rs

@@ -18,12 +18,12 @@ use lance_io::{
     scheduler::{ScanScheduler, SchedulerConfig},
     utils::CachedFileSize,
 };
-use rand::{Rng, RngCore, seq::SliceRandom};
+use rand::{seq::SliceRandom, Rng, RngCore};
 
 use crate::{
-    Error, Result,
     arrow::{SendableRecordBatchStream, SimpleRecordBatchStream},
-    dataloader::permutation::util::{TemporaryDirectory, non_crypto_rng},
+    dataloader::permutation::util::{non_crypto_rng, TemporaryDirectory},
+    Error, Result,
 };
 
 #[derive(Debug, Clone)]
@@ -281,7 +281,7 @@ mod tests {
     use datafusion_expr::col;
     use futures::TryStreamExt;
     use lance_datagen::{BatchCount, BatchGeneratorBuilder, ByteCount, RowCount, Seed};
-    use rand::{SeedableRng, rngs::SmallRng};
+    use rand::{rngs::SmallRng, SeedableRng};
 
     fn test_gen() -> BatchGeneratorBuilder {
         lance_datagen::gen_batch()

rust/lancedb/src/dataloader/permutation/split.rs

@@ -2,8 +2,8 @@
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 use std::sync::{
-    Arc,
     atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
+    Arc,
 };
 
 use arrow_array::{Array, BooleanArray, RecordBatch, UInt64Array};
@@ -15,22 +15,21 @@ use lance_arrow::SchemaExt;
 use lance_core::ROW_ID;
 
 use crate::{
-    Error, Result,
     arrow::{SendableRecordBatchStream, SimpleRecordBatchStream},
     dataloader::{
         permutation::shuffle::{Shuffler, ShufflerConfig},
         permutation::util::TemporaryDirectory,
     },
     query::{Query, QueryBase, Select},
+    Error, Result,
 };
 
 pub const SPLIT_ID_COLUMN: &str = "split_id";
 
 /// Strategy for assigning rows to splits
-#[derive(Debug, Clone, Default)]
+#[derive(Debug, Clone)]
 pub enum SplitStrategy {
     /// All rows will have split id 0
-    #[default]
     NoSplit,
     /// Rows will be randomly assigned to splits
     ///
@@ -74,6 +73,15 @@ pub enum SplitStrategy {
     Calculated { calculation: String },
 }
 
+// The default is not to split the data
+//
+// All data will be assigned to a single split.
+impl Default for SplitStrategy {
+    fn default() -> Self {
+        Self::NoSplit
+    }
+}
+
 impl SplitStrategy {
     pub fn validate(&self, num_rows: u64) -> Result<()> {
         match self {

rust/lancedb/src/dataloader/permutation/util.rs

@@ -7,12 +7,12 @@ use arrow_array::RecordBatch;
 use arrow_schema::{Fields, Schema};
 use datafusion_execution::disk_manager::DiskManagerMode;
 use futures::TryStreamExt;
-use rand::{RngCore, SeedableRng, rngs::SmallRng};
+use rand::{rngs::SmallRng, RngCore, SeedableRng};
 use tempfile::TempDir;
 
 use crate::{
-    Error, Result,
     arrow::{SendableRecordBatchStream, SimpleRecordBatchStream},
+    Error, Result,
 };
 
 /// Directory to use for temporary files

rust/lancedb/src/embeddings.rs

@@ -18,14 +18,14 @@ use std::{
 };
 
 use arrow_array::{Array, RecordBatch, RecordBatchReader};
-use arrow_schema::{DataType, Field, SchemaBuilder, SchemaRef};
+use arrow_schema::{DataType, Field, SchemaBuilder};
 // use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
 
 use crate::{
-    Error,
     error::Result,
     table::{ColumnDefinition, ColumnKind, TableDefinition},
+    Error,
 };
 
 /// Trait for embedding functions
@@ -190,112 +190,6 @@ impl<R: RecordBatchReader> WithEmbeddings<R> {
     }
 }
 
-/// Compute embedding arrays for a batch.
-///
-/// When multiple embedding functions are defined, they are computed in parallel using
-/// scoped threads. For a single embedding function, computation is done inline.
-fn compute_embedding_arrays(
-    batch: &RecordBatch,
-    embeddings: &[(EmbeddingDefinition, Arc<dyn EmbeddingFunction>)],
-) -> Result<Vec<Arc<dyn Array>>> {
-    if embeddings.len() == 1 {
-        let (fld, func) = &embeddings[0];
-        let src_column =
-            batch
-                .column_by_name(&fld.source_column)
-                .ok_or_else(|| Error::InvalidInput {
-                    message: format!("Source column '{}' not found", fld.source_column),
-                })?;
-        return Ok(vec![func.compute_source_embeddings(src_column.clone())?]);
-    }
-
-    // Parallel path: multiple embeddings
-    std::thread::scope(|s| {
-        let handles: Vec<_> = embeddings
-            .iter()
-            .map(|(fld, func)| {
-                let src_column = batch.column_by_name(&fld.source_column).ok_or_else(|| {
-                    Error::InvalidInput {
-                        message: format!("Source column '{}' not found", fld.source_column),
-                    }
-                })?;
-
-                let handle = s.spawn(move || func.compute_source_embeddings(src_column.clone()));
-
-                Ok(handle)
-            })
-            .collect::<Result<_>>()?;
-
-        handles
-            .into_iter()
-            .map(|h| {
-                h.join().map_err(|e| Error::Runtime {
-                    message: format!("Thread panicked during embedding computation: {:?}", e),
-                })?
-            })
-            .collect()
-    })
-}
-
-/// Compute the output schema when embeddings are applied to a base schema.
-///
-/// This returns the schema with embedding columns appended.
-pub fn compute_output_schema(
-    base_schema: &SchemaRef,
-    embeddings: &[(EmbeddingDefinition, Arc<dyn EmbeddingFunction>)],
-) -> Result<SchemaRef> {
-    let mut sb: SchemaBuilder = base_schema.as_ref().into();
-
-    for (ed, func) in embeddings {
-        let src_field = base_schema
-            .field_with_name(&ed.source_column)
-            .map_err(|_| Error::InvalidInput {
-                message: format!("Source column '{}' not found in schema", ed.source_column),
-            })?;
-
-        let field_name = ed
-            .dest_column
-            .clone()
-            .unwrap_or_else(|| format!("{}_embedding", &ed.source_column));
-
-        sb.push(Field::new(
-            field_name,
-            func.dest_type()?.into_owned(),
-            src_field.is_nullable(),
-        ));
-    }
-
-    Ok(Arc::new(sb.finish()))
-}
-
-/// Compute embeddings for a batch and append as new columns.
-///
-/// This function computes embeddings using the provided embedding functions and
-/// appends them as new columns to the batch.
-pub fn compute_embeddings_for_batch(
-    batch: RecordBatch,
-    embeddings: &[(EmbeddingDefinition, Arc<dyn EmbeddingFunction>)],
-) -> Result<RecordBatch> {
-    let embedding_arrays = compute_embedding_arrays(&batch, embeddings)?;
-
-    let mut result = batch;
-    for ((fld, _), embedding) in embeddings.iter().zip(embedding_arrays.iter()) {
-        let dst_field_name = fld
-            .dest_column
-            .clone()
-            .unwrap_or_else(|| format!("{}_embedding", &fld.source_column));
-
-        let dst_field = Field::new(
-            dst_field_name,
-            embedding.data_type().clone(),
-            embedding.nulls().is_some(),
-        );
-
-        result = result.try_with_column(dst_field, embedding.clone())?;
-    }
-    Ok(result)
-}
-
 impl<R: RecordBatchReader> WithEmbeddings<R> {
     fn dest_fields(&self) -> Result<Vec<Field>> {
         let schema = self.inner.schema();
@@ -346,6 +240,48 @@ impl<R: RecordBatchReader> WithEmbeddings<R> {
             column_definitions,
         })
     }
+
+    fn compute_embeddings_parallel(&self, batch: &RecordBatch) -> Result<Vec<Arc<dyn Array>>> {
+        if self.embeddings.len() == 1 {
+            let (fld, func) = &self.embeddings[0];
+            let src_column =
+                batch
+                    .column_by_name(&fld.source_column)
+                    .ok_or_else(|| Error::InvalidInput {
+                        message: format!("Source column '{}' not found", fld.source_column),
+                    })?;
+            return Ok(vec![func.compute_source_embeddings(src_column.clone())?]);
+        }
+
+        // Parallel path: multiple embeddings
+        std::thread::scope(|s| {
+            let handles: Vec<_> = self
+                .embeddings
+                .iter()
+                .map(|(fld, func)| {
+                    let src_column = batch.column_by_name(&fld.source_column).ok_or_else(|| {
+                        Error::InvalidInput {
+                            message: format!("Source column '{}' not found", fld.source_column),
+                        }
+                    })?;
+
+                    let handle =
+                        s.spawn(move || func.compute_source_embeddings(src_column.clone()));
+
+                    Ok(handle)
+                })
+                .collect::<Result<_>>()?;
+
+            handles
+                .into_iter()
+                .map(|h| {
+                    h.join().map_err(|e| Error::Runtime {
+                        message: format!("Thread panicked during embedding computation: {:?}", e),
+                    })?
+                })
+                .collect()
+        })
+    }
 }
 
 impl<R: RecordBatchReader> Iterator for MaybeEmbedded<R> {
@@ -373,13 +309,37 @@ impl<R: RecordBatchReader> Iterator for WithEmbeddings<R> {
     fn next(&mut self) -> Option<Self::Item> {
         let batch = self.inner.next()?;
         match batch {
-            Ok(batch) => match compute_embeddings_for_batch(batch, &self.embeddings) {
-                Ok(batch_with_embeddings) => Some(Ok(batch_with_embeddings)),
-                Err(e) => Some(Err(arrow_schema::ArrowError::ComputeError(format!(
-                    "Error computing embedding: {}",
-                    e
-                )))),
-            },
+            Ok(batch) => {
+                let embeddings = match self.compute_embeddings_parallel(&batch) {
+                    Ok(emb) => emb,
+                    Err(e) => {
+                        return Some(Err(arrow_schema::ArrowError::ComputeError(format!(
+                            "Error computing embedding: {}",
+                            e
+                        ))))
+                    }
+                };
+
+                let mut batch = batch;
+                for ((fld, _), embedding) in self.embeddings.iter().zip(embeddings.iter()) {
+                    let dst_field_name = fld
+                        .dest_column
+                        .clone()
+                        .unwrap_or_else(|| format!("{}_embedding", &fld.source_column));
+
+                    let dst_field = Field::new(
+                        dst_field_name,
+                        embedding.data_type().clone(),
+                        embedding.nulls().is_some(),
+                    );
+
+                    match batch.try_with_column(dst_field.clone(), embedding.clone()) {
+                        Ok(b) => batch = b,
+                        Err(e) => return Some(Err(e)),
+                    };
+                }
+                Some(Ok(batch))
+            }
             Err(e) => Some(Err(e)),
         }
     }

rust/lancedb/src/embeddings/bedrock.rs

@@ -8,7 +8,7 @@ use arrow::array::{AsArray, Float32Builder};
 use arrow_array::{Array, ArrayRef, FixedSizeListArray, Float32Array};
 use arrow_data::ArrayData;
 use arrow_schema::DataType;
-use serde_json::{Value, json};
+use serde_json::{json, Value};
 
 use super::EmbeddingFunction;
 use crate::{Error, Result};

rust/lancedb/src/embeddings/openai.rs

@@ -8,9 +8,9 @@ use arrow_array::{Array, ArrayRef, FixedSizeListArray, Float32Array};
 use arrow_data::ArrayData;
 use arrow_schema::DataType;
 use async_openai::{
-    Client,
     config::OpenAIConfig,
     types::{CreateEmbeddingRequest, Embedding, EmbeddingInput, EncodingFormat},
+    Client,
 };
 use tokio::{runtime::Handle, task};