chore: update lance dependency to v4.0.0-beta.12

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.28.0-beta.11"
+current_version = "0.27.0-beta.5"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

.github/ISSUE_TEMPLATE/documentation.yml

@@ -18,6 +18,6 @@ body:
       label: Link
       description: >
         Provide a link to the existing documentation, if applicable.
-      placeholder: ex. https://docs.lancedb.com/tables/...
+      placeholder: ex. https://lancedb.com/docs/tables/...
     validations:
       required: false

.github/dependabot.yml

@@ -1,18 +0,0 @@
-version: 2
-
-# Scope: the root Cargo workspace, which produces the Rust binaries we
-# ship to users (the Node.js and Python native extensions). The
-# `rust/lancedb` library crate shares the same lockfile; its consumers
-# pick their own dependency versions, but bumping transitive deps here
-# keeps the binaries we ship current.
-updates:
-  - package-ecosystem: cargo
-    directory: /
-    schedule:
-      interval: weekly
-    open-pull-requests-limit: 10
-    groups:
-      rust-minor-patch:
-        update-types:
-          - minor
-          - patch

.github/workflows/build_linux_wheel/action.yml

@@ -23,10 +23,8 @@ runs:
   steps:
     - name: CONFIRM ARM BUILD
       shell: bash
-      env:
-        ARM_BUILD: ${{ inputs.arm-build }}
       run: |
-        echo "ARM BUILD: $ARM_BUILD"
+        echo "ARM BUILD: ${{ inputs.arm-build }}"
     - name: Build x86_64 Manylinux wheel
       if: ${{ inputs.arm-build == 'false' }}
       uses: PyO3/maturin-action@v1

.github/workflows/build_windows_wheel/action.yml

@@ -29,3 +29,7 @@ runs:
         args: ${{ inputs.args }}
         docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
         working-directory: python
+    - uses: actions/upload-artifact@v4
+      with:
+        name: windows-wheels
+        path: python\target\wheels

.github/workflows/dev.yml

@@ -8,9 +8,6 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
-permissions:
-  contents: read
-
 jobs:
   labeler:
     permissions:

.github/workflows/java-publish.yml

@@ -19,9 +19,6 @@ on:
     paths:
       - .github/workflows/java-publish.yml
 
-permissions:
-  contents: read
-
 jobs:
   publish:
     name: Build and Publish
@@ -43,7 +40,7 @@ jobs:
           server-username: SONATYPE_USER
           server-password: SONATYPE_TOKEN
           gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
-          gpg-passphrase: MAVEN_GPG_PASSPHRASE
+          gpg-passphrase: ${{ secrets.GPG_PASSPHRASE }}
       - name: Set git config
         run: |
           git config --global user.email "dev+gha@lancedb.com"
@@ -58,11 +55,10 @@ jobs:
           echo "use-agent" >> ~/.gnupg/gpg.conf
           echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
           export GPG_TTY=$(tty)
-          ./mvnw --batch-mode -DskipTests -DpushChanges=false deploy -pl lancedb-core -am -P deploy-to-ossrh
+          ./mvnw --batch-mode -DskipTests -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -pl lancedb-core -am -P deploy-to-ossrh
         env:
           SONATYPE_USER: ${{ secrets.SONATYPE_USER }}
           SONATYPE_TOKEN: ${{ secrets.SONATYPE_TOKEN }}
-          MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
 
   report-failure:
     name: Report Workflow Failure

.github/workflows/java.yml

@@ -16,7 +16,6 @@ on:
   push:
     branches:
       - main
-      - release/**
     paths:
       - java/**
       - .github/workflows/java.yml
@@ -25,9 +24,6 @@ on:
       - java/**
       - .github/workflows/java.yml
 
-permissions:
-  contents: read
-
 jobs:
   build-java:
     runs-on: ubuntu-24.04

.github/workflows/license-header-check.yml

@@ -3,7 +3,6 @@ on:
   push:
     branches:
       - main
-      - release/**
   pull_request:
     paths:
       - rust/**
@@ -11,10 +10,6 @@ on:
       - nodejs/**
       - java/**
       - .github/workflows/license-header-check.yml
-
-permissions:
-  contents: read
-
 jobs:
   check-licenses:
     runs-on: ubuntu-latest

.github/workflows/nodejs.yml

@@ -4,21 +4,15 @@ on:
   push:
     branches:
       - main
-      - release/**
   pull_request:
     paths:
       - Cargo.toml
-      - Cargo.lock
-      - rust-toolchain.toml
       - nodejs/**
       - rust/**
       - docs/src/js/**
       - .github/workflows/nodejs.yml
       - docker-compose.yml
 
-permissions:
-  contents: read
-
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true

.github/workflows/npm-publish.yml

@@ -19,7 +19,6 @@ on:
     paths:
       - .github/workflows/npm-publish.yml
       - Cargo.toml # Change in dependency frequently breaks builds
-      - Cargo.lock
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -125,12 +124,7 @@ jobs:
             pre_build: |-
               set -e &&
               apt-get update &&
-              apt-get install -y protobuf-compiler pkg-config &&
+              apt-get install -y protobuf-compiler pkg-config
-              # The base image (manylinux2014-cross) sets TARGET_CC to the old
-              # GCC 4.8 cross-compiler. aws-lc-sys checks TARGET_CC before CC,
-              # so it picks up GCC even though the napi-rs image sets CC=clang.
-              # Override to use the image's clang-18 which supports -fuse-ld=lld.
-              export TARGET_CC=clang TARGET_CXX=clang++
           - target: x86_64-unknown-linux-musl
             # This one seems to need some extra memory
             host: ubuntu-2404-8x-x64
@@ -150,10 +144,9 @@ jobs:
               set -e &&
               apt-get update &&
               apt-get install -y protobuf-compiler pkg-config &&
-              export TARGET_CC=clang TARGET_CXX=clang++ &&
+              # https://github.com/aws/aws-lc-rs/issues/737#issuecomment-2725918627
-              # The manylinux2014 sysroot has glibc 2.17 headers which lack
+              ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc/aarch64-unknown-linux-gnu/4.8.5/crtbeginS.o /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/crtbeginS.o &&
-              # AT_HWCAP2 (added in Linux 3.17). Define it for aws-lc-sys.
+              ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/gcc &&
-              export CFLAGS="$CFLAGS -DAT_HWCAP2=26" &&
               rustup target add aarch64-unknown-linux-gnu
           - target: aarch64-unknown-linux-musl
             host: ubuntu-2404-8x-x64
@@ -273,7 +266,7 @@ jobs:
           - target: x86_64-unknown-linux-gnu
             host: ubuntu-latest
           - target: aarch64-unknown-linux-gnu
-            host: ubuntu-2404-8x-arm64
+            host: buildjet-16vcpu-ubuntu-2204-arm
         node:
           - '20'
     runs-on: ${{ matrix.settings.host }}

.github/workflows/pypi-publish.yml

@@ -8,18 +8,11 @@ on:
     # This should trigger a dry run (we skip the final publish step)
     paths:
       - .github/workflows/pypi-publish.yml
-      - .github/workflows/build_linux_wheel/action.yml
-      - .github/workflows/build_mac_wheel/action.yml
-      - .github/workflows/build_windows_wheel/action.yml
       - Cargo.toml # Change in dependency frequently breaks builds
-      - Cargo.lock
 
 env:
   PIP_EXTRA_INDEX_URL: "https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/"
 
-permissions:
-  contents: read
-
 jobs:
   linux:
     name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
@@ -46,7 +39,7 @@ jobs:
             runner: ubuntu-2404-8x-arm64
     runs-on: ${{ matrix.config.runner }}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           lfs: true
@@ -60,12 +53,11 @@ jobs:
           args: "--release --strip ${{ matrix.config.extra_args }}"
           arm-build: ${{ matrix.config.platform == 'aarch64' }}
           manylinux: ${{ matrix.config.manylinux }}
-      - uses: actions/upload-artifact@v7
+      - uses: ./.github/workflows/upload_wheel
         if: startsWith(github.ref, 'refs/tags/python-v')
         with:
-          name: wheels-linux-${{ matrix.config.platform }}-${{ matrix.config.manylinux }}
+          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
-          path: target/wheels/lancedb-*.whl
+          fury_token: ${{ secrets.FURY_TOKEN }}
-          if-no-files-found: error
   mac:
     timeout-minutes: 90
     runs-on: ${{ matrix.config.runner }}
@@ -77,7 +69,7 @@ jobs:
     env:
       MACOSX_DEPLOYMENT_TARGET: 10.15
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           lfs: true
@@ -89,21 +81,16 @@ jobs:
         with:
           python-minor-version: 10
           args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
-      - uses: actions/upload-artifact@v7
+      - uses: ./.github/workflows/upload_wheel
         if: startsWith(github.ref, 'refs/tags/python-v')
         with:
-          name: wheels-mac-${{ matrix.config.target }}
+          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
-          path: target/wheels/lancedb-*.whl
+          fury_token: ${{ secrets.FURY_TOKEN }}
-          if-no-files-found: error
   windows:
-    timeout-minutes: 90
+    timeout-minutes: 60
     runs-on: windows-latest
-    env:
-      # link.exe is single-threaded and the long pole on Windows builds. Use
-      # rustc's bundled lld-link instead.
-      CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           lfs: true
@@ -115,70 +102,19 @@ jobs:
         with:
           python-minor-version: 10
           args: "--release --strip"
-      - uses: actions/upload-artifact@v7
+          vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }}
+      - uses: ./.github/workflows/upload_wheel
         if: startsWith(github.ref, 'refs/tags/python-v')
         with:
-          name: wheels-windows
+          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
-          path: target/wheels/lancedb-*.whl
+          fury_token: ${{ secrets.FURY_TOKEN }}
-          if-no-files-found: error
-  publish:
-    name: Publish wheels
-    if: startsWith(github.ref, 'refs/tags/python-v')
-    needs: [linux, mac, windows]
-    runs-on: ubuntu-latest
-    permissions:
-      id-token: write
-      contents: read
-    steps:
-      - uses: actions/checkout@v6
-      - name: Download wheel artifacts
-        uses: actions/download-artifact@v8
-        with:
-          pattern: wheels-*
-          path: target/wheels
-          merge-multiple: true
-      - name: List wheels
-        run: ls -la target/wheels
-      - name: Choose repo
-        id: choose_repo
-        run: |
-          if [[ ${{ github.ref }} == *beta* ]]; then
-            echo "repo=fury" >> $GITHUB_OUTPUT
-          else
-            echo "repo=pypi" >> $GITHUB_OUTPUT
-          fi
-      - name: Publish to Fury
-        if: steps.choose_repo.outputs.repo == 'fury'
-        env:
-          FURY_TOKEN: ${{ secrets.FURY_TOKEN }}
-        run: |
-          shopt -s nullglob
-          WHEELS=(target/wheels/lancedb-*.whl)
-          if [[ ${#WHEELS[@]} -eq 0 ]]; then
-            echo "No wheels found in target/wheels/" >&2
-            exit 1
-          fi
-          for WHEEL in "${WHEELS[@]}"; do
-            echo "Uploading $WHEEL to Fury"
-            curl -f -F package=@"$WHEEL" "https://$FURY_TOKEN@push.fury.io/lancedb/"
-          done
-      # NOTE: pypa/gh-action-pypi-publish must be invoked directly from a
-      # workflow file, not from inside a composite action. When called from a
-      # composite, `github.action_repository` is empty (actions/runner#2473)
-      # and the action falls back to `github.repository`, producing a bogus
-      # `docker://ghcr.io/<repo>:<ref>` image reference that GHA tries to pull.
-      - name: Publish to PyPI
-        if: steps.choose_repo.outputs.repo == 'pypi'
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          packages-dir: target/wheels/
   gh-release:
     if: startsWith(github.ref, 'refs/tags/python-v')
     runs-on: ubuntu-latest
     permissions:
       contents: write
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           lfs: true
@@ -241,13 +177,13 @@ jobs:
   report-failure:
     name: Report Workflow Failure
     runs-on: ubuntu-latest
-    needs: [linux, mac, windows, publish]
+    needs: [linux, mac, windows]
     permissions:
       contents: read
       issues: write
     if: always() && failure() && startsWith(github.ref, 'refs/tags/python-v')
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/create-failure-issue
         with:
           job-results: ${{ toJSON(needs) }}

.github/workflows/python.yml

@@ -4,12 +4,9 @@ on:
   push:
     branches:
       - main
-      - release/**
   pull_request:
     paths:
       - Cargo.toml
-      - Cargo.lock
-      - rust-toolchain.toml
       - python/**
       - rust/**
       - .github/workflows/python.yml
@@ -18,9 +15,6 @@ on:
       - .github/workflows/build_windows_wheel/**
       - .github/workflows/run_tests/**
 
-permissions:
-  contents: read
-
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -112,6 +106,7 @@ jobs:
       - name: Install
         run: |
           pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings]
+          pip install tantivy
           pip install mlx
       - name: Doctest
         run: pytest --doctest-modules python/lancedb
@@ -230,5 +225,6 @@ jobs:
           pip install "pydantic<2"
           pip install pyarrow==16
           pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
+          pip install tantivy
       - name: Run tests
         run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests

.github/workflows/rust.yml

@@ -4,21 +4,12 @@ on:
   push:
     branches:
       - main
-      - release/**
   pull_request:
     paths:
       - Cargo.toml
-      - Cargo.lock
-      - rust-toolchain.toml
-      - deny.toml
       - rust/**
-      - nodejs/Cargo.toml
-      - python/Cargo.toml
       - .github/workflows/rust.yml
 
-permissions:
-  contents: read
-
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -60,17 +51,6 @@ jobs:
       - name: Run clippy (without remote feature)
         run: cargo clippy --profile ci --workspace --tests -- -D warnings
 
-  deny:
-    # Supply-chain checks: advisories, licenses, banned crates, and source
-    # restrictions. Configuration lives in `deny.toml` at the workspace root.
-    timeout-minutes: 10
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@v4
-      - uses: EmbarkStudios/cargo-deny-action@v2
-        with:
-          command: check advisories bans licenses sources
-
   build-no-lock:
     runs-on: ubuntu-24.04
     timeout-minutes: 30
@@ -226,14 +206,14 @@ jobs:
       - name: Downgrade  dependencies
         # These packages have newer requirements for MSRV
         run: |
-          cargo update -p aws-sdk-bedrockruntime --precise 1.77.0
+          cargo update -p aws-sdk-bedrockruntime --precise 1.64.0
-          cargo update -p aws-sdk-dynamodb --precise 1.68.0
+          cargo update -p aws-sdk-dynamodb --precise 1.55.0
-          cargo update -p aws-config --precise 1.6.0
+          cargo update -p aws-config --precise 1.5.10
-          cargo update -p aws-sdk-kms --precise 1.63.0
+          cargo update -p aws-sdk-kms --precise 1.51.0
-          cargo update -p aws-sdk-s3 --precise 1.79.0
+          cargo update -p aws-sdk-s3 --precise 1.65.0
-          cargo update -p aws-sdk-sso --precise 1.62.0
+          cargo update -p aws-sdk-sso --precise 1.50.0
-          cargo update -p aws-sdk-ssooidc --precise 1.63.0
+          cargo update -p aws-sdk-ssooidc --precise 1.51.0
-          cargo update -p aws-sdk-sts --precise 1.63.0
+          cargo update -p aws-sdk-sts --precise 1.51.0
           cargo update -p home --precise 0.5.9
       - name: cargo +${{ matrix.msrv }} check
         env:

.github/workflows/update_package_lock_run.yml

@@ -3,9 +3,6 @@ name: Update package-lock.json
 on:
   workflow_dispatch:
 
-permissions:
-  contents: read
-
 jobs:
   publish:
     runs-on: ubuntu-latest

.github/workflows/update_package_lock_run_nodejs.yml

@@ -3,9 +3,6 @@ name: Update NodeJs package-lock.json
 on:
   workflow_dispatch:
 
-permissions:
-  contents: read
-
 jobs:
   publish:
     runs-on: ubuntu-latest

.github/workflows/upload_wheel/action.yml

@@ -0,0 +1,45 @@
+name: upload-wheel
+
+description: "Upload wheels to Pypi"
+inputs:
+  pypi_token:
+    required: true
+    description: "release token for the repo"
+  fury_token:
+    required: true
+    description: "release token for the fury repo"
+
+runs:
+  using: "composite"
+  steps:
+  - name: Install dependencies
+    shell: bash
+    run: |
+      python -m pip install --upgrade pip
+      pip install twine
+      python3 -m pip install --upgrade pkginfo
+  - name: Choose repo
+    shell: bash
+    id: choose_repo
+    run: |
+      if [[ ${{ github.ref }} == *beta* ]]; then
+        echo "repo=fury" >> $GITHUB_OUTPUT
+      else
+        echo "repo=pypi" >> $GITHUB_OUTPUT
+      fi
+  - name: Publish to PyPI
+    shell: bash
+    env:
+      FURY_TOKEN: ${{ inputs.fury_token }}
+      PYPI_TOKEN: ${{ inputs.pypi_token }}
+    run: |
+      if [[ ${{ steps.choose_repo.outputs.repo }} == fury ]]; then
+        WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1)
+        echo "Uploading $WHEEL to Fury"
+        curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/
+      else
+        twine upload --repository ${{ steps.choose_repo.outputs.repo }} \
+          --username __token__ \
+          --password $PYPI_TOKEN \
+          target/wheels/lancedb-*.whl
+      fi

Cargo.toml

@@ -1,5 +1,7 @@
 [workspace]
 members = ["rust/lancedb", "nodejs", "python"]
+# Python package needs to be built by maturin.
+exclude = ["python"]
 resolver = "2"
 
 [workspace.package]
@@ -13,40 +15,40 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"
 
 [workspace.dependencies]
-lance = { "version" = "=6.0.0", default-features = false }
+lance = { "version" = "=4.0.0-beta.12", default-features = false, "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = "=6.0.0"
+lance-core = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = "=6.0.0"
+lance-datagen = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = "=6.0.0"
+lance-file = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=6.0.0", default-features = false }
+lance-io = { "version" = "=4.0.0-beta.12", default-features = false, "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = "=6.0.0"
+lance-index = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = "=6.0.0"
+lance-linalg = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = "=6.0.0"
+lance-namespace = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=6.0.0", default-features = false }
+lance-namespace-impls = { "version" = "=4.0.0-beta.12", default-features = false, "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = "=6.0.0"
+lance-table = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = "=6.0.0"
+lance-testing = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = "=6.0.0"
+lance-datafusion = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = "=6.0.0"
+lance-encoding = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = "=6.0.0"
+lance-arrow = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
-arrow = { version = "58.0.0", optional = false }
+arrow = { version = "57.2", optional = false }
-arrow-array = "58.0.0"
+arrow-array = "57.2"
-arrow-data = "58.0.0"
+arrow-data = "57.2"
-arrow-ipc = "58.0.0"
+arrow-ipc = "57.2"
-arrow-ord = "58.0.0"
+arrow-ord = "57.2"
-arrow-schema = "58.0.0"
+arrow-schema = "57.2"
-arrow-select = "58.0.0"
+arrow-select = "57.2"
-arrow-cast = "58.0.0"
+arrow-cast = "57.2"
 async-trait = "0"
-datafusion = { version = "53.0.0", default-features = false }
+datafusion = { version = "52.1", default-features = false }
-datafusion-catalog = "53.0.0"
+datafusion-catalog = "52.1"
-datafusion-common = { version = "53.0.0", default-features = false }
+datafusion-common = { version = "52.1", default-features = false }
-datafusion-execution = "53.0.0"
+datafusion-execution = "52.1"
-datafusion-expr = "53.0.0"
+datafusion-expr = "52.1"
-datafusion-functions = "53.0.0"
+datafusion-functions = "52.1"
-datafusion-physical-plan = "53.0.0"
+datafusion-physical-plan = "52.1"
-datafusion-physical-expr = "53.0.0"
+datafusion-physical-expr = "52.1"
-datafusion-sql = "53.0.0"
+datafusion-sql = "52.1"
 env_logger = "0.11"
 half = { "version" = "2.7.1", default-features = false, features = [
     "num-traits",

README.md

@@ -15,7 +15,7 @@
 
 # **The Multimodal AI Lakehouse**
 
-[**How to Install** ](#how-to-install) ✦ [**Detailed Documentation**](https://docs.lancedb.com) ✦ [**Tutorials and Recipes**](https://github.com/lancedb/vectordb-recipes/tree/main) ✦  [**Contributors**](#contributors) 
+[**How to Install** ](#how-to-install) ✦ [**Detailed Documentation**](https://lancedb.com/docs) ✦ [**Tutorials and Recipes**](https://github.com/lancedb/vectordb-recipes/tree/main) ✦  [**Contributors**](#contributors) 
 
 **The ultimate multimodal data platform for AI/ML applications.** 
 
@@ -57,7 +57,7 @@ LanceDB is a central location where developers can build, train and analyze thei
 
 ## **How to Install**:
 
-Follow the [Quickstart](https://docs.lancedb.com/quickstart) doc to set up LanceDB locally. 
+Follow the [Quickstart](https://lancedb.com/docs/quickstart/) doc to set up LanceDB locally. 
 
 **API & SDK:** We also support Python, Typescript and Rust SDKs
 

ci/check_lance_release.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import argparse
-import functools
 import json
 import os
 import re
@@ -27,7 +26,6 @@ SEMVER_RE = re.compile(
 )
 
 
-@functools.total_ordering
 @dataclass(frozen=True)
 class SemVer:
     major: int
@@ -158,9 +156,7 @@ def read_current_version(repo_root: Path) -> str:
 
 
 def determine_latest_tag(tags: Iterable[TagInfo]) -> TagInfo:
-    # Stable releases (no prerelease) are always preferred over pre-releases.
+    return max(tags, key=lambda tag: tag.semver)
-    # Within each group, standard semver ordering applies.
-    return max(tags, key=lambda tag: (not tag.semver.prerelease, tag.semver))
 
 
 def write_outputs(args: argparse.Namespace, payload: dict) -> None:

deny.toml

@@ -1,196 +0,0 @@
-# cargo-deny configuration for LanceDB.
-#
-# Run locally with `cargo deny check`. See
-# https://embarkstudios.github.io/cargo-deny/ for the full reference.
-
-# The set of target triples we care about. cargo-deny will only consider
-# dependencies that are used on at least one of these targets. Keeping this
-# explicit avoids noise from platform-specific crates (e.g. wasm, android,
-# ios) that we never actually ship.
-[graph]
-targets = [
-    "x86_64-unknown-linux-gnu",
-    "aarch64-unknown-linux-gnu",
-    "x86_64-apple-darwin",
-    "aarch64-apple-darwin",
-    "x86_64-pc-windows-msvc",
-    "aarch64-pc-windows-msvc",
-]
-all-features = true
-
-[output]
-feature-depth = 1
-
-# ---------------------------------------------------------------------------
-# Advisories: security vulnerabilities and yanked crates.
-# ---------------------------------------------------------------------------
-[advisories]
-version = 2
-# Fail the check if any crate in the lockfile has been yanked from crates.io.
-# Yanked crates are a signal the author retracted the release (often due to
-# bugs or security issues) and should not be depended on.
-yanked = "deny"
-# Advisory IDs we have explicitly reviewed and chosen to accept. Every
-# entry must include a rationale and, where possible, an upstream issue
-# pointing to a fix. Revisit this list whenever dependencies are updated.
-ignore = [
-    # rsa: Marvin Attack timing side-channel in PKCS#1 v1.5 decryption.
-    # Reached only through opendal → reqsign → rsa. We do not use RSA
-    # decryption in LanceDB ourselves; this is dormant in the signing path.
-    # No fixed release exists upstream as of this writing.
-    # https://rustsec.org/advisories/RUSTSEC-2023-0071
-    { id = "RUSTSEC-2023-0071", reason = "rsa crate via opendal/reqsign; no fixed upstream release" },
-
-    # instant: unmaintained. Pulled in via backoff → instant. Upstream
-    # recommends switching to `web-time`; fix has to come from backoff.
-    # https://rustsec.org/advisories/RUSTSEC-2024-0384
-    { id = "RUSTSEC-2024-0384", reason = "transitive via backoff; waiting on backoff replacement" },
-
-    # paste: unmaintained (author archived the repo). Used transitively by
-    # datafusion and the arrow ecosystem; widespread, no drop-in replacement.
-    # https://rustsec.org/advisories/RUSTSEC-2024-0436
-    { id = "RUSTSEC-2024-0436", reason = "transitive via datafusion; awaiting ecosystem migration" },
-
-    # encoding: unmaintained. Reached through lindera-dictionary, which is
-    # required by the native Lindera tokenizer path. Lindera has not migrated
-    # off this crate yet.
-    # https://rustsec.org/advisories/RUSTSEC-2021-0153
-    { id = "RUSTSEC-2021-0153", reason = "transitive via lindera-dictionary for native Lindera tokenizer" },
-
-    # fast-float: unsound and unmaintained. Reached only through polars-arrow
-    # from the optional Polars integration; replacement requires a Polars
-    # dependency upgrade.
-    # https://rustsec.org/advisories/RUSTSEC-2024-0379
-    { id = "RUSTSEC-2024-0379", reason = "transitive via polars-arrow; waiting on Polars migration" },
-
-    # tantivy: segfault on malformed input due to missing bounds check.
-    # Pulled in via lance for full-text search. We only feed tantivy
-    # documents we construct ourselves, not attacker-controlled bytes.
-    # Tracked for a lance dependency bump.
-    # https://rustsec.org/advisories/RUSTSEC-2025-0003
-    { id = "RUSTSEC-2025-0003", reason = "tantivy via lance; inputs are internally produced, not user-supplied bytes" },
-
-    # backoff: unmaintained. Reached only via async-openai. Replacement
-    # requires async-openai to migrate (or us to drop async-openai).
-    # https://rustsec.org/advisories/RUSTSEC-2025-0012
-    { id = "RUSTSEC-2025-0012", reason = "transitive via async-openai; waiting on upstream migration" },
-
-    # number_prefix: unmaintained. Transitive via indicatif → hf-hub.
-    # No security impact, just maintenance status.
-    # https://rustsec.org/advisories/RUSTSEC-2025-0119
-    { id = "RUSTSEC-2025-0119", reason = "transitive via hf-hub/indicatif; cosmetic formatting crate" },
-
-    # bincode: unmaintained. Reached through lindera and lindera-dictionary,
-    # which are required by the native Lindera tokenizer path. Lindera has not
-    # migrated to another serialization format yet.
-    # https://rustsec.org/advisories/RUSTSEC-2025-0141
-    { id = "RUSTSEC-2025-0141", reason = "transitive via lindera/lindera-dictionary for native Lindera tokenizer" },
-
-    # lru: soundness issue in IterMut. Reached only through aws-sdk-s3 in
-    # LanceDB's dev-dependency graph; LanceDB does not use that iterator
-    # directly. Clearing this requires the AWS SDK chain to update lru.
-    # https://rustsec.org/advisories/RUSTSEC-2026-0002
-    { id = "RUSTSEC-2026-0002", reason = "transitive via aws-sdk-s3 dev-dependency; waiting on AWS SDK lru upgrade" },
-
-    # rustls-webpki 0.101.7 (old major line): name-constraint checks for
-    # URI / wildcard names. Pulled in only via the legacy rustls 0.21 chain
-    # from aws-smithy-http-client. The 0.103 line we actively use is patched.
-    # Clearing the 0.101 copy requires the aws-sdk chain to migrate off
-    # rustls 0.21.
-    # https://rustsec.org/advisories/RUSTSEC-2026-0098
-    # https://rustsec.org/advisories/RUSTSEC-2026-0099
-    { id = "RUSTSEC-2026-0098", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
-    { id = "RUSTSEC-2026-0099", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
-
-    # rustls-webpki 0.101.7: reachable panic in CRL parsing. Same legacy
-    # rustls 0.21 chain from aws-smithy-http-client as above. The 0.103 line
-    # we actively use is upgraded to 0.103.13 which contains the fix.
-    # https://rustsec.org/advisories/RUSTSEC-2026-0104
-    { id = "RUSTSEC-2026-0104", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
-
-    # rand 0.8.5: soundness issue only when ThreadRng reseeds inside a custom
-    # logger. Reached through several transitive chains. LanceDB does not use
-    # rand from a custom logger; upgrade once all pinned chains accept 0.8.6+.
-    # https://rustsec.org/advisories/RUSTSEC-2026-0097
-    { id = "RUSTSEC-2026-0097", reason = "transitive rand 0.8.5; LanceDB does not call ThreadRng from custom logging" },
-]
-
-# ---------------------------------------------------------------------------
-# Licenses: only allow licenses we've reviewed as compatible with Apache-2.0.
-# ---------------------------------------------------------------------------
-[licenses]
-version = 2
-# SPDX identifiers for licenses that are compatible with our Apache-2.0
-# distribution. Additions require legal review.
-allow = [
-    "Apache-2.0",
-    "Apache-2.0 WITH LLVM-exception",
-    "MIT",
-    "BSD-2-Clause",
-    "BSD-3-Clause",
-    "ISC",
-    "Unicode-3.0",
-    "Unicode-DFS-2016",
-    "Zlib",
-    "CC0-1.0",
-    "MPL-2.0",
-    "BSL-1.0",
-    "OpenSSL",
-    # 0BSD ("BSD Zero Clause") is effectively public domain — no attribution
-    # required. Pulled in by `mock_instant`.
-    "0BSD",
-    # bzip2-1.0.6 is the permissive upstream bzip2 license (BSD-like). Pulled
-    # in by `libbz2-rs-sys`, the pure-Rust bzip2 implementation.
-    "bzip2-1.0.6",
-    # CDLA-Permissive-2.0 is a permissive data license used by `webpki-roots`
-    # for the Mozilla CA root bundle. Data-only, distribution-compatible.
-    "CDLA-Permissive-2.0",
-]
-confidence-threshold = 0.8
-# Crates whose license cannot be determined from Cargo metadata but whose
-# license we've manually confirmed from upstream. Keep this list minimal.
-[[licenses.clarify]]
-# polars-arrow-format omits the `license` field in its Cargo.toml, but the
-# upstream repo (pola-rs/polars-arrow-format) is dual-licensed Apache-2.0 OR
-# MIT. See https://github.com/pola-rs/polars-arrow-format/blob/main/LICENSE
-crate = "polars-arrow-format"
-expression = "Apache-2.0 OR MIT"
-license-files = []
-
-# ---------------------------------------------------------------------------
-# Bans: disallow specific crates and flag dependency hygiene issues.
-# ---------------------------------------------------------------------------
-[bans]
-# Warn (not deny) on duplicate versions of the same crate. In a large
-# workspace like this one, duplicates are common and often unavoidable
-# transitively. We surface them to discourage growth, but don't fail CI.
-multiple-versions = "warn"
-# Wildcard version requirements (`foo = "*"`) are a footgun — they let any
-# future release in without review. Ban them outright.
-wildcards = "deny"
-# Internal workspace crates reference each other via `path = "..."`, which
-# cargo-deny sees as a wildcard version. That's fine for private workspace
-# members (not published to crates.io), so allow it specifically for paths.
-allow-wildcard-paths = true
-# Features that, if enabled, should cause the check to fail.
-deny = []
-# Crates to skip when checking for duplicate versions.
-skip = []
-# Similar to `skip`, but also skips the entire transitive subtree.
-skip-tree = []
-
-# ---------------------------------------------------------------------------
-# Sources: restrict where crates can come from.
-# ---------------------------------------------------------------------------
-[sources]
-# Deny any registry other than the ones explicitly listed below.
-unknown-registry = "deny"
-# Deny any git dependency whose host isn't in the allow-list below. This
-# prevents accidental pulls from arbitrary forks.
-unknown-git = "deny"
-allow-registry = ["https://github.com/rust-lang/crates.io-index"]
-# Lance is developed in a sibling repo and pulled as a git dependency until
-# releases are cut to crates.io. Allow that specific host.
-allow-git = [
-    "https://github.com/lance-format/lance",
-]

docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3.9"
 services:
   localstack:
-    image: localstack/localstack:4.0
+    image: localstack/localstack:3.3
     ports:
       - 4566:4566
     environment:

dockerfiles/Dockerfile

@@ -24,4 +24,4 @@ RUN python --version && \
   rustc --version && \
   protoc --version
 
-RUN pip install --no-cache-dir lancedb
+RUN pip install --no-cache-dir tantivy lancedb

docs/README.md

@@ -1,6 +1,6 @@
 # LanceDB Documentation
 
-LanceDB docs are available at [docs.lancedb.com](https://docs.lancedb.com).
+LanceDB docs are available at [lancedb.com/docs](https://lancedb.com/docs).
 
 The SDK docs are built and deployed automatically by [Github Actions](../.github/workflows/docs.yml)
 whenever a commit is pushed to the `main` branch. So it is possible for the docs to show

docs/requirements.txt

@@ -1,8 +1,8 @@
-mkdocs==1.6.1
+mkdocs==1.5.3
 mkdocs-jupyter==0.24.1
-mkdocs-material==9.6.23
+mkdocs-material==9.5.3
 mkdocs-autorefs>=0.5,<=1.0
-mkdocstrings[python]>=0.24,<1.0
+mkdocstrings[python]==0.25.2
 griffe>=0.40,<1.0
 mkdocs-render-swagger-plugin>=0.1.0
 pydantic>=2.0,<3.0

docs/src/java/java.md

@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-core</artifactId>
-    <version>0.28.0-beta.11</version>
+    <version>0.27.0-beta.5</version>
 </dependency>
 ```
 
@@ -57,32 +57,32 @@ LanceNamespace namespaceClient = LanceDbNamespaceClientBuilder.newBuilder()
 
 ## Metadata Operations
 
-### Creating a Namespace Path
+### Creating a Namespace
 
-Namespace paths organize tables hierarchically. Create the desired namespace path before creating tables within it:
+Namespaces organize tables hierarchically. Create a namespace before creating tables within it:
 
 ```java
 import org.lance.namespace.model.CreateNamespaceRequest;
 import org.lance.namespace.model.CreateNamespaceResponse;
 
-// Create a child namespace path
+// Create a child namespace
 CreateNamespaceRequest request = new CreateNamespaceRequest();
 request.setId(Arrays.asList("my_namespace"));
 
 CreateNamespaceResponse response = namespaceClient.createNamespace(request);
 ```
 
-You can also create nested namespace paths:
+You can also create nested namespaces:
 
 ```java
-// Create a nested namespace path: parent/child
+// Create a nested namespace: parent/child
 CreateNamespaceRequest request = new CreateNamespaceRequest();
 request.setId(Arrays.asList("parent_namespace", "child_namespace"));
 
 CreateNamespaceResponse response = namespaceClient.createNamespace(request);
 ```
 
-### Describing a Namespace Path
+### Describing a Namespace
 
 ```java
 import org.lance.namespace.model.DescribeNamespaceRequest;
@@ -95,22 +95,22 @@ DescribeNamespaceResponse response = namespaceClient.describeNamespace(request);
 System.out.println("Namespace properties: " + response.getProperties());
 ```
 
-### Listing Namespace Paths
+### Listing Namespaces
 
 ```java
 import org.lance.namespace.model.ListNamespacesRequest;
 import org.lance.namespace.model.ListNamespacesResponse;
 
-// List all namespace paths at the root level
+// List all namespaces at root level
 ListNamespacesRequest request = new ListNamespacesRequest();
 request.setId(Arrays.asList());  // Empty for root
 
 ListNamespacesResponse response = namespaceClient.listNamespaces(request);
 for (String ns : response.getNamespaces()) {
-    System.out.println("Namespace path: " + ns);
+    System.out.println("Namespace: " + ns);
 }
 
-// List child namespace paths under a parent path
+// List child namespaces under a parent
 ListNamespacesRequest childRequest = new ListNamespacesRequest();
 childRequest.setId(Arrays.asList("parent_namespace"));
 
@@ -123,7 +123,7 @@ ListNamespacesResponse childResponse = namespaceClient.listNamespaces(childReque
 import org.lance.namespace.model.ListTablesRequest;
 import org.lance.namespace.model.ListTablesResponse;
 
-// List tables in a namespace path
+// List tables in a namespace
 ListTablesRequest request = new ListTablesRequest();
 request.setId(Arrays.asList("my_namespace"));
 
@@ -133,7 +133,7 @@ for (String table : response.getTables()) {
 }
 ```
 
-### Dropping a Namespace Path
+### Dropping a Namespace
 
 ```java
 import org.lance.namespace.model.DropNamespaceRequest;
@@ -175,7 +175,7 @@ DropTableResponse response = namespaceClient.dropTable(request);
 
 ### Creating a Table
 
-Tables are created within a namespace path by providing data in Apache Arrow IPC format:
+Tables are created within a namespace by providing data in Apache Arrow IPC format:
 
 ```java
 import org.lance.namespace.LanceNamespace;
@@ -242,7 +242,7 @@ try (BufferAllocator allocator = new RootAllocator();
     }
     byte[] tableData = out.toByteArray();
 
-    // Create a table in a namespace path
+    // Create table in a namespace
     CreateTableRequest request = new CreateTableRequest();
     request.setId(Arrays.asList("my_namespace", "my_table"));
     CreateTableResponse response = namespaceClient.createTable(request, tableData);

docs/src/js/README.md

@@ -34,7 +34,7 @@ const results = await table.vectorSearch([0.1, 0.3]).limit(20).toArray();
 console.log(results);
 ```
 
-The [quickstart](https://docs.lancedb.com/quickstart/) contains more complete examples.
+The [quickstart](https://lancedb.com/docs/quickstart/basic-usage/) contains more complete examples.
 
 ## Development
 

docs/src/js/classes/Connection.md

@@ -61,8 +61,8 @@ sharing the same data, deletion, and index files.
 * **options.sourceVersion?**: `number`
     The version of the source table to clone.
 
-* **options.targetNamespacePath?**: `string`[]
+* **options.targetNamespace?**: `string`[]
-    The namespace path for the target table (defaults to root namespace).
+    The namespace for the target table (defaults to root namespace).
 
 #### Returns
 
@@ -116,13 +116,13 @@ Creates a new empty Table
 
 `Promise`<[`Table`](Table.md)>
 
-#### createEmptyTable(name, schema, namespacePath, options)
+#### createEmptyTable(name, schema, namespace, options)
 
 ```ts
 abstract createEmptyTable(
    name,
    schema,
-   namespacePath?,
+   namespace?,
    options?): Promise<Table>
 ```
 
@@ -136,8 +136,8 @@ Creates a new empty Table
 * **schema**: [`SchemaLike`](../type-aliases/SchemaLike.md)
     The schema of the table
 
-* **namespacePath?**: `string`[]
+* **namespace?**: `string`[]
-    The namespace path to create the table in (defaults to root namespace)
+    The namespace to create the table in (defaults to root namespace)
 
 * **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
     Additional options
@@ -150,10 +150,10 @@ Creates a new empty Table
 
 ### createTable()
 
-#### createTable(options, namespacePath)
+#### createTable(options, namespace)
 
 ```ts
-abstract createTable(options, namespacePath?): Promise<Table>
+abstract createTable(options, namespace?): Promise<Table>
 ```
 
 Creates a new Table and initialize it with new data.
@@ -163,8 +163,8 @@ Creates a new Table and initialize it with new data.
 * **options**: `object` & `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
     The options object.
 
-* **namespacePath?**: `string`[]
+* **namespace?**: `string`[]
-    The namespace path to create the table in (defaults to root namespace)
+    The namespace to create the table in (defaults to root namespace)
 
 ##### Returns
 
@@ -197,13 +197,13 @@ Creates a new Table and initialize it with new data.
 
 `Promise`&lt;[`Table`](Table.md)&gt;
 
-#### createTable(name, data, namespacePath, options)
+#### createTable(name, data, namespace, options)
 
 ```ts
 abstract createTable(
    name,
    data,
-   namespacePath?,
+   namespace?,
    options?): Promise<Table>
 ```
 
@@ -218,8 +218,8 @@ Creates a new Table and initialize it with new data.
     Non-empty Array of Records
     to be inserted into the table
 
-* **namespacePath?**: `string`[]
+* **namespace?**: `string`[]
-    The namespace path to create the table in (defaults to root namespace)
+    The namespace to create the table in (defaults to root namespace)
 
 * **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
     Additional options
@@ -247,15 +247,15 @@ Return a brief description of the connection
 ### dropAllTables()
 
 ```ts
-abstract dropAllTables(namespacePath?): Promise<void>
+abstract dropAllTables(namespace?): Promise<void>
 ```
 
 Drop all tables in the database.
 
 #### Parameters
 
-* **namespacePath?**: `string`[]
+* **namespace?**: `string`[]
-    The namespace path to drop tables from (defaults to root namespace).
+    The namespace to drop tables from (defaults to root namespace).
 
 #### Returns
 
@@ -266,7 +266,7 @@ Drop all tables in the database.
 ### dropTable()
 
 ```ts
-abstract dropTable(name, namespacePath?): Promise<void>
+abstract dropTable(name, namespace?): Promise<void>
 ```
 
 Drop an existing table.
@@ -276,8 +276,8 @@ Drop an existing table.
 * **name**: `string`
     The name of the table to drop.
 
-* **namespacePath?**: `string`[]
+* **namespace?**: `string`[]
-    The namespace path of the table (defaults to root namespace).
+    The namespace of the table (defaults to root namespace).
 
 #### Returns
 
@@ -304,7 +304,7 @@ Return true if the connection has not been closed
 ```ts
 abstract openTable(
    name,
-   namespacePath?,
+   namespace?,
    options?): Promise<Table>
 ```
 
@@ -315,8 +315,8 @@ Open a table in the database.
 * **name**: `string`
     The name of the table
 
-* **namespacePath?**: `string`[]
+* **namespace?**: `string`[]
-    The namespace path of the table (defaults to root namespace)
+    The namespace of the table (defaults to root namespace)
 
 * **options?**: `Partial`&lt;[`OpenTableOptions`](../interfaces/OpenTableOptions.md)&gt;
     Additional options
@@ -349,10 +349,10 @@ Tables will be returned in lexicographical order.
 
 `Promise`&lt;`string`[]&gt;
 
-#### tableNames(namespacePath, options)
+#### tableNames(namespace, options)
 
 ```ts
-abstract tableNames(namespacePath?, options?): Promise<string[]>
+abstract tableNames(namespace?, options?): Promise<string[]>
 ```
 
 List all the table names in this database.
@@ -361,8 +361,8 @@ Tables will be returned in lexicographical order.
 
 ##### Parameters
 
-* **namespacePath?**: `string`[]
+* **namespace?**: `string`[]
-    The namespace path to list tables from (defaults to root namespace)
+    The namespace to list tables from (defaults to root namespace)
 
 * **options?**: `Partial`&lt;[`TableNamesOptions`](../interfaces/TableNamesOptions.md)&gt;
     options to control the

docs/src/js/classes/Table.md

@@ -485,7 +485,19 @@ Modeled after ``VACUUM`` in PostgreSQL.
  - Prune: Removes old versions of the dataset
  - Index: Optimizes the indices, adding new data to existing indices
 
- The frequency an application should call optimize is based on the frequency of
+ Experimental API
+ ----------------
+
+ The optimization process is undergoing active development and may change.
+ Our goal with these changes is to improve the performance of optimization and
+ reduce the complexity.
+
+ That being said, it is essential today to run optimize if you want the best
+ performance.  It should be stable and safe to use in production, but it our
+ hope that the API may be simplified (or not even need to be called) in the
+ future.
+
+ The frequency an application shoudl call optimize is based on the frequency of
  data modifications.  If data is frequently added, deleted, or updated then
  optimize should be run frequently.  A good rule of thumb is to run optimize if
  you have added or modified 100,000 or more records or run more than 20 data
@@ -501,34 +513,6 @@ Modeled after ``VACUUM`` in PostgreSQL.
 
 ***
 
-### prewarmData()
-
-```ts
-abstract prewarmData(columns?): Promise<void>
-```
-
-Prewarm one or more columns of data in the table.
-
-#### Parameters
-
-* **columns?**: `string`[]
-    The columns to prewarm. If undefined, all columns are prewarmed.
-    This will load the column data into the page cache so that future queries that
-    read those columns avoid the initial cold-start latency.  This call initiates
-    prewarming and returns once the request is accepted; the warming itself may
-    continue in the background.  Calling it on already-prewarmed columns is a
-    no-op on the server.
-    Prewarming is generally useful for columns used in filters or projections.
-    Large columns (e.g. high-dimensional vectors or binary data) may not be
-    practical to prewarm.
-    This feature is currently only supported on remote tables.
-
-#### Returns
-
-`Promise`&lt;`void`&gt;
-
-***
-
 ### prewarmIndex()
 
 ```ts

docs/src/js/interfaces/ClientConfig.md

@@ -53,18 +53,3 @@ optional tlsConfig: TlsConfig;
 ```ts
 optional userAgent: string;
 ```
-
-***
-
-### userId?
-
-```ts
-optional userId: string;
-```
-
-User identifier for tracking purposes.
-
-This is sent as the `x-lancedb-user-id` header in requests to LanceDB Cloud/Enterprise.
-It can be set directly, or via the `LANCEDB_USER_ID` environment variable.
-Alternatively, set `LANCEDB_USER_ID_ENV_KEY` to specify another environment
-variable that contains the user ID value.

docs/src/js/interfaces/ConnectionOptions.md

@@ -41,29 +41,6 @@ for testing purposes.
 
 ***
 
-### manifestEnabled?
-
-```ts
-optional manifestEnabled: boolean;
-```
-
-(For LanceDB OSS only): use directory namespace manifests as the source
-of truth for table metadata. Existing directory-listed root tables are
-migrated into the manifest on access.
-
-***
-
-### namespaceClientProperties?
-
-```ts
-optional namespaceClientProperties: Record<string, string>;
-```
-
-(For LanceDB OSS only): extra properties for the backing namespace
-client used by manifest-enabled native connections.
-
-***
-
 ### readConsistencyInterval?
 
 ```ts
@@ -112,4 +89,4 @@ optional storageOptions: Record<string, string>;
 
 (For LanceDB OSS only): configuration for object storage.
 
-The available options are described at https://docs.lancedb.com/storage/
+The available options are described at https://lancedb.com/docs/storage/

docs/src/js/interfaces/CreateTableOptions.md

@@ -97,4 +97,4 @@ Configuration for object storage.
 Options already set on the connection will be inherited by the table,
 but can be overridden here.
 
-The available options are described at https://docs.lancedb.com/storage/
+The available options are described at https://lancedb.com/docs/storage/

docs/src/js/interfaces/OpenTableOptions.md

@@ -42,4 +42,4 @@ Configuration for object storage.
 Options already set on the connection will be inherited by the table,
 but can be overridden here.
 
-The available options are described at https://docs.lancedb.com/storage/
+The available options are described at https://lancedb.com/docs/storage/

docs/src/js/interfaces/OptimizeOptions.md

@@ -37,12 +37,3 @@ tbl.optimize({cleanupOlderThan: new Date()});
 ```ts
 deleteUnverified: boolean;
 ```
-
-Because they may be part of an in-progress transaction, files newer than
-7 days old are not deleted by default. If you are sure that there are no
-in-progress transactions, then you can set this to true to delete all
-files older than `cleanupOlderThan`.
-
-**WARNING**: This should only be set to true if you can guarantee that
-no other process is currently working on this dataset. Otherwise the
-dataset could be put into a corrupted state.

docs/src/js/namespaces/embedding/classes/EmbeddingFunction.md

@@ -52,7 +52,7 @@ new EmbeddingFunction<T, M>(): EmbeddingFunction<T, M>
 ### computeQueryEmbeddings()
 
 ```ts
-computeQueryEmbeddings(data): Promise<number[] | Uint8Array | Float32Array | Float64Array>
+computeQueryEmbeddings(data): Promise<number[] | Float32Array | Float64Array>
 ```
 
 Compute the embeddings for a single query
@@ -63,7 +63,7 @@ Compute the embeddings for a single query
 
 #### Returns
 
-`Promise`&lt;`number`[] \| `Uint8Array` \| `Float32Array` \| `Float64Array`&gt;
+`Promise`&lt;`number`[] \| `Float32Array` \| `Float64Array`&gt;
 
 ***
 

docs/src/js/namespaces/embedding/classes/TextEmbeddingFunction.md

@@ -37,7 +37,7 @@ new TextEmbeddingFunction<M>(): TextEmbeddingFunction<M>
 ### computeQueryEmbeddings()
 
 ```ts
-computeQueryEmbeddings(data): Promise<number[] | Uint8Array | Float32Array | Float64Array>
+computeQueryEmbeddings(data): Promise<number[] | Float32Array | Float64Array>
 ```
 
 Compute the embeddings for a single query
@@ -48,7 +48,7 @@ Compute the embeddings for a single query
 
 #### Returns
 
-`Promise`&lt;`number`[] \| `Uint8Array` \| `Float32Array` \| `Float64Array`&gt;
+`Promise`&lt;`number`[] \| `Float32Array` \| `Float64Array`&gt;
 
 #### Overrides
 

docs/src/js/type-aliases/IntoVector.md

@@ -7,10 +7,5 @@
 # Type Alias: IntoVector
 
 ```ts
-type IntoVector:
+type IntoVector: Float32Array | Float64Array | number[] | Promise<Float32Array | Float64Array | number[]>;
-  | Float32Array
-  | Float64Array
-  | Uint8Array
-  | number[]
-  | Promise<Float32Array | Float64Array | Uint8Array | number[]>;
 ```

docs/src/python/python.md

@@ -36,20 +36,6 @@ is also an [asynchronous API client](#connections-asynchronous).
 
 ::: lancedb.table.Tags
 
-## Expressions
-
-Type-safe expression builder for filters and projections. Use these instead
-of raw SQL strings with [where][lancedb.query.LanceQueryBuilder.where] and
-[select][lancedb.query.LanceQueryBuilder.select].
-
-::: lancedb.expr.Expr
-
-::: lancedb.expr.col
-
-::: lancedb.expr.lit
-
-::: lancedb.expr.func
-
 ## Querying (Synchronous)
 
 ::: lancedb.query.Query
@@ -94,11 +80,11 @@ of raw SQL strings with [where][lancedb.query.LanceQueryBuilder.where] and
 
 ## Full text search
 
-Use [lancedb.table.Table.create_fts_index][] for the synchronous API or
+::: lancedb.fts.create_index
-[lancedb.table.AsyncTable.create_index][] with [lancedb.index.FTS][] for the
-asynchronous API.
 
-::: lancedb.index.FTS
+::: lancedb.fts.populate_index
+
+::: lancedb.fts.search_index
 
 ## Utilities
 

java/lancedb-core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
       <groupId>com.lancedb</groupId>
       <artifactId>lancedb-parent</artifactId>
-      <version>0.28.0-beta.11</version>
+      <version>0.27.0-beta.5</version>
       <relativePath>../pom.xml</relativePath>
     </parent>
 

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.28.0-beta.11</version>
+    <version>0.27.0-beta.5</version>
     <packaging>pom</packaging>
     <name>${project.artifactId}</name>
     <description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>6.0.0</lance-core.version>
+        <lance-core.version>4.0.0-beta.12</lance-core.version>
         <spotless.skip>false</spotless.skip>
         <spotless.version>2.30.0</spotless.version>
         <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>

nodejs/Cargo.toml

@@ -1,8 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.28.0-beta.11"
+version = "0.27.0-beta.5"
-publish = false
 license.workspace = true
 description.workspace = true
 repository.workspace = true
@@ -16,8 +15,6 @@ crate-type = ["cdylib"]
 async-trait.workspace = true
 arrow-ipc.workspace = true
 arrow-array.workspace = true
-arrow-buffer = "58.0.0"
-half.workspace = true
 arrow-schema.workspace = true
 env_logger.workspace = true
 futures.workspace = true
@@ -32,8 +29,8 @@ lzma-sys = { version = "0.1", features = ["static"] }
 log.workspace = true
 
 # Pin to resolve build failures; update periodically for security patches.
-aws-lc-sys = "=0.40.0"
+aws-lc-sys = "=0.38.0"
-aws-lc-rs = "=1.16.3"
+aws-lc-rs = "=1.16.1"
 
 [build-dependencies]
 napi-build = "2.3.1"

nodejs/README.md

@@ -30,7 +30,7 @@ const results = await table.vectorSearch([0.1, 0.3]).limit(20).toArray();
 console.log(results);
 ```
 
-The [quickstart](https://docs.lancedb.com/quickstart/) contains more complete examples.
+The [quickstart](https://lancedb.com/docs/quickstart/basic-usage/) contains more complete examples.
 
 ## Development
 

nodejs/__test__/rerankers.test.ts

@@ -1,8 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-import { spawn } from "node:child_process";
-import * as path from "node:path";
 import { RecordBatch } from "apache-arrow";
 import * as tmp from "tmp";
 import { Connection, Index, Table, connect, makeArrowTable } from "../lancedb";
@@ -78,91 +76,4 @@ describe("rerankers", function () {
 
     expect(result).toHaveLength(2);
   });
-
-  it("does not keep process alive after rerank query", async function () {
-    const script = `
-import * as lancedb from "./dist/index.js";
-import * as os from "node:os";
-import * as path from "node:path";
-import * as fs from "node:fs/promises";
-
-const dir = await fs.mkdtemp(path.join(os.tmpdir(), "lancedb-rerank-exit-"));
-const db = await lancedb.connect(dir);
-const table = await db.createTable("test", [{ text: "hello", vector: [1, 2, 3] }], {
-  mode: "overwrite",
-});
-await table.createIndex("text", { config: lancedb.Index.fts() });
-await table.waitForIndex(["text_idx"], 30);
-
-const reranker = await lancedb.rerankers.RRFReranker.create();
-await table
-  .query()
-  .nearestTo([1, 2, 3])
-  .fullTextSearch("hello")
-  .rerank(reranker)
-  .toArray();
-
-table.close();
-db.close();
-`;
-
-    await new Promise<void>((resolve, reject) => {
-      const child = spawn(
-        process.execPath,
-        ["--input-type=module", "-e", script],
-        {
-          cwd: path.resolve(__dirname, ".."),
-          stdio: ["ignore", "pipe", "pipe"],
-        },
-      );
-
-      let stdout = "";
-      let stderr = "";
-
-      child.stdout.on("data", (chunk) => {
-        stdout += chunk.toString();
-      });
-
-      child.stderr.on("data", (chunk) => {
-        stderr += chunk.toString();
-      });
-
-      const timeout = setTimeout(() => {
-        child.kill();
-        reject(
-          new Error(
-            `child process did not exit in time\nstdout:\n${stdout}\nstderr:\n${stderr}`,
-          ),
-        );
-      }, 20_000);
-
-      child.on("error", (err) => {
-        clearTimeout(timeout);
-        reject(err);
-      });
-
-      child.on("exit", (code, signal) => {
-        clearTimeout(timeout);
-        if (signal !== null) {
-          reject(
-            new Error(
-              `child process exited with signal ${signal}\nstdout:\n${stdout}\nstderr:\n${stderr}`,
-            ),
-          );
-          return;
-        }
-
-        if (code !== 0) {
-          reject(
-            new Error(
-              `child process exited with code ${code}\nstdout:\n${stdout}\nstderr:\n${stderr}`,
-            ),
-          );
-          return;
-        }
-
-        resolve();
-      });
-    });
-  });
 });

nodejs/__test__/table.test.ts

@@ -103,7 +103,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
         },
         numIndices: 0,
         numRows: 3,
-        totalBytes: 44,
+        totalBytes: 24,
       });
     });
 
@@ -1870,25 +1870,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       expect(results.length).toBe(3);
     });
 
-    test("prewarmData errors on local tables", async () => {
-      const db = await connect(tmpDir.name);
-      const data = [
-        { text: "alpha", vector: [0.1, 0.2, 0.3] },
-        { text: "beta", vector: [0.4, 0.5, 0.6] },
-      ];
-      const table = await db.createTable("prewarm_data_test", data);
-
-      // prewarmData is only supported on remote tables. We verify the call
-      // is wired through napi and surfaces the expected error for both
-      // arg shapes (undefined and string[]).
-      await expect(table.prewarmData()).rejects.toThrow(
-        "prewarm_data is currently only supported on remote tables",
-      );
-      await expect(table.prewarmData(["text"])).rejects.toThrow(
-        "prewarm_data is currently only supported on remote tables",
-      );
-    });
-
     test("full text index on list", async () => {
       const db = await connect(tmpDir.name);
       const data = [

nodejs/__test__/vector_types.test.ts

@@ -1,110 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-import * as tmp from "tmp";
-
-import { type Table, connect } from "../lancedb";
-import {
-  Field,
-  FixedSizeList,
-  Float32,
-  Int64,
-  Schema,
-  makeArrowTable,
-} from "../lancedb/arrow";
-
-describe("Vector query with different typed arrays", () => {
-  let tmpDir: tmp.DirResult;
-
-  afterEach(() => {
-    tmpDir?.removeCallback();
-  });
-
-  async function createFloat32Table(): Promise<Table> {
-    tmpDir = tmp.dirSync({ unsafeCleanup: true });
-    const db = await connect(tmpDir.name);
-    const schema = new Schema([
-      new Field("id", new Int64(), true),
-      new Field(
-        "vec",
-        new FixedSizeList(2, new Field("item", new Float32())),
-        true,
-      ),
-    ]);
-    const data = makeArrowTable(
-      [
-        { id: 1n, vec: [1.0, 0.0] },
-        { id: 2n, vec: [0.0, 1.0] },
-        { id: 3n, vec: [1.0, 1.0] },
-      ],
-      { schema },
-    );
-    return db.createTable("test_f32", data);
-  }
-
-  it("should search with Float32Array (baseline)", async () => {
-    const table = await createFloat32Table();
-    const results = await table
-      .query()
-      .nearestTo(new Float32Array([1.0, 0.0]))
-      .limit(1)
-      .toArray();
-
-    expect(results.length).toBe(1);
-    expect(Number(results[0].id)).toBe(1);
-  });
-
-  it("should search with number[] (backward compat)", async () => {
-    const table = await createFloat32Table();
-    const results = await table
-      .query()
-      .nearestTo([1.0, 0.0])
-      .limit(1)
-      .toArray();
-
-    expect(results.length).toBe(1);
-    expect(Number(results[0].id)).toBe(1);
-  });
-
-  it("should search with Float64Array via raw path", async () => {
-    const table = await createFloat32Table();
-    const results = await table
-      .query()
-      .nearestTo(new Float64Array([1.0, 0.0]))
-      .limit(1)
-      .toArray();
-
-    expect(results.length).toBe(1);
-    expect(Number(results[0].id)).toBe(1);
-  });
-
-  it("should add multiple query vectors with Float64Array", async () => {
-    const table = await createFloat32Table();
-    const results = await table
-      .query()
-      .nearestTo(new Float64Array([1.0, 0.0]))
-      .addQueryVector(new Float64Array([0.0, 1.0]))
-      .limit(2)
-      .toArray();
-
-    expect(results.length).toBeGreaterThanOrEqual(2);
-  });
-
-  // Float16Array is only available in Node 22+; not in TypeScript's standard lib yet
-  const float16ArrayCtor = (globalThis as unknown as Record<string, unknown>)
-    .Float16Array as (new (values: number[]) => unknown) | undefined;
-  const hasFloat16 = float16ArrayCtor !== undefined;
-  const f16it = hasFloat16 ? it : it.skip;
-
-  f16it("should search with Float16Array via raw path", async () => {
-    const table = await createFloat32Table();
-    const results = await table
-      .query()
-      .nearestTo(new float16ArrayCtor!([1.0, 0.0]) as Float32Array)
-      .limit(1)
-      .toArray();
-
-    expect(results.length).toBe(1);
-    expect(Number(results[0].id)).toBe(1);
-  });
-});

nodejs/lancedb/arrow.ts

@@ -117,9 +117,8 @@ export type TableLike =
 export type IntoVector =
   | Float32Array
   | Float64Array
-  | Uint8Array
   | number[]
-  | Promise<Float32Array | Float64Array | Uint8Array | number[]>;
+  | Promise<Float32Array | Float64Array | number[]>;
 
 export type MultiVector = IntoVector[];
 
@@ -127,48 +126,14 @@ export function isMultiVector(value: unknown): value is MultiVector {
   return Array.isArray(value) && isIntoVector(value[0]);
 }
 
-// Float16Array is not in TypeScript's standard lib yet; access dynamically
-type Float16ArrayCtor = new (
-  ...args: unknown[]
-) => { buffer: ArrayBuffer; byteOffset: number; byteLength: number };
-const float16ArrayCtor = (globalThis as unknown as Record<string, unknown>)
-  .Float16Array as Float16ArrayCtor | undefined;
-
 export function isIntoVector(value: unknown): value is IntoVector {
   return (
     value instanceof Float32Array ||
     value instanceof Float64Array ||
-    value instanceof Uint8Array ||
-    (float16ArrayCtor !== undefined && value instanceof float16ArrayCtor) ||
     (Array.isArray(value) && !Array.isArray(value[0]))
   );
 }
 
-/**
- * Extract the underlying byte buffer and data type from a typed array
- * for passing to the Rust NAPI layer without precision loss.
- */
-export function extractVectorBuffer(
-  vector: Float32Array | Float64Array | Uint8Array,
-): { data: Uint8Array; dtype: string } | null {
-  if (float16ArrayCtor !== undefined && vector instanceof float16ArrayCtor) {
-    return {
-      data: new Uint8Array(vector.buffer, vector.byteOffset, vector.byteLength),
-      dtype: "float16",
-    };
-  }
-  if (vector instanceof Float64Array) {
-    return {
-      data: new Uint8Array(vector.buffer, vector.byteOffset, vector.byteLength),
-      dtype: "float64",
-    };
-  }
-  if (vector instanceof Uint8Array && !(vector instanceof Float32Array)) {
-    return { data: vector, dtype: "uint8" };
-  }
-  return null;
-}
-
 export function isArrowTable(value: object): value is TableLike {
   if (value instanceof ArrowTable) return true;
   return "schema" in value && "batches" in value;

nodejs/lancedb/connection.ts

@@ -42,7 +42,7 @@ export interface CreateTableOptions {
    * Options already set on the connection will be inherited by the table,
    * but can be overridden here.
    *
-   * The available options are described at https://docs.lancedb.com/storage/
+   * The available options are described at https://lancedb.com/docs/storage/
    */
   storageOptions?: Record<string, string>;
 
@@ -78,7 +78,7 @@ export interface OpenTableOptions {
    * Options already set on the connection will be inherited by the table,
    * but can be overridden here.
    *
-   * The available options are described at https://docs.lancedb.com/storage/
+   * The available options are described at https://lancedb.com/docs/storage/
    */
   storageOptions?: Record<string, string>;
   /**
@@ -166,25 +166,25 @@ export abstract class Connection {
    * List all the table names in this database.
    *
    * Tables will be returned in lexicographical order.
-   * @param {string[]} namespacePath - The namespace path to list tables from (defaults to root namespace)
+   * @param {string[]} namespace - The namespace to list tables from (defaults to root namespace)
    * @param {Partial<TableNamesOptions>} options - options to control the
    * paging / start point
    *
    */
   abstract tableNames(
-    namespacePath?: string[],
+    namespace?: string[],
     options?: Partial<TableNamesOptions>,
   ): Promise<string[]>;
 
   /**
    * Open a table in the database.
    * @param {string} name - The name of the table
-   * @param {string[]} namespacePath - The namespace path of the table (defaults to root namespace)
+   * @param {string[]} namespace - The namespace of the table (defaults to root namespace)
    * @param {Partial<OpenTableOptions>} options - Additional options
    */
   abstract openTable(
     name: string,
-    namespacePath?: string[],
+    namespace?: string[],
     options?: Partial<OpenTableOptions>,
   ): Promise<Table>;
 
@@ -193,7 +193,7 @@ export abstract class Connection {
    * @param {object} options - The options object.
    * @param {string} options.name - The name of the table.
    * @param {Data} options.data - Non-empty Array of Records to be inserted into the table
-   * @param {string[]} namespacePath - The namespace path to create the table in (defaults to root namespace)
+   * @param {string[]} namespace - The namespace to create the table in (defaults to root namespace)
    *
    */
   abstract createTable(
@@ -201,7 +201,7 @@ export abstract class Connection {
       name: string;
       data: Data;
     } & Partial<CreateTableOptions>,
-    namespacePath?: string[],
+    namespace?: string[],
   ): Promise<Table>;
   /**
    * Creates a new Table and initialize it with new data.
@@ -220,13 +220,13 @@ export abstract class Connection {
    * @param {string} name - The name of the table.
    * @param {Record<string, unknown>[] | TableLike} data - Non-empty Array of Records
    * to be inserted into the table
-   * @param {string[]} namespacePath - The namespace path to create the table in (defaults to root namespace)
+   * @param {string[]} namespace - The namespace to create the table in (defaults to root namespace)
    * @param {Partial<CreateTableOptions>} options - Additional options
    */
   abstract createTable(
     name: string,
     data: Record<string, unknown>[] | TableLike,
-    namespacePath?: string[],
+    namespace?: string[],
     options?: Partial<CreateTableOptions>,
   ): Promise<Table>;
 
@@ -245,28 +245,28 @@ export abstract class Connection {
    * Creates a new empty Table
    * @param {string} name - The name of the table.
    * @param {Schema} schema - The schema of the table
-   * @param {string[]} namespacePath - The namespace path to create the table in (defaults to root namespace)
+   * @param {string[]} namespace - The namespace to create the table in (defaults to root namespace)
    * @param {Partial<CreateTableOptions>} options - Additional options
    */
   abstract createEmptyTable(
     name: string,
     schema: import("./arrow").SchemaLike,
-    namespacePath?: string[],
+    namespace?: string[],
     options?: Partial<CreateTableOptions>,
   ): Promise<Table>;
 
   /**
    * Drop an existing table.
    * @param {string} name The name of the table to drop.
-   * @param {string[]} namespacePath The namespace path of the table (defaults to root namespace).
+   * @param {string[]} namespace The namespace of the table (defaults to root namespace).
    */
-  abstract dropTable(name: string, namespacePath?: string[]): Promise<void>;
+  abstract dropTable(name: string, namespace?: string[]): Promise<void>;
 
   /**
    * Drop all tables in the database.
-   * @param {string[]} namespacePath The namespace path to drop tables from (defaults to root namespace).
+   * @param {string[]} namespace The namespace to drop tables from (defaults to root namespace).
    */
-  abstract dropAllTables(namespacePath?: string[]): Promise<void>;
+  abstract dropAllTables(namespace?: string[]): Promise<void>;
 
   /**
    * Clone a table from a source table.
@@ -279,7 +279,7 @@ export abstract class Connection {
    * @param {string} targetTableName - The name of the target table to create.
    * @param {string} sourceUri - The URI of the source table to clone from.
    * @param {object} options - Clone options.
-   * @param {string[]} options.targetNamespacePath - The namespace path for the target table (defaults to root namespace).
+   * @param {string[]} options.targetNamespace - The namespace for the target table (defaults to root namespace).
    * @param {number} options.sourceVersion - The version of the source table to clone.
    * @param {string} options.sourceTag - The tag of the source table to clone.
    * @param {boolean} options.isShallow - Whether to perform a shallow clone (defaults to true).
@@ -288,7 +288,7 @@ export abstract class Connection {
     targetTableName: string,
     sourceUri: string,
     options?: {
-      targetNamespacePath?: string[];
+      targetNamespace?: string[];
       sourceVersion?: number;
       sourceTag?: string;
       isShallow?: boolean;
@@ -319,25 +319,25 @@ export class LocalConnection extends Connection {
   }
 
   async tableNames(
-    namespacePathOrOptions?: string[] | Partial<TableNamesOptions>,
+    namespaceOrOptions?: string[] | Partial<TableNamesOptions>,
     options?: Partial<TableNamesOptions>,
   ): Promise<string[]> {
-    // Detect if first argument is namespacePath array or options object
+    // Detect if first argument is namespace array or options object
-    let namespacePath: string[] | undefined;
+    let namespace: string[] | undefined;
     let tableNamesOptions: Partial<TableNamesOptions> | undefined;
 
-    if (Array.isArray(namespacePathOrOptions)) {
+    if (Array.isArray(namespaceOrOptions)) {
-      // First argument is namespacePath array
+      // First argument is namespace array
-      namespacePath = namespacePathOrOptions;
+      namespace = namespaceOrOptions;
       tableNamesOptions = options;
     } else {
       // First argument is options object (backwards compatibility)
-      namespacePath = undefined;
+      namespace = undefined;
-      tableNamesOptions = namespacePathOrOptions;
+      tableNamesOptions = namespaceOrOptions;
     }
 
     return this.inner.tableNames(
-      namespacePath ?? [],
+      namespace ?? [],
       tableNamesOptions?.startAfter,
       tableNamesOptions?.limit,
     );
@@ -345,12 +345,12 @@ export class LocalConnection extends Connection {
 
   async openTable(
     name: string,
-    namespacePath?: string[],
+    namespace?: string[],
     options?: Partial<OpenTableOptions>,
   ): Promise<Table> {
     const innerTable = await this.inner.openTable(
       name,
-      namespacePath ?? [],
+      namespace ?? [],
       cleanseStorageOptions(options?.storageOptions),
       options?.indexCacheSize,
     );
@@ -362,7 +362,7 @@ export class LocalConnection extends Connection {
     targetTableName: string,
     sourceUri: string,
     options?: {
-      targetNamespacePath?: string[];
+      targetNamespace?: string[];
       sourceVersion?: number;
       sourceTag?: string;
       isShallow?: boolean;
@@ -371,7 +371,7 @@ export class LocalConnection extends Connection {
     const innerTable = await this.inner.cloneTable(
       targetTableName,
       sourceUri,
-      options?.targetNamespacePath ?? [],
+      options?.targetNamespace ?? [],
       options?.sourceVersion ?? null,
       options?.sourceTag ?? null,
       options?.isShallow ?? true,
@@ -406,42 +406,42 @@ export class LocalConnection extends Connection {
     nameOrOptions:
       | string
       | ({ name: string; data: Data } & Partial<CreateTableOptions>),
-    dataOrNamespacePath?: Record<string, unknown>[] | TableLike | string[],
+    dataOrNamespace?: Record<string, unknown>[] | TableLike | string[],
-    namespacePathOrOptions?: string[] | Partial<CreateTableOptions>,
+    namespaceOrOptions?: string[] | Partial<CreateTableOptions>,
     options?: Partial<CreateTableOptions>,
   ): Promise<Table> {
     if (typeof nameOrOptions !== "string" && "name" in nameOrOptions) {
-      // First overload: createTable(options, namespacePath?)
+      // First overload: createTable(options, namespace?)
       const { name, data, ...createOptions } = nameOrOptions;
-      const namespacePath = dataOrNamespacePath as string[] | undefined;
+      const namespace = dataOrNamespace as string[] | undefined;
-      return this._createTableImpl(name, data, namespacePath, createOptions);
+      return this._createTableImpl(name, data, namespace, createOptions);
     }
 
-    // Second overload: createTable(name, data, namespacePath?, options?)
+    // Second overload: createTable(name, data, namespace?, options?)
     const name = nameOrOptions;
-    const data = dataOrNamespacePath as Record<string, unknown>[] | TableLike;
+    const data = dataOrNamespace as Record<string, unknown>[] | TableLike;
 
-    // Detect if third argument is namespacePath array or options object
+    // Detect if third argument is namespace array or options object
-    let namespacePath: string[] | undefined;
+    let namespace: string[] | undefined;
     let createOptions: Partial<CreateTableOptions> | undefined;
 
-    if (Array.isArray(namespacePathOrOptions)) {
+    if (Array.isArray(namespaceOrOptions)) {
-      // Third argument is namespacePath array
+      // Third argument is namespace array
-      namespacePath = namespacePathOrOptions;
+      namespace = namespaceOrOptions;
       createOptions = options;
     } else {
       // Third argument is options object (backwards compatibility)
-      namespacePath = undefined;
+      namespace = undefined;
-      createOptions = namespacePathOrOptions;
+      createOptions = namespaceOrOptions;
     }
 
-    return this._createTableImpl(name, data, namespacePath, createOptions);
+    return this._createTableImpl(name, data, namespace, createOptions);
   }
 
   private async _createTableImpl(
     name: string,
     data: Data,
-    namespacePath?: string[],
+    namespace?: string[],
     options?: Partial<CreateTableOptions>,
   ): Promise<Table> {
     if (data === undefined) {
@@ -455,7 +455,7 @@ export class LocalConnection extends Connection {
       name,
       buf,
       mode,
-      namespacePath ?? [],
+      namespace ?? [],
       storageOptions,
     );
 
@@ -465,21 +465,21 @@ export class LocalConnection extends Connection {
   async createEmptyTable(
     name: string,
     schema: import("./arrow").SchemaLike,
-    namespacePathOrOptions?: string[] | Partial<CreateTableOptions>,
+    namespaceOrOptions?: string[] | Partial<CreateTableOptions>,
     options?: Partial<CreateTableOptions>,
   ): Promise<Table> {
-    // Detect if third argument is namespacePath array or options object
+    // Detect if third argument is namespace array or options object
-    let namespacePath: string[] | undefined;
+    let namespace: string[] | undefined;
     let createOptions: Partial<CreateTableOptions> | undefined;
 
-    if (Array.isArray(namespacePathOrOptions)) {
+    if (Array.isArray(namespaceOrOptions)) {
-      // Third argument is namespacePath array
+      // Third argument is namespace array
-      namespacePath = namespacePathOrOptions;
+      namespace = namespaceOrOptions;
       createOptions = options;
     } else {
       // Third argument is options object (backwards compatibility)
-      namespacePath = undefined;
+      namespace = undefined;
-      createOptions = namespacePathOrOptions;
+      createOptions = namespaceOrOptions;
     }
 
     let mode: string = createOptions?.mode ?? "create";
@@ -502,18 +502,18 @@ export class LocalConnection extends Connection {
       name,
       buf,
       mode,
-      namespacePath ?? [],
+      namespace ?? [],
       storageOptions,
     );
     return new LocalTable(innerTable);
   }
 
-  async dropTable(name: string, namespacePath?: string[]): Promise<void> {
+  async dropTable(name: string, namespace?: string[]): Promise<void> {
-    return this.inner.dropTable(name, namespacePath ?? []);
+    return this.inner.dropTable(name, namespace ?? []);
   }
 
-  async dropAllTables(namespacePath?: string[]): Promise<void> {
+  async dropAllTables(namespace?: string[]): Promise<void> {
-    return this.inner.dropAllTables(namespacePath ?? []);
+    return this.inner.dropAllTables(namespace ?? []);
   }
 }
 

nodejs/lancedb/query.ts

@@ -5,7 +5,6 @@ import {
   Table as ArrowTable,
   type IntoVector,
   RecordBatch,
-  extractVectorBuffer,
   fromBufferToRecordBatch,
   fromRecordBatchToBuffer,
   tableFromIPC,
@@ -662,8 +661,10 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
       const res = (async () => {
         try {
           const v = await vector;
+          const arr = Float32Array.from(v);
+          //
           // biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
-          const value: any = this.addQueryVector(v);
+          const value: any = this.addQueryVector(arr);
           const inner = value.inner as
             | NativeVectorQuery
             | Promise<NativeVectorQuery>;
@@ -675,12 +676,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
       return new VectorQuery(res);
     } else {
       super.doCall((inner) => {
-        const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector);
+        inner.addQueryVector(Float32Array.from(vector));
-        if (raw) {
-          inner.addQueryVectorRaw(raw.data, raw.dtype);
-        } else {
-          inner.addQueryVector(Float32Array.from(vector as number[]));
-        }
       });
       return this;
     }
@@ -769,23 +765,14 @@ export class Query extends StandardQueryBase<NativeQuery> {
    * a default `limit` of 10 will be used.  @see {@link Query#limit}
    */
   nearestTo(vector: IntoVector): VectorQuery {
-    const callNearestTo = (
-      inner: NativeQuery,
-      resolved: Float32Array | Float64Array | Uint8Array | number[],
-    ): NativeVectorQuery => {
-      const raw = Array.isArray(resolved)
-        ? null
-        : extractVectorBuffer(resolved);
-      if (raw) {
-        return inner.nearestToRaw(raw.data, raw.dtype);
-      }
-      return inner.nearestTo(Float32Array.from(resolved as number[]));
-    };
-
     if (this.inner instanceof Promise) {
       const nativeQuery = this.inner.then(async (inner) => {
-        const resolved = vector instanceof Promise ? await vector : vector;
+        if (vector instanceof Promise) {
-        return callNearestTo(inner, resolved);
+          const arr = await vector.then((v) => Float32Array.from(v));
+          return inner.nearestTo(arr);
+        } else {
+          return inner.nearestTo(Float32Array.from(vector));
+        }
       });
       return new VectorQuery(nativeQuery);
     }
@@ -793,8 +780,10 @@ export class Query extends StandardQueryBase<NativeQuery> {
       const res = (async () => {
         try {
           const v = await vector;
+          const arr = Float32Array.from(v);
+          //
           // biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
-          const value: any = this.nearestTo(v);
+          const value: any = this.nearestTo(arr);
           const inner = value.inner as
             | NativeVectorQuery
             | Promise<NativeVectorQuery>;
@@ -805,7 +794,7 @@ export class Query extends StandardQueryBase<NativeQuery> {
       })();
       return new VectorQuery(res);
     } else {
-      const vectorQuery = callNearestTo(this.inner, vector);
+      const vectorQuery = this.inner.nearestTo(Float32Array.from(vector));
       return new VectorQuery(vectorQuery);
     }
   }

nodejs/lancedb/table.ts

@@ -285,25 +285,6 @@ export abstract class Table {
    */
   abstract prewarmIndex(name: string): Promise<void>;
 
-  /**
-   * Prewarm one or more columns of data in the table.
-   *
-   * @param columns The columns to prewarm. If undefined, all columns are prewarmed.
-   *
-   * This will load the column data into the page cache so that future queries that
-   * read those columns avoid the initial cold-start latency.  This call initiates
-   * prewarming and returns once the request is accepted; the warming itself may
-   * continue in the background.  Calling it on already-prewarmed columns is a
-   * no-op on the server.
-   *
-   * Prewarming is generally useful for columns used in filters or projections.
-   * Large columns (e.g. high-dimensional vectors or binary data) may not be
-   * practical to prewarm.
-   *
-   * This feature is currently only supported on remote tables.
-   */
-  abstract prewarmData(columns?: string[]): Promise<void>;
-
   /**
    * Waits for asynchronous indexing to complete on the table.
    *
@@ -729,10 +710,6 @@ export class LocalTable extends Table {
     await this.inner.prewarmIndex(name);
   }
 
-  async prewarmData(columns?: string[]): Promise<void> {
-    await this.inner.prewarmData(columns);
-  }
-
   async waitForIndex(
     indexNames: string[],
     timeoutSeconds: number,

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.28.0-beta.11",
+	"version": "0.27.0-beta.5",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.28.0-beta.11",
+	"version": "0.27.0-beta.5",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.28.0-beta.11",
+	"version": "0.27.0-beta.5",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.28.0-beta.11",
+	"version": "0.27.0-beta.5",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.28.0-beta.11",
+	"version": "0.27.0-beta.5",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.28.0-beta.11",
+  "version": "0.27.0-beta.5",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.28.0-beta.11",
+	"version": "0.27.0-beta.5",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@lancedb/lancedb",
-  "version": "0.28.0-beta.11",
+  "version": "0.27.0-beta.5",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@lancedb/lancedb",
-      "version": "0.28.0-beta.11",
+      "version": "0.27.0-beta.5",
       "cpu": [
         "x64",
         "arm64"

nodejs/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.28.0-beta.11",
+  "version": "0.27.0-beta.5",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -75,6 +75,7 @@
     "build:debug": "napi build --platform --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir lancedb",
     "postbuild:debug": "shx mkdir -p dist && shx cp lancedb/*.node dist/",
     "build:release": "napi build --platform --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir dist",
+    "postbuild:release": "shx mkdir -p dist && shx cp lancedb/*.node dist/",
     "build": "npm run build:debug && npm run tsc",
     "build-release": "npm run build:release && npm run tsc",
     "tsc": "tsc -b",

nodejs/src/connection.rs

@@ -67,12 +67,6 @@ impl Connection {
                 builder = builder.storage_option(key, value);
             }
         }
-        if let Some(manifest_enabled) = options.manifest_enabled {
-            builder = builder.manifest_enabled(manifest_enabled);
-        }
-        if let Some(namespace_client_properties) = options.namespace_client_properties {
-            builder = builder.namespace_client_properties(namespace_client_properties);
-        }
 
         // Create client config, optionally with header provider
         let client_config = options.client_config.unwrap_or_default();
@@ -125,12 +119,12 @@ impl Connection {
     #[napi(catch_unwind)]
     pub async fn table_names(
         &self,
-        namespace_path: Option<Vec<String>>,
+        namespace: Vec<String>,
         start_after: Option<String>,
         limit: Option<u32>,
     ) -> napi::Result<Vec<String>> {
         let mut op = self.get_inner()?.table_names();
-        op = op.namespace(namespace_path.unwrap_or_default());
+        op = op.namespace(namespace);
         if let Some(start_after) = start_after {
             op = op.start_after(start_after);
         }
@@ -152,7 +146,7 @@ impl Connection {
         name: String,
         buf: Buffer,
         mode: String,
-        namespace_path: Option<Vec<String>>,
+        namespace: Vec<String>,
         storage_options: Option<HashMap<String, String>>,
     ) -> napi::Result<Table> {
         let batches = ipc_file_to_batches(buf.to_vec())
@@ -160,7 +154,7 @@ impl Connection {
         let mode = Self::parse_create_mode_str(&mode)?;
         let mut builder = self.get_inner()?.create_table(&name, batches).mode(mode);
 
-        builder = builder.namespace(namespace_path.unwrap_or_default());
+        builder = builder.namespace(namespace);
 
         if let Some(storage_options) = storage_options {
             for (key, value) in storage_options {
@@ -177,7 +171,7 @@ impl Connection {
         name: String,
         schema_buf: Buffer,
         mode: String,
-        namespace_path: Option<Vec<String>>,
+        namespace: Vec<String>,
         storage_options: Option<HashMap<String, String>>,
     ) -> napi::Result<Table> {
         let schema = ipc_file_to_schema(schema_buf.to_vec()).map_err(|e| {
@@ -189,7 +183,7 @@ impl Connection {
             .create_empty_table(&name, schema)
             .mode(mode);
 
-        builder = builder.namespace(namespace_path.unwrap_or_default());
+        builder = builder.namespace(namespace);
 
         if let Some(storage_options) = storage_options {
             for (key, value) in storage_options {
@@ -204,13 +198,13 @@ impl Connection {
     pub async fn open_table(
         &self,
         name: String,
-        namespace_path: Option<Vec<String>>,
+        namespace: Vec<String>,
         storage_options: Option<HashMap<String, String>>,
         index_cache_size: Option<u32>,
     ) -> napi::Result<Table> {
         let mut builder = self.get_inner()?.open_table(&name);
 
-        builder = builder.namespace(namespace_path.unwrap_or_default());
+        builder = builder.namespace(namespace);
 
         if let Some(storage_options) = storage_options {
             for (key, value) in storage_options {
@@ -229,7 +223,7 @@ impl Connection {
         &self,
         target_table_name: String,
         source_uri: String,
-        target_namespace_path: Option<Vec<String>>,
+        target_namespace: Vec<String>,
         source_version: Option<i64>,
         source_tag: Option<String>,
         is_shallow: bool,
@@ -238,7 +232,7 @@ impl Connection {
             .get_inner()?
             .clone_table(&target_table_name, &source_uri);
 
-        builder = builder.target_namespace(target_namespace_path.unwrap_or_default());
+        builder = builder.target_namespace(target_namespace);
 
         if let Some(version) = source_version {
             builder = builder.source_version(version as u64);
@@ -256,21 +250,18 @@ impl Connection {
 
     /// Drop table with the name. Or raise an error if the table does not exist.
     #[napi(catch_unwind)]
-    pub async fn drop_table(
+    pub async fn drop_table(&self, name: String, namespace: Vec<String>) -> napi::Result<()> {
-        &self,
-        name: String,
-        namespace_path: Option<Vec<String>>,
-    ) -> napi::Result<()> {
-        let ns = namespace_path.unwrap_or_default();
         self.get_inner()?
-            .drop_table(&name, &ns)
+            .drop_table(&name, &namespace)
             .await
             .default_error()
     }
 
     #[napi(catch_unwind)]
-    pub async fn drop_all_tables(&self, namespace_path: Option<Vec<String>>) -> napi::Result<()> {
+    pub async fn drop_all_tables(&self, namespace: Vec<String>) -> napi::Result<()> {
-        let ns = namespace_path.unwrap_or_default();
+        self.get_inner()?
-        self.get_inner()?.drop_all_tables(&ns).await.default_error()
+            .drop_all_tables(&namespace)
+            .await
+            .default_error()
     }
 }

nodejs/src/lib.rs

@@ -35,15 +35,8 @@ pub struct ConnectionOptions {
     pub read_consistency_interval: Option<f64>,
     /// (For LanceDB OSS only): configuration for object storage.
     ///
-    /// The available options are described at https://docs.lancedb.com/storage/
+    /// The available options are described at https://lancedb.com/docs/storage/
     pub storage_options: Option<HashMap<String, String>>,
-    /// (For LanceDB OSS only): use directory namespace manifests as the source
-    /// of truth for table metadata. Existing directory-listed root tables are
-    /// migrated into the manifest on access.
-    pub manifest_enabled: Option<bool>,
-    /// (For LanceDB OSS only): extra properties for the backing namespace
-    /// client used by manifest-enabled native connections.
-    pub namespace_client_properties: Option<HashMap<String, String>>,
     /// (For LanceDB OSS only): the session to use for this connection. Holds
     /// shared caches and other session-specific state.
     pub session: Option<session::Session>,

nodejs/src/query.rs

@@ -3,12 +3,6 @@
 
 use std::sync::Arc;
 
-use arrow_array::{
-    Array, Float16Array as ArrowFloat16Array, Float32Array as ArrowFloat32Array,
-    Float64Array as ArrowFloat64Array, UInt8Array as ArrowUInt8Array,
-};
-use arrow_buffer::ScalarBuffer;
-use half::f16;
 use lancedb::index::scalar::{
     BooleanQuery, BoostQuery, FtsQuery, FullTextSearchQuery, MatchQuery, MultiMatchQuery, Occur,
     Operator, PhraseQuery,
@@ -30,33 +24,6 @@ use crate::rerankers::RerankHybridCallbackArgs;
 use crate::rerankers::Reranker;
 use crate::util::{parse_distance_type, schema_to_buffer};
 
-fn bytes_to_arrow_array(data: Uint8Array, dtype: String) -> napi::Result<Arc<dyn Array>> {
-    let buf = arrow_buffer::Buffer::from(data.to_vec());
-    let num_bytes = buf.len();
-    match dtype.as_str() {
-        "float16" => {
-            let scalar_buf = ScalarBuffer::<f16>::new(buf, 0, num_bytes / 2);
-            Ok(Arc::new(ArrowFloat16Array::new(scalar_buf, None)))
-        }
-        "float32" => {
-            let scalar_buf = ScalarBuffer::<f32>::new(buf, 0, num_bytes / 4);
-            Ok(Arc::new(ArrowFloat32Array::new(scalar_buf, None)))
-        }
-        "float64" => {
-            let scalar_buf = ScalarBuffer::<f64>::new(buf, 0, num_bytes / 8);
-            Ok(Arc::new(ArrowFloat64Array::new(scalar_buf, None)))
-        }
-        "uint8" => {
-            let scalar_buf = ScalarBuffer::<u8>::new(buf, 0, num_bytes);
-            Ok(Arc::new(ArrowUInt8Array::new(scalar_buf, None)))
-        }
-        _ => Err(napi::Error::from_reason(format!(
-            "Unsupported vector dtype: {}. Expected one of: float16, float32, float64, uint8",
-            dtype
-        ))),
-    }
-}
-
 #[napi]
 pub struct Query {
     inner: LanceDbQuery,
@@ -111,13 +78,6 @@ impl Query {
         Ok(VectorQuery { inner })
     }
 
-    #[napi]
-    pub fn nearest_to_raw(&mut self, data: Uint8Array, dtype: String) -> Result<VectorQuery> {
-        let array = bytes_to_arrow_array(data, dtype)?;
-        let inner = self.inner.clone().nearest_to(array).default_error()?;
-        Ok(VectorQuery { inner })
-    }
-
     #[napi]
     pub fn fast_search(&mut self) {
         self.inner = self.inner.clone().fast_search();
@@ -203,13 +163,6 @@ impl VectorQuery {
         Ok(())
     }
 
-    #[napi]
-    pub fn add_query_vector_raw(&mut self, data: Uint8Array, dtype: String) -> Result<()> {
-        let array = bytes_to_arrow_array(data, dtype)?;
-        self.inner = self.inner.clone().add_query_vector(array).default_error()?;
-        Ok(())
-    }
-
     #[napi]
     pub fn distance_type(&mut self, distance_type: String) -> napi::Result<()> {
         let distance_type = parse_distance_type(distance_type)?;

nodejs/src/remote.rs

@@ -92,13 +92,6 @@ pub struct ClientConfig {
     pub extra_headers: Option<HashMap<String, String>>,
     pub id_delimiter: Option<String>,
     pub tls_config: Option<TlsConfig>,
-    /// User identifier for tracking purposes.
-    ///
-    /// This is sent as the `x-lancedb-user-id` header in requests to LanceDB Cloud/Enterprise.
-    /// It can be set directly, or via the `LANCEDB_USER_ID` environment variable.
-    /// Alternatively, set `LANCEDB_USER_ID_ENV_KEY` to specify another environment
-    /// variable that contains the user ID value.
-    pub user_id: Option<String>,
 }
 
 impl From<TimeoutConfig> for lancedb::remote::TimeoutConfig {
@@ -152,7 +145,6 @@ impl From<ClientConfig> for lancedb::remote::ClientConfig {
             id_delimiter: config.id_delimiter,
             tls_config: config.tls_config.map(Into::into),
             header_provider: None, // the header provider is set separately later
-            user_id: config.user_id,
         }
     }
 }

nodejs/src/rerankers.rs

@@ -18,7 +18,6 @@ type RerankHybridFn = ThreadsafeFunction<
     RerankHybridCallbackArgs,
     Status,
     false,
-    true,
 >;
 
 /// Reranker implementation that "wraps" a NodeJS Reranker implementation.
@@ -33,10 +32,7 @@ impl Reranker {
     pub fn new(
         rerank_hybrid: Function<RerankHybridCallbackArgs, Promise<Buffer>>,
     ) -> napi::Result<Self> {
-        let rerank_hybrid = rerank_hybrid
+        let rerank_hybrid = rerank_hybrid.build_threadsafe_function().build()?;
-            .build_threadsafe_function()
-            .weak::<true>()
-            .build()?;
         Ok(Self { rerank_hybrid })
     }
 }

nodejs/src/table.rs

@@ -159,14 +159,6 @@ impl Table {
             .default_error()
     }
 
-    #[napi(catch_unwind)]
-    pub async fn prewarm_data(&self, columns: Option<Vec<String>>) -> napi::Result<()> {
-        self.inner_ref()?
-            .prewarm_data(columns)
-            .await
-            .default_error()
-    }
-
     #[napi(catch_unwind)]
     pub async fn wait_for_index(&self, index_names: Vec<String>, timeout_s: i64) -> Result<()> {
         let timeout = std::time::Duration::from_secs(timeout_s.try_into().unwrap());

python/.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.32.0"
+current_version = "0.30.0-beta.5"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

python/.gitignore

@@ -1,5 +1,3 @@
 # Test data created by some example tests
 data/
 _lancedb.pyd
-# macOS debug symbols bundle generated during build
-*.dSYM/

python/Cargo.toml

@@ -1,7 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.32.0"
+version = "0.30.0-beta.5"
-publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"
 license.workspace = true
@@ -15,7 +14,7 @@ name = "_lancedb"
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow = { version = "58.0.0", features = ["pyarrow"] }
+arrow = { version = "57.2", features = ["pyarrow"] }
 async-trait = "0.1"
 bytes = "1"
 lancedb = { path = "../rust/lancedb", default-features = false }
@@ -24,9 +23,8 @@ lance-namespace.workspace = true
 lance-namespace-impls.workspace = true
 lance-io.workspace = true
 env_logger.workspace = true
-log.workspace = true
+pyo3 = { version = "0.26", features = ["extension-module", "abi3-py39"] }
-pyo3 = { version = "0.28", features = ["extension-module", "abi3-py39"] }
+pyo3-async-runtimes = { version = "0.26", features = [
-pyo3-async-runtimes = { version = "0.28", features = [
     "attributes",
     "tokio-runtime",
 ] }
@@ -35,11 +33,10 @@ futures.workspace = true
 serde = "1"
 serde_json = "1"
 snafu.workspace = true
-tokio = { version = "1.40", features = ["sync", "rt-multi-thread"] }
+tokio = { version = "1.40", features = ["sync"] }
-libc = "0.2"
 
 [build-dependencies]
-pyo3-build-config = { version = "0.28", features = [
+pyo3-build-config = { version = "0.26", features = [
     "extension-module",
     "abi3-py39",
 ] }

python/PYTHON_THIRD_PARTY_LICENSES.md

@@ -183,6 +183,7 @@
 | stack-data                     | 0.6.3           | MIT License                                                                                      | http://github.com/alexmojaki/stack_data                                                           |
 | sympy                          | 1.14.0          | BSD License                                                                                      | https://sympy.org                                                                                 |
 | tabulate                       | 0.9.0           | MIT License                                                                                      | https://github.com/astanin/python-tabulate                                                        |
+| tantivy                        | 0.25.1          | UNKNOWN                                                                                          | UNKNOWN                                                                                           |
 | threadpoolctl                  | 3.6.0           | BSD License                                                                                      | https://github.com/joblib/threadpoolctl                                                           |
 | timm                           | 1.0.24          | Apache Software License                                                                          | https://github.com/huggingface/pytorch-image-models                                               |
 | tinycss2                       | 1.4.0           | BSD License                                                                                      | https://www.courtbouillon.org/tinycss2                                                            |

python/pyproject.toml

@@ -45,7 +45,7 @@ repository = "https://github.com/lancedb/lancedb"
 
 [project.optional-dependencies]
 pylance = [
-    "pylance>=6.0.0",
+    "pylance>=4.0.0b7",
 ]
 tests = [
     "aiohttp>=3.9.0",
@@ -57,8 +57,9 @@ tests = [
     "duckdb>=0.9.0",
     "pytz>=2023.3",
     "polars>=0.19, <=1.3.0",
+    "tantivy>=0.20.0",
     "pyarrow-stubs>=16.0",
-    "pylance>=6.0.0",
+    "pylance>=4.0.0b7",
     "requests>=2.31.0",
     "datafusion>=52,<53",
 ]
@@ -82,7 +83,7 @@ embeddings = [
     "colpali-engine>=0.3.10",
     "huggingface_hub>=0.19.0",
     "InstructorEmbedding>=1.0.1",
-    "google-genai>=1.0.0",
+    "google.generativeai>=0.3.0",
     "boto3>=1.28.57",
     "awscli>=1.44.38",
     "botocore>=1.31.57",

python/python/lancedb/__init__.py

@@ -6,7 +6,8 @@ import importlib.metadata
 import os
 from concurrent.futures import ThreadPoolExecutor
 from datetime import timedelta
-from typing import Dict, Optional, Union, Any, List
+from typing import Dict, Optional, Union, Any
+import warnings
 
 __version__ = importlib.metadata.version("lancedb")
 
@@ -14,9 +15,9 @@ from ._lancedb import connect as lancedb_connect
 from .common import URI, sanitize_uri
 from urllib.parse import urlparse
 from .db import AsyncConnection, DBConnection, LanceDBConnection
+from .io import StorageOptionsProvider
 from .remote import ClientConfig
 from .remote.db import RemoteDBConnection
-from .expr import Expr, col, lit, func
 from .schema import vector
 from .table import AsyncTable, Table
 from ._lancedb import Session
@@ -62,7 +63,7 @@ def _check_s3_bucket_with_dots(
 
 
 def connect(
-    uri: Optional[URI] = None,
+    uri: URI,
     *,
     api_key: Optional[str] = None,
     region: str = "us-east-1",
@@ -72,19 +73,14 @@ def connect(
     client_config: Union[ClientConfig, Dict[str, Any], None] = None,
     storage_options: Optional[Dict[str, str]] = None,
     session: Optional[Session] = None,
-    manifest_enabled: bool = False,
-    namespace_client_impl: Optional[str] = None,
-    namespace_client_properties: Optional[Dict[str, str]] = None,
-    namespace_client_pushdown_operations: Optional[List[str]] = None,
     **kwargs: Any,
 ) -> DBConnection:
     """Connect to a LanceDB database.
 
     Parameters
     ----------
-    uri: str or Path, optional
+    uri: str or Path
-        The uri of the database. When ``namespace_client_impl`` is provided you may
+        The uri of the database.
-        omit ``uri`` and connect through a namespace client instead.
     api_key: str, optional
         If presented, connect to LanceDB cloud.
         Otherwise, connect to a database on file system or cloud storage.
@@ -110,29 +106,13 @@ def connect(
         default configuration is used.
     storage_options: dict, optional
         Additional options for the storage backend. See available options at
-        <https://docs.lancedb.com/storage/>
+        <https://lancedb.com/docs/storage/>
-    manifest_enabled : bool, default False
-        When true for local/native connections, use directory namespace
-        manifests as the source of truth for table metadata. Existing
-        directory-listed root tables are migrated into the manifest on access.
     session: Session, optional
         (For LanceDB OSS only)
         A session to use for this connection. Sessions allow you to configure
         cache sizes for index and metadata caches, which can significantly
         impact memory use and performance. They can also be re-used across
         multiple connections to share the same cache state.
-    namespace_client_impl : str, optional
-        When provided along with ``namespace_client_properties``, ``connect``
-        returns a namespace-backed connection by delegating to
-        :func:`connect_namespace`. The value identifies which namespace
-        implementation to load (e.g., ``"dir"`` or ``"rest"``).
-    namespace_client_properties : dict, optional
-        Configuration to pass to the namespace client implementation. Required
-        when ``namespace_client_impl`` is set.
-    namespace_client_pushdown_operations : list[str], optional
-        Only used when ``namespace_client_properties`` is provided. Forwards to
-        :func:`connect_namespace` to control which operations are executed on the
-        namespace service (e.g., ``["QueryTable", "CreateTable"]``).
 
     Examples
     --------
@@ -152,48 +132,11 @@ def connect(
     >>> db = lancedb.connect("db://my_database", api_key="ldb_...",
     ...                      client_config={"retry_config": {"retries": 5}})
 
-    Connect to a namespace-backed database:
-
-    >>> db = lancedb.connect(namespace_client_impl="dir",
-    ...                      namespace_client_properties={"root": "/tmp/ns"})
-
     Returns
     -------
     conn : DBConnection
         A connection to a LanceDB database.
     """
-    if namespace_client_impl is not None:
-        if namespace_client_properties is None:
-            raise ValueError(
-                "namespace_client_properties must be provided when "
-                "namespace_client_impl is set"
-            )
-        if kwargs:
-            raise ValueError(f"Unknown keyword arguments: {kwargs}")
-        return connect_namespace(
-            namespace_client_impl,
-            namespace_client_properties,
-            read_consistency_interval=read_consistency_interval,
-            storage_options=storage_options,
-            session=session,
-            namespace_client_pushdown_operations=namespace_client_pushdown_operations,
-        )
-
-    if namespace_client_properties is not None and not manifest_enabled:
-        raise ValueError(
-            "namespace_client_impl must be provided when using "
-            "namespace_client_properties unless manifest_enabled=True"
-        )
-
-    if namespace_client_pushdown_operations is not None:
-        raise ValueError(
-            "namespace_client_pushdown_operations is only valid when "
-            "connecting through a namespace"
-        )
-    if uri is None:
-        raise ValueError(
-            "uri is required when not connecting through a namespace client"
-        )
     if isinstance(uri, str) and uri.startswith("db://"):
         if api_key is None:
             api_key = os.environ.get("LANCEDB_API_KEY")
@@ -222,92 +165,9 @@ def connect(
         read_consistency_interval=read_consistency_interval,
         storage_options=storage_options,
         session=session,
-        manifest_enabled=manifest_enabled,
-        namespace_client_properties=namespace_client_properties,
     )
 
 
-WORKER_PROPERTY_PREFIX = "_lancedb_worker_"
-
-
-def _apply_worker_overrides(props: dict[str, str]) -> dict[str, str]:
-    """Apply worker property overrides.
-
-    Any key starting with ``_lancedb_worker_`` is extracted, the prefix
-    is stripped, and the resulting key-value pair is put back into the
-    map (overriding the existing value if present).  The original
-    prefixed key is removed.
-    """
-    worker_keys = [k for k in props if k.startswith(WORKER_PROPERTY_PREFIX)]
-    if not worker_keys:
-        return props
-    result = dict(props)
-    for key in worker_keys:
-        value = result.pop(key)
-        real_key = key[len(WORKER_PROPERTY_PREFIX) :]
-        result[real_key] = value
-    return result
-
-
-def deserialize_conn(
-    data: str,
-    *,
-    for_worker: bool = False,
-) -> DBConnection:
-    """Reconstruct a DBConnection from a serialized string.
-
-    The string must have been produced by
-    :meth:`DBConnection.serialize`.
-
-    Parameters
-    ----------
-    data : str
-        String produced by ``serialize()``.
-    for_worker : bool, default False
-        When ``True``, any namespace client property whose key starts
-        with ``_lancedb_worker_`` has that prefix stripped and the
-        value overrides the corresponding property.  For example,
-        ``_lancedb_worker_uri`` replaces ``uri``.
-
-    Returns
-    -------
-    DBConnection
-        A new connection matching the serialized state.
-    """
-    import json
-
-    parsed = json.loads(data)
-    connection_type = parsed.get("connection_type")
-
-    rci_secs = parsed.get("read_consistency_interval_seconds")
-    rci = timedelta(seconds=rci_secs) if rci_secs is not None else None
-    storage_options = parsed.get("storage_options")
-
-    if connection_type == "namespace":
-        props = dict(parsed.get("namespace_client_properties") or {})
-        if for_worker:
-            props = _apply_worker_overrides(props)
-        return connect_namespace(
-            namespace_client_impl=parsed["namespace_client_impl"],
-            namespace_client_properties=props,
-            read_consistency_interval=rci,
-            storage_options=storage_options,
-            namespace_client_pushdown_operations=parsed.get(
-                "namespace_client_pushdown_operations"
-            ),
-        )
-    elif connection_type == "local":
-        return LanceDBConnection(
-            parsed["uri"],
-            read_consistency_interval=rci,
-            storage_options=storage_options,
-            manifest_enabled=parsed.get("manifest_enabled", False),
-            namespace_client_properties=parsed.get("namespace_client_properties"),
-        )
-    else:
-        raise ValueError(f"Unknown connection_type: {connection_type}")
-
-
 async def connect_async(
     uri: URI,
     *,
@@ -318,8 +178,6 @@ async def connect_async(
     client_config: Optional[Union[ClientConfig, Dict[str, Any]]] = None,
     storage_options: Optional[Dict[str, str]] = None,
     session: Optional[Session] = None,
-    manifest_enabled: bool = False,
-    namespace_client_properties: Optional[Dict[str, str]] = None,
 ) -> AsyncConnection:
     """Connect to a LanceDB database.
 
@@ -352,20 +210,13 @@ async def connect_async(
         default configuration is used.
     storage_options: dict, optional
         Additional options for the storage backend. See available options at
-        <https://docs.lancedb.com/storage/>
+        <https://lancedb.com/docs/storage/>
     session: Session, optional
         (For LanceDB OSS only)
         A session to use for this connection. Sessions allow you to configure
         cache sizes for index and metadata caches, which can significantly
         impact memory use and performance. They can also be re-used across
         multiple connections to share the same cache state.
-    manifest_enabled : bool, default False
-        When true for local/native connections, use directory namespace
-        manifests as the source of truth for table metadata. Existing
-        directory-listed root tables are migrated into the manifest on access.
-    namespace_client_properties : dict, optional
-        Additional directory namespace client properties to use with
-        ``manifest_enabled=True``.
 
     Examples
     --------
@@ -408,8 +259,6 @@ async def connect_async(
             client_config,
             storage_options,
             session,
-            manifest_enabled,
-            namespace_client_properties,
         )
     )
 
@@ -422,10 +271,6 @@ __all__ = [
     "AsyncConnection",
     "AsyncLanceNamespaceDBConnection",
     "AsyncTable",
-    "col",
-    "Expr",
-    "func",
-    "lit",
     "URI",
     "sanitize_uri",
     "vector",
@@ -434,6 +279,17 @@ __all__ = [
     "LanceNamespaceDBConnection",
     "RemoteDBConnection",
     "Session",
+    "StorageOptionsProvider",
     "Table",
     "__version__",
 ]
+
+
+def __warn_on_fork():
+    warnings.warn(
+        "lance is not fork-safe. If you are using multiprocessing, use spawn instead.",
+    )
+
+
+if hasattr(os, "register_at_fork"):
+    os.register_at_fork(before=__warn_on_fork)  # type: ignore[attr-defined]

python/python/lancedb/_lancedb.pyi

@@ -12,9 +12,9 @@ from .index import (
     LabelList,
     HnswPq,
     HnswSq,
-    HnswFlat,
     FTS,
 )
+from .io import StorageOptionsProvider
 from lance_namespace import (
     ListNamespacesResponse,
     CreateNamespaceResponse,
@@ -26,33 +26,6 @@ from .remote import ClientConfig
 
 IvfHnswPq: type[HnswPq] = HnswPq
 IvfHnswSq: type[HnswSq] = HnswSq
-IvfHnswFlat: type[HnswFlat] = HnswFlat
-
-class PyExpr:
-    """A type-safe DataFusion expression node (Rust-side handle)."""
-
-    def eq(self, other: "PyExpr") -> "PyExpr": ...
-    def ne(self, other: "PyExpr") -> "PyExpr": ...
-    def lt(self, other: "PyExpr") -> "PyExpr": ...
-    def lte(self, other: "PyExpr") -> "PyExpr": ...
-    def gt(self, other: "PyExpr") -> "PyExpr": ...
-    def gte(self, other: "PyExpr") -> "PyExpr": ...
-    def and_(self, other: "PyExpr") -> "PyExpr": ...
-    def or_(self, other: "PyExpr") -> "PyExpr": ...
-    def not_(self) -> "PyExpr": ...
-    def add(self, other: "PyExpr") -> "PyExpr": ...
-    def sub(self, other: "PyExpr") -> "PyExpr": ...
-    def mul(self, other: "PyExpr") -> "PyExpr": ...
-    def div(self, other: "PyExpr") -> "PyExpr": ...
-    def lower(self) -> "PyExpr": ...
-    def upper(self) -> "PyExpr": ...
-    def contains(self, substr: "PyExpr") -> "PyExpr": ...
-    def cast(self, data_type: pa.DataType) -> "PyExpr": ...
-    def to_sql(self) -> str: ...
-
-def expr_col(name: str) -> PyExpr: ...
-def expr_lit(value: Union[bool, int, float, str]) -> PyExpr: ...
-def expr_func(name: str, args: List[PyExpr]) -> PyExpr: ...
 
 class Session:
     def __init__(
@@ -73,35 +46,35 @@ class Connection(object):
     async def close(self): ...
     async def list_namespaces(
         self,
-        namespace_path: Optional[List[str]] = None,
+        namespace: Optional[List[str]] = None,
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
     ) -> ListNamespacesResponse: ...
     async def create_namespace(
         self,
-        namespace_path: List[str],
+        namespace: List[str],
         mode: Optional[str] = None,
         properties: Optional[Dict[str, str]] = None,
     ) -> CreateNamespaceResponse: ...
     async def drop_namespace(
         self,
-        namespace_path: List[str],
+        namespace: List[str],
         mode: Optional[str] = None,
         behavior: Optional[str] = None,
     ) -> DropNamespaceResponse: ...
     async def describe_namespace(
         self,
-        namespace_path: List[str],
+        namespace: List[str],
     ) -> DescribeNamespaceResponse: ...
     async def list_tables(
         self,
-        namespace_path: Optional[List[str]] = None,
+        namespace: Optional[List[str]] = None,
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
     ) -> ListTablesResponse: ...
     async def table_names(
         self,
-        namespace_path: Optional[List[str]],
+        namespace: Optional[List[str]],
         start_after: Optional[str],
         limit: Optional[int],
     ) -> list[str]: ...  # Deprecated: Use list_tables instead
@@ -110,8 +83,9 @@ class Connection(object):
         name: str,
         mode: str,
         data: pa.RecordBatchReader,
-        namespace_path: Optional[List[str]] = None,
+        namespace: Optional[List[str]] = None,
         storage_options: Optional[Dict[str, str]] = None,
+        storage_options_provider: Optional[StorageOptionsProvider] = None,
         location: Optional[str] = None,
     ) -> Table: ...
     async def create_empty_table(
@@ -119,15 +93,17 @@ class Connection(object):
         name: str,
         mode: str,
         schema: pa.Schema,
-        namespace_path: Optional[List[str]] = None,
+        namespace: Optional[List[str]] = None,
         storage_options: Optional[Dict[str, str]] = None,
+        storage_options_provider: Optional[StorageOptionsProvider] = None,
         location: Optional[str] = None,
     ) -> Table: ...
     async def open_table(
         self,
         name: str,
-        namespace_path: Optional[List[str]] = None,
+        namespace: Optional[List[str]] = None,
         storage_options: Optional[Dict[str, str]] = None,
+        storage_options_provider: Optional[StorageOptionsProvider] = None,
         index_cache_size: Optional[int] = None,
         location: Optional[str] = None,
     ) -> Table: ...
@@ -135,7 +111,7 @@ class Connection(object):
         self,
         target_table_name: str,
         source_uri: str,
-        target_namespace_path: Optional[List[str]] = None,
+        target_namespace: Optional[List[str]] = None,
         source_version: Optional[int] = None,
         source_tag: Optional[str] = None,
         is_shallow: bool = True,
@@ -144,18 +120,13 @@ class Connection(object):
         self,
         cur_name: str,
         new_name: str,
-        cur_namespace_path: Optional[List[str]] = None,
+        cur_namespace: Optional[List[str]] = None,
-        new_namespace_path: Optional[List[str]] = None,
+        new_namespace: Optional[List[str]] = None,
     ) -> None: ...
     async def drop_table(
-        self, name: str, namespace_path: Optional[List[str]] = None
+        self, name: str, namespace: Optional[List[str]] = None
     ) -> None: ...
-    async def drop_all_tables(
+    async def drop_all_tables(self, namespace: Optional[List[str]] = None) -> None: ...
-        self, namespace_path: Optional[List[str]] = None
-    ) -> None: ...
-    async def namespace_client_config(
-        self,
-    ) -> Dict[str, Any]: ...
 
 class Table:
     def name(self) -> str: ...
@@ -164,10 +135,7 @@ class Table:
     def close(self) -> None: ...
     async def schema(self) -> pa.Schema: ...
     async def add(
-        self,
+        self, data: pa.RecordBatchReader, mode: Literal["append", "overwrite"]
-        data: pa.RecordBatchReader,
-        mode: Literal["append", "overwrite"],
-        progress: Optional[Any] = None,
     ) -> AddResult: ...
     async def update(
         self, updates: Dict[str, str], where: Optional[str]
@@ -182,7 +150,6 @@ class Table:
             IvfPq,
             HnswPq,
             HnswSq,
-            HnswFlat,
             BTree,
             Bitmap,
             LabelList,
@@ -245,8 +212,6 @@ async def connect(
     client_config: Optional[Union[ClientConfig, Dict[str, Any]]],
     storage_options: Optional[Dict[str, str]],
     session: Optional[Session],
-    manifest_enabled: bool = False,
-    namespace_client_properties: Optional[Dict[str, str]] = None,
 ) -> Connection: ...
 
 class RecordBatchStream:
@@ -257,9 +222,7 @@ class RecordBatchStream:
 
 class Query:
     def where(self, filter: str): ...
-    def where_expr(self, expr: PyExpr): ...
+    def select(self, columns: Tuple[str, str]): ...
-    def select(self, columns: List[Tuple[str, str]]): ...
-    def select_expr(self, columns: List[Tuple[str, PyExpr]]): ...
     def select_columns(self, columns: List[str]): ...
     def limit(self, limit: int): ...
     def offset(self, offset: int): ...
@@ -285,9 +248,7 @@ class TakeQuery:
 
 class FTSQuery:
     def where(self, filter: str): ...
-    def where_expr(self, expr: PyExpr): ...
+    def select(self, columns: List[str]): ...
-    def select(self, columns: List[Tuple[str, str]]): ...
-    def select_expr(self, columns: List[Tuple[str, PyExpr]]): ...
     def limit(self, limit: int): ...
     def offset(self, offset: int): ...
     def fast_search(self): ...
@@ -306,9 +267,7 @@ class VectorQuery:
     async def output_schema(self) -> pa.Schema: ...
     async def execute(self) -> RecordBatchStream: ...
     def where(self, filter: str): ...
-    def where_expr(self, expr: PyExpr): ...
+    def select(self, columns: List[str]): ...
-    def select(self, columns: List[Tuple[str, str]]): ...
-    def select_expr(self, columns: List[Tuple[str, PyExpr]]): ...
     def select_with_projection(self, columns: Tuple[str, str]): ...
     def limit(self, limit: int): ...
     def offset(self, offset: int): ...
@@ -325,9 +284,7 @@ class VectorQuery:
 
 class HybridQuery:
     def where(self, filter: str): ...
-    def where_expr(self, expr: PyExpr): ...
+    def select(self, columns: List[str]): ...
-    def select(self, columns: List[Tuple[str, str]]): ...
-    def select_expr(self, columns: List[Tuple[str, PyExpr]]): ...
     def limit(self, limit: int): ...
     def offset(self, offset: int): ...
     def fast_search(self): ...
@@ -445,7 +402,7 @@ class AsyncPermutationBuilder:
     async def execute(self) -> Table: ...
 
 def async_permutation_builder(
-    table: Table,
+    table: Table, dest_table_name: str
 ) -> AsyncPermutationBuilder: ...
 def fts_query_to_json(query: Any) -> str: ...
 

python/python/lancedb/background_loop.py

@@ -2,9 +2,7 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 import asyncio
-import os
 import threading
-import warnings
 
 
 class BackgroundEventLoop:
@@ -15,9 +13,6 @@ class BackgroundEventLoop:
     """
 
     def __init__(self):
-        self._start()
-
-    def _start(self):
         self.loop = asyncio.new_event_loop()
         self.thread = threading.Thread(
             target=self.loop.run_forever,
@@ -36,30 +31,3 @@ class BackgroundEventLoop:
 
 
 LOOP = BackgroundEventLoop()
-
-_FORK_WARNED = False
-
-
-def _reset_after_fork():
-    # Threads do not survive fork(), so the asyncio loop in LOOP.thread is
-    # dead in the child. Re-initialize the singleton in place so existing
-    # `from .background_loop import LOOP` references in other modules see
-    # the new state. The Rust-side tokio runtime is reset analogously by a
-    # pthread_atfork hook installed in the _lancedb extension.
-    LOOP._start()
-    global _FORK_WARNED
-    if not _FORK_WARNED:
-        _FORK_WARNED = True
-        warnings.warn(
-            "lancedb fork support is experimental: the internal async "
-            "runtime has been reset in the forked child, but a small chance "
-            "of deadlock remains if other state was mid-operation at fork "
-            "time. The 'forkserver' or 'spawn' multiprocessing start method "
-            "is likely a safer alternative.",
-            RuntimeWarning,
-            stacklevel=2,
-        )
-
-
-if hasattr(os, "register_at_fork"):
-    os.register_at_fork(after_in_child=_reset_after_fork)

python/python/lancedb/common.py

@@ -96,7 +96,7 @@ def data_to_reader(
             f"Unknown data type {type(data)}. "
             "Supported types: list of dicts, pandas DataFrame, polars DataFrame, "
             "pyarrow Table/RecordBatch, or Pydantic models. "
-            "See https://docs.lancedb.com/tables/ for examples."
+            "See https://lancedb.com/docs/tables/ for examples."
         )
 
 

python/python/lancedb/embeddings/gemini_text.py

@@ -19,10 +19,10 @@ from .utils import TEXT, api_key_not_found_help
 @register("gemini-text")
 class GeminiText(TextEmbeddingFunction):
     """
-    An embedding function that uses Google's Gemini API. Requires GOOGLE_API_KEY to
+    An embedding function that uses the Google's Gemini API. Requires GOOGLE_API_KEY to
     be set.
 
-    https://ai.google.dev/gemini-api/docs/embeddings
+    https://ai.google.dev/docs/embeddings_guide
 
     Supports various tasks types:
     | Task Type               | Description                                            |
@@ -46,12 +46,9 @@ class GeminiText(TextEmbeddingFunction):
 
     Parameters
     ----------
-    name: str, default "gemini-embedding-001"
+    name: str, default "models/embedding-001"
-        The name of the model to use. Supported models include:
+        The name of the model to use. See the Gemini documentation for a list of
-        - "gemini-embedding-001" (768 dimensions)
+        available models.
-
-        Note: The legacy "models/embedding-001" format is also supported but
-        "gemini-embedding-001" is recommended.
 
     query_task_type: str, default "retrieval_query"
         Sets the task type for the queries.
@@ -80,7 +77,7 @@ class GeminiText(TextEmbeddingFunction):
 
     """
 
-    name: str = "gemini-embedding-001"
+    name: str = "models/embedding-001"
     query_task_type: str = "retrieval_query"
     source_task_type: str = "retrieval_document"
 
@@ -117,48 +114,23 @@ class GeminiText(TextEmbeddingFunction):
         texts: list[str] or np.ndarray (of str)
             The texts to embed
         """
-        from google.genai import types
+        if (
+            kwargs.get("task_type") == "retrieval_document"
+        ):  # Provide a title to use existing API design
+            title = "Embedding of a document"
+            kwargs["title"] = title
 
-        task_type = kwargs.get("task_type")
+        return [
-
+            self.client.embed_content(model=self.name, content=text, **kwargs)[
-        # Build content objects for embed_content
+                "embedding"
-        contents = []
+            ]
-        for text in texts:
+            for text in texts
-            if task_type == "retrieval_document":
+        ]
-                # Provide a title for retrieval_document task
-                contents.append(
-                    {"parts": [{"text": "Embedding of a document"}, {"text": text}]}
-                )
-            else:
-                contents.append({"parts": [{"text": text}]})
-
-        # Build config
-        config_kwargs = {}
-        if task_type:
-            config_kwargs["task_type"] = task_type.upper()  # API expects uppercase
-
-        # Call embed_content for each content
-        embeddings = []
-        for content in contents:
-            config = (
-                types.EmbedContentConfig(**config_kwargs) if config_kwargs else None
-            )
-            response = self.client.models.embed_content(
-                model=self.name,
-                contents=content,
-                config=config,
-            )
-            embeddings.append(response.embeddings[0].values)
-
-        return embeddings
 
     @cached_property
     def client(self):
-        attempt_import_or_raise("google.genai", "google-genai")
+        genai = attempt_import_or_raise("google.generativeai", "google.generativeai")
 
         if not os.environ.get("GOOGLE_API_KEY"):
             api_key_not_found_help("google")
-
+        return genai
-        from google import genai as genai_module
-
-        return genai_module.Client(api_key=os.environ.get("GOOGLE_API_KEY"))

python/python/lancedb/embeddings/utils.py

@@ -10,7 +10,6 @@ import sys
 import threading
 import time
 import urllib.error
-import urllib.request
 import weakref
 import logging
 from functools import wraps

python/python/lancedb/expr.py

@@ -1,298 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-"""Type-safe expression builder for filters and projections.
-
-Instead of writing raw SQL strings you can build expressions with Python
-operators::
-
-    from lancedb.expr import col, lit
-
-    # filter: age > 18 AND status = 'active'
-    filt = (col("age") > lit(18)) & (col("status") == lit("active"))
-
-    # projection: compute a derived column
-    proj = {"score": col("raw_score") * lit(1.5)}
-
-    table.search().where(filt).select(proj).to_list()
-"""
-
-from __future__ import annotations
-
-from typing import Union
-
-import pyarrow as pa
-
-from lancedb._lancedb import PyExpr, expr_col, expr_lit, expr_func
-
-__all__ = ["Expr", "col", "lit", "func"]
-
-_STR_TO_PA_TYPE: dict = {
-    "bool": pa.bool_(),
-    "boolean": pa.bool_(),
-    "int8": pa.int8(),
-    "int16": pa.int16(),
-    "int32": pa.int32(),
-    "int64": pa.int64(),
-    "uint8": pa.uint8(),
-    "uint16": pa.uint16(),
-    "uint32": pa.uint32(),
-    "uint64": pa.uint64(),
-    "float16": pa.float16(),
-    "float32": pa.float32(),
-    "float": pa.float32(),
-    "float64": pa.float64(),
-    "double": pa.float64(),
-    "string": pa.string(),
-    "utf8": pa.string(),
-    "str": pa.string(),
-    "large_string": pa.large_utf8(),
-    "large_utf8": pa.large_utf8(),
-    "date32": pa.date32(),
-    "date": pa.date32(),
-    "date64": pa.date64(),
-}
-
-
-def _coerce(value: "ExprLike") -> "Expr":
-    """Return *value* as an :class:`Expr`, wrapping plain Python values via
-    :func:`lit` if needed."""
-    if isinstance(value, Expr):
-        return value
-    return lit(value)
-
-
-# Type alias used in annotations.
-ExprLike = Union["Expr", bool, int, float, str]
-
-
-class Expr:
-    """A type-safe expression node.
-
-    Construct instances with :func:`col` and :func:`lit`, then combine them
-    using Python operators or the named methods below.
-
-    Examples
-    --------
-    >>> from lancedb.expr import col, lit
-    >>> filt = (col("age") > lit(18)) & (col("name").lower() == lit("alice"))
-    >>> proj = {"double": col("x") * lit(2)}
-    """
-
-    # Make Expr unhashable so that == returns an Expr rather than being used
-    # for dict keys / set membership.
-    __hash__ = None  # type: ignore[assignment]
-
-    def __init__(self, inner: PyExpr) -> None:
-        self._inner = inner
-
-    # ── comparisons ──────────────────────────────────────────────────────────
-
-    def __eq__(self, other: ExprLike) -> "Expr":  # type: ignore[override]
-        """Equal to (``col("x") == 1``)."""
-        return Expr(self._inner.eq(_coerce(other)._inner))
-
-    def __ne__(self, other: ExprLike) -> "Expr":  # type: ignore[override]
-        """Not equal to (``col("x") != 1``)."""
-        return Expr(self._inner.ne(_coerce(other)._inner))
-
-    def __lt__(self, other: ExprLike) -> "Expr":
-        """Less than (``col("x") < 1``)."""
-        return Expr(self._inner.lt(_coerce(other)._inner))
-
-    def __le__(self, other: ExprLike) -> "Expr":
-        """Less than or equal to (``col("x") <= 1``)."""
-        return Expr(self._inner.lte(_coerce(other)._inner))
-
-    def __gt__(self, other: ExprLike) -> "Expr":
-        """Greater than (``col("x") > 1``)."""
-        return Expr(self._inner.gt(_coerce(other)._inner))
-
-    def __ge__(self, other: ExprLike) -> "Expr":
-        """Greater than or equal to (``col("x") >= 1``)."""
-        return Expr(self._inner.gte(_coerce(other)._inner))
-
-    # ── logical ──────────────────────────────────────────────────────────────
-
-    def __and__(self, other: "Expr") -> "Expr":
-        """Logical AND (``expr_a & expr_b``)."""
-        return Expr(self._inner.and_(_coerce(other)._inner))
-
-    def __or__(self, other: "Expr") -> "Expr":
-        """Logical OR (``expr_a | expr_b``)."""
-        return Expr(self._inner.or_(_coerce(other)._inner))
-
-    def __invert__(self) -> "Expr":
-        """Logical NOT (``~expr``)."""
-        return Expr(self._inner.not_())
-
-    # ── arithmetic ───────────────────────────────────────────────────────────
-
-    def __add__(self, other: ExprLike) -> "Expr":
-        """Add (``col("x") + 1``)."""
-        return Expr(self._inner.add(_coerce(other)._inner))
-
-    def __radd__(self, other: ExprLike) -> "Expr":
-        """Right-hand add (``1 + col("x")``)."""
-        return Expr(_coerce(other)._inner.add(self._inner))
-
-    def __sub__(self, other: ExprLike) -> "Expr":
-        """Subtract (``col("x") - 1``)."""
-        return Expr(self._inner.sub(_coerce(other)._inner))
-
-    def __rsub__(self, other: ExprLike) -> "Expr":
-        """Right-hand subtract (``1 - col("x")``)."""
-        return Expr(_coerce(other)._inner.sub(self._inner))
-
-    def __mul__(self, other: ExprLike) -> "Expr":
-        """Multiply (``col("x") * 2``)."""
-        return Expr(self._inner.mul(_coerce(other)._inner))
-
-    def __rmul__(self, other: ExprLike) -> "Expr":
-        """Right-hand multiply (``2 * col("x")``)."""
-        return Expr(_coerce(other)._inner.mul(self._inner))
-
-    def __truediv__(self, other: ExprLike) -> "Expr":
-        """Divide (``col("x") / 2``)."""
-        return Expr(self._inner.div(_coerce(other)._inner))
-
-    def __rtruediv__(self, other: ExprLike) -> "Expr":
-        """Right-hand divide (``1 / col("x")``)."""
-        return Expr(_coerce(other)._inner.div(self._inner))
-
-    # ── string methods ───────────────────────────────────────────────────────
-
-    def lower(self) -> "Expr":
-        """Convert string column values to lowercase."""
-        return Expr(self._inner.lower())
-
-    def upper(self) -> "Expr":
-        """Convert string column values to uppercase."""
-        return Expr(self._inner.upper())
-
-    def contains(self, substr: "ExprLike") -> "Expr":
-        """Return True where the string contains *substr*."""
-        return Expr(self._inner.contains(_coerce(substr)._inner))
-
-    # ── type cast ────────────────────────────────────────────────────────────
-
-    def cast(self, data_type: Union[str, "pa.DataType"]) -> "Expr":
-        """Cast values to *data_type*.
-
-        Parameters
-        ----------
-        data_type:
-            A PyArrow ``DataType`` (e.g. ``pa.int32()``) or one of the type
-            name strings: ``"bool"``, ``"int8"``, ``"int16"``, ``"int32"``,
-            ``"int64"``, ``"uint8"``–``"uint64"``, ``"float32"``,
-            ``"float64"``, ``"string"``, ``"date32"``, ``"date64"``.
-        """
-        if isinstance(data_type, str):
-            try:
-                data_type = _STR_TO_PA_TYPE[data_type]
-            except KeyError:
-                raise ValueError(
-                    f"unsupported data type: '{data_type}'. Supported: "
-                    f"{', '.join(_STR_TO_PA_TYPE)}"
-                )
-        return Expr(self._inner.cast(data_type))
-
-    # ── named comparison helpers (alternative to operators) ──────────────────
-
-    def eq(self, other: ExprLike) -> "Expr":
-        """Equal to."""
-        return self.__eq__(other)
-
-    def ne(self, other: ExprLike) -> "Expr":
-        """Not equal to."""
-        return self.__ne__(other)
-
-    def lt(self, other: ExprLike) -> "Expr":
-        """Less than."""
-        return self.__lt__(other)
-
-    def lte(self, other: ExprLike) -> "Expr":
-        """Less than or equal to."""
-        return self.__le__(other)
-
-    def gt(self, other: ExprLike) -> "Expr":
-        """Greater than."""
-        return self.__gt__(other)
-
-    def gte(self, other: ExprLike) -> "Expr":
-        """Greater than or equal to."""
-        return self.__ge__(other)
-
-    def and_(self, other: "Expr") -> "Expr":
-        """Logical AND."""
-        return self.__and__(other)
-
-    def or_(self, other: "Expr") -> "Expr":
-        """Logical OR."""
-        return self.__or__(other)
-
-    # ── utilities ────────────────────────────────────────────────────────────
-
-    def to_sql(self) -> str:
-        """Render the expression as a SQL string (useful for debugging)."""
-        return self._inner.to_sql()
-
-    def __repr__(self) -> str:
-        return f"Expr({self._inner.to_sql()})"
-
-
-# ── free functions ────────────────────────────────────────────────────────────
-
-
-def col(name: str) -> Expr:
-    """Reference a table column by name.
-
-    Parameters
-    ----------
-    name:
-        The column name.
-
-    Examples
-    --------
-    >>> from lancedb.expr import col, lit
-    >>> col("age") > lit(18)
-    Expr((age > 18))
-    """
-    return Expr(expr_col(name))
-
-
-def lit(value: Union[bool, int, float, str]) -> Expr:
-    """Create a literal (constant) value expression.
-
-    Parameters
-    ----------
-    value:
-        A Python ``bool``, ``int``, ``float``, or ``str``.
-
-    Examples
-    --------
-    >>> from lancedb.expr import col, lit
-    >>> col("price") * lit(1.1)
-    Expr((price * 1.1))
-    """
-    return Expr(expr_lit(value))
-
-
-def func(name: str, *args: ExprLike) -> Expr:
-    """Call an arbitrary SQL function by name.
-
-    Parameters
-    ----------
-    name:
-        The SQL function name (e.g. ``"lower"``, ``"upper"``).
-    *args:
-        The function arguments as :class:`Expr` or plain Python literals.
-
-    Examples
-    --------
-    >>> from lancedb.expr import col, func
-    >>> func("lower", col("name"))
-    Expr(lower(name))
-    """
-    inner_args = [_coerce(a)._inner for a in args]
-    return Expr(expr_func(name, inner_args))

python/python/lancedb/fts.py

@@ -0,0 +1,201 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+"""Full text search index using tantivy-py"""
+
+import os
+from typing import List, Tuple, Optional
+
+import pyarrow as pa
+
+try:
+    import tantivy
+except ImportError:
+    raise ImportError(
+        "Please install tantivy-py `pip install tantivy` to use the full text search feature."  # noqa: E501
+    )
+
+from .table import LanceTable
+
+
+def create_index(
+    index_path: str,
+    text_fields: List[str],
+    ordering_fields: Optional[List[str]] = None,
+    tokenizer_name: str = "default",
+) -> tantivy.Index:
+    """
+    Create a new Index (not populated)
+
+    Parameters
+    ----------
+    index_path : str
+        Path to the index directory
+    text_fields : List[str]
+        List of text fields to index
+    ordering_fields: List[str]
+        List of unsigned type fields to order by at search time
+    tokenizer_name : str, default "default"
+        The tokenizer to use
+
+    Returns
+    -------
+    index : tantivy.Index
+        The index object (not yet populated)
+    """
+    if ordering_fields is None:
+        ordering_fields = []
+    # Declaring our schema.
+    schema_builder = tantivy.SchemaBuilder()
+    # special field that we'll populate with row_id
+    schema_builder.add_integer_field("doc_id", stored=True)
+    # data fields
+    for name in text_fields:
+        schema_builder.add_text_field(name, stored=True, tokenizer_name=tokenizer_name)
+    if ordering_fields:
+        for name in ordering_fields:
+            schema_builder.add_unsigned_field(name, fast=True)
+    schema = schema_builder.build()
+    os.makedirs(index_path, exist_ok=True)
+    index = tantivy.Index(schema, path=index_path)
+    return index
+
+
+def populate_index(
+    index: tantivy.Index,
+    table: LanceTable,
+    fields: List[str],
+    writer_heap_size: Optional[int] = None,
+    ordering_fields: Optional[List[str]] = None,
+) -> int:
+    """
+    Populate an index with data from a LanceTable
+
+    Parameters
+    ----------
+    index : tantivy.Index
+        The index object
+    table : LanceTable
+        The table to index
+    fields : List[str]
+        List of fields to index
+    writer_heap_size : int
+        The writer heap size in bytes, defaults to 1GB
+
+    Returns
+    -------
+    int
+        The number of rows indexed
+    """
+    if ordering_fields is None:
+        ordering_fields = []
+    writer_heap_size = writer_heap_size or 1024 * 1024 * 1024
+    # first check the fields exist and are string or large string type
+    nested = []
+
+    for name in fields:
+        try:
+            f = table.schema.field(name)  # raises KeyError if not found
+        except KeyError:
+            f = resolve_path(table.schema, name)
+            nested.append(name)
+
+        if not pa.types.is_string(f.type) and not pa.types.is_large_string(f.type):
+            raise TypeError(f"Field {name} is not a string type")
+
+    # create a tantivy writer
+    writer = index.writer(heap_size=writer_heap_size)
+    # write data into index
+    dataset = table.to_lance()
+    row_id = 0
+
+    max_nested_level = 0
+    if len(nested) > 0:
+        max_nested_level = max([len(name.split(".")) for name in nested])
+
+    for b in dataset.to_batches(columns=fields + ordering_fields):
+        if max_nested_level > 0:
+            b = pa.Table.from_batches([b])
+            for _ in range(max_nested_level - 1):
+                b = b.flatten()
+        for i in range(b.num_rows):
+            doc = tantivy.Document()
+            for name in fields:
+                value = b[name][i].as_py()
+                if value is not None:
+                    doc.add_text(name, value)
+            for name in ordering_fields:
+                value = b[name][i].as_py()
+                if value is not None:
+                    doc.add_unsigned(name, value)
+            if not doc.is_empty:
+                doc.add_integer("doc_id", row_id)
+                writer.add_document(doc)
+            row_id += 1
+    # commit changes
+    writer.commit()
+    return row_id
+
+
+def resolve_path(schema, field_name: str) -> pa.Field:
+    """
+    Resolve a nested field path to a list of field names
+
+    Parameters
+    ----------
+    field_name : str
+        The field name to resolve
+
+    Returns
+    -------
+    List[str]
+        The resolved path
+    """
+    path = field_name.split(".")
+    field = schema.field(path.pop(0))
+    for segment in path:
+        if pa.types.is_struct(field.type):
+            field = field.type.field(segment)
+        else:
+            raise KeyError(f"field {field_name} not found in schema {schema}")
+    return field
+
+
+def search_index(
+    index: tantivy.Index, query: str, limit: int = 10, ordering_field=None
+) -> Tuple[Tuple[int], Tuple[float]]:
+    """
+    Search an index for a query
+
+    Parameters
+    ----------
+    index : tantivy.Index
+        The index object
+    query : str
+        The query string
+    limit : int
+        The maximum number of results to return
+
+    Returns
+    -------
+    ids_and_score: list[tuple[int], tuple[float]]
+        A tuple of two tuples, the first containing the document ids
+        and the second containing the scores
+    """
+    searcher = index.searcher()
+    query = index.parse_query(query)
+    # get top results
+    if ordering_field:
+        results = searcher.search(query, limit, order_by_field=ordering_field)
+    else:
+        results = searcher.search(query, limit)
+    if results.count == 0:
+        return tuple(), tuple()
+    return tuple(
+        zip(
+            *[
+                (searcher.doc(doc_address)["doc_id"][0], score)
+                for score, doc_address in results.hits
+            ]
+        )
+    )

python/python/lancedb/index.py

@@ -7,7 +7,6 @@ from typing import Literal, Optional
 from ._lancedb import (
     IndexConfig,
 )
-from .types import BaseTokenizerType
 
 lang_mapping = {
     "ar": "Arabic",
@@ -112,12 +111,8 @@ class FTS:
         - "simple": Splits text by whitespace and punctuation.
         - "whitespace": Split text by whitespace, but not punctuation.
         - "raw": No tokenization. The entire text is treated as a single token.
-        - "ngram": N-gram tokenizer for substring-style matching.
-        - "jieba/*": Jieba tokenizer loaded from Lance's language model home.
-        - "lindera/*": Lindera tokenizer loaded from Lance's language model home.
     language : str, default "English"
-        The language to use for stemming and stop-word removal. This is not the
+        The language to use for tokenization.
-        primary way to enable CJK tokenization.
     max_token_length : int, default 40
         The maximum token length to index. Tokens longer than this length will be
         ignored.
@@ -132,17 +127,10 @@ class FTS:
     ascii_folding : bool, default True
         Whether to fold ASCII characters. This converts accented characters to
         their ASCII equivalent. For example, "café" would be converted to "cafe".
-
-    Notes
-    -----
-    Model-backed tokenizers such as ``jieba/default`` and ``lindera/ipadic``
-    require tokenizer models in Lance's language model home. Set
-    ``LANCE_LANGUAGE_MODEL_HOME`` to override the default platform data
-    directory under ``lance/language_models``.
     """
 
     with_position: bool = False
-    base_tokenizer: BaseTokenizerType = "simple"
+    base_tokenizer: Literal["simple", "raw", "whitespace"] = "simple"
     language: str = "English"
     max_token_length: Optional[int] = 40
     lower_case: bool = True
@@ -388,98 +376,9 @@ class HnswSq:
     target_partition_size: Optional[int] = None
 
 
-@dataclass
-class HnswFlat:
-    """Describe a HNSW-FLAT index configuration.
-
-    HNSW-FLAT stands for Hierarchical Navigable Small World without quantization.
-    It stores raw vectors in the HNSW graph, providing the highest recall among
-    the IVF_HNSW family at the cost of more memory and disk space compared to
-    :class:`HnswSq` or :class:`HnswPq`.
-
-    Parameters
-    ----------
-
-    distance_type: str, default "l2"
-
-        The distance metric used to train the index.
-
-        The following distance types are available:
-
-        "l2" - Euclidean distance. This is a very common distance metric that
-        accounts for both magnitude and direction when determining the distance
-        between vectors. l2 distance has a range of [0, ∞).
-
-        "cosine" - Cosine distance.  Cosine distance is a distance metric
-        calculated from the cosine similarity between two vectors. Cosine
-        similarity is a measure of similarity between two non-zero vectors of an
-        inner product space. It is defined to equal the cosine of the angle
-        between them.  Unlike l2, the cosine distance is not affected by the
-        magnitude of the vectors.  Cosine distance has a range of [0, 2].
-
-        "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
-        distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
-        l2 norm is 1), then dot distance is equivalent to the cosine distance.
-
-    num_partitions, default sqrt(num_rows)
-
-        The number of IVF partitions to create.
-
-        For HNSW, we recommend a small number of partitions. Setting this to 1
-        works well for most tables. For very large tables, training just one HNSW
-        graph will require too much memory. Each partition becomes its own HNSW
-        graph, so setting this value higher reduces the peak memory use of
-        training.
-
-    max_iterations, default 50
-
-        Max iterations to train kmeans.
-
-        When training an IVF index we use kmeans to calculate the partitions.
-        This parameter controls how many iterations of kmeans to run.
-
-    sample_rate, default 256
-
-        The rate used to calculate the number of training vectors for kmeans.
-
-    m, default 20
-
-        The number of neighbors to select for each vector in the HNSW graph.
-
-        This value controls the tradeoff between search speed and accuracy.
-        The higher the value the more accurate the search but the slower it
-        will be.
-
-    ef_construction, default 300
-
-        The number of candidates to evaluate during the construction of the HNSW
-        graph.
-
-        This value controls the tradeoff between build speed and accuracy.
-        The higher the value the more accurate the build but the slower it will
-        be.  150 to 300 is the typical range. 100 is a minimum for good quality
-        search results. In most cases, there is no benefit to setting this higher
-        than 500.  This value should be set to a value that is not less than `ef`
-        in the search phase.
-
-    target_partition_size, default is 1,048,576
-
-        The target size of each partition.
-    """
-
-    distance_type: Literal["l2", "cosine", "dot"] = "l2"
-    num_partitions: Optional[int] = None
-    max_iterations: int = 50
-    sample_rate: int = 256
-    m: int = 20
-    ef_construction: int = 300
-    target_partition_size: Optional[int] = None
-
-
 # Backwards-compatible aliases
 IvfHnswPq = HnswPq
 IvfHnswSq = HnswSq
-IvfHnswFlat = HnswFlat
 
 
 @dataclass
@@ -799,13 +698,11 @@ __all__ = [
     "IvfPq",
     "IvfHnswPq",
     "IvfHnswSq",
-    "IvfHnswFlat",
     "IvfSq",
     "IvfRq",
     "IvfFlat",
     "HnswPq",
     "HnswSq",
-    "HnswFlat",
     "IndexConfig",
     "FTS",
     "Bitmap",

python/python/lancedb/io.py

@@ -2,3 +2,70 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 """I/O utilities and interfaces for LanceDB."""
+
+from abc import ABC, abstractmethod
+from typing import Dict
+
+
+class StorageOptionsProvider(ABC):
+    """Abstract base class for providing storage options to LanceDB tables.
+
+    Storage options providers enable automatic credential refresh for cloud
+    storage backends (e.g., AWS S3, Azure Blob Storage, GCS). When credentials
+    have an expiration time, the provider's fetch_storage_options() method will
+    be called periodically to get fresh credentials before they expire.
+
+    Example
+    -------
+    >>> class MyProvider(StorageOptionsProvider):
+    ...     def fetch_storage_options(self) -> Dict[str, str]:
+    ...         # Fetch fresh credentials from your credential manager
+    ...         return {
+    ...             "aws_access_key_id": "...",
+    ...             "aws_secret_access_key": "...",
+    ...             "expires_at_millis": "1234567890000"  # Optional
+    ...         }
+    """
+
+    @abstractmethod
+    def fetch_storage_options(self) -> Dict[str, str]:
+        """Fetch fresh storage credentials.
+
+        This method is called by LanceDB when credentials need to be refreshed.
+        If the returned dictionary contains an "expires_at_millis" key with a
+        Unix timestamp in milliseconds, LanceDB will automatically refresh the
+        credentials before that time. If the key is not present, credentials
+        are assumed to not expire.
+
+        Returns
+        -------
+        Dict[str, str]
+            Dictionary containing cloud storage credentials and optionally an
+            expiration time:
+            - "expires_at_millis" (optional): Unix timestamp in milliseconds when
+              credentials expire
+            - Provider-specific credential keys (e.g., aws_access_key_id,
+              aws_secret_access_key, etc.)
+
+        Raises
+        ------
+        RuntimeError
+            If credentials cannot be fetched or are invalid
+        """
+        pass
+
+    def provider_id(self) -> str:
+        """Return a human-readable unique identifier for this provider instance.
+
+        This identifier is used for caching and equality comparison. Two providers
+        with the same ID will share the same cached object store connection.
+
+        The default implementation uses the class name and string representation.
+        Override this method if you need custom identification logic.
+
+        Returns
+        -------
+        str
+            A unique identifier for this provider instance
+        """
+        return f"{self.__class__.__name__} {{ repr: {str(self)!r} }}"

python/python/lancedb/permutation.py

@@ -1,11 +1,10 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-import copy
-import json
-
 from deprecation import deprecated
+from lancedb import AsyncConnection, DBConnection
 import pyarrow as pa
+import json
 
 from ._lancedb import async_permutation_builder, PermutationReader
 from .table import LanceTable
@@ -37,7 +36,10 @@ class PermutationBuilder:
     be referenced by name in the future.  If names are not provided then they can only
     be referenced by their ordinal index.  There is no requirement to name every split.
 
-    The permutation is stored in memory and will be lost when the program exits.
+    By default, the permutation will be stored in memory and will be lost when the
+    program exits.  To persist the permutation (for very large datasets or to share
+    the permutation across multiple workers) use the [persist](#persist) method to
+    create a permanent table.
     """
 
     def __init__(self, table: LanceTable):
@@ -49,6 +51,15 @@ class PermutationBuilder:
         """
         self._async = async_permutation_builder(table)
 
+    def persist(
+        self, database: Union[DBConnection, AsyncConnection], table_name: str
+    ) -> "PermutationBuilder":
+        """
+        Persist the permutation to the given database.
+        """
+        self._async.persist(database, table_name)
+        return self
+
     def split_random(
         self,
         *,
@@ -273,8 +284,9 @@ class Permutations:
         self.permutation_table = permutation_table
 
         if permutation_table.schema.metadata is not None:
-            raw = permutation_table.schema.metadata.get(b"split_names")
+            split_names = permutation_table.schema.metadata.get(
-            split_names = raw.decode("utf-8") if raw is not None else None
+                b"split_names", None
+            ).decode("utf-8")
             if split_names is not None:
                 self.split_names = json.loads(split_names)
                 self.split_dict = {
@@ -369,44 +381,20 @@ class Permutation:
 
     def __init__(
         self,
-        base_table: LanceTable,
+        reader: PermutationReader,
-        permutation_table: Optional[LanceTable],
-        split: int,
         selection: dict[str, str],
         batch_size: int,
         transform_fn: Callable[pa.RecordBatch, Any],
-        offset: Optional[int] = None,
-        limit: Optional[int] = None,
-        connection_factory: Optional[Callable[[str], LanceTable]] = None,
-        _reader: Optional[PermutationReader] = None,
     ):
         """
         Internal constructor.  Use [from_tables](#from_tables) instead.
         """
-        assert base_table is not None, "base_table is required"
+        assert reader is not None, "reader is required"
         assert selection is not None, "selection is required"
-        self.base_table = base_table
+        self.reader = reader
-        self.permutation_table = permutation_table
-        self.split = split
         self.selection = selection
         self.transform_fn = transform_fn
         self.batch_size = batch_size
-        self.offset = offset
-        self.limit = limit
-        self.connection_factory = connection_factory
-        if _reader is None:
-            _reader = LOOP.run(self._build_reader())
-        self.reader: PermutationReader = _reader
-
-    async def _build_reader(self) -> PermutationReader:
-        reader = await PermutationReader.from_tables(
-            self.base_table, self.permutation_table, self.split
-        )
-        if self.offset is not None:
-            reader = await reader.with_offset(self.offset)
-        if self.limit is not None:
-            reader = await reader.with_limit(self.limit)
-        return reader
 
     def _with_selection(self, selection: dict[str, str]) -> "Permutation":
         """
@@ -415,97 +403,21 @@ class Permutation:
         Does not validation of the selection and it replaces it entirely.  This is not
         intended for public use.
         """
-        new = copy.copy(self)
+        return Permutation(self.reader, selection, self.batch_size, self.transform_fn)
-        new.selection = selection
+
-        return new
+    def _with_reader(self, reader: PermutationReader) -> "Permutation":
+        """
+        Creates a new permutation with the given reader
+
+        This is an internal method and should not be used directly.
+        """
+        return Permutation(reader, self.selection, self.batch_size, self.transform_fn)
 
     def with_batch_size(self, batch_size: int) -> "Permutation":
         """
         Creates a new permutation with the given batch size
         """
-        new = copy.copy(self)
+        return Permutation(self.reader, self.selection, batch_size, self.transform_fn)
-        new.batch_size = batch_size
-        return new
-
-    def with_connection_factory(
-        self, connection_factory: Callable[[str], LanceTable]
-    ) -> "Permutation":
-        """
-        Creates a new permutation that will use ``connection_factory`` to reopen
-        the base table when this permutation is unpickled in a worker process.
-
-        The factory is a callable that takes a single argument — the base table
-        name — and returns a [LanceTable]. It must be picklable; the worker
-        will pickle it via standard ``pickle`` and call it to recover the base
-        table. Picklable callables in practice means top-level (module-level)
-        functions, ``functools.partial`` of such functions, or instances of
-        picklable classes implementing ``__call__``. Lambdas and closures over
-        local variables don't pickle with the default protocol.
-
-        Setting a factory is necessary when the URI alone is not enough to
-        re-open the connection — most importantly for LanceDB Cloud (``db://``)
-        connections, where ``api_key`` and ``region`` aren't recoverable from
-        the connection object after construction.
-
-        For local file or cloud-storage paths the factory is optional: if not
-        set, ``__getstate__`` falls back to capturing
-        ``(uri, storage_options, namespace_path)`` and re-opening via
-        ``lancedb.connect(uri, storage_options=...)``.
-
-        Examples
-        --------
-        Basic native (file-system path), parameterized via ``functools.partial``::
-
-            import functools, lancedb
-            from lancedb.permutation import Permutation
-
-            def open_native_table(uri: str, table_name: str):
-                return lancedb.connect(uri).open_table(table_name)
-
-            factory = functools.partial(open_native_table, "/data/lance_db")
-            permutation = Permutation.identity(
-                factory("training")
-            ).with_connection_factory(factory)
-
-        Native via :func:`lancedb.connect_namespace` (e.g. a directory- or
-        REST-backed namespace client). The factory takes the
-        implementation name and properties dict as partial-bound args so
-        the worker can rebuild the same namespace connection::
-
-            def open_via_namespace(
-                impl: str, properties: dict[str, str], table_name: str,
-            ):
-                return lancedb.connect_namespace(impl, properties).open_table(
-                    table_name,
-                )
-
-            factory = functools.partial(
-                open_via_namespace,
-                "dir",
-                {"root": "/data/lance_db"},
-            )
-
-        LanceDB Cloud, reading credentials from env vars at worker startup
-        so secrets aren't pickled into the dataset::
-
-            import os, lancedb
-
-            def open_remote_table(table_name: str):
-                db = lancedb.connect(
-                    "db://my-database",
-                    api_key=os.environ["LANCEDB_API_KEY"],
-                    region=os.environ.get("LANCEDB_REGION", "us-east-1"),
-                )
-                return db.open_table(table_name)
-
-            permutation = Permutation.identity(
-                open_remote_table("training")
-            ).with_connection_factory(open_remote_table)
-        """
-        assert connection_factory is not None, "connection_factory is required"
-        new = copy.copy(self)
-        new.connection_factory = connection_factory
-        return new
 
     @classmethod
     def identity(cls, table: LanceTable) -> "Permutation":
@@ -548,8 +460,9 @@ class Permutation:
                         f"Cannot create a permutation on split `{split}`"
                         " because no split names are defined in the permutation table"
                     )
-                raw = permutation_table.schema.metadata.get(b"split_names")
+                split_names = permutation_table.schema.metadata.get(
-                split_names = raw.decode("utf-8") if raw is not None else None
+                    b"split_names", None
+                ).decode("utf-8")
                 if split_names is None:
                     raise ValueError(
                         f"Cannot create a permutation on split `{split}`"
@@ -578,126 +491,11 @@ class Permutation:
             schema = await reader.output_schema(None)
             initial_selection = {name: name for name in schema.names}
             return cls(
-                base_table,
+                reader, initial_selection, DEFAULT_BATCH_SIZE, Transforms.arrow2python
-                permutation_table,
-                split,
-                initial_selection,
-                DEFAULT_BATCH_SIZE,
-                Transforms.arrow2python,
-                _reader=reader,
             )
 
         return LOOP.run(do_from_tables())
 
-    def __getstate__(self) -> dict[str, Any]:
-        """Build a picklable state dict for this permutation.
-
-        The base table is captured either via a user-supplied
-        ``connection_factory`` (see [with_connection_factory]) or, as a
-        fallback, by introspecting ``(uri, storage_options, namespace_path)``
-        on the connection. The permutation table — always an in-memory
-        LanceDB table — is captured as a pyarrow Table (which pickles via
-        Arrow IPC natively). The reader is dropped from the wire format;
-        ``__setstate__`` rebuilds it from the restored tables.
-        """
-        permutation_data: Optional[pa.Table] = None
-        if self.permutation_table is not None:
-            permutation_data = self.permutation_table.to_arrow()
-
-        common = {
-            "base_table_name": self.base_table.name,
-            "permutation_data": permutation_data,
-            "split": self.split,
-            "selection": self.selection,
-            "batch_size": self.batch_size,
-            "transform_fn": self.transform_fn,
-            "offset": self.offset,
-            "limit": self.limit,
-            "connection_factory": self.connection_factory,
-        }
-
-        if self.connection_factory is not None:
-            # The factory carries enough state to recover the base table on
-            # its own; we don't need to capture the URI / storage options /
-            # namespace from the existing connection.
-            return common
-
-        # URI-introspection fallback: only viable for native (OSS) connections
-        # where (uri, storage_options) is enough to reopen. Remote / cloud
-        # connections don't expose recoverable api_key / region — those users
-        # must call with_connection_factory().
-        try:
-            base_uri = self.base_table._conn.uri
-            storage_options = self.base_table._conn.storage_options
-        except AttributeError as e:
-            raise ValueError(
-                "Cannot pickle this Permutation: the base table's connection "
-                "does not expose a uri/storage_options, which usually means it "
-                "is a remote (LanceDB Cloud) connection. Call "
-                "Permutation.with_connection_factory(...) first to provide a "
-                "picklable callable that re-opens the base table from a worker "
-                "process."
-            ) from e
-
-        if base_uri.startswith("memory://"):
-            # In-memory base tables don't exist in any worker process by
-            # default, so dump the entire base table into the pickle. This
-            # can be expensive for large datasets — users with large
-            # in-memory base tables should either persist them or set a
-            # connection_factory.
-            return {
-                **common,
-                "base_table_data": self.base_table.to_arrow(),
-            }
-
-        return {
-            **common,
-            "base_table_uri": base_uri,
-            "base_table_namespace": self.base_table._namespace_path,
-            "base_table_storage_options": storage_options,
-        }
-
-    def __setstate__(self, state: dict[str, Any]) -> None:
-        from . import connect
-
-        connection_factory = state["connection_factory"]
-        if connection_factory is not None:
-            base_table = connection_factory(state["base_table_name"])
-        elif "base_table_data" in state:
-            # In-memory base table inlined into the pickle; rebuild the same
-            # way we rebuild the in-memory permutation table.
-            mem_db = connect("memory://")
-            base_table = mem_db.create_table(
-                state["base_table_name"], state["base_table_data"]
-            )
-        else:
-            base_db = connect(
-                state["base_table_uri"],
-                storage_options=state["base_table_storage_options"],
-            )
-            base_table = base_db.open_table(
-                state["base_table_name"],
-                namespace_path=state["base_table_namespace"] or None,
-            )
-
-        permutation_table: Optional[LanceTable] = None
-        if state["permutation_data"] is not None:
-            mem_db = connect("memory://")
-            permutation_table = mem_db.create_table(
-                "permutation", state["permutation_data"]
-            )
-
-        self.base_table = base_table
-        self.permutation_table = permutation_table
-        self.split = state["split"]
-        self.selection = state["selection"]
-        self.batch_size = state["batch_size"]
-        self.transform_fn = state["transform_fn"]
-        self.offset = state["offset"]
-        self.limit = state["limit"]
-        self.connection_factory = connection_factory
-        self.reader = LOOP.run(self._build_reader())
-
     @property
     def schema(self) -> pa.Schema:
         async def do_output_schema():
@@ -964,9 +762,7 @@ class Permutation:
         for expensive operations such as image decoding.
         """
         assert transform is not None, "transform is required"
-        new = copy.copy(self)
+        return Permutation(self.reader, self.selection, self.batch_size, transform)
-        new.transform_fn = transform
-        return new
 
     def __getitem__(self, index: int) -> Any:
         """
@@ -1001,10 +797,12 @@ class Permutation:
         """
         Skip the first `skip` rows of the permutation
         """
-        new = copy.copy(self)
+
-        new.offset = skip
+        async def do_with_skip():
-        new.reader = LOOP.run(new._build_reader())
+            reader = await self.reader.with_offset(skip)
-        return new
+            return self._with_reader(reader)
+
+        return LOOP.run(do_with_skip())
 
     @deprecated(details="Use with_take instead")
     def take(self, limit: int) -> "Permutation":
@@ -1022,10 +820,12 @@ class Permutation:
         """
         Limit the permutation to `limit` rows (following any `skip`)
         """
-        new = copy.copy(self)
+
-        new.limit = limit
+        async def do_with_take():
-        new.reader = LOOP.run(new._build_reader())
+            reader = await self.reader.with_limit(limit)
-        return new
+            return self._with_reader(reader)
+
+        return LOOP.run(do_with_take())
 
     @deprecated(details="Use with_repeat instead")
     def repeat(self, times: int) -> "Permutation":

python/python/lancedb/pydantic.py

@@ -10,7 +10,6 @@ import sys
 import types
 from abc import ABC, abstractmethod
 from datetime import date, datetime
-from enum import Enum
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -315,19 +314,6 @@ def _pydantic_type_to_arrow_type(tp: Any, field: FieldInfo) -> pa.DataType:
                 return pa.list_(pa.list_(tp.value_arrow_type(), tp.dim()))
             # For regular Vector
             return pa.list_(tp.value_arrow_type(), tp.dim())
-        if _safe_issubclass(tp, Enum):
-            # Map Enum to the Arrow type of its value.
-            # For string-valued enums, use dictionary encoding for efficiency.
-            # For integer enums, use the native type.
-            # Fall back to utf8 for mixed-type or empty enums.
-            value_types = {type(m.value) for m in tp}
-            if len(value_types) == 1:
-                value_type = value_types.pop()
-                if value_type is str:
-                    # Use dictionary encoding for string enums
-                    return pa.dictionary(pa.int32(), pa.utf8())
-                return _py_type_to_arrow_type(value_type, field)
-            return pa.utf8()
     return _py_type_to_arrow_type(tp, field)
 
 

python/python/lancedb/query.py

@@ -25,6 +25,7 @@ import deprecation
 import numpy as np
 import pyarrow as pa
 import pyarrow.compute as pc
+import pyarrow.fs as pa_fs
 import pydantic
 
 from lancedb.pydantic import PYDANTIC_VERSION
@@ -37,7 +38,6 @@ from .rerankers.base import Reranker
 from .rerankers.rrf import RRFReranker
 from .rerankers.util import check_reranker_result
 from .util import flatten_columns
-from .expr import Expr
 from lancedb._lancedb import fts_query_to_json
 from typing_extensions import Annotated
 
@@ -70,7 +70,7 @@ def ensure_vector_query(
 ) -> Union[List[float], List[List[float]], pa.Array, List[pa.Array]]:
     if isinstance(val, list):
         if len(val) == 0:
-            raise ValueError("Vector query must be a non-empty list")
+            return ValueError("Vector query must be a non-empty list")
         sample = val[0]
     else:
         if isinstance(val, float):
@@ -83,7 +83,7 @@ def ensure_vector_query(
         return val
     if isinstance(sample, list):
         if len(sample) == 0:
-            raise ValueError("Vector query must be a non-empty list")
+            return ValueError("Vector query must be a non-empty list")
         if isinstance(sample[0], float):
             # val is list of list of floats
             return val
@@ -449,8 +449,8 @@ class Query(pydantic.BaseModel):
         ensure_vector_query,
     ] = None
 
-    # sql filter or type-safe Expr to refine the query with
+    # sql filter to refine the query with
-    filter: Optional[Union[str, Expr]] = None
+    filter: Optional[str] = None
 
     # if True then apply the filter after vector search
     postfilter: Optional[bool] = None
@@ -464,8 +464,8 @@ class Query(pydantic.BaseModel):
     # distance type to use for vector search
     distance_type: Optional[str] = None
 
-    # which columns to return in the results (dict values may be str or Expr)
+    # which columns to return in the results
-    columns: Optional[Union[List[str], Dict[str, Union[str, Expr]]]] = None
+    columns: Optional[Union[List[str], Dict[str, str]]] = None
 
     # minimum number of IVF partitions to search
     #
@@ -856,15 +856,14 @@ class LanceQueryBuilder(ABC):
             self._offset = offset
         return self
 
-    def select(self, columns: Union[list[str], dict[str, Union[str, Expr]]]) -> Self:
+    def select(self, columns: Union[list[str], dict[str, str]]) -> Self:
         """Set the columns to return.
 
         Parameters
         ----------
-        columns: list of str, or dict of str to str or Expr
+        columns: list of str, or dict of str to str default None
             List of column names to be fetched.
-            Or a dictionary of column names to SQL expressions or
+            Or a dictionary of column names to SQL expressions.
-            :class:`~lancedb.expr.Expr` objects.
             All columns are fetched if None or unspecified.
 
         Returns
@@ -878,15 +877,15 @@ class LanceQueryBuilder(ABC):
             raise ValueError("columns must be a list or a dictionary")
         return self
 
-    def where(self, where: Union[str, Expr], prefilter: bool = True) -> Self:
+    def where(self, where: str, prefilter: bool = True) -> Self:
         """Set the where clause.
 
         Parameters
         ----------
-        where: str or :class:`~lancedb.expr.Expr`
+        where: str
-            The filter condition.  Can be a SQL string or a type-safe
+            The where clause which is a valid SQL where clause. See
-            :class:`~lancedb.expr.Expr` built with :func:`~lancedb.expr.col`
+            `Lance filter pushdown <https://lance.org/guide/read_and_write#filter-push-down>`_
-            and :func:`~lancedb.expr.lit`.
+            for valid SQL expressions.
         prefilter: bool, default True
             If True, apply the filter before vector search, otherwise the
             filter is applied on the result of vector search.
@@ -1356,17 +1355,15 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
 
         return result_set
 
-    def where(
+    def where(self, where: str, prefilter: bool = None) -> LanceVectorQueryBuilder:
-        self, where: Union[str, Expr], prefilter: bool = None
-    ) -> LanceVectorQueryBuilder:
         """Set the where clause.
 
         Parameters
         ----------
-        where: str or :class:`~lancedb.expr.Expr`
+        where: str
-            The filter condition.  Can be a SQL string or a type-safe
+            The where clause which is a valid SQL where clause. See
-            :class:`~lancedb.expr.Expr` built with :func:`~lancedb.expr.col`
+            `Lance filter pushdown <https://lance.org/guide/read_and_write#filter-push-down>`_
-            and :func:`~lancedb.expr.lit`.
+            for valid SQL expressions.
         prefilter: bool, default True
             If True, apply the filter before vector search, otherwise the
             filter is applied on the result of vector search.
@@ -1525,7 +1522,9 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
         return self._table._output_schema(self.to_query_object())
 
     def to_arrow(self, *, timeout: Optional[timedelta] = None) -> pa.Table:
-        self._table._ensure_no_legacy_fts_index()
+        path, fs, exist = self._table._get_fts_index_path()
+        if exist:
+            return self.tantivy_to_arrow()
 
         query = self._query
         if self._phrase_query:
@@ -1549,6 +1548,90 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
     ):
         raise NotImplementedError("to_batches on an FTS query")
 
+    def tantivy_to_arrow(self) -> pa.Table:
+        try:
+            import tantivy
+        except ImportError:
+            raise ImportError(
+                "Please install tantivy-py `pip install tantivy` to use the full text search feature."  # noqa: E501
+            )
+
+        from .fts import search_index
+
+        # get the index path
+        path, fs, exist = self._table._get_fts_index_path()
+
+        # check if the index exist
+        if not exist:
+            raise FileNotFoundError(
+                "Fts index does not exist. "
+                "Please first call table.create_fts_index(['<field_names>']) to "
+                "create the fts index."
+            )
+
+        # Check that we are on local filesystem
+        if not isinstance(fs, pa_fs.LocalFileSystem):
+            raise NotImplementedError(
+                "Tantivy-based full text search "
+                "is only supported on the local filesystem"
+            )
+        # open the index
+        index = tantivy.Index.open(path)
+        # get the scores and doc ids
+        query = self._query
+        if self._phrase_query:
+            query = query.replace('"', "'")
+            query = f'"{query}"'
+        limit = self._limit if self._limit is not None else 10
+        row_ids, scores = search_index(
+            index, query, limit, ordering_field=self.ordering_field_name
+        )
+        if len(row_ids) == 0:
+            empty_schema = pa.schema([pa.field("_score", pa.float32())])
+            return pa.Table.from_batches([], schema=empty_schema)
+        scores = pa.array(scores)
+        output_tbl = self._table.to_lance().take(row_ids, columns=self._columns)
+        output_tbl = output_tbl.append_column("_score", scores)
+        # this needs to match vector search results which are uint64
+        row_ids = pa.array(row_ids, type=pa.uint64())
+
+        if self._where is not None:
+            tmp_name = "__lancedb__duckdb__indexer__"
+            output_tbl = output_tbl.append_column(
+                tmp_name, pa.array(range(len(output_tbl)))
+            )
+            try:
+                # TODO would be great to have Substrait generate pyarrow compute
+                # expressions or conversely have pyarrow support SQL expressions
+                # using Substrait
+                import duckdb
+
+                indexer = duckdb.sql(
+                    f"SELECT {tmp_name} FROM output_tbl WHERE {self._where}"
+                ).to_arrow_table()[tmp_name]
+                output_tbl = output_tbl.take(indexer).drop([tmp_name])
+                row_ids = row_ids.take(indexer)
+
+            except ImportError:
+                import tempfile
+
+                import lance
+
+                # TODO Use "memory://" instead once that's supported
+                with tempfile.TemporaryDirectory() as tmp:
+                    ds = lance.write_dataset(output_tbl, tmp)
+                    output_tbl = ds.to_table(filter=self._where)
+                    indexer = output_tbl[tmp_name]
+                    row_ids = row_ids.take(indexer)
+                    output_tbl = output_tbl.drop([tmp_name])
+
+        if self._with_row_id:
+            output_tbl = output_tbl.append_column("_rowid", row_ids)
+
+        if self._reranker is not None:
+            output_tbl = self._reranker.rerank_fts(self._query, output_tbl)
+        return output_tbl
+
     def rerank(self, reranker: Reranker) -> LanceFtsQueryBuilder:
         """Rerank the results using the specified reranker.
 
@@ -1643,7 +1726,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
     def _validate_query(self, query, vector=None, text=None):
         if query is not None and (vector is not None or text is not None):
             raise ValueError(
-                "You can either provide a string query in search() method "
+                "You can either provide a string query in search() method"
                 "or set `vector()` and `text()` explicitly for hybrid search."
                 "But not both."
             )
@@ -2122,8 +2205,8 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
             self._vector_query.select(self._columns)
             self._fts_query.select(self._columns)
         if self._where:
-            self._vector_query.where(self._where, not self._postfilter)
+            self._vector_query.where(self._where, self._postfilter)
-            self._fts_query.where(self._where, not self._postfilter)
+            self._fts_query.where(self._where, self._postfilter)
         if self._with_row_id:
             self._vector_query.with_row_id(True)
             self._fts_query.with_row_id(True)
@@ -2203,20 +2286,10 @@ class AsyncQueryBase(object):
         """
         if isinstance(columns, list) and all(isinstance(c, str) for c in columns):
             self._inner.select_columns(columns)
-        elif isinstance(columns, dict) and all(isinstance(k, str) for k in columns):
+        elif isinstance(columns, dict) and all(
-            if any(isinstance(v, Expr) for v in columns.values()):
+            isinstance(k, str) and isinstance(v, str) for k, v in columns.items()
-                # At least one value is an Expr — use the type-safe path.
+        ):
-                from .expr import _coerce
+            self._inner.select(list(columns.items()))
-
-                pairs = [(k, _coerce(v)._inner) for k, v in columns.items()]
-                self._inner.select_expr(pairs)
-            elif all(isinstance(v, str) for v in columns.values()):
-                self._inner.select(list(columns.items()))
-            else:
-                raise TypeError(
-                    "dict values must be str or Expr, got "
-                    + str({k: type(v) for k, v in columns.items()})
-                )
         else:
             raise TypeError("columns must be a list of column names or a dict")
         return self
@@ -2456,13 +2529,11 @@ class AsyncStandardQuery(AsyncQueryBase):
         """
         super().__init__(inner)
 
-    def where(self, predicate: Union[str, Expr]) -> Self:
+    def where(self, predicate: str) -> Self:
         """
         Only return rows matching the given predicate
 
-        The predicate can be a SQL string or a type-safe
+        The predicate should be supplied as an SQL query string.
-        :class:`~lancedb.expr.Expr` built with :func:`~lancedb.expr.col`
-        and :func:`~lancedb.expr.lit`.
 
         Examples
         --------
@@ -2474,10 +2545,7 @@ class AsyncStandardQuery(AsyncQueryBase):
         Filtering performance can often be improved by creating a scalar index
         on the filter column(s).
         """
-        if isinstance(predicate, Expr):
+        self._inner.where(predicate)
-            self._inner.where_expr(predicate._inner)
-        else:
-            self._inner.where(predicate)
         return self
 
     def limit(self, limit: int) -> Self:

python/python/lancedb/remote/__init__.py

@@ -145,33 +145,6 @@ class TlsConfig:
 
 @dataclass
 class ClientConfig:
-    """Configuration for the LanceDB Cloud HTTP client.
-
-    Attributes
-    ----------
-    user_agent: str
-        User agent string sent with requests.
-    retry_config: RetryConfig
-        Configuration for retrying failed requests.
-    timeout_config: Optional[TimeoutConfig]
-        Configuration for request timeouts.
-    extra_headers: Optional[dict]
-        Additional headers to include in requests.
-    id_delimiter: Optional[str]
-        The delimiter to use when constructing object identifiers.
-    tls_config: Optional[TlsConfig]
-        TLS/mTLS configuration for secure connections.
-    header_provider: Optional[HeaderProvider]
-        Provider for dynamic headers to be added to each request.
-    user_id: Optional[str]
-        User identifier for tracking purposes. This is sent as the
-        `x-lancedb-user-id` header in requests to LanceDB Cloud/Enterprise.
-
-        This can also be set via the `LANCEDB_USER_ID` environment variable.
-        Alternatively, set `LANCEDB_USER_ID_ENV_KEY` to specify another
-        environment variable that contains the user ID value.
-    """
-
     user_agent: str = f"LanceDB-Python-Client/{__version__}"
     retry_config: RetryConfig = field(default_factory=RetryConfig)
     timeout_config: Optional[TimeoutConfig] = field(default_factory=TimeoutConfig)
@@ -179,7 +152,6 @@ class ClientConfig:
     id_delimiter: Optional[str] = None
     tls_config: Optional[TlsConfig] = None
     header_provider: Optional["HeaderProvider"] = None
-    user_id: Optional[str] = None
 
     def __post_init__(self):
         if isinstance(self.retry_config, dict):

python/python/lancedb/remote/db.py

@@ -24,7 +24,6 @@ from ..common import DATA
 from ..db import DBConnection, LOOP
 from ..embeddings import EmbeddingFunctionConfig
 from lance_namespace import (
-    LanceNamespace,
     CreateNamespaceResponse,
     DescribeNamespaceResponse,
     DropNamespaceResponse,
@@ -112,7 +111,7 @@ class RemoteDBConnection(DBConnection):
     @override
     def list_namespaces(
         self,
-        namespace_path: Optional[List[str]] = None,
+        namespace: Optional[List[str]] = None,
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
     ) -> ListNamespacesResponse:
@@ -120,7 +119,7 @@ class RemoteDBConnection(DBConnection):
 
         Parameters
         ----------
-        namespace_path: List[str], optional
+        namespace: List[str], optional
             The parent namespace to list namespaces in.
             None or empty list represents root namespace.
         page_token: str, optional
@@ -134,18 +133,18 @@ class RemoteDBConnection(DBConnection):
         ListNamespacesResponse
             Response containing namespace names and optional page_token for pagination.
         """
-        if namespace_path is None:
+        if namespace is None:
-            namespace_path = []
+            namespace = []
         return LOOP.run(
             self._conn.list_namespaces(
-                namespace_path=namespace_path, page_token=page_token, limit=limit
+                namespace=namespace, page_token=page_token, limit=limit
             )
         )
 
     @override
     def create_namespace(
         self,
-        namespace_path: List[str],
+        namespace: List[str],
         mode: Optional[str] = None,
         properties: Optional[Dict[str, str]] = None,
     ) -> CreateNamespaceResponse:
@@ -153,7 +152,7 @@ class RemoteDBConnection(DBConnection):
 
         Parameters
         ----------
-        namespace_path: List[str]
+        namespace: List[str]
             The namespace identifier to create.
         mode: str, optional
             Creation mode - "create" (fail if exists), "exist_ok" (skip if exists),
@@ -168,14 +167,14 @@ class RemoteDBConnection(DBConnection):
         """
         return LOOP.run(
             self._conn.create_namespace(
-                namespace_path=namespace_path, mode=mode, properties=properties
+                namespace=namespace, mode=mode, properties=properties
             )
         )
 
     @override
     def drop_namespace(
         self,
-        namespace_path: List[str],
+        namespace: List[str],
         mode: Optional[str] = None,
         behavior: Optional[str] = None,
     ) -> DropNamespaceResponse:
@@ -183,7 +182,7 @@ class RemoteDBConnection(DBConnection):
 
         Parameters
         ----------
-        namespace_path: List[str]
+        namespace: List[str]
             The namespace identifier to drop.
         mode: str, optional
             Whether to skip if not exists ("SKIP") or fail ("FAIL"). Case insensitive.
@@ -197,20 +196,16 @@ class RemoteDBConnection(DBConnection):
             Response containing properties and transaction_id if applicable.
         """
         return LOOP.run(
-            self._conn.drop_namespace(
+            self._conn.drop_namespace(namespace=namespace, mode=mode, behavior=behavior)
-                namespace_path=namespace_path, mode=mode, behavior=behavior
-            )
         )
 
     @override
-    def describe_namespace(
+    def describe_namespace(self, namespace: List[str]) -> DescribeNamespaceResponse:
-        self, namespace_path: List[str]
-    ) -> DescribeNamespaceResponse:
         """Describe a namespace.
 
         Parameters
         ----------
-        namespace_path: List[str]
+        namespace: List[str]
             The namespace identifier to describe.
 
         Returns
@@ -218,12 +213,12 @@ class RemoteDBConnection(DBConnection):
         DescribeNamespaceResponse
             Response containing the namespace properties.
         """
-        return LOOP.run(self._conn.describe_namespace(namespace_path=namespace_path))
+        return LOOP.run(self._conn.describe_namespace(namespace=namespace))
 
     @override
     def list_tables(
         self,
-        namespace_path: Optional[List[str]] = None,
+        namespace: Optional[List[str]] = None,
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
     ) -> ListTablesResponse:
@@ -231,7 +226,7 @@ class RemoteDBConnection(DBConnection):
 
         Parameters
         ----------
-        namespace_path: List[str], optional
+        namespace: List[str], optional
             The namespace to list tables in.
             None or empty list represents root namespace.
         page_token: str, optional
@@ -245,11 +240,11 @@ class RemoteDBConnection(DBConnection):
         ListTablesResponse
             Response containing table names and optional page_token for pagination.
         """
-        if namespace_path is None:
+        if namespace is None:
-            namespace_path = []
+            namespace = []
         return LOOP.run(
             self._conn.list_tables(
-                namespace_path=namespace_path, page_token=page_token, limit=limit
+                namespace=namespace, page_token=page_token, limit=limit
             )
         )
 
@@ -259,7 +254,7 @@ class RemoteDBConnection(DBConnection):
         page_token: Optional[str] = None,
         limit: int = 10,
         *,
-        namespace_path: Optional[List[str]] = None,
+        namespace: Optional[List[str]] = None,
     ) -> Iterable[str]:
         """List the names of all tables in the database.
 
@@ -268,7 +263,7 @@ class RemoteDBConnection(DBConnection):
 
         Parameters
         ----------
-        namespace_path: List[str], default []
+        namespace: List[str], default []
             The namespace to list tables in.
             Empty list represents root namespace.
         page_token: str
@@ -287,11 +282,11 @@ class RemoteDBConnection(DBConnection):
             DeprecationWarning,
             stacklevel=2,
         )
-        if namespace_path is None:
+        if namespace is None:
-            namespace_path = []
+            namespace = []
         return LOOP.run(
             self._conn.table_names(
-                namespace_path=namespace_path, start_after=page_token, limit=limit
+                namespace=namespace, start_after=page_token, limit=limit
             )
         )
 
@@ -300,7 +295,7 @@ class RemoteDBConnection(DBConnection):
         self,
         name: str,
         *,
-        namespace_path: Optional[List[str]] = None,
+        namespace: Optional[List[str]] = None,
         storage_options: Optional[Dict[str, str]] = None,
         index_cache_size: Optional[int] = None,
     ) -> Table:
@@ -310,7 +305,7 @@ class RemoteDBConnection(DBConnection):
         ----------
         name: str
             The name of the table.
-        namespace_path: List[str], optional
+        namespace: List[str], optional
             The namespace to open the table from.
             None or empty list represents root namespace.
 
@@ -320,15 +315,15 @@ class RemoteDBConnection(DBConnection):
         """
         from .table import RemoteTable
 
-        if namespace_path is None:
+        if namespace is None:
-            namespace_path = []
+            namespace = []
         if index_cache_size is not None:
             logging.info(
                 "index_cache_size is ignored in LanceDb Cloud"
                 " (there is no local cache to configure)"
             )
 
-        table = LOOP.run(self._conn.open_table(name, namespace_path=namespace_path))
+        table = LOOP.run(self._conn.open_table(name, namespace=namespace))
         return RemoteTable(table, self.db_name)
 
     def clone_table(
@@ -336,7 +331,7 @@ class RemoteDBConnection(DBConnection):
         target_table_name: str,
         source_uri: str,
         *,
-        target_namespace_path: Optional[List[str]] = None,
+        target_namespace: Optional[List[str]] = None,
         source_version: Optional[int] = None,
         source_tag: Optional[str] = None,
         is_shallow: bool = True,
@@ -349,7 +344,7 @@ class RemoteDBConnection(DBConnection):
             The name of the target table to create.
         source_uri: str
             The URI of the source table to clone from.
-        target_namespace_path: List[str], optional
+        target_namespace: List[str], optional
             The namespace for the target table.
             None or empty list represents root namespace.
         source_version: int, optional
@@ -366,13 +361,13 @@ class RemoteDBConnection(DBConnection):
         """
         from .table import RemoteTable
 
-        if target_namespace_path is None:
+        if target_namespace is None:
-            target_namespace_path = []
+            target_namespace = []
         table = LOOP.run(
             self._conn.clone_table(
                 target_table_name,
                 source_uri,
-                target_namespace_path=target_namespace_path,
+                target_namespace=target_namespace,
                 source_version=source_version,
                 source_tag=source_tag,
                 is_shallow=is_shallow,
@@ -392,7 +387,7 @@ class RemoteDBConnection(DBConnection):
         exist_ok: bool = False,
         embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
         *,
-        namespace_path: Optional[List[str]] = None,
+        namespace: Optional[List[str]] = None,
     ) -> Table:
         """Create a [Table][lancedb.table.Table] in the database.
 
@@ -400,7 +395,7 @@ class RemoteDBConnection(DBConnection):
         ----------
         name: str
             The name of the table.
-        namespace_path: List[str], optional
+        namespace: List[str], optional
             The namespace to create the table in.
             None or empty list represents root namespace.
         data: The data to initialize the table, *optional*
@@ -500,8 +495,8 @@ class RemoteDBConnection(DBConnection):
                 mode = "exist_ok"
             elif not mode:
                 mode = "exist_ok"
-        if namespace_path is None:
+        if namespace is None:
-            namespace_path = []
+            namespace = []
         validate_table_name(name)
         if embedding_functions is not None:
             logging.warning(
@@ -516,7 +511,7 @@ class RemoteDBConnection(DBConnection):
             self._conn.create_table(
                 name,
                 data,
-                namespace_path=namespace_path,
+                namespace=namespace,
                 mode=mode,
                 schema=schema,
                 on_bad_vectors=on_bad_vectors,
@@ -526,28 +521,28 @@ class RemoteDBConnection(DBConnection):
         return RemoteTable(table, self.db_name)
 
     @override
-    def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
+    def drop_table(self, name: str, namespace: Optional[List[str]] = None):
         """Drop a table from the database.
 
         Parameters
         ----------
         name: str
             The name of the table.
-        namespace_path: List[str], optional
+        namespace: List[str], optional
             The namespace to drop the table from.
             None or empty list represents root namespace.
         """
-        if namespace_path is None:
+        if namespace is None:
-            namespace_path = []
+            namespace = []
-        LOOP.run(self._conn.drop_table(name, namespace_path=namespace_path))
+        LOOP.run(self._conn.drop_table(name, namespace=namespace))
 
     @override
     def rename_table(
         self,
         cur_name: str,
         new_name: str,
-        cur_namespace_path: Optional[List[str]] = None,
+        cur_namespace: Optional[List[str]] = None,
-        new_namespace_path: Optional[List[str]] = None,
+        new_namespace: Optional[List[str]] = None,
     ):
         """Rename a table in the database.
 
@@ -558,32 +553,19 @@ class RemoteDBConnection(DBConnection):
         new_name: str
             The new name of the table.
         """
-        if cur_namespace_path is None:
+        if cur_namespace is None:
-            cur_namespace_path = []
+            cur_namespace = []
-        if new_namespace_path is None:
+        if new_namespace is None:
-            new_namespace_path = []
+            new_namespace = []
         LOOP.run(
             self._conn.rename_table(
                 cur_name,
                 new_name,
-                cur_namespace_path=cur_namespace_path,
+                cur_namespace=cur_namespace,
-                new_namespace_path=new_namespace_path,
+                new_namespace=new_namespace,
             )
         )
 
-    @override
-    def namespace_client(self) -> LanceNamespace:
-        """Get the equivalent namespace client for this connection.
-
-        Returns a RestNamespace with the same URI and authentication headers.
-
-        Returns
-        -------
-        LanceNamespace
-            The namespace client for this connection.
-        """
-        return LOOP.run(self._conn.namespace_client())
-
     async def close(self):
         """Close the connection to the database."""
-        self._conn.close()
+        self._client.close()

python/python/lancedb/remote/table.py

@@ -4,7 +4,7 @@
 from datetime import timedelta
 import logging
 from functools import cached_property
-from typing import Any, Callable, Dict, Iterable, List, Optional, Union, Literal
+from typing import Dict, Iterable, List, Optional, Union, Literal
 import warnings
 
 from lancedb._lancedb import (
@@ -22,7 +22,6 @@ from lancedb.index import (
     FTS,
     BTree,
     Bitmap,
-    HnswFlat,
     HnswSq,
     IvfFlat,
     IvfPq,
@@ -36,11 +35,9 @@ import pyarrow as pa
 from lancedb.common import DATA, VEC, VECTOR_COLUMN_NAME
 from lancedb.merge import LanceMergeInsertBuilder
 from lancedb.embeddings import EmbeddingFunctionRegistry
-from lancedb.table import _normalize_progress
 
 from ..query import LanceVectorQueryBuilder, LanceQueryBuilder, LanceTakeQueryBuilder
 from ..table import AsyncTable, IndexStatistics, Query, Table, Tags
-from ..types import BaseTokenizerType
 
 
 class RemoteTable(Table):
@@ -169,7 +166,7 @@ class RemoteTable(Table):
         wait_timeout: Optional[timedelta] = None,
         with_position: bool = False,
         # tokenizer configs:
-        base_tokenizer: BaseTokenizerType = "simple",
+        base_tokenizer: str = "simple",
         language: str = "English",
         max_token_length: Optional[int] = 40,
         lower_case: bool = True,
@@ -286,15 +283,13 @@ class RemoteTable(Table):
             )
         elif index_type == "IVF_HNSW_SQ":
             config = HnswSq(distance_type=metric, num_partitions=num_partitions)
-        elif index_type == "IVF_HNSW_FLAT":
-            config = HnswFlat(distance_type=metric, num_partitions=num_partitions)
         elif index_type == "IVF_FLAT":
             config = IvfFlat(distance_type=metric, num_partitions=num_partitions)
         else:
             raise ValueError(
                 f"Unknown vector index type: {index_type}. Valid options are"
                 " 'IVF_FLAT', 'IVF_PQ', 'IVF_RQ', 'IVF_SQ',"
-                " 'IVF_HNSW_PQ', 'IVF_HNSW_SQ', 'IVF_HNSW_FLAT'"
+                " 'IVF_HNSW_PQ', 'IVF_HNSW_SQ'"
             )
 
         LOOP.run(
@@ -313,7 +308,6 @@ class RemoteTable(Table):
         mode: str = "append",
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
-        progress: Optional[Union[bool, Callable, Any]] = None,
     ) -> AddResult:
         """Add more data to the [Table](Table). It has the same API signature as
         the OSS version.
@@ -336,29 +330,17 @@ class RemoteTable(Table):
             One of "error", "drop", "fill".
         fill_value: float, default 0.
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
-        progress: bool, callable, or tqdm-like, optional
-            A callback or tqdm-compatible progress bar. See
-            :meth:`Table.add` for details.
 
         Returns
         -------
         AddResult
             An object containing the new version number of the table after adding data.
         """
-        progress, owns = _normalize_progress(progress)
+        return LOOP.run(
-        try:
+            self._table.add(
-            return LOOP.run(
+                data, mode=mode, on_bad_vectors=on_bad_vectors, fill_value=fill_value
-                self._table.add(
-                    data,
-                    mode=mode,
-                    on_bad_vectors=on_bad_vectors,
-                    fill_value=fill_value,
-                    progress=progress,
-                )
             )
-        finally:
+        )
-            if owns:
-                progress.close()
 
     def search(
         self,

python/python/lancedb/types.py

@@ -24,7 +24,6 @@ VectorIndexType = Literal[
     "IVF_PQ",
     "IVF_HNSW_SQ",
     "IVF_HNSW_PQ",
-    "IVF_HNSW_FLAT",
     "IVF_RQ",
 ]
 ScalarIndexType = Literal["BTREE", "BITMAP", "LABEL_LIST"]
@@ -32,7 +31,6 @@ IndexType = Literal[
     "IVF_PQ",
     "IVF_HNSW_PQ",
     "IVF_HNSW_SQ",
-    "IVF_HNSW_FLAT",
     "IVF_SQ",
     "FTS",
     "BTREE",
@@ -42,5 +40,4 @@ IndexType = Literal[
 ]
 
 # Tokenizer literals
-BuiltinTokenizerType = Literal["simple", "raw", "whitespace", "ngram"]
+BaseTokenizerType = Literal["simple", "raw", "whitespace", "ngram"]
-BaseTokenizerType = BuiltinTokenizerType | str

python/python/tests/docs/test_search.py

@@ -180,7 +180,7 @@ def test_fts_fuzzy_query():
         ),
         mode="overwrite",
     )
-    table.create_fts_index("text", replace=True)
+    table.create_fts_index("text", use_tantivy=False, replace=True)
 
     results = table.search(MatchQuery("foo", "text", fuzziness=1)).to_pandas()
     assert len(results) == 4
@@ -230,7 +230,7 @@ def test_fts_boost_query():
         ),
         mode="overwrite",
     )
-    table.create_fts_index("desc", replace=True)
+    table.create_fts_index("desc", use_tantivy=False, replace=True)
 
     results = table.search(
         BoostQuery(
@@ -265,7 +265,7 @@ def test_fts_boolean_query(tmp_path):
         ],
         mode="overwrite",
     )
-    table.create_fts_index("text", replace=True)
+    table.create_fts_index("text", use_tantivy=False, replace=True)
 
     # SHOULD
     results = table.search(
@@ -319,7 +319,9 @@ def test_fts_native():
         ],
     )
 
-    table.create_fts_index("text")
+    # passing `use_tantivy=False` to use lance FTS index
+    # `use_tantivy=True` by default
+    table.create_fts_index("text", use_tantivy=False)
     table.search("puppy").limit(10).select(["text"]).to_list()
     # [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
     # ...
@@ -330,6 +332,7 @@ def test_fts_native():
     # --8<-- [start:fts_config_folding]
     table.create_fts_index(
         "text",
+        use_tantivy=False,
         language="French",
         stem=True,
         ascii_folding=True,
@@ -343,7 +346,7 @@ def test_fts_native():
     table.search("puppy").limit(10).where("text='foo'", prefilter=False).to_list()
     # --8<-- [end:fts_postfiltering]
     # --8<-- [start:fts_with_position]
-    table.create_fts_index("text", with_position=True, replace=True)
+    table.create_fts_index("text", use_tantivy=False, with_position=True, replace=True)
     # --8<-- [end:fts_with_position]
     # --8<-- [start:fts_incremental_index]
     table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])

python/python/tests/models/jieba/default/dict.txt

@@ -1,8 +0,0 @@
-我们 98740 r
-都 202780 d
-有 423765 v
-光明 1219 n
-的 318825 uj
-前途 1263 n
-前 62779 f
-途 857 n

python/python/tests/models/lindera/ipadic/config.yml

@@ -1,4 +0,0 @@
-segmenter:
-  mode: "normal"
-  dictionary:
-    path: "./python/tests/models/lindera/ipadic/main"

python/python/tests/test_db.py

@@ -3,7 +3,6 @@
 
 
 import re
-import sys
 from datetime import timedelta
 import os
 
@@ -15,7 +14,8 @@ import pytest
 from lancedb.pydantic import LanceModel, Vector
 
 
-def test_basic(tmp_path):
+@pytest.mark.parametrize("use_tantivy", [True, False])
+def test_basic(tmp_path, use_tantivy):
     db = lancedb.connect(tmp_path)
 
     assert db.uri == str(tmp_path)
@@ -48,7 +48,7 @@ def test_basic(tmp_path):
     assert len(rs) == 1
     assert rs["item"].iloc[0] == "foo"
 
-    table.create_fts_index("item")
+    table.create_fts_index("item", use_tantivy=use_tantivy)
     rs = table.search("bar", query_type="fts").to_pandas()
     assert len(rs) == 1
     assert rs["item"].iloc[0] == "bar"
@@ -183,8 +183,8 @@ def test_table_names(tmp_db: lancedb.DBConnection):
     result = list(tmp_db.table_names("test2", limit=2))
     assert result == ["test3"], f"Expected ['test3'], got {result}"
 
-    # Test that namespace_path parameter can be passed as keyword
+    # Test that namespace parameter can be passed as keyword
-    result = list(tmp_db.table_names(namespace_path=[]))
+    result = list(tmp_db.table_names(namespace=[]))
     assert len(result) == 3
 
 
@@ -896,22 +896,42 @@ def test_bypass_vector_index_sync(tmp_db: lancedb.DBConnection):
 
 
 def test_local_namespace_operations(tmp_path):
-    """Test that local mode namespace operations work via directory namespace."""
+    """Test that local mode namespace operations behave as expected."""
+    # Create a local database connection
     db = lancedb.connect(tmp_path)
 
-    # Root namespace starts empty
+    # Test list_namespaces returns empty list for root namespace
-    assert db.list_namespaces().namespaces == []
+    namespaces = db.list_namespaces().namespaces
+    assert namespaces == []
 
-    # Create and list child namespace
+    # Test list_namespaces with non-empty namespace raises NotImplementedError
-    db.create_namespace(["child"])
+    with pytest.raises(
-    assert "child" in db.list_namespaces().namespaces
+        NotImplementedError,
+        match="Namespace operations are not supported for listing database",
+    ):
+        db.list_namespaces(namespace=["test"])
 
-    # List namespaces under child
-    assert db.list_namespaces(namespace_path=["child"]).namespaces == []
 
-    # Drop namespace
+def test_local_create_namespace_not_supported(tmp_path):
-    db.drop_namespace(["child"])
+    """Test that create_namespace is not supported in local mode."""
-    assert db.list_namespaces().namespaces == []
+    db = lancedb.connect(tmp_path)
+
+    with pytest.raises(
+        NotImplementedError,
+        match="Namespace operations are not supported for listing database",
+    ):
+        db.create_namespace(["test_namespace"])
+
+
+def test_local_drop_namespace_not_supported(tmp_path):
+    """Test that drop_namespace is not supported in local mode."""
+    db = lancedb.connect(tmp_path)
+
+    with pytest.raises(
+        NotImplementedError,
+        match="Namespace operations are not supported for listing database",
+    ):
+        db.drop_namespace(["test_namespace"])
 
 
 def test_clone_table_latest_version(tmp_path):
@@ -1028,59 +1048,3 @@ def test_clone_table_deep_clone_fails(tmp_path):
     source_uri = os.path.join(tmp_path, "source.lance")
     with pytest.raises(Exception, match="Deep clone is not yet implemented"):
         db.clone_table("cloned", source_uri, is_shallow=False)
-
-
-@pytest.mark.skipif(sys.platform == "win32", reason="Namespace client issues")
-def test_namespace_client_native_storage(tmp_path):
-    """Test namespace_client() returns DirectoryNamespace for native storage."""
-    from lance.namespace import DirectoryNamespace
-
-    db = lancedb.connect(tmp_path)
-    ns_client = db.namespace_client()
-
-    assert isinstance(ns_client, DirectoryNamespace)
-    assert str(tmp_path) in ns_client.namespace_id()
-
-
-@pytest.mark.skipif(sys.platform == "win32", reason="Namespace client issues")
-def test_namespace_client_with_storage_options(tmp_path):
-    """Test namespace_client() preserves storage options."""
-    from lance.namespace import DirectoryNamespace
-
-    storage_options = {"timeout": "10s"}
-    db = lancedb.connect(tmp_path, storage_options=storage_options)
-    ns_client = db.namespace_client()
-
-    assert isinstance(ns_client, DirectoryNamespace)
-
-
-@pytest.mark.skipif(sys.platform == "win32", reason="Namespace client issues")
-def test_namespace_client_operations(tmp_path):
-    """Test that namespace_client() returns a functional namespace client."""
-    db = lancedb.connect(tmp_path)
-    ns_client = db.namespace_client()
-
-    # Create a table through the main db connection
-    data = [{"id": 1, "text": "hello", "vector": [1.0, 2.0]}]
-    db.create_table("test_table", data=data)
-
-    # Verify the namespace client can see the table
-    from lance_namespace import ListTablesRequest
-
-    # id=[] means root namespace
-    response = ns_client.list_tables(ListTablesRequest(id=[]))
-    # Tables can be strings or objects with name attribute
-    table_names = [t.name if hasattr(t, "name") else t for t in response.tables]
-    assert "test_table" in table_names
-
-
-@pytest.mark.skipif(sys.platform == "win32", reason="Namespace client issues")
-def test_namespace_client_namespace_connection(tmp_path):
-    """Test namespace_client() returns the backing client for namespace connections."""
-    from lance.namespace import DirectoryNamespace
-
-    db = lancedb.connect_namespace("dir", {"root": str(tmp_path)})
-    ns_client = db.namespace_client()
-
-    assert isinstance(ns_client, DirectoryNamespace)
-    assert str(tmp_path) in ns_client.namespace_id()

python/python/tests/test_embeddings.py

@@ -546,24 +546,3 @@ def test_openai_no_retry_on_401(mock_sleep):
     assert mock_func.call_count == 1
     # Verify that sleep was never called (no retries)
     assert mock_sleep.call_count == 0
-
-
-def test_url_retrieve_downloads_image():
-    """
-    Embedding functions like open-clip, siglip, and jinaai use url_retrieve()
-    to download images from HTTP URLs. For example, open_clip._to_pil() calls:
-
-        PIL_Image.open(io.BytesIO(url_retrieve(image)))
-
-    Verify that url_retrieve() can download an image and open it as PIL Image,
-    matching the real usage pattern in embedding functions.
-    """
-    import io
-
-    Image = pytest.importorskip("PIL.Image")
-    from lancedb.embeddings.utils import url_retrieve
-
-    image_url = "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg"
-    image_bytes = url_retrieve(image_url)
-    img = Image.open(io.BytesIO(image_bytes))
-    assert img.size[0] > 0 and img.size[1] > 0

python/python/tests/test_fts.py

@@ -15,10 +15,7 @@
 #  limitations under the License.
 import os
 import random
-import shutil
 from unittest import mock
-from pathlib import Path
-import zipfile
 
 import lancedb as ldb
 from lancedb.db import DBConnection
@@ -39,7 +36,8 @@ import pytest
 import pytest_asyncio
 from utils import exception_output
 
-TEST_LANGUAGE_MODEL_HOME = Path(__file__).parent / "models"
+pytest.importorskip("lancedb.fts")
+tantivy = pytest.importorskip("tantivy")
 
 
 @pytest.fixture
@@ -94,40 +92,6 @@ def table(tmp_path) -> ldb.table.LanceTable:
     return table
 
 
-@pytest.fixture
-def language_model_home(monkeypatch, tmp_path):
-    model_home = tmp_path / "language-models"
-    shutil.copytree(TEST_LANGUAGE_MODEL_HOME, model_home)
-    monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(model_home))
-    return model_home
-
-
-@pytest.fixture
-def lindera_ipadic(language_model_home):
-    model_path = language_model_home / "lindera" / "ipadic"
-    extracted_model = model_path / "main"
-    config_path = model_path / "config.yml"
-
-    if extracted_model.exists():
-        shutil.rmtree(extracted_model)
-
-    with zipfile.ZipFile(model_path / "main.zip", "r") as zip_ref:
-        zip_ref.extractall(model_path)
-    config_path.write_text(
-        "segmenter:\n"
-        '  mode: "normal"\n'
-        "  dictionary:\n"
-        f'    path: "{extracted_model.resolve().as_posix()}"\n',
-        encoding="utf-8",
-    )
-
-    try:
-        yield
-    finally:
-        if extracted_model.exists():
-            shutil.rmtree(extracted_model)
-
-
 @pytest_asyncio.fixture
 async def async_table(tmp_path) -> ldb.table.AsyncTable:
     # Use local random state to avoid affecting other tests
@@ -180,53 +144,58 @@ async def async_table(tmp_path) -> ldb.table.AsyncTable:
     return table
 
 
-@pytest.mark.parametrize(
+def test_create_index(tmp_path):
-    ("kwargs", "match"),
+    index = ldb.fts.create_index(str(tmp_path / "index"), ["text"])
-    [
+    assert isinstance(index, tantivy.Index)
-        (
+    assert os.path.exists(str(tmp_path / "index"))
-            {"use_tantivy": True},
-            "Tantivy-based FTS has been removed",
-        ),
-        (
-            {"ordering_field_names": ["count"]},
-            "ordering_field_names was only supported",
-        ),
-        (
-            {"writer_heap_size": 128},
-            "writer_heap_size was only supported",
-        ),
-    ],
-)
-def test_reject_removed_tantivy_parameters(table, kwargs, match):
-    with pytest.raises(ValueError, match=match):
-        table.create_fts_index("text", **kwargs)
 
 
-def test_reject_legacy_tantivy_index(table):
+def test_create_index_with_stemming(tmp_path, table):
-    path, _, _ = table._get_fts_index_path()
+    index = ldb.fts.create_index(
-    os.makedirs(path, exist_ok=True)
+        str(tmp_path / "index"), ["text"], tokenizer_name="en_stem"
+    )
+    assert isinstance(index, tantivy.Index)
+    assert os.path.exists(str(tmp_path / "index"))
 
-    with pytest.raises(ValueError, match="Legacy Tantivy FTS index detected"):
+    # Check stemming by running tokenizer on non empty table
-        table.search("puppy").limit(5).to_list()
+    table.create_fts_index("text", tokenizer_name="en_stem", use_tantivy=True)
-
-    with pytest.raises(ValueError, match="Legacy Tantivy FTS index detected"):
-        table.create_fts_index("text")
 
 
+@pytest.mark.parametrize("use_tantivy", [True, False])
 @pytest.mark.parametrize("with_position", [True, False])
-def test_create_inverted_index(table, with_position):
+def test_create_inverted_index(table, use_tantivy, with_position):
+    if use_tantivy and not with_position:
+        pytest.skip("we don't support building a tantivy index without position")
     table.create_fts_index(
         "text",
+        use_tantivy=use_tantivy,
         with_position=with_position,
         name="custom_fts_index",
     )
-    indices = table.list_indices()
+    if not use_tantivy:
-    fts_indices = [i for i in indices if i.index_type == "FTS"]
+        indices = table.list_indices()
-    assert any(i.name == "custom_fts_index" for i in fts_indices)
+        fts_indices = [i for i in indices if i.index_type == "FTS"]
+        assert any(i.name == "custom_fts_index" for i in fts_indices)
 
 
-def test_search_fts(table):
+def test_populate_index(tmp_path, table):
-    table.create_fts_index("text")
+    index = ldb.fts.create_index(str(tmp_path / "index"), ["text"])
+    assert ldb.fts.populate_index(index, table, ["text"]) == len(table)
+
+
+def test_search_index(tmp_path, table):
+    index = ldb.fts.create_index(str(tmp_path / "index"), ["text"])
+    ldb.fts.populate_index(index, table, ["text"])
+    index.reload()
+    results = ldb.fts.search_index(index, query="puppy", limit=5)
+    assert len(results) == 2
+    assert len(results[0]) == 5  # row_ids
+    assert len(results[1]) == 5  # _score
+
+
+@pytest.mark.parametrize("use_tantivy", [True, False])
+def test_search_fts(table, use_tantivy):
+    table.create_fts_index("text", use_tantivy=use_tantivy)
     results = table.search("puppy").select(["id", "text"]).limit(5).to_list()
     assert len(results) == 5
     assert len(results[0]) == 3  # id, text, _score
@@ -235,52 +204,53 @@ def test_search_fts(table):
     results = table.search("puppy").select(["id", "text"]).to_list()
     assert len(results) == 10
 
-    # Test with a query
+    if not use_tantivy:
-    results = (
+        # Test with a query
-        table.search(MatchQuery("puppy", "text"))
+        results = (
-        .select(["id", "text"])
+            table.search(MatchQuery("puppy", "text"))
-        .limit(5)
+            .select(["id", "text"])
-        .to_list()
+            .limit(5)
-    )
+            .to_list()
-    assert len(results) == 5
-
-    # Test boost query
-    results = (
-        table.search(
-            BoostQuery(
-                MatchQuery("puppy", "text"),
-                MatchQuery("runs", "text"),
-            )
         )
-        .select(["id", "text"])
+        assert len(results) == 5
-        .limit(5)
-        .to_list()
-    )
-    assert len(results) == 5
 
-    # Test multi match query
+        # Test boost query
-    table.create_fts_index("text2")
+        results = (
-    results = (
+            table.search(
-        table.search(MultiMatchQuery("puppy", ["text", "text2"]))
+                BoostQuery(
-        .select(["id", "text"])
+                    MatchQuery("puppy", "text"),
-        .limit(5)
+                    MatchQuery("runs", "text"),
-        .to_list()
+                )
-    )
+            )
-    assert len(results) == 5
+            .select(["id", "text"])
-    assert len(results[0]) == 3  # id, text, _score
+            .limit(5)
+            .to_list()
+        )
+        assert len(results) == 5
 
-    # Test boolean query
+        # Test multi match query
-    results = (
+        table.create_fts_index("text2", use_tantivy=use_tantivy)
-        table.search(MatchQuery("puppy", "text") & MatchQuery("runs", "text"))
+        results = (
-        .select(["id", "text"])
+            table.search(MultiMatchQuery("puppy", ["text", "text2"]))
-        .limit(5)
+            .select(["id", "text"])
-        .to_list()
+            .limit(5)
-    )
+            .to_list()
-    assert len(results) == 5
+        )
-    assert len(results[0]) == 3  # id, text, _score
+        assert len(results) == 5
-    for r in results:
+        assert len(results[0]) == 3  # id, text, _score
-        assert "puppy" in r["text"]
+
-        assert "runs" in r["text"]
+        # Test boolean query
+        results = (
+            table.search(MatchQuery("puppy", "text") & MatchQuery("runs", "text"))
+            .select(["id", "text"])
+            .limit(5)
+            .to_list()
+        )
+        assert len(results) == 5
+        assert len(results[0]) == 3  # id, text, _score
+        for r in results:
+            assert "puppy" in r["text"]
+            assert "runs" in r["text"]
 
 
 @pytest.mark.asyncio
@@ -348,13 +318,13 @@ async def test_fts_select_async(async_table):
 
 
 def test_search_fts_phrase_query(table):
-    table.create_fts_index("text", with_position=False)
+    table.create_fts_index("text", use_tantivy=False, with_position=False)
     try:
         phrase_results = table.search('"puppy runs"').limit(100).to_list()
         assert False
     except Exception:
         pass
-    table.create_fts_index("text", with_position=True, replace=True)
+    table.create_fts_index("text", use_tantivy=False, with_position=True, replace=True)
     results = table.search("puppy").limit(100).to_list()
 
     # Test with quotation marks
@@ -405,8 +375,8 @@ async def test_search_fts_phrase_query_async(async_table):
 
 
 def test_search_fts_specify_column(table):
-    table.create_fts_index("text")
+    table.create_fts_index("text", use_tantivy=False)
-    table.create_fts_index("text2")
+    table.create_fts_index("text2", use_tantivy=False)
 
     results = table.search("puppy", fts_columns="text").limit(5).to_list()
     assert len(results) == 5
@@ -500,8 +470,42 @@ async def test_search_fts_specify_column_async(async_table):
         pass
 
 
-def test_create_index_from_table(tmp_path, table):
+def test_search_ordering_field_index_table(tmp_path, table):
-    table.create_fts_index("text")
+    table.create_fts_index("text", ordering_field_names=["count"], use_tantivy=True)
+    rows = (
+        table.search("puppy", ordering_field_name="count")
+        .limit(20)
+        .select(["text", "count"])
+        .to_list()
+    )
+    for r in rows:
+        assert "puppy" in r["text"]
+    assert sorted(rows, key=lambda x: x["count"], reverse=True) == rows
+
+
+def test_search_ordering_field_index(tmp_path, table):
+    index = ldb.fts.create_index(
+        str(tmp_path / "index"), ["text"], ordering_fields=["count"]
+    )
+
+    ldb.fts.populate_index(index, table, ["text"], ordering_fields=["count"])
+    index.reload()
+    results = ldb.fts.search_index(
+        index, query="puppy", limit=5, ordering_field="count"
+    )
+    assert len(results) == 2
+    assert len(results[0]) == 5  # row_ids
+    assert len(results[1]) == 5  # _distance
+    rows = table.to_lance().take(results[0]).to_pylist()
+
+    for r in rows:
+        assert "puppy" in r["text"]
+    assert sorted(rows, key=lambda x: x["count"], reverse=True) == rows
+
+
+@pytest.mark.parametrize("use_tantivy", [True, False])
+def test_create_index_from_table(tmp_path, table, use_tantivy):
+    table.create_fts_index("text", use_tantivy=use_tantivy)
     df = table.search("puppy").limit(5).select(["text"]).to_pandas()
     assert len(df) <= 5
     assert "text" in df.columns
@@ -521,24 +525,36 @@ def test_create_index_from_table(tmp_path, table):
     )
 
     with pytest.raises(Exception, match="already exists"):
-        table.create_fts_index("text")
+        table.create_fts_index("text", use_tantivy=use_tantivy)
 
-    table.create_fts_index("text", replace=True)
+    table.create_fts_index("text", replace=True, use_tantivy=use_tantivy)
     assert len(table.search("gorilla").limit(1).to_pandas()) == 1
 
 
 def test_create_index_multiple_columns(tmp_path, table):
-    with pytest.raises(ValueError, match="Native FTS indexes can only be created"):
+    table.create_fts_index(["text", "text2"], use_tantivy=True)
-        table.create_fts_index(["text", "text2"])
+    df = table.search("puppy").limit(5).to_pandas()
+    assert len(df) == 5
+    assert "text" in df.columns
+    assert "text2" in df.columns
+
+
+def test_empty_rs(tmp_path, table, mocker):
+    table.create_fts_index(["text", "text2"], use_tantivy=True)
+    mocker.patch("lancedb.fts.search_index", return_value=([], []))
+    df = table.search("puppy").limit(5).to_pandas()
+    assert len(df) == 0
 
 
 def test_nested_schema(tmp_path, table):
-    with pytest.raises(ValueError, match="top-level fields"):
+    table.create_fts_index("nested.text", use_tantivy=True)
-        table.create_fts_index("nested.text")
+    rs = table.search("puppy").limit(5).to_list()
+    assert len(rs) == 5
 
 
-def test_search_index_with_filter(table):
+@pytest.mark.parametrize("use_tantivy", [True, False])
-    table.create_fts_index("text")
+def test_search_index_with_filter(table, use_tantivy):
+    table.create_fts_index("text", use_tantivy=use_tantivy)
     orig_import = __import__
 
     def import_mock(name, *args):
@@ -568,7 +584,8 @@ def test_search_index_with_filter(table):
         assert r["_rowid"] is not None
 
 
-def test_null_input(table):
+@pytest.mark.parametrize("use_tantivy", [True, False])
+def test_null_input(table, use_tantivy):
     table.add(
         [
             {
@@ -581,13 +598,14 @@ def test_null_input(table):
             }
         ]
     )
-    table.create_fts_index("text")
+    table.create_fts_index("text", use_tantivy=use_tantivy)
 
 
 def test_syntax(table):
     # https://github.com/lancedb/lancedb/issues/769
-    table.create_fts_index("text")
+    table.create_fts_index("text", use_tantivy=True)
-    table.search("they could have been dogs OR").limit(10).to_list()
+    with pytest.raises(ValueError, match="Syntax Error"):
+        table.search("they could have been dogs OR").limit(10).to_list()
 
     # these should work
 
@@ -598,7 +616,6 @@ def test_syntax(table):
     ).to_list()
 
     # phrase queries
-    table.create_fts_index("text", with_position=True, replace=True)
     table.search("they could have been dogs OR cats").phrase_query().limit(10).to_list()
     table.search('"they could have been dogs OR cats"').limit(10).to_list()
     table.search('''"the cats OR dogs were not really 'pets' at all"''').limit(
@@ -622,7 +639,7 @@ def test_language(mem_db: DBConnection):
     table = mem_db.create_table("test", data=data)
 
     with pytest.raises(ValueError) as e:
-        table.create_fts_index("text", language="klingon")
+        table.create_fts_index("text", use_tantivy=False, language="klingon")
 
     assert exception_output(e) == (
         "ValueError: LanceDB does not support the requested language: 'klingon'\n"
@@ -633,6 +650,7 @@ def test_language(mem_db: DBConnection):
 
     table.create_fts_index(
         "text",
+        use_tantivy=False,
         language="French",
         stem=True,
         ascii_folding=True,
@@ -672,7 +690,7 @@ def test_fts_on_list(mem_db: DBConnection):
         }
     )
     table = mem_db.create_table("test", data=data)
-    table.create_fts_index("text", with_position=True)
+    table.create_fts_index("text", use_tantivy=False, with_position=True)
 
     res = table.search("lance").limit(5).to_list()
     assert len(res) == 3
@@ -684,7 +702,7 @@ def test_fts_on_list(mem_db: DBConnection):
 def test_fts_ngram(mem_db: DBConnection):
     data = pa.table({"text": ["hello world", "lance database", "lance is cool"]})
     table = mem_db.create_table("test", data=data)
-    table.create_fts_index("text", base_tokenizer="ngram")
+    table.create_fts_index("text", use_tantivy=False, base_tokenizer="ngram")
 
     results = table.search("lan", query_type="fts").limit(10).to_list()
     assert len(results) == 2
@@ -703,6 +721,7 @@ def test_fts_ngram(mem_db: DBConnection):
     # test setting min_ngram_length and prefix_only
     table.create_fts_index(
         "text",
+        use_tantivy=False,
         base_tokenizer="ngram",
         replace=True,
         ngram_min_length=2,
@@ -723,90 +742,6 @@ def test_fts_ngram(mem_db: DBConnection):
     assert set(r["text"] for r in results) == {"lance database", "lance is cool"}
 
 
-def test_fts_jieba_tokenizer(mem_db: DBConnection, language_model_home):
-    data = pa.table({"text": ["我们都有光明的前途", "光明的前途"]})
-    table = mem_db.create_table("test_jieba", data=data)
-    table.create_fts_index(
-        "text",
-        base_tokenizer="jieba/default",
-        stem=False,
-        remove_stop_words=False,
-        ascii_folding=False,
-    )
-
-    results = table.search("我们", query_type="fts").limit(10).to_list()
-    assert [row["text"] for row in results] == ["我们都有光明的前途"]
-
-
-def test_fts_jieba_missing_language_model_note(
-    mem_db: DBConnection, monkeypatch, tmp_path
-):
-    missing_root = tmp_path / "missing-language-models"
-    monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(missing_root))
-    table = mem_db.create_table(
-        "test_missing_jieba_model",
-        data=pa.table({"text": ["我们都有光明的前途"]}),
-    )
-
-    with pytest.raises((ValueError, RuntimeError)) as e:
-        table.create_fts_index(
-            "text",
-            base_tokenizer="jieba/default",
-            stem=False,
-            remove_stop_words=False,
-            ascii_folding=False,
-        )
-
-    output = exception_output(e)
-    assert "Invalid directory path:" in output
-    assert "LANCE_LANGUAGE_MODEL_HOME" in output
-    assert "jieba/default" in output
-
-
-@pytest.mark.asyncio
-async def test_fts_jieba_missing_language_model_note_async(monkeypatch, tmp_path):
-    missing_root = tmp_path / "missing-language-models"
-    monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(missing_root))
-    db = await ldb.connect_async(tmp_path / "async-db")
-    table = await db.create_table(
-        "test_missing_jieba_model_async",
-        data=pa.table({"text": ["我们都有光明的前途"]}),
-    )
-
-    with pytest.raises((ValueError, RuntimeError)) as e:
-        await table.create_index(
-            "text",
-            config=FTS(
-                base_tokenizer="jieba/default",
-                stem=False,
-                remove_stop_words=False,
-                ascii_folding=False,
-            ),
-        )
-
-    output = exception_output(e)
-    assert "Invalid directory path:" in output
-    assert "LANCE_LANGUAGE_MODEL_HOME" in output
-    assert "jieba/default" in output
-
-
-def test_fts_lindera_tokenizer(
-    mem_db: DBConnection, language_model_home, lindera_ipadic
-):
-    data = pa.table({"text": ["成田国際空港", "東京国際空港", "羽田空港"]})
-    table = mem_db.create_table("test_lindera", data=data)
-    table.create_fts_index(
-        "text",
-        base_tokenizer="lindera/ipadic",
-        stem=False,
-        remove_stop_words=False,
-        ascii_folding=False,
-    )
-
-    results = table.search("成田", query_type="fts").limit(10).to_list()
-    assert [row["text"] for row in results] == ["成田国際空港"]
-
-
 def test_fts_query_to_json():
     """Test that FTS query to_json() produces valid JSON strings with exact format."""
 
@@ -951,7 +886,7 @@ def test_fts_query_to_json():
 
 
 def test_fts_fast_search(table):
-    table.create_fts_index("text")
+    table.create_fts_index("text", use_tantivy=False)
 
     # Insert some unindexed data
     table.add(

python/python/tests/test_hybrid_query.py

@@ -28,7 +28,7 @@ def sync_table(tmpdir_factory) -> Table:
         }
     )
     table = db.create_table("test", data)
-    table.create_fts_index("text", with_position=False)
+    table.create_fts_index("text", with_position=False, use_tantivy=False)
     return table
 
 
@@ -177,60 +177,6 @@ async def test_analyze_plan(table: AsyncTable):
     assert "metrics=" in res
 
 
-@pytest.fixture
-def table_with_id(tmpdir_factory) -> Table:
-    tmp_path = str(tmpdir_factory.mktemp("data"))
-    db = lancedb.connect(tmp_path)
-    data = pa.table(
-        {
-            "id": pa.array([1, 2, 3, 4], type=pa.int64()),
-            "text": pa.array(["a", "b", "cat", "dog"]),
-            "vector": pa.array(
-                [[0.1, 0.1], [2, 2], [-0.1, -0.1], [0.5, -0.5]],
-                type=pa.list_(pa.float32(), list_size=2),
-            ),
-        }
-    )
-    table = db.create_table("test_with_id", data)
-    table.create_fts_index("text", with_position=False)
-    return table
-
-
-def test_hybrid_prefilter_explain_plan(table_with_id: Table):
-    """
-    Verify that the prefilter logic is not inverted in LanceHybridQueryBuilder.
-    """
-    plan_prefilter = (
-        table_with_id.search(query_type="hybrid")
-        .vector([0.0, 0.0])
-        .text("dog")
-        .where("id = 1", prefilter=True)
-        .limit(2)
-        .explain_plan(verbose=True)
-    )
-
-    plan_postfilter = (
-        table_with_id.search(query_type="hybrid")
-        .vector([0.0, 0.0])
-        .text("dog")
-        .where("id = 1", prefilter=False)
-        .limit(2)
-        .explain_plan(verbose=True)
-    )
-
-    # prefilter=True: filter is pushed into the LanceRead scan.
-    # The FTS sub-plan exposes this as "full_filter=id = Int64(1)" inside LanceRead.
-    assert "full_filter=id = Int64(1)" in plan_prefilter, (
-        f"Should push the filter into the scan.\nPlan:\n{plan_prefilter}"
-    )
-
-    # prefilter=False: filter is applied as a separate FilterExec after the search.
-    # The filter must NOT be embedded in the scan.
-    assert "full_filter=id = Int64(1)" not in plan_postfilter, (
-        f"Should NOT push the filter into the scan.\nPlan:\n{plan_postfilter}"
-    )
-
-
 def test_normalize_scores():
     cases = [
         (pa.array([0.1, 0.4]), pa.array([0.0, 1.0])),

python/python/tests/test_index.py

@@ -3,7 +3,6 @@
 
 from datetime import timedelta
 import random
-from typing import get_args, get_type_hints
 
 import pyarrow as pa
 import pytest
@@ -16,16 +15,13 @@ from lancedb.index import (
     IvfSq,
     IvfHnswPq,
     IvfHnswSq,
-    IvfHnswFlat,
     IvfRq,
     Bitmap,
     LabelList,
     HnswPq,
     HnswSq,
-    HnswFlat,
     FTS,
 )
-from lancedb.table import IndexStatistics
 
 
 @pytest_asyncio.fixture
@@ -252,21 +248,6 @@ async def test_create_hnswpq_alias_index(some_table: AsyncTable):
     assert indices[0].index_type in {"HnswPq", "IvfHnswPq"}
 
 
-@pytest.mark.asyncio
-async def test_create_hnswflat_index(some_table: AsyncTable):
-    await some_table.create_index("vector", config=HnswFlat(num_partitions=10))
-    indices = await some_table.list_indices()
-    assert len(indices) == 1
-
-
-@pytest.mark.asyncio
-async def test_create_hnswflat_alias_index(some_table: AsyncTable):
-    await some_table.create_index("vector", config=IvfHnswFlat(num_partitions=5))
-    indices = await some_table.list_indices()
-    assert len(indices) == 1
-    assert indices[0].index_type in {"HnswFlat", "IvfHnswFlat"}
-
-
 @pytest.mark.asyncio
 async def test_create_ivfsq_index(some_table: AsyncTable):
     await some_table.create_index("vector", config=IvfSq(num_partitions=10))
@@ -302,24 +283,3 @@ async def test_create_index_with_binary_vectors(binary_table: AsyncTable):
     for v in range(256):
         res = await binary_table.query().nearest_to([v] * 128).to_arrow()
         assert res["id"][0].as_py() == v
-
-
-def test_index_statistics_index_type_lists_all_supported_values():
-    expected_index_types = {
-        "IVF_FLAT",
-        "IVF_SQ",
-        "IVF_PQ",
-        "IVF_RQ",
-        "IVF_HNSW_SQ",
-        "IVF_HNSW_PQ",
-        "IVF_HNSW_FLAT",
-        "FTS",
-        "BTREE",
-        "BITMAP",
-        "LABEL_LIST",
-    }
-
-    assert (
-        set(get_args(get_type_hints(IndexStatistics)["index_type"]))
-        == expected_index_types
-    )

python/python/tests/test_namespace.py

@@ -8,7 +8,6 @@ import shutil
 import pytest
 import pyarrow as pa
 import lancedb
-from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
 
 
 class TestNamespaceConnection:
@@ -33,16 +32,6 @@ class TestNamespaceConnection:
         # Initially no tables in root
         assert len(list(db.table_names())) == 0
 
-    def test_connect_via_connect_helper(self):
-        """Connecting via lancedb.connect should delegate to namespace connection."""
-        db = lancedb.connect(
-            namespace_client_impl="dir",
-            namespace_client_properties={"root": self.temp_dir},
-        )
-
-        assert isinstance(db, lancedb.LanceNamespaceDBConnection)
-        assert len(list(db.table_names())) == 0
-
     def test_create_table_through_namespace(self):
         """Test creating a table through namespace."""
         db = lancedb.connect_namespace("dir", {"root": self.temp_dir})
@@ -60,14 +49,14 @@ class TestNamespaceConnection:
         )
 
         # Create empty table in child namespace
-        table = db.create_table("test_table", schema=schema, namespace_path=["test_ns"])
+        table = db.create_table("test_table", schema=schema, namespace=["test_ns"])
         assert table is not None
         assert table.name == "test_table"
         assert table.namespace == ["test_ns"]
         assert table.id == "test_ns$test_table"
 
         # Table should appear in child namespace
-        table_names = list(db.table_names(namespace_path=["test_ns"]))
+        table_names = list(db.table_names(namespace=["test_ns"]))
         assert "test_table" in table_names
         assert len(table_names) == 1
 
@@ -90,10 +79,10 @@ class TestNamespaceConnection:
                 pa.field("vector", pa.list_(pa.float32(), 2)),
             ]
         )
-        db.create_table("test_table", schema=schema, namespace_path=["test_ns"])
+        db.create_table("test_table", schema=schema, namespace=["test_ns"])
 
         # Open the table
-        table = db.open_table("test_table", namespace_path=["test_ns"])
+        table = db.open_table("test_table", namespace=["test_ns"])
         assert table is not None
         assert table.name == "test_table"
         assert table.namespace == ["test_ns"]
@@ -118,31 +107,31 @@ class TestNamespaceConnection:
                 pa.field("vector", pa.list_(pa.float32(), 2)),
             ]
         )
-        db.create_table("table1", schema=schema, namespace_path=["test_ns"])
+        db.create_table("table1", schema=schema, namespace=["test_ns"])
-        db.create_table("table2", schema=schema, namespace_path=["test_ns"])
+        db.create_table("table2", schema=schema, namespace=["test_ns"])
 
         # Verify both tables exist in child namespace
-        table_names = list(db.table_names(namespace_path=["test_ns"]))
+        table_names = list(db.table_names(namespace=["test_ns"]))
         assert "table1" in table_names
         assert "table2" in table_names
         assert len(table_names) == 2
 
         # Drop one table
-        db.drop_table("table1", namespace_path=["test_ns"])
+        db.drop_table("table1", namespace=["test_ns"])
 
         # Verify only table2 remains
-        table_names = list(db.table_names(namespace_path=["test_ns"]))
+        table_names = list(db.table_names(namespace=["test_ns"]))
         assert "table1" not in table_names
         assert "table2" in table_names
         assert len(table_names) == 1
 
         # Drop the second table
-        db.drop_table("table2", namespace_path=["test_ns"])
+        db.drop_table("table2", namespace=["test_ns"])
-        assert len(list(db.table_names(namespace_path=["test_ns"]))) == 0
+        assert len(list(db.table_names(namespace=["test_ns"]))) == 0
 
         # Should not be able to open dropped table
-        with pytest.raises(TableNotFoundError):
+        with pytest.raises(RuntimeError):
-            db.open_table("table1", namespace_path=["test_ns"])
+            db.open_table("table1", namespace=["test_ns"])
 
     def test_create_table_with_schema(self):
         """Test creating a table with explicit schema through namespace."""
@@ -161,7 +150,7 @@ class TestNamespaceConnection:
         )
 
         # Create table with schema in child namespace
-        table = db.create_table("test_table", schema=schema, namespace_path=["test_ns"])
+        table = db.create_table("test_table", schema=schema, namespace=["test_ns"])
         assert table is not None
         assert table.namespace == ["test_ns"]
 
@@ -185,7 +174,7 @@ class TestNamespaceConnection:
                 pa.field("vector", pa.list_(pa.float32(), 2)),
             ]
         )
-        db.create_table("old_name", schema=schema, namespace_path=["test_ns"])
+        db.create_table("old_name", schema=schema, namespace=["test_ns"])
 
         # Rename should raise NotImplementedError
         with pytest.raises(NotImplementedError, match="rename_table is not supported"):
@@ -206,20 +195,20 @@ class TestNamespaceConnection:
             ]
         )
         for i in range(3):
-            db.create_table(f"table{i}", schema=schema, namespace_path=["test_ns"])
+            db.create_table(f"table{i}", schema=schema, namespace=["test_ns"])
 
         # Verify tables exist in child namespace
-        assert len(list(db.table_names(namespace_path=["test_ns"]))) == 3
+        assert len(list(db.table_names(namespace=["test_ns"]))) == 3
 
         # Drop all tables in child namespace
-        db.drop_all_tables(namespace_path=["test_ns"])
+        db.drop_all_tables(namespace=["test_ns"])
 
         # Verify all tables are gone from child namespace
-        assert len(list(db.table_names(namespace_path=["test_ns"]))) == 0
+        assert len(list(db.table_names(namespace=["test_ns"]))) == 0
 
         # Test that table_names works with keyword-only namespace parameter
-        db.create_table("test_table", schema=schema, namespace_path=["test_ns"])
+        db.create_table("test_table", schema=schema, namespace=["test_ns"])
-        result = list(db.table_names(namespace_path=["test_ns"]))
+        result = list(db.table_names(namespace=["test_ns"]))
         assert "test_table" in result
 
     def test_table_operations(self):
@@ -237,7 +226,7 @@ class TestNamespaceConnection:
                 pa.field("text", pa.string()),
             ]
         )
-        table = db.create_table("test_table", schema=schema, namespace_path=["test_ns"])
+        table = db.create_table("test_table", schema=schema, namespace=["test_ns"])
 
         # Verify empty table was created
         result = table.to_pandas()
@@ -308,25 +297,25 @@ class TestNamespaceConnection:
             ]
         )
         table = db.create_table(
-            "test_table", schema=schema, namespace_path=["test_namespace"]
+            "test_table", schema=schema, namespace=["test_namespace"]
         )
         assert table is not None
 
         # Verify table exists in namespace
-        tables_in_namespace = list(db.table_names(namespace_path=["test_namespace"]))
+        tables_in_namespace = list(db.table_names(namespace=["test_namespace"]))
         assert "test_table" in tables_in_namespace
         assert len(tables_in_namespace) == 1
 
         # Open table from namespace
-        table = db.open_table("test_table", namespace_path=["test_namespace"])
+        table = db.open_table("test_table", namespace=["test_namespace"])
         assert table is not None
         assert table.name == "test_table"
 
         # Drop table from namespace
-        db.drop_table("test_table", namespace_path=["test_namespace"])
+        db.drop_table("test_table", namespace=["test_namespace"])
 
         # Verify table no longer exists in namespace
-        tables_in_namespace = list(db.table_names(namespace_path=["test_namespace"]))
+        tables_in_namespace = list(db.table_names(namespace=["test_namespace"]))
         assert len(tables_in_namespace) == 0
 
         # Drop namespace
@@ -348,14 +337,14 @@ class TestNamespaceConnection:
                 pa.field("vector", pa.list_(pa.float32(), 2)),
             ]
         )
-        db.create_table("test_table", schema=schema, namespace_path=["test_namespace"])
+        db.create_table("test_table", schema=schema, namespace=["test_namespace"])
 
         # Try to drop namespace with tables - should fail
-        with pytest.raises(NamespaceNotEmptyError):
+        with pytest.raises(RuntimeError, match="is not empty"):
             db.drop_namespace(["test_namespace"])
 
         # Drop table first
-        db.drop_table("test_table", namespace_path=["test_namespace"])
+        db.drop_table("test_table", namespace=["test_namespace"])
 
         # Now dropping namespace should work
         db.drop_namespace(["test_namespace"])
@@ -378,10 +367,10 @@ class TestNamespaceConnection:
 
         # Create table with same name in both namespaces
         table_a = db.create_table(
-            "same_name_table", schema=schema, namespace_path=["namespace_a"]
+            "same_name_table", schema=schema, namespace=["namespace_a"]
         )
         table_b = db.create_table(
-            "same_name_table", schema=schema, namespace_path=["namespace_b"]
+            "same_name_table", schema=schema, namespace=["namespace_b"]
         )
 
         # Add different data to each table
@@ -399,9 +388,7 @@ class TestNamespaceConnection:
         table_b.add(data_b)
 
         # Verify data in namespace_a table
-        opened_table_a = db.open_table(
+        opened_table_a = db.open_table("same_name_table", namespace=["namespace_a"])
-            "same_name_table", namespace_path=["namespace_a"]
-        )
         result_a = opened_table_a.to_pandas().sort_values("id").reset_index(drop=True)
         assert len(result_a) == 2
         assert result_a["id"].tolist() == [1, 2]
@@ -412,9 +399,7 @@ class TestNamespaceConnection:
         assert [v.tolist() for v in result_a["vector"]] == [[1.0, 2.0], [3.0, 4.0]]
 
         # Verify data in namespace_b table
-        opened_table_b = db.open_table(
+        opened_table_b = db.open_table("same_name_table", namespace=["namespace_b"])
-            "same_name_table", namespace_path=["namespace_b"]
-        )
         result_b = opened_table_b.to_pandas().sort_values("id").reset_index(drop=True)
         assert len(result_b) == 3
         assert result_b["id"].tolist() == [10, 20, 30]
@@ -434,8 +419,8 @@ class TestNamespaceConnection:
         assert "same_name_table" not in root_tables
 
         # Clean up
-        db.drop_table("same_name_table", namespace_path=["namespace_a"])
+        db.drop_table("same_name_table", namespace=["namespace_a"])
-        db.drop_table("same_name_table", namespace_path=["namespace_b"])
+        db.drop_table("same_name_table", namespace=["namespace_b"])
         db.drop_namespace(["namespace_a"])
         db.drop_namespace(["namespace_b"])
 
@@ -463,8 +448,6 @@ class TestAsyncNamespaceConnection:
         table_names = await db.table_names()
         assert len(list(table_names)) == 0
 
-    # Async connect via namespace helper is not enabled yet.
-
     async def test_create_table_async(self):
         """Test creating a table asynchronously through namespace."""
         db = lancedb.connect_namespace_async("dir", {"root": self.temp_dir})
@@ -483,13 +466,13 @@ class TestAsyncNamespaceConnection:
 
         # Create empty table in child namespace
         table = await db.create_table(
-            "test_table", schema=schema, namespace_path=["test_ns"]
+            "test_table", schema=schema, namespace=["test_ns"]
         )
         assert table is not None
         assert isinstance(table, lancedb.AsyncTable)
 
         # Table should appear in child namespace
-        table_names = await db.table_names(namespace_path=["test_ns"])
+        table_names = await db.table_names(namespace=["test_ns"])
         assert "test_table" in list(table_names)
 
     async def test_open_table_async(self):
@@ -506,10 +489,10 @@ class TestAsyncNamespaceConnection:
                 pa.field("vector", pa.list_(pa.float32(), 2)),
             ]
         )
-        await db.create_table("test_table", schema=schema, namespace_path=["test_ns"])
+        await db.create_table("test_table", schema=schema, namespace=["test_ns"])
 
         # Open the table
-        table = await db.open_table("test_table", namespace_path=["test_ns"])
+        table = await db.open_table("test_table", namespace=["test_ns"])
         assert table is not None
         assert isinstance(table, lancedb.AsyncTable)
 
@@ -563,20 +546,20 @@ class TestAsyncNamespaceConnection:
                 pa.field("vector", pa.list_(pa.float32(), 2)),
             ]
         )
-        await db.create_table("table1", schema=schema, namespace_path=["test_ns"])
+        await db.create_table("table1", schema=schema, namespace=["test_ns"])
-        await db.create_table("table2", schema=schema, namespace_path=["test_ns"])
+        await db.create_table("table2", schema=schema, namespace=["test_ns"])
 
         # Verify both tables exist in child namespace
-        table_names = list(await db.table_names(namespace_path=["test_ns"]))
+        table_names = list(await db.table_names(namespace=["test_ns"]))
         assert "table1" in table_names
         assert "table2" in table_names
         assert len(table_names) == 2
 
         # Drop one table
-        await db.drop_table("table1", namespace_path=["test_ns"])
+        await db.drop_table("table1", namespace=["test_ns"])
 
         # Verify only table2 remains
-        table_names = list(await db.table_names(namespace_path=["test_ns"]))
+        table_names = list(await db.table_names(namespace=["test_ns"]))
         assert "table1" not in table_names
         assert "table2" in table_names
         assert len(table_names) == 1
@@ -605,24 +588,20 @@ class TestAsyncNamespaceConnection:
             ]
         )
         table = await db.create_table(
-            "test_table", schema=schema, namespace_path=["test_namespace"]
+            "test_table", schema=schema, namespace=["test_namespace"]
         )
         assert table is not None
 
         # Verify table exists in namespace
-        tables_in_namespace = list(
+        tables_in_namespace = list(await db.table_names(namespace=["test_namespace"]))
-            await db.table_names(namespace_path=["test_namespace"])
-        )
         assert "test_table" in tables_in_namespace
         assert len(tables_in_namespace) == 1
 
         # Drop table from namespace
-        await db.drop_table("test_table", namespace_path=["test_namespace"])
+        await db.drop_table("test_table", namespace=["test_namespace"])
 
         # Verify table no longer exists in namespace
-        tables_in_namespace = list(
+        tables_in_namespace = list(await db.table_names(namespace=["test_namespace"]))
-            await db.table_names(namespace_path=["test_namespace"])
-        )
         assert len(tables_in_namespace) == 0
 
         # Drop namespace
@@ -647,98 +626,15 @@ class TestAsyncNamespaceConnection:
             ]
         )
         for i in range(3):
-            await db.create_table(
+            await db.create_table(f"table{i}", schema=schema, namespace=["test_ns"])
-                f"table{i}", schema=schema, namespace_path=["test_ns"]
-            )
 
         # Verify tables exist in child namespace
-        table_names = await db.table_names(namespace_path=["test_ns"])
+        table_names = await db.table_names(namespace=["test_ns"])
         assert len(list(table_names)) == 3
 
         # Drop all tables in child namespace
-        await db.drop_all_tables(namespace_path=["test_ns"])
+        await db.drop_all_tables(namespace=["test_ns"])
 
         # Verify all tables are gone from child namespace
-        table_names = await db.table_names(namespace_path=["test_ns"])
+        table_names = await db.table_names(namespace=["test_ns"])
         assert len(list(table_names)) == 0
-
-
-class TestPushdownOperations:
-    """Test pushdown operations on namespace connections."""
-
-    def setup_method(self):
-        """Set up test fixtures."""
-        self.temp_dir = tempfile.mkdtemp()
-
-    def teardown_method(self):
-        """Clean up test fixtures."""
-        shutil.rmtree(self.temp_dir, ignore_errors=True)
-
-    def test_query_table_pushdown_stored(self):
-        """Test that QueryTable pushdown is stored on sync connection."""
-        db = lancedb.connect_namespace(
-            "dir",
-            {"root": self.temp_dir},
-            namespace_client_pushdown_operations=["QueryTable"],
-        )
-        assert "QueryTable" in db._namespace_client_pushdown_operations
-
-    def test_create_table_pushdown_stored(self):
-        """Test that CreateTable pushdown is stored on sync connection."""
-        db = lancedb.connect_namespace(
-            "dir",
-            {"root": self.temp_dir},
-            namespace_client_pushdown_operations=["CreateTable"],
-        )
-        assert "CreateTable" in db._namespace_client_pushdown_operations
-
-    def test_both_pushdowns_stored(self):
-        """Test that both pushdown operations can be set together."""
-        db = lancedb.connect_namespace(
-            "dir",
-            {"root": self.temp_dir},
-            namespace_client_pushdown_operations=["QueryTable", "CreateTable"],
-        )
-        assert "QueryTable" in db._namespace_client_pushdown_operations
-        assert "CreateTable" in db._namespace_client_pushdown_operations
-
-    def test_pushdown_defaults_to_empty(self):
-        """Test that pushdown operations default to empty."""
-        db = lancedb.connect_namespace("dir", {"root": self.temp_dir})
-        assert len(db._namespace_client_pushdown_operations) == 0
-
-
-@pytest.mark.asyncio
-class TestAsyncPushdownOperations:
-    """Test pushdown operations on async namespace connections."""
-
-    def setup_method(self):
-        """Set up test fixtures."""
-        self.temp_dir = tempfile.mkdtemp()
-
-    def teardown_method(self):
-        """Clean up test fixtures."""
-        shutil.rmtree(self.temp_dir, ignore_errors=True)
-
-    async def test_async_query_table_pushdown_stored(self):
-        """Test that QueryTable pushdown is stored on async connection."""
-        db = lancedb.connect_namespace_async(
-            "dir",
-            {"root": self.temp_dir},
-            namespace_client_pushdown_operations=["QueryTable"],
-        )
-        assert "QueryTable" in db._namespace_client_pushdown_operations
-
-    async def test_async_create_table_pushdown_stored(self):
-        """Test that CreateTable pushdown is stored on async connection."""
-        db = lancedb.connect_namespace_async(
-            "dir",
-            {"root": self.temp_dir},
-            namespace_client_pushdown_operations=["CreateTable"],
-        )
-        assert "CreateTable" in db._namespace_client_pushdown_operations
-
-    async def test_async_pushdown_defaults_to_empty(self):
-        """Test that pushdown operations default to empty on async connection."""
-        db = lancedb.connect_namespace_async("dir", {"root": self.temp_dir})
-        assert len(db._namespace_client_pushdown_operations) == 0