chore: update lance dependency to v7.0.0-beta.2

2026-08-30 18:08:24 +00:00 · 2026-05-03 20:14:08 +00:00
53 changed files with 291 additions and 2143 deletions
@@ -29,3 +29,7 @@ runs:
        args: ${{ inputs.args }}
        docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
        working-directory: python
+    - uses: actions/upload-artifact@v4
+      with:
+        name: windows-wheels
+        path: python\target\wheels
@@ -43,7 +43,7 @@ jobs:
          server-username: SONATYPE_USER
          server-password: SONATYPE_TOKEN
          gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
-          gpg-passphrase: MAVEN_GPG_PASSPHRASE
+          gpg-passphrase: ${{ secrets.GPG_PASSPHRASE }}
      - name: Set git config
        run: |
          git config --global user.email "dev+gha@lancedb.com"
@@ -58,11 +58,10 @@ jobs:
          echo "use-agent" >> ~/.gnupg/gpg.conf
          echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
          export GPG_TTY=$(tty)
-          ./mvnw --batch-mode -DskipTests -DpushChanges=false deploy -pl lancedb-core -am -P deploy-to-ossrh
+          ./mvnw --batch-mode -DskipTests -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -pl lancedb-core -am -P deploy-to-ossrh
        env:
          SONATYPE_USER: ${{ secrets.SONATYPE_USER }}
          SONATYPE_TOKEN: ${{ secrets.SONATYPE_TOKEN }}
-          MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}

  report-failure:
    name: Report Workflow Failure
@@ -16,7 +16,6 @@ on:
  push:
    branches:
      - main
-      - release/**
    paths:
      - java/**
      - .github/workflows/java.yml
@@ -3,7 +3,6 @@ on:
  push:
    branches:
      - main
-      - release/**
  pull_request:
    paths:
      - rust/**
@@ -4,7 +4,6 @@ on:
  push:
    branches:
      - main
-      - release/**
  pull_request:
    paths:
      - Cargo.toml
@@ -8,9 +8,6 @@ on:
    # This should trigger a dry run (we skip the final publish step)
    paths:
      - .github/workflows/pypi-publish.yml
-      - .github/workflows/build_linux_wheel/action.yml
-      - .github/workflows/build_mac_wheel/action.yml
-      - .github/workflows/build_windows_wheel/action.yml
      - Cargo.toml # Change in dependency frequently breaks builds
      - Cargo.lock

@@ -24,6 +21,9 @@ jobs:
  linux:
    name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
    timeout-minutes: 60
+    permissions:
+      id-token: write
+      contents: read
    strategy:
      matrix:
        config:
@@ -46,7 +46,7 @@ jobs:
            runner: ubuntu-2404-8x-arm64
    runs-on: ${{ matrix.config.runner }}
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
@@ -60,14 +60,15 @@ jobs:
          args: "--release --strip ${{ matrix.config.extra_args }}"
          arm-build: ${{ matrix.config.platform == 'aarch64' }}
          manylinux: ${{ matrix.config.manylinux }}
-      - uses: actions/upload-artifact@v7
+      - uses: ./.github/workflows/upload_wheel
        if: startsWith(github.ref, 'refs/tags/python-v')
        with:
-          name: wheels-linux-${{ matrix.config.platform }}-${{ matrix.config.manylinux }}
-          path: target/wheels/lancedb-*.whl
-          if-no-files-found: error
+          fury_token: ${{ secrets.FURY_TOKEN }}
  mac:
    timeout-minutes: 90
+    permissions:
+      id-token: write
+      contents: read
    runs-on: ${{ matrix.config.runner }}
    strategy:
      matrix:
@@ -77,7 +78,7 @@ jobs:
    env:
      MACOSX_DEPLOYMENT_TARGET: 10.15
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
@@ -89,21 +90,18 @@ jobs:
        with:
          python-minor-version: 10
          args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
-      - uses: actions/upload-artifact@v7
+      - uses: ./.github/workflows/upload_wheel
        if: startsWith(github.ref, 'refs/tags/python-v')
        with:
-          name: wheels-mac-${{ matrix.config.target }}
-          path: target/wheels/lancedb-*.whl
-          if-no-files-found: error
+          fury_token: ${{ secrets.FURY_TOKEN }}
  windows:
-    timeout-minutes: 90
+    timeout-minutes: 60
+    permissions:
+      id-token: write
+      contents: read
    runs-on: windows-latest
-    env:
-      # link.exe is single-threaded and the long pole on Windows builds. Use
-      # rustc's bundled lld-link instead.
-      CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
@@ -115,70 +113,18 @@ jobs:
        with:
          python-minor-version: 10
          args: "--release --strip"
-      - uses: actions/upload-artifact@v7
+          vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }}
+      - uses: ./.github/workflows/upload_wheel
        if: startsWith(github.ref, 'refs/tags/python-v')
        with:
-          name: wheels-windows
-          path: target/wheels/lancedb-*.whl
-          if-no-files-found: error
-  publish:
-    name: Publish wheels
-    if: startsWith(github.ref, 'refs/tags/python-v')
-    needs: [linux, mac, windows]
-    runs-on: ubuntu-latest
-    permissions:
-      id-token: write
-      contents: read
-    steps:
-      - uses: actions/checkout@v6
-      - name: Download wheel artifacts
-        uses: actions/download-artifact@v8
-        with:
-          pattern: wheels-*
-          path: target/wheels
-          merge-multiple: true
-      - name: List wheels
-        run: ls -la target/wheels
-      - name: Choose repo
-        id: choose_repo
-        run: |
-          if [[ ${{ github.ref }} == *beta* ]]; then
-            echo "repo=fury" >> $GITHUB_OUTPUT
-          else
-            echo "repo=pypi" >> $GITHUB_OUTPUT
-          fi
-      - name: Publish to Fury
-        if: steps.choose_repo.outputs.repo == 'fury'
-        env:
-          FURY_TOKEN: ${{ secrets.FURY_TOKEN }}
-        run: |
-          shopt -s nullglob
-          WHEELS=(target/wheels/lancedb-*.whl)
-          if [[ ${#WHEELS[@]} -eq 0 ]]; then
-            echo "No wheels found in target/wheels/" >&2
-            exit 1
-          fi
-          for WHEEL in "${WHEELS[@]}"; do
-            echo "Uploading $WHEEL to Fury"
-            curl -f -F package=@"$WHEEL" "https://$FURY_TOKEN@push.fury.io/lancedb/"
-          done
-      # NOTE: pypa/gh-action-pypi-publish must be invoked directly from a
-      # workflow file, not from inside a composite action. When called from a
-      # composite, `github.action_repository` is empty (actions/runner#2473)
-      # and the action falls back to `github.repository`, producing a bogus
-      # `docker://ghcr.io/<repo>:<ref>` image reference that GHA tries to pull.
-      - name: Publish to PyPI
-        if: steps.choose_repo.outputs.repo == 'pypi'
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          packages-dir: target/wheels/
+          fury_token: ${{ secrets.FURY_TOKEN }}
  gh-release:
    if: startsWith(github.ref, 'refs/tags/python-v')
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
@@ -241,13 +187,13 @@ jobs:
  report-failure:
    name: Report Workflow Failure
    runs-on: ubuntu-latest
-    needs: [linux, mac, windows, publish]
+    needs: [linux, mac, windows]
    permissions:
      contents: read
      issues: write
    if: always() && failure() && startsWith(github.ref, 'refs/tags/python-v')
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
      - uses: ./.github/actions/create-failure-issue
        with:
          job-results: ${{ toJSON(needs) }}
@@ -4,7 +4,6 @@ on:
  push:
    branches:
      - main
-      - release/**
  pull_request:
    paths:
      - Cargo.toml
@@ -4,7 +4,6 @@ on:
  push:
    branches:
      - main
-      - release/**
  pull_request:
    paths:
      - Cargo.toml
@@ -0,0 +1,34 @@
+name: upload-wheel
+
+description: "Upload wheels to Pypi"
+inputs:
+  fury_token:
+    required: true
+    description: "release token for the fury repo"
+
+runs:
+  using: "composite"
+  steps:
+  - name: Choose repo
+    shell: bash
+    id: choose_repo
+    run: |
+      if [[ ${{ github.ref }} == *beta* ]]; then
+        echo "repo=fury" >> $GITHUB_OUTPUT
+      else
+        echo "repo=pypi" >> $GITHUB_OUTPUT
+      fi
+  - name: Publish to Fury
+    if: steps.choose_repo.outputs.repo == 'fury'
+    shell: bash
+    env:
+      FURY_TOKEN: ${{ inputs.fury_token }}
+    run: |
+      WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1)
+      echo "Uploading $WHEEL to Fury"
+      curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/
+  - name: Publish to PyPI
+    if: steps.choose_repo.outputs.repo == 'pypi'
+    uses: pypa/gh-action-pypi-publish@release/v1
+    with:
+      packages-dir: target/wheels/
@@ -365,7 +365,7 @@ version = "58.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags",
 "serde_core",
 "serde_json",
 ]
@@ -1186,26 +1186,6 @@ dependencies = [
 "num-traits",
 ]

-[[package]]
-name = "bincode"
-version = "2.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740"
-dependencies = [
- "bincode_derive",
- "serde",
- "unty",
-]
-
-[[package]]
-name = "bincode_derive"
-version = "2.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09"
-dependencies = [
- "virtue",
-]
-
 [[package]]
 name = "bit-set"
 version = "0.8.0"
@@ -1221,12 +1201,6 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"

-[[package]]
-name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-
 [[package]]
 name = "bitflags"
 version = "2.11.0"
@@ -1464,15 +1438,6 @@ dependencies = [
 "shlex",
 ]

-[[package]]
-name = "cedarwood"
-version = "0.4.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d910bedd62c24733263d0bed247460853c9d22e8956bd4cd964302095e04e90"
-dependencies = [
- "smallvec",
-]
-
 [[package]]
 name = "cfg-if"
 version = "1.0.4"
@@ -1528,7 +1493,7 @@ checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f"
 dependencies = [
 "parse-zoneinfo",
 "phf 0.11.3",
- "phf_codegen 0.11.3",
+ "phf_codegen",
 ]

 [[package]]
@@ -1786,7 +1751,7 @@ version = "0.29.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags",
 "crossterm_winapi",
 "document-features",
 "parking_lot",
@@ -2807,70 +2772,6 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"

-[[package]]
-name = "encoding"
-version = "0.2.33"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec"
-dependencies = [
- "encoding-index-japanese",
- "encoding-index-korean",
- "encoding-index-simpchinese",
- "encoding-index-singlebyte",
- "encoding-index-tradchinese",
-]
-
-[[package]]
-name = "encoding-index-japanese"
-version = "1.20141219.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91"
-dependencies = [
- "encoding_index_tests",
-]
-
-[[package]]
-name = "encoding-index-korean"
-version = "1.20141219.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81"
-dependencies = [
- "encoding_index_tests",
-]
-
-[[package]]
-name = "encoding-index-simpchinese"
-version = "1.20141219.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7"
-dependencies = [
- "encoding_index_tests",
-]
-
-[[package]]
-name = "encoding-index-singlebyte"
-version = "1.20141219.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a"
-dependencies = [
- "encoding_index_tests",
-]
-
-[[package]]
-name = "encoding-index-tradchinese"
-version = "1.20141219.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18"
-dependencies = [
- "encoding_index_tests",
-]
-
-[[package]]
-name = "encoding_index_tests"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569"
-
 [[package]]
 name = "encoding_rs"
 version = "0.8.35"
@@ -2880,15 +2781,6 @@ dependencies = [
 "cfg-if",
 ]

-[[package]]
-name = "encoding_rs_io"
-version = "0.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cc3c5651fb62ab8aa3103998dade57efdd028544bd300516baa31840c252a83"
-dependencies = [
- "encoding_rs",
-]
-
 [[package]]
 name = "enum-as-inner"
 version = "0.6.1"
@@ -3044,17 +2936,6 @@ dependencies = [
 "subtle",
 ]

-[[package]]
-name = "filetime"
-version = "0.2.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db"
-dependencies = [
- "cfg-if",
- "libc",
- "libredox",
-]
-
 [[package]]
 name = "find-msvc-tools"
 version = "0.1.9"
@@ -3073,7 +2954,7 @@ version = "25.12.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags",
 "rustc_version",
 ]

@@ -3140,9 +3021,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"

 [[package]]
 name = "fsst"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65f4a8c268d4d18be0f79a897c758f4eb6b074cc5c4bab56e828f0657d6bd521"
+version = "7.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.2#75ddf97f15eda7e8380e9856bcfe3959b0614153"
 dependencies = [
 "arrow-array",
 "rand 0.9.4",
@@ -4015,7 +3895,7 @@ version = "0.7.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fdd7bddefd0a8833b88a4b68f90dae22c7450d11b354198baee3874fd811b344"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags",
 "cfg-if",
 "libc",
 ]
@@ -4090,28 +3970,6 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9028f49264629065d057f340a86acb84867925865f73bbf8d47b4d149a7e88b8"

-[[package]]
-name = "jieba-macros"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a29cfc5dcd898604c6f80363411fa6b6b08e27d1d253d6225b9cb6702ea02fc0"
-dependencies = [
- "phf_codegen 0.13.1",
-]
-
-[[package]]
-name = "jieba-rs"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3245d6e9d1d5facbd6a23848d6b67e3439738ccbb4fa5a3d65da315ba1a910a2"
-dependencies = [
- "cedarwood",
- "jieba-macros",
- "phf 0.13.1",
- "regex",
- "rustc-hash",
-]
-
 [[package]]
 name = "jiff"
 version = "0.2.23"
@@ -4208,20 +4066,10 @@ dependencies = [
 "simple_asn1",
 ]

-[[package]]
-name = "kanaria"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0f9d9652540055ac4fded998a73aca97d965899077ab1212587437da44196ff"
-dependencies = [
- "bitflags 1.3.2",
-]
-
 [[package]]
 name = "lance"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1dec6c13a1b9a608cc8275bb7e967250aba18b1dfde65ea07a78c6d442e8bf5c"
+version = "7.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.2#75ddf97f15eda7e8380e9856bcfe3959b0614153"
 dependencies = [
 "arrow",
 "arrow-arith",
@@ -4289,9 +4137,8 @@ dependencies = [

 [[package]]
 name = "lance-arrow"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32f0efc3607bac297f1b41ac4dc3d6ffbadd36f61a24bec46b70c1f9bda1feda"
+version = "7.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.2#75ddf97f15eda7e8380e9856bcfe3959b0614153"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4312,9 +4159,8 @@ dependencies = [

 [[package]]
 name = "lance-bitpacking"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c04b043c89e46ed3548f3b6423dcbdfac485e636c6e670d642424e954cfab66"
+version = "7.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.2#75ddf97f15eda7e8380e9856bcfe3959b0614153"
 dependencies = [
 "arrayref",
 "paste",
@@ -4323,9 +4169,8 @@ dependencies = [

 [[package]]
 name = "lance-core"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62e954fbffdf484587e956d34613b187a4d4785f6a52b0d89b0c04f1d75d6246"
+version = "7.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.2#75ddf97f15eda7e8380e9856bcfe3959b0614153"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4362,9 +4207,8 @@ dependencies = [

 [[package]]
 name = "lance-datafusion"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ac79df93aa8050f1c33fdc76bd1e86da714c309c08170be5eb1e353a67dc29b"
+version = "7.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.2#75ddf97f15eda7e8380e9856bcfe3959b0614153"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -4395,9 +4239,8 @@ dependencies = [

 [[package]]
 name = "lance-datagen"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52d904119ec5682fdf5729dd943bfdeea78e43585d07a13cce34b2bcee41328b"
+version = "7.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.2#75ddf97f15eda7e8380e9856bcfe3959b0614153"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -4415,9 +4258,8 @@ dependencies = [

 [[package]]
 name = "lance-encoding"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4386ea75123fe6194e166f0f1d375b6041d44507fe7b7c63ac0da493de78a2f"
+version = "7.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.2#75ddf97f15eda7e8380e9856bcfe3959b0614153"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -4445,7 +4287,7 @@ dependencies = [
 "prost-types",
 "rand 0.9.4",
 "snafu 0.9.0",
- "strum 0.26.3",
+ "strum",
 "tokio",
 "tracing",
 "xxhash-rust",
@@ -4454,9 +4296,8 @@ dependencies = [

 [[package]]
 name = "lance-file"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "422ad760cadf7c3e8e23aefc60d4f08eff553f4ad899c40c4a5bb9a976888c44"
+version = "7.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.2#75ddf97f15eda7e8380e9856bcfe3959b0614153"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -4488,9 +4329,8 @@ dependencies = [

 [[package]]
 name = "lance-index"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a653b7dc78a171f0692ddd15afc458e296667c481228171d39f5d9a1f2a1faf0"
+version = "7.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.2#75ddf97f15eda7e8380e9856bcfe3959b0614153"
 dependencies = [
 "arrow",
 "arrow-arith",
@@ -4517,7 +4357,6 @@ dependencies = [
 "futures",
 "half",
 "itertools 0.13.0",
- "jieba-rs",
 "jsonb",
 "lance-arrow",
 "lance-core",
@@ -4555,9 +4394,8 @@ dependencies = [

 [[package]]
 name = "lance-io"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "313f83fe349a5df2a7a24ad7707696ae0437d0270b0a1b78b340ee553eef7a6c"
+version = "7.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.2#75ddf97f15eda7e8380e9856bcfe3959b0614153"
 dependencies = [
 "arrow",
 "arrow-arith",
@@ -4601,9 +4439,8 @@ dependencies = [

 [[package]]
 name = "lance-linalg"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "66bb2b89cc84b4fdb96c2436224394c1bb32d38e1e38a9c96e22e6e3918b0458"
+version = "7.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.2#75ddf97f15eda7e8380e9856bcfe3959b0614153"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4619,9 +4456,8 @@ dependencies = [

 [[package]]
 name = "lance-namespace"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbfd6ad2e589bd8fc00965f808f16895b50d6cbd6405bd7a972db82fe62e2e24"
+version = "7.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.2#75ddf97f15eda7e8380e9856bcfe3959b0614153"
 dependencies = [
 "arrow",
 "async-trait",
@@ -4634,9 +4470,8 @@ dependencies = [

 [[package]]
 name = "lance-namespace-impls"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08c9d4a7b0dceb1a4d2697bab8ce62d52f8b8fd0eec4b4ae6d40017f0ccad0cc"
+version = "7.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.2#75ddf97f15eda7e8380e9856bcfe3959b0614153"
 dependencies = [
 "arrow",
 "arrow-ipc",
@@ -4672,23 +4507,21 @@ dependencies = [

 [[package]]
 name = "lance-namespace-reqwest-client"
-version = "0.7.6"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f65e31bdaa13e01dab6e7cf566da31df243c34a542f0d915d3601ec0e01e61d2"
+checksum = "0f061dd6fe63e3ba4052702a9d40973ee4ac57f612f04222897a149576213832"
 dependencies = [
 "reqwest",
 "serde",
 "serde_json",
 "serde_repr",
- "serde_with",
 "url",
 ]

 [[package]]
 name = "lance-table"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a427dea3c93535c65f3c44985742c08ec4bf4f5f817779157b1dc3c6a2a3288"
+version = "7.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.2#75ddf97f15eda7e8380e9856bcfe3959b0614153"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -4727,9 +4560,8 @@ dependencies = [

 [[package]]
 name = "lance-testing"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf17e430caece4d5eb72e8ed76a61b05afc56ae66e79f2abe0945a9794a861d9"
+version = "7.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.2#75ddf97f15eda7e8380e9856bcfe3959b0614153"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -4740,12 +4572,9 @@ dependencies = [

 [[package]]
 name = "lance-tokenizer"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "810ae0847a16b92629c2d894843b9666d2d975d8a7c3bd3ce21ad19c7900b140"
+version = "7.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.2#75ddf97f15eda7e8380e9856bcfe3959b0614153"
 dependencies = [
- "jieba-rs",
- "lindera",
 "rust-stemmers",
 "serde",
 "unicode-normalization",
@@ -4869,7 +4698,6 @@ dependencies = [
 "lance-namespace",
 "lance-namespace-impls",
 "lancedb",
- "libc",
 "log",
 "pin-project",
 "pyo3",
@@ -4981,134 +4809,7 @@ version = "0.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08"
 dependencies = [
- "bitflags 2.11.0",
 "libc",
- "plain",
- "redox_syscall 0.7.4",
-]
-
-[[package]]
-name = "lindera"
-version = "0.44.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50aba4ef41052280722f2120f65606b9218e8718032a3c752b953c4d8091f02e"
-dependencies = [
- "anyhow",
- "bincode",
- "byteorder",
- "csv",
- "kanaria",
- "lindera-cc-cedict",
- "lindera-dictionary",
- "lindera-ipadic",
- "lindera-ipadic-neologd",
- "lindera-ko-dic",
- "lindera-unidic",
- "once_cell",
- "regex",
- "serde",
- "serde_json",
- "serde_yaml",
- "strum 0.27.2",
- "strum_macros 0.27.2",
- "unicode-blocks",
- "unicode-normalization",
- "unicode-segmentation",
- "yada",
-]
-
-[[package]]
-name = "lindera-cc-cedict"
-version = "0.44.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d77e7a0830fd60f23828ad914439997288c1d2cdd9e269be67f967c27b56350"
-dependencies = [
- "bincode",
- "byteorder",
- "lindera-dictionary",
- "once_cell",
- "tokio",
-]
-
-[[package]]
-name = "lindera-dictionary"
-version = "0.44.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "489cc70922782af3fd397c0e130846caefe1c15b27c2211aac8f88a9f4590aaf"
-dependencies = [
- "anyhow",
- "bincode",
- "byteorder",
- "csv",
- "derive_builder",
- "encoding",
- "encoding_rs",
- "encoding_rs_io",
- "flate2",
- "glob",
- "log",
- "md5",
- "memmap2 0.9.10",
- "once_cell",
- "rand 0.9.4",
- "reqwest",
- "serde",
- "tar",
- "thiserror 2.0.18",
- "tokio",
- "yada",
-]
-
-[[package]]
-name = "lindera-ipadic"
-version = "0.44.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78870521431dfaf0f94ddd3484fa08367e9d354fc8c708572f2f00007225ddfa"
-dependencies = [
- "bincode",
- "byteorder",
- "lindera-dictionary",
- "once_cell",
- "tokio",
-]
-
-[[package]]
-name = "lindera-ipadic-neologd"
-version = "0.44.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abcb3dc3056e5c683e12c2c5e8d40076f7ecfd7bd46f5fc0e4ae9e58152b5d85"
-dependencies = [
- "bincode",
- "byteorder",
- "lindera-dictionary",
- "once_cell",
- "tokio",
-]
-
-[[package]]
-name = "lindera-ko-dic"
-version = "0.44.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e99316158bab14f0256d912055521ca784f76c63e7460db8a74775c5dc1f8bc2"
-dependencies = [
- "bincode",
- "byteorder",
- "lindera-dictionary",
- "once_cell",
- "tokio",
-]
-
-[[package]]
-name = "lindera-unidic"
-version = "0.44.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52672945166c14276bbba25e4ec79d7e126db1b503c0a6aa07ffc0141ae15cfa"
-dependencies = [
- "bincode",
- "byteorder",
- "lindera-dictionary",
- "once_cell",
- "tokio",
 ]

 [[package]]
@@ -5265,12 +4966,6 @@ dependencies = [
 "digest",
 ]

-[[package]]
-name = "md5"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0"
-
 [[package]]
 name = "memchr"
 version = "2.8.0"
@@ -5421,7 +5116,7 @@ version = "3.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6944d0bf100571cd6e1a98a316cdca262deb6fccf8d93f5ae1502ca3fc88bd3"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags",
 "ctor",
 "futures",
 "napi-build",
@@ -5735,7 +5430,7 @@ version = "6.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags",
 "libc",
 "once_cell",
 "onig_sys",
@@ -5853,7 +5548,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
 dependencies = [
 "cfg-if",
 "libc",
- "redox_syscall 0.5.18",
+ "redox_syscall",
 "smallvec",
 "windows-link",
 ]
@@ -5962,36 +5657,16 @@ dependencies = [
 "phf_shared 0.12.1",
 ]

-[[package]]
-name = "phf"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
-dependencies = [
- "phf_shared 0.13.1",
- "serde",
-]
-
 [[package]]
 name = "phf_codegen"
 version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"
 dependencies = [
- "phf_generator 0.11.3",
+ "phf_generator",
 "phf_shared 0.11.3",
 ]

-[[package]]
-name = "phf_codegen"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1"
-dependencies = [
- "phf_generator 0.13.1",
- "phf_shared 0.13.1",
-]
-
 [[package]]
 name = "phf_generator"
 version = "0.11.3"
@@ -6002,16 +5677,6 @@ dependencies = [
 "rand 0.8.5",
 ]

-[[package]]
-name = "phf_generator"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737"
-dependencies = [
- "fastrand",
- "phf_shared 0.13.1",
-]
-
 [[package]]
 name = "phf_shared"
 version = "0.11.3"
@@ -6030,15 +5695,6 @@ dependencies = [
 "siphasher",
 ]

-[[package]]
-name = "phf_shared"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266"
-dependencies = [
- "siphasher",
-]
-
 [[package]]
 name = "pin-project"
 version = "1.1.11"
@@ -6125,12 +5781,6 @@ version = "0.3.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"

-[[package]]
-name = "plain"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
-
 [[package]]
 name = "planus"
 version = "0.3.1"
@@ -6228,7 +5878,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "465f70d3e96b6d0b1a43c358ba451286b8c8bd56696feff020d65702aa33e35c"
 dependencies = [
 "ahash",
- "bitflags 2.11.0",
+ "bitflags",
 "bytemuck",
 "chrono",
 "chrono-tz 0.8.6",
@@ -6302,7 +5952,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "89b2632b1af668e2058d5f8f916d8fbde3cac63d03ae29a705f598e41dcfeb7f"
 dependencies = [
 "ahash",
- "bitflags 2.11.0",
+ "bitflags",
 "glob",
 "once_cell",
 "polars-arrow",
@@ -6953,7 +6603,7 @@ version = "11.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags",
 ]

 [[package]]
@@ -7025,16 +6675,7 @@ version = "0.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
 dependencies = [
- "bitflags 2.11.0",
-]
-
-[[package]]
-name = "redox_syscall"
-version = "0.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a"
-dependencies = [
- "bitflags 2.11.0",
+ "bitflags",
 ]

 [[package]]
@@ -7230,9 +6871,9 @@ dependencies = [

 [[package]]
 name = "roaring"
-version = "0.11.4"
+version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9"
+checksum = "8ba9ce64a8f45d7fc86358410bb1a82e8c987504c0d4900e9141d69a9f26c885"
 dependencies = [
 "bytemuck",
 "byteorder",
@@ -7330,7 +6971,7 @@ version = "1.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags",
 "errno",
 "libc",
 "linux-raw-sys",
@@ -7555,7 +7196,7 @@ version = "3.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags",
 "core-foundation 0.10.1",
 "core-foundation-sys",
 "libc",
@@ -7701,19 +7342,6 @@ dependencies = [
 "syn 2.0.117",
 ]

-[[package]]
-name = "serde_yaml"
-version = "0.9.34+deprecated"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
-dependencies = [
- "indexmap 2.13.0",
- "itoa",
- "ryu",
- "serde",
- "unsafe-libyaml",
-]
-
 [[package]]
 name = "sha1"
 version = "0.10.6"
@@ -8054,15 +7682,6 @@ dependencies = [
 "strum_macros 0.26.4",
 ]

-[[package]]
-name = "strum"
-version = "0.27.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf"
-dependencies = [
- "strum_macros 0.27.2",
-]
-
 [[package]]
 name = "strum_macros"
 version = "0.25.3"
@@ -8089,18 +7708,6 @@ dependencies = [
 "syn 2.0.117",
 ]

-[[package]]
-name = "strum_macros"
-version = "0.27.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"
-dependencies = [
- "heck 0.5.0",
- "proc-macro2",
- "quote",
- "syn 2.0.117",
-]
-
 [[package]]
 name = "subtle"
 version = "2.6.1"
@@ -8155,7 +7762,7 @@ version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags",
 "byteorder",
 "enum-as-inner",
 "libc",
@@ -8183,7 +7790,7 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags",
 "core-foundation 0.9.4",
 "system-configuration-sys",
 ]
@@ -8210,17 +7817,6 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"

-[[package]]
-name = "tar"
-version = "0.4.45"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973"
-dependencies = [
- "filetime",
- "libc",
- "xattr",
-]
-
 [[package]]
 name = "target-features"
 version = "0.1.6"
@@ -8548,7 +8144,7 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags",
 "bytes",
 "http 1.4.0",
 "http-body 1.0.1",
@@ -8566,7 +8162,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8"
 dependencies = [
 "async-compression",
- "bitflags 2.11.0",
+ "bitflags",
 "bytes",
 "futures-core",
 "futures-util",
@@ -8689,12 +8285,6 @@ version = "2.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"

-[[package]]
-name = "unicode-blocks"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b12e05d9e06373163a9bb6bb8c263c261b396643a99445fe6b9811fd376581b"
-
 [[package]]
 name = "unicode-ident"
 version = "1.0.24"
@@ -8752,12 +8342,6 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"

-[[package]]
-name = "unsafe-libyaml"
-version = "0.2.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
-
 [[package]]
 name = "untrusted"
 version = "0.7.1"
@@ -8770,12 +8354,6 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"

-[[package]]
-name = "unty"
-version = "0.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae"
-
 [[package]]
 name = "ureq"
 version = "2.12.1"
@@ -8855,12 +8433,6 @@ version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"

-[[package]]
-name = "virtue"
-version = "0.0.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1"
-
 [[package]]
 name = "vsimd"
 version = "0.8.0"
@@ -9010,7 +8582,7 @@ version = "0.244.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
 dependencies = [
- "bitflags 2.11.0",
+ "bitflags",
 "hashbrown 0.15.5",
 "indexmap 2.13.0",
 "semver",
@@ -9406,7 +8978,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
 dependencies = [
 "anyhow",
- "bitflags 2.11.0",
+ "bitflags",
 "indexmap 2.13.0",
 "log",
 "serde",
@@ -9451,16 +9023,6 @@ dependencies = [
 "tap",
 ]

-[[package]]
-name = "xattr"
-version = "1.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
-dependencies = [
- "libc",
- "rustix",
-]
-
 [[package]]
 name = "xmlparser"
 version = "0.13.6"
@@ -9473,12 +9035,6 @@ version = "0.8.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"

-[[package]]
-name = "yada"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aed111bd9e48a802518765906cbdadf0b45afb72b9c81ab049a3b86252adffdd"
-
 [[package]]
 name = "yoke"
 version = "0.8.1"
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"

 [workspace.dependencies]
-lance = { "version" = "=6.0.0", default-features = false }
-lance-core = "=6.0.0"
-lance-datagen = "=6.0.0"
-lance-file = "=6.0.0"
-lance-io = { "version" = "=6.0.0", default-features = false }
-lance-index = "=6.0.0"
-lance-linalg = "=6.0.0"
-lance-namespace = "=6.0.0"
-lance-namespace-impls = { "version" = "=6.0.0", default-features = false }
-lance-table = "=6.0.0"
-lance-testing = "=6.0.0"
-lance-datafusion = "=6.0.0"
-lance-encoding = "=6.0.0"
-lance-arrow = "=6.0.0"
+lance = { "version" = "=7.0.0-beta.2", default-features = false, "tag" = "v7.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=7.0.0-beta.2", "tag" = "v7.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=7.0.0-beta.2", "tag" = "v7.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=7.0.0-beta.2", "tag" = "v7.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=7.0.0-beta.2", default-features = false, "tag" = "v7.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=7.0.0-beta.2", "tag" = "v7.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=7.0.0-beta.2", "tag" = "v7.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=7.0.0-beta.2", "tag" = "v7.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=7.0.0-beta.2", default-features = false, "tag" = "v7.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=7.0.0-beta.2", "tag" = "v7.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=7.0.0-beta.2", "tag" = "v7.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=7.0.0-beta.2", "tag" = "v7.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=7.0.0-beta.2", "tag" = "v7.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=7.0.0-beta.2", "tag" = "v7.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
 arrow = { version = "58.0.0", optional = false }
@@ -51,18 +51,6 @@ ignore = [
    # https://rustsec.org/advisories/RUSTSEC-2024-0436
    { id = "RUSTSEC-2024-0436", reason = "transitive via datafusion; awaiting ecosystem migration" },

-    # encoding: unmaintained. Reached through lindera-dictionary, which is
-    # required by the native Lindera tokenizer path. Lindera has not migrated
-    # off this crate yet.
-    # https://rustsec.org/advisories/RUSTSEC-2021-0153
-    { id = "RUSTSEC-2021-0153", reason = "transitive via lindera-dictionary for native Lindera tokenizer" },
-
-    # fast-float: unsound and unmaintained. Reached only through polars-arrow
-    # from the optional Polars integration; replacement requires a Polars
-    # dependency upgrade.
-    # https://rustsec.org/advisories/RUSTSEC-2024-0379
-    { id = "RUSTSEC-2024-0379", reason = "transitive via polars-arrow; waiting on Polars migration" },
-
    # tantivy: segfault on malformed input due to missing bounds check.
    # Pulled in via lance for full-text search. We only feed tantivy
    # documents we construct ourselves, not attacker-controlled bytes.
@@ -80,17 +68,11 @@ ignore = [
    # https://rustsec.org/advisories/RUSTSEC-2025-0119
    { id = "RUSTSEC-2025-0119", reason = "transitive via hf-hub/indicatif; cosmetic formatting crate" },

-    # bincode: unmaintained. Reached through lindera and lindera-dictionary,
-    # which are required by the native Lindera tokenizer path. Lindera has not
-    # migrated to another serialization format yet.
-    # https://rustsec.org/advisories/RUSTSEC-2025-0141
-    { id = "RUSTSEC-2025-0141", reason = "transitive via lindera/lindera-dictionary for native Lindera tokenizer" },
-
-    # lru: soundness issue in IterMut. Reached only through aws-sdk-s3 in
-    # LanceDB's dev-dependency graph; LanceDB does not use that iterator
-    # directly. Clearing this requires the AWS SDK chain to update lru.
-    # https://rustsec.org/advisories/RUSTSEC-2026-0002
-    { id = "RUSTSEC-2026-0002", reason = "transitive via aws-sdk-s3 dev-dependency; waiting on AWS SDK lru upgrade" },
+    # rustls-pemfile: unmaintained. Reached from two separate chains:
+    # rustls-native-certs 0.6 (via hyper-rustls 0.24) and object_store 0.12.
+    # Both upstream dependencies need to move before we can drop it.
+    # https://rustsec.org/advisories/RUSTSEC-2025-0134
+    { id = "RUSTSEC-2025-0134", reason = "transitive via rustls-native-certs/object_store; waiting on upstream migration" },

    # rustls-webpki 0.101.7 (old major line): name-constraint checks for
    # URI / wildcard names. Pulled in only via the legacy rustls 0.21 chain
@@ -107,12 +89,6 @@ ignore = [
    # we actively use is upgraded to 0.103.13 which contains the fix.
    # https://rustsec.org/advisories/RUSTSEC-2026-0104
    { id = "RUSTSEC-2026-0104", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
-
-    # rand 0.8.5: soundness issue only when ThreadRng reseeds inside a custom
-    # logger. Reached through several transitive chains. LanceDB does not use
-    # rand from a custom logger; upgrade once all pinned chains accept 0.8.6+.
-    # https://rustsec.org/advisories/RUSTSEC-2026-0097
-    { id = "RUSTSEC-2026-0097", reason = "transitive rand 0.8.5; LanceDB does not call ThreadRng from custom logging" },
 ]

 # ---------------------------------------------------------------------------
@@ -501,34 +501,6 @@ Modeled after ``VACUUM`` in PostgreSQL.

 ***

-### prewarmData()
-
-```ts
-abstract prewarmData(columns?): Promise<void>
-```
-
-Prewarm one or more columns of data in the table.
-
-#### Parameters
-
-* **columns?**: `string`[]
-    The columns to prewarm. If undefined, all columns are prewarmed.
-    This will load the column data into the page cache so that future queries that
-    read those columns avoid the initial cold-start latency.  This call initiates
-    prewarming and returns once the request is accepted; the warming itself may
-    continue in the background.  Calling it on already-prewarmed columns is a
-    no-op on the server.
-    Prewarming is generally useful for columns used in filters or projections.
-    Large columns (e.g. high-dimensional vectors or binary data) may not be
-    practical to prewarm.
-    This feature is currently only supported on remote tables.
-
-#### Returns
-
-`Promise`&lt;`void`&gt;
-
-***
-
 ### prewarmIndex()

 ```ts
@@ -94,11 +94,11 @@ of raw SQL strings with [where][lancedb.query.LanceQueryBuilder.where] and

 ## Full text search

-Use [lancedb.table.Table.create_fts_index][] for the synchronous API or
-[lancedb.table.AsyncTable.create_index][] with [lancedb.index.FTS][] for the
-asynchronous API.
+::: lancedb.fts.create_index

-::: lancedb.index.FTS
+::: lancedb.fts.populate_index
+
+::: lancedb.fts.search_index

 ## Utilities

@@ -28,7 +28,7 @@
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>6.0.0</lance-core.version>
+        <lance-core.version>7.0.0-beta.2</lance-core.version>
        <spotless.skip>false</spotless.skip>
        <spotless.version>2.30.0</spotless.version>
        <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
@@ -1870,25 +1870,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
      expect(results.length).toBe(3);
    });

-    test("prewarmData errors on local tables", async () => {
-      const db = await connect(tmpDir.name);
-      const data = [
-        { text: "alpha", vector: [0.1, 0.2, 0.3] },
-        { text: "beta", vector: [0.4, 0.5, 0.6] },
-      ];
-      const table = await db.createTable("prewarm_data_test", data);
-
-      // prewarmData is only supported on remote tables. We verify the call
-      // is wired through napi and surfaces the expected error for both
-      // arg shapes (undefined and string[]).
-      await expect(table.prewarmData()).rejects.toThrow(
-        "prewarm_data is currently only supported on remote tables",
-      );
-      await expect(table.prewarmData(["text"])).rejects.toThrow(
-        "prewarm_data is currently only supported on remote tables",
-      );
-    });
-
    test("full text index on list", async () => {
      const db = await connect(tmpDir.name);
      const data = [
@@ -285,25 +285,6 @@ export abstract class Table {
   */
  abstract prewarmIndex(name: string): Promise<void>;

-  /**
-   * Prewarm one or more columns of data in the table.
-   *
-   * @param columns The columns to prewarm. If undefined, all columns are prewarmed.
-   *
-   * This will load the column data into the page cache so that future queries that
-   * read those columns avoid the initial cold-start latency.  This call initiates
-   * prewarming and returns once the request is accepted; the warming itself may
-   * continue in the background.  Calling it on already-prewarmed columns is a
-   * no-op on the server.
-   *
-   * Prewarming is generally useful for columns used in filters or projections.
-   * Large columns (e.g. high-dimensional vectors or binary data) may not be
-   * practical to prewarm.
-   *
-   * This feature is currently only supported on remote tables.
-   */
-  abstract prewarmData(columns?: string[]): Promise<void>;
-
  /**
   * Waits for asynchronous indexing to complete on the table.
   *
@@ -729,10 +710,6 @@ export class LocalTable extends Table {
    await this.inner.prewarmIndex(name);
  }

-  async prewarmData(columns?: string[]): Promise<void> {
-    await this.inner.prewarmData(columns);
-  }
-
  async waitForIndex(
    indexNames: string[],
    timeoutSeconds: number,
@@ -75,6 +75,7 @@
    "build:debug": "napi build --platform --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir lancedb",
    "postbuild:debug": "shx mkdir -p dist && shx cp lancedb/*.node dist/",
    "build:release": "napi build --platform --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir dist",
+    "postbuild:release": "shx mkdir -p dist && shx cp lancedb/*.node dist/",
    "build": "npm run build:debug && npm run tsc",
    "build-release": "npm run build:release && npm run tsc",
    "tsc": "tsc -b",
@@ -159,14 +159,6 @@ impl Table {
            .default_error()
    }

-    #[napi(catch_unwind)]
-    pub async fn prewarm_data(&self, columns: Option<Vec<String>>) -> napi::Result<()> {
-        self.inner_ref()?
-            .prewarm_data(columns)
-            .await
-            .default_error()
-    }
-
    #[napi(catch_unwind)]
    pub async fn wait_for_index(&self, index_names: Vec<String>, timeout_s: i64) -> Result<()> {
        let timeout = std::time::Duration::from_secs(timeout_s.try_into().unwrap());
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.32.0"
+current_version = "0.31.0-beta.11"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.32.0"
+version = "0.31.0-beta.11"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"
@@ -35,8 +35,7 @@ futures.workspace = true
 serde = "1"
 serde_json = "1"
 snafu.workspace = true
-tokio = { version = "1.40", features = ["sync", "rt-multi-thread"] }
-libc = "0.2"
+tokio = { version = "1.40", features = ["sync"] }

 [build-dependencies]
 pyo3-build-config = { version = "0.28", features = [
@@ -45,7 +45,7 @@ repository = "https://github.com/lancedb/lancedb"

 [project.optional-dependencies]
 pylance = [
-    "pylance>=6.0.0",
+    "pylance>=5.0.0b5",
 ]
 tests = [
    "aiohttp>=3.9.0",
@@ -58,7 +58,7 @@ tests = [
    "pytz>=2023.3",
    "polars>=0.19, <=1.3.0",
    "pyarrow-stubs>=16.0",
-    "pylance>=6.0.0",
+    "pylance>=5.0.0b5",
    "requests>=2.31.0",
    "datafusion>=52,<53",
 ]
@@ -7,6 +7,7 @@ import os
 from concurrent.futures import ThreadPoolExecutor
 from datetime import timedelta
 from typing import Dict, Optional, Union, Any, List
+import warnings

 __version__ = importlib.metadata.version("lancedb")

@@ -437,3 +438,13 @@ __all__ = [
    "Table",
    "__version__",
 ]
+
+
+def __warn_on_fork():
+    warnings.warn(
+        "lance is not fork-safe. If you are using multiprocessing, use spawn instead.",
+    )
+
+
+if hasattr(os, "register_at_fork"):
+    os.register_at_fork(before=__warn_on_fork)  # type: ignore[attr-defined]
@@ -12,7 +12,6 @@ from .index import (
    LabelList,
    HnswPq,
    HnswSq,
-    HnswFlat,
    FTS,
 )
 from lance_namespace import (
@@ -26,7 +25,6 @@ from .remote import ClientConfig

 IvfHnswPq: type[HnswPq] = HnswPq
 IvfHnswSq: type[HnswSq] = HnswSq
-IvfHnswFlat: type[HnswFlat] = HnswFlat

 class PyExpr:
    """A type-safe DataFusion expression node (Rust-side handle)."""
@@ -182,7 +180,6 @@ class Table:
            IvfPq,
            HnswPq,
            HnswSq,
-            HnswFlat,
            BTree,
            Bitmap,
            LabelList,
@@ -445,7 +442,7 @@ class AsyncPermutationBuilder:
    async def execute(self) -> Table: ...

 def async_permutation_builder(
-    table: Table,
+    table: Table, dest_table_name: str
 ) -> AsyncPermutationBuilder: ...
 def fts_query_to_json(query: Any) -> str: ...

@@ -2,9 +2,7 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors

 import asyncio
-import os
 import threading
-import warnings


 class BackgroundEventLoop:
@@ -15,9 +13,6 @@ class BackgroundEventLoop:
    """

    def __init__(self):
-        self._start()
-
-    def _start(self):
        self.loop = asyncio.new_event_loop()
        self.thread = threading.Thread(
            target=self.loop.run_forever,
@@ -36,30 +31,3 @@ class BackgroundEventLoop:


 LOOP = BackgroundEventLoop()
-
-_FORK_WARNED = False
-
-
-def _reset_after_fork():
-    # Threads do not survive fork(), so the asyncio loop in LOOP.thread is
-    # dead in the child. Re-initialize the singleton in place so existing
-    # `from .background_loop import LOOP` references in other modules see
-    # the new state. The Rust-side tokio runtime is reset analogously by a
-    # pthread_atfork hook installed in the _lancedb extension.
-    LOOP._start()
-    global _FORK_WARNED
-    if not _FORK_WARNED:
-        _FORK_WARNED = True
-        warnings.warn(
-            "lancedb fork support is experimental: the internal async "
-            "runtime has been reset in the forked child, but a small chance "
-            "of deadlock remains if other state was mid-operation at fork "
-            "time. The 'forkserver' or 'spawn' multiprocessing start method "
-            "is likely a safer alternative.",
-            RuntimeWarning,
-            stacklevel=2,
-        )
-
-
-if hasattr(os, "register_at_fork"):
-    os.register_at_fork(after_in_child=_reset_after_fork)
@@ -7,7 +7,6 @@ from typing import Literal, Optional
 from ._lancedb import (
    IndexConfig,
 )
-from .types import BaseTokenizerType

 lang_mapping = {
    "ar": "Arabic",
@@ -112,12 +111,8 @@ class FTS:
        - "simple": Splits text by whitespace and punctuation.
        - "whitespace": Split text by whitespace, but not punctuation.
        - "raw": No tokenization. The entire text is treated as a single token.
-        - "ngram": N-gram tokenizer for substring-style matching.
-        - "jieba/*": Jieba tokenizer loaded from Lance's language model home.
-        - "lindera/*": Lindera tokenizer loaded from Lance's language model home.
    language : str, default "English"
-        The language to use for stemming and stop-word removal. This is not the
-        primary way to enable CJK tokenization.
+        The language to use for tokenization.
    max_token_length : int, default 40
        The maximum token length to index. Tokens longer than this length will be
        ignored.
@@ -132,17 +127,10 @@ class FTS:
    ascii_folding : bool, default True
        Whether to fold ASCII characters. This converts accented characters to
        their ASCII equivalent. For example, "café" would be converted to "cafe".
-
-    Notes
-    -----
-    Model-backed tokenizers such as ``jieba/default`` and ``lindera/ipadic``
-    require tokenizer models in Lance's language model home. Set
-    ``LANCE_LANGUAGE_MODEL_HOME`` to override the default platform data
-    directory under ``lance/language_models``.
    """

    with_position: bool = False
-    base_tokenizer: BaseTokenizerType = "simple"
+    base_tokenizer: Literal["simple", "raw", "whitespace"] = "simple"
    language: str = "English"
    max_token_length: Optional[int] = 40
    lower_case: bool = True
@@ -388,98 +376,9 @@ class HnswSq:
    target_partition_size: Optional[int] = None


-@dataclass
-class HnswFlat:
-    """Describe a HNSW-FLAT index configuration.
-
-    HNSW-FLAT stands for Hierarchical Navigable Small World without quantization.
-    It stores raw vectors in the HNSW graph, providing the highest recall among
-    the IVF_HNSW family at the cost of more memory and disk space compared to
-    :class:`HnswSq` or :class:`HnswPq`.
-
-    Parameters
-    ----------
-
-    distance_type: str, default "l2"
-
-        The distance metric used to train the index.
-
-        The following distance types are available:
-
-        "l2" - Euclidean distance. This is a very common distance metric that
-        accounts for both magnitude and direction when determining the distance
-        between vectors. l2 distance has a range of [0, ∞).
-
-        "cosine" - Cosine distance.  Cosine distance is a distance metric
-        calculated from the cosine similarity between two vectors. Cosine
-        similarity is a measure of similarity between two non-zero vectors of an
-        inner product space. It is defined to equal the cosine of the angle
-        between them.  Unlike l2, the cosine distance is not affected by the
-        magnitude of the vectors.  Cosine distance has a range of [0, 2].
-
-        "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
-        distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
-        l2 norm is 1), then dot distance is equivalent to the cosine distance.
-
-    num_partitions, default sqrt(num_rows)
-
-        The number of IVF partitions to create.
-
-        For HNSW, we recommend a small number of partitions. Setting this to 1
-        works well for most tables. For very large tables, training just one HNSW
-        graph will require too much memory. Each partition becomes its own HNSW
-        graph, so setting this value higher reduces the peak memory use of
-        training.
-
-    max_iterations, default 50
-
-        Max iterations to train kmeans.
-
-        When training an IVF index we use kmeans to calculate the partitions.
-        This parameter controls how many iterations of kmeans to run.
-
-    sample_rate, default 256
-
-        The rate used to calculate the number of training vectors for kmeans.
-
-    m, default 20
-
-        The number of neighbors to select for each vector in the HNSW graph.
-
-        This value controls the tradeoff between search speed and accuracy.
-        The higher the value the more accurate the search but the slower it
-        will be.
-
-    ef_construction, default 300
-
-        The number of candidates to evaluate during the construction of the HNSW
-        graph.
-
-        This value controls the tradeoff between build speed and accuracy.
-        The higher the value the more accurate the build but the slower it will
-        be.  150 to 300 is the typical range. 100 is a minimum for good quality
-        search results. In most cases, there is no benefit to setting this higher
-        than 500.  This value should be set to a value that is not less than `ef`
-        in the search phase.
-
-    target_partition_size, default is 1,048,576
-
-        The target size of each partition.
-    """
-
-    distance_type: Literal["l2", "cosine", "dot"] = "l2"
-    num_partitions: Optional[int] = None
-    max_iterations: int = 50
-    sample_rate: int = 256
-    m: int = 20
-    ef_construction: int = 300
-    target_partition_size: Optional[int] = None
-
-
 # Backwards-compatible aliases
 IvfHnswPq = HnswPq
 IvfHnswSq = HnswSq
-IvfHnswFlat = HnswFlat


@dataclass
@@ -799,13 +698,11 @@ __all__ = [
    "IvfPq",
    "IvfHnswPq",
    "IvfHnswSq",
-    "IvfHnswFlat",
    "IvfSq",
    "IvfRq",
    "IvfFlat",
    "HnswPq",
    "HnswSq",
-    "HnswFlat",
    "IndexConfig",
    "FTS",
    "Bitmap",
@@ -1,11 +1,10 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors

-import copy
-import json
-
 from deprecation import deprecated
+from lancedb import AsyncConnection, DBConnection
 import pyarrow as pa
+import json

 from ._lancedb import async_permutation_builder, PermutationReader
 from .table import LanceTable
@@ -37,7 +36,10 @@ class PermutationBuilder:
    be referenced by name in the future.  If names are not provided then they can only
    be referenced by their ordinal index.  There is no requirement to name every split.

-    The permutation is stored in memory and will be lost when the program exits.
+    By default, the permutation will be stored in memory and will be lost when the
+    program exits.  To persist the permutation (for very large datasets or to share
+    the permutation across multiple workers) use the [persist](#persist) method to
+    create a permanent table.
    """

    def __init__(self, table: LanceTable):
@@ -49,6 +51,15 @@ class PermutationBuilder:
        """
        self._async = async_permutation_builder(table)

+    def persist(
+        self, database: Union[DBConnection, AsyncConnection], table_name: str
+    ) -> "PermutationBuilder":
+        """
+        Persist the permutation to the given database.
+        """
+        self._async.persist(database, table_name)
+        return self
+
    def split_random(
        self,
        *,
@@ -369,44 +380,20 @@ class Permutation:

    def __init__(
        self,
-        base_table: LanceTable,
-        permutation_table: Optional[LanceTable],
-        split: int,
+        reader: PermutationReader,
        selection: dict[str, str],
        batch_size: int,
        transform_fn: Callable[pa.RecordBatch, Any],
-        offset: Optional[int] = None,
-        limit: Optional[int] = None,
-        connection_factory: Optional[Callable[[str], LanceTable]] = None,
-        _reader: Optional[PermutationReader] = None,
    ):
        """
        Internal constructor.  Use [from_tables](#from_tables) instead.
        """
-        assert base_table is not None, "base_table is required"
+        assert reader is not None, "reader is required"
        assert selection is not None, "selection is required"
-        self.base_table = base_table
-        self.permutation_table = permutation_table
-        self.split = split
+        self.reader = reader
        self.selection = selection
        self.transform_fn = transform_fn
        self.batch_size = batch_size
-        self.offset = offset
-        self.limit = limit
-        self.connection_factory = connection_factory
-        if _reader is None:
-            _reader = LOOP.run(self._build_reader())
-        self.reader: PermutationReader = _reader
-
-    async def _build_reader(self) -> PermutationReader:
-        reader = await PermutationReader.from_tables(
-            self.base_table, self.permutation_table, self.split
-        )
-        if self.offset is not None:
-            reader = await reader.with_offset(self.offset)
-        if self.limit is not None:
-            reader = await reader.with_limit(self.limit)
-        return reader

    def _with_selection(self, selection: dict[str, str]) -> "Permutation":
        """
@@ -415,97 +402,21 @@ class Permutation:
        Does not validation of the selection and it replaces it entirely.  This is not
        intended for public use.
        """
-        new = copy.copy(self)
-        new.selection = selection
-        return new
+        return Permutation(self.reader, selection, self.batch_size, self.transform_fn)
+
+    def _with_reader(self, reader: PermutationReader) -> "Permutation":
+        """
+        Creates a new permutation with the given reader
+
+        This is an internal method and should not be used directly.
+        """
+        return Permutation(reader, self.selection, self.batch_size, self.transform_fn)

    def with_batch_size(self, batch_size: int) -> "Permutation":
        """
        Creates a new permutation with the given batch size
        """
-        new = copy.copy(self)
-        new.batch_size = batch_size
-        return new
-
-    def with_connection_factory(
-        self, connection_factory: Callable[[str], LanceTable]
-    ) -> "Permutation":
-        """
-        Creates a new permutation that will use ``connection_factory`` to reopen
-        the base table when this permutation is unpickled in a worker process.
-
-        The factory is a callable that takes a single argument — the base table
-        name — and returns a [LanceTable]. It must be picklable; the worker
-        will pickle it via standard ``pickle`` and call it to recover the base
-        table. Picklable callables in practice means top-level (module-level)
-        functions, ``functools.partial`` of such functions, or instances of
-        picklable classes implementing ``__call__``. Lambdas and closures over
-        local variables don't pickle with the default protocol.
-
-        Setting a factory is necessary when the URI alone is not enough to
-        re-open the connection — most importantly for LanceDB Cloud (``db://``)
-        connections, where ``api_key`` and ``region`` aren't recoverable from
-        the connection object after construction.
-
-        For local file or cloud-storage paths the factory is optional: if not
-        set, ``__getstate__`` falls back to capturing
-        ``(uri, storage_options, namespace_path)`` and re-opening via
-        ``lancedb.connect(uri, storage_options=...)``.
-
-        Examples
-        --------
-        Basic native (file-system path), parameterized via ``functools.partial``::
-
-            import functools, lancedb
-            from lancedb.permutation import Permutation
-
-            def open_native_table(uri: str, table_name: str):
-                return lancedb.connect(uri).open_table(table_name)
-
-            factory = functools.partial(open_native_table, "/data/lance_db")
-            permutation = Permutation.identity(
-                factory("training")
-            ).with_connection_factory(factory)
-
-        Native via :func:`lancedb.connect_namespace` (e.g. a directory- or
-        REST-backed namespace client). The factory takes the
-        implementation name and properties dict as partial-bound args so
-        the worker can rebuild the same namespace connection::
-
-            def open_via_namespace(
-                impl: str, properties: dict[str, str], table_name: str,
-            ):
-                return lancedb.connect_namespace(impl, properties).open_table(
-                    table_name,
-                )
-
-            factory = functools.partial(
-                open_via_namespace,
-                "dir",
-                {"root": "/data/lance_db"},
-            )
-
-        LanceDB Cloud, reading credentials from env vars at worker startup
-        so secrets aren't pickled into the dataset::
-
-            import os, lancedb
-
-            def open_remote_table(table_name: str):
-                db = lancedb.connect(
-                    "db://my-database",
-                    api_key=os.environ["LANCEDB_API_KEY"],
-                    region=os.environ.get("LANCEDB_REGION", "us-east-1"),
-                )
-                return db.open_table(table_name)
-
-            permutation = Permutation.identity(
-                open_remote_table("training")
-            ).with_connection_factory(open_remote_table)
-        """
-        assert connection_factory is not None, "connection_factory is required"
-        new = copy.copy(self)
-        new.connection_factory = connection_factory
-        return new
+        return Permutation(self.reader, self.selection, batch_size, self.transform_fn)

    @classmethod
    def identity(cls, table: LanceTable) -> "Permutation":
@@ -578,126 +489,11 @@ class Permutation:
            schema = await reader.output_schema(None)
            initial_selection = {name: name for name in schema.names}
            return cls(
-                base_table,
-                permutation_table,
-                split,
-                initial_selection,
-                DEFAULT_BATCH_SIZE,
-                Transforms.arrow2python,
-                _reader=reader,
+                reader, initial_selection, DEFAULT_BATCH_SIZE, Transforms.arrow2python
            )

        return LOOP.run(do_from_tables())

-    def __getstate__(self) -> dict[str, Any]:
-        """Build a picklable state dict for this permutation.
-
-        The base table is captured either via a user-supplied
-        ``connection_factory`` (see [with_connection_factory]) or, as a
-        fallback, by introspecting ``(uri, storage_options, namespace_path)``
-        on the connection. The permutation table — always an in-memory
-        LanceDB table — is captured as a pyarrow Table (which pickles via
-        Arrow IPC natively). The reader is dropped from the wire format;
-        ``__setstate__`` rebuilds it from the restored tables.
-        """
-        permutation_data: Optional[pa.Table] = None
-        if self.permutation_table is not None:
-            permutation_data = self.permutation_table.to_arrow()
-
-        common = {
-            "base_table_name": self.base_table.name,
-            "permutation_data": permutation_data,
-            "split": self.split,
-            "selection": self.selection,
-            "batch_size": self.batch_size,
-            "transform_fn": self.transform_fn,
-            "offset": self.offset,
-            "limit": self.limit,
-            "connection_factory": self.connection_factory,
-        }
-
-        if self.connection_factory is not None:
-            # The factory carries enough state to recover the base table on
-            # its own; we don't need to capture the URI / storage options /
-            # namespace from the existing connection.
-            return common
-
-        # URI-introspection fallback: only viable for native (OSS) connections
-        # where (uri, storage_options) is enough to reopen. Remote / cloud
-        # connections don't expose recoverable api_key / region — those users
-        # must call with_connection_factory().
-        try:
-            base_uri = self.base_table._conn.uri
-            storage_options = self.base_table._conn.storage_options
-        except AttributeError as e:
-            raise ValueError(
-                "Cannot pickle this Permutation: the base table's connection "
-                "does not expose a uri/storage_options, which usually means it "
-                "is a remote (LanceDB Cloud) connection. Call "
-                "Permutation.with_connection_factory(...) first to provide a "
-                "picklable callable that re-opens the base table from a worker "
-                "process."
-            ) from e
-
-        if base_uri.startswith("memory://"):
-            # In-memory base tables don't exist in any worker process by
-            # default, so dump the entire base table into the pickle. This
-            # can be expensive for large datasets — users with large
-            # in-memory base tables should either persist them or set a
-            # connection_factory.
-            return {
-                **common,
-                "base_table_data": self.base_table.to_arrow(),
-            }
-
-        return {
-            **common,
-            "base_table_uri": base_uri,
-            "base_table_namespace": self.base_table._namespace_path,
-            "base_table_storage_options": storage_options,
-        }
-
-    def __setstate__(self, state: dict[str, Any]) -> None:
-        from . import connect
-
-        connection_factory = state["connection_factory"]
-        if connection_factory is not None:
-            base_table = connection_factory(state["base_table_name"])
-        elif "base_table_data" in state:
-            # In-memory base table inlined into the pickle; rebuild the same
-            # way we rebuild the in-memory permutation table.
-            mem_db = connect("memory://")
-            base_table = mem_db.create_table(
-                state["base_table_name"], state["base_table_data"]
-            )
-        else:
-            base_db = connect(
-                state["base_table_uri"],
-                storage_options=state["base_table_storage_options"],
-            )
-            base_table = base_db.open_table(
-                state["base_table_name"],
-                namespace_path=state["base_table_namespace"] or None,
-            )
-
-        permutation_table: Optional[LanceTable] = None
-        if state["permutation_data"] is not None:
-            mem_db = connect("memory://")
-            permutation_table = mem_db.create_table(
-                "permutation", state["permutation_data"]
-            )
-
-        self.base_table = base_table
-        self.permutation_table = permutation_table
-        self.split = state["split"]
-        self.selection = state["selection"]
-        self.batch_size = state["batch_size"]
-        self.transform_fn = state["transform_fn"]
-        self.offset = state["offset"]
-        self.limit = state["limit"]
-        self.connection_factory = connection_factory
-        self.reader = LOOP.run(self._build_reader())
-
    @property
    def schema(self) -> pa.Schema:
        async def do_output_schema():
@@ -964,9 +760,7 @@ class Permutation:
        for expensive operations such as image decoding.
        """
        assert transform is not None, "transform is required"
-        new = copy.copy(self)
-        new.transform_fn = transform
-        return new
+        return Permutation(self.reader, self.selection, self.batch_size, transform)

    def __getitem__(self, index: int) -> Any:
        """
@@ -1001,10 +795,12 @@ class Permutation:
        """
        Skip the first `skip` rows of the permutation
        """
-        new = copy.copy(self)
-        new.offset = skip
-        new.reader = LOOP.run(new._build_reader())
-        return new
+
+        async def do_with_skip():
+            reader = await self.reader.with_offset(skip)
+            return self._with_reader(reader)
+
+        return LOOP.run(do_with_skip())

    @deprecated(details="Use with_take instead")
    def take(self, limit: int) -> "Permutation":
@@ -1022,10 +818,12 @@ class Permutation:
        """
        Limit the permutation to `limit` rows (following any `skip`)
        """
-        new = copy.copy(self)
-        new.limit = limit
-        new.reader = LOOP.run(new._build_reader())
-        return new
+
+        async def do_with_take():
+            reader = await self.reader.with_limit(limit)
+            return self._with_reader(reader)
+
+        return LOOP.run(do_with_take())

    @deprecated(details="Use with_repeat instead")
    def repeat(self, times: int) -> "Permutation":
@@ -22,7 +22,6 @@ from lancedb.index import (
    FTS,
    BTree,
    Bitmap,
-    HnswFlat,
    HnswSq,
    IvfFlat,
    IvfPq,
@@ -40,7 +39,6 @@ from lancedb.table import _normalize_progress

 from ..query import LanceVectorQueryBuilder, LanceQueryBuilder, LanceTakeQueryBuilder
 from ..table import AsyncTable, IndexStatistics, Query, Table, Tags
-from ..types import BaseTokenizerType


 class RemoteTable(Table):
@@ -169,7 +167,7 @@ class RemoteTable(Table):
        wait_timeout: Optional[timedelta] = None,
        with_position: bool = False,
        # tokenizer configs:
-        base_tokenizer: BaseTokenizerType = "simple",
+        base_tokenizer: str = "simple",
        language: str = "English",
        max_token_length: Optional[int] = 40,
        lower_case: bool = True,
@@ -286,15 +284,13 @@ class RemoteTable(Table):
            )
        elif index_type == "IVF_HNSW_SQ":
            config = HnswSq(distance_type=metric, num_partitions=num_partitions)
-        elif index_type == "IVF_HNSW_FLAT":
-            config = HnswFlat(distance_type=metric, num_partitions=num_partitions)
        elif index_type == "IVF_FLAT":
            config = IvfFlat(distance_type=metric, num_partitions=num_partitions)
        else:
            raise ValueError(
                f"Unknown vector index type: {index_type}. Valid options are"
                " 'IVF_FLAT', 'IVF_PQ', 'IVF_RQ', 'IVF_SQ',"
-                " 'IVF_HNSW_PQ', 'IVF_HNSW_SQ', 'IVF_HNSW_FLAT'"
+                " 'IVF_HNSW_PQ', 'IVF_HNSW_SQ'"
            )

        LOOP.run(
@@ -57,7 +57,6 @@ from .index import (
    LabelList,
    HnswPq,
    HnswSq,
-    HnswFlat,
    FTS,
 )
 from .merge import LanceMergeInsertBuilder
@@ -87,59 +86,6 @@ from .util import (
 )
 from .index import lang_mapping

-_MODEL_BACKED_TOKENIZER_PREFIXES = ("jieba", "lindera")
-_MODEL_BACKED_TOKENIZER_ERRORS = (
-    "unknown base tokenizer",
-    "Invalid directory path:",
-    "Failed to load Jieba",
-    "Failed to load tokenizer config",
-    "Failed to initialize default tokenizer",
-)
-
-
-def _add_unique_note(exception: BaseException, note: str) -> None:
-    existing_notes = getattr(exception, "__notes__", ()) or ()
-    message = (
-        exception.args[0]
-        if exception.args and isinstance(exception.args[0], str)
-        else ""
-    )
-    if note not in existing_notes and note not in message:
-        add_note(exception, note)
-
-
-def _is_model_backed_tokenizer(base_tokenizer: str) -> bool:
-    return any(
-        base_tokenizer == prefix or base_tokenizer.startswith(f"{prefix}/")
-        for prefix in _MODEL_BACKED_TOKENIZER_PREFIXES
-    )
-
-
-def _maybe_add_fts_error_note(
-    exception: BaseException, *, base_tokenizer: str, language: Optional[str] = None
-) -> None:
-    message = str(exception)
-    if language is not None and "not support the requested language" in message:
-        supported_langs = ", ".join(lang_mapping.values())
-        _add_unique_note(exception, f"Supported languages: {supported_langs}")
-        return
-
-    if not _is_model_backed_tokenizer(base_tokenizer):
-        return
-
-    if not any(marker in message for marker in _MODEL_BACKED_TOKENIZER_ERRORS):
-        return
-
-    _add_unique_note(
-        exception,
-        "Model-backed tokenizers such as 'jieba/default' and 'lindera/ipadic' "
-        "require tokenizer models in Lance's language model home. Set "
-        "LANCE_LANGUAGE_MODEL_HOME to override the default platform data "
-        "directory under 'lance/language_models'. Expected layouts include "
-        "'<model-home>/jieba/default/...' and "
-        "'<model-home>/lindera/ipadic/...'.",
-    )
-

 if TYPE_CHECKING:
    from .db import LanceDBConnection
@@ -1012,10 +958,7 @@ class Table(ABC):
        tokenizer_name: str, default "default"
            A compatibility alias for native tokenizer configs. Can be "raw",
            "default" or the 2 letter language code followed by "_stem". So
-            for english it would be "en_stem". For new native FTS indexes, use
-            ``base_tokenizer`` directly; ``tokenizer_name`` is a legacy
-            compatibility alias and does not expose model-backed tokenizer names
-            such as ``jieba/default`` or ``lindera/ipadic``.
+            for english it would be "en_stem".
        use_tantivy: bool, default False
            Deprecated legacy Tantivy parameter. Setting this to True raises an
            error.
@@ -1029,11 +972,8 @@ class Table(ABC):
            - "whitespace": Split text by whitespace, but not punctuation.
            - "raw": No tokenization. The entire text is treated as a single token.
            - "ngram": N-Gram tokenizer.
-            - "jieba/*": Jieba tokenizer loaded from Lance's language model home.
-            - "lindera/*": Lindera tokenizer loaded from Lance's language model home.
        language : str, default "English"
-            The language to use for stemming and stop-word removal. This is not
-            the primary way to enable CJK tokenization.
+            The language to use for tokenization.
        max_token_length : int, default 40
            The maximum token length to index. Tokens longer than this length will be
            ignored.
@@ -1059,13 +999,6 @@ class Table(ABC):
            The timeout to wait if indexing is asynchronous.
        name: str, optional
            The name of the index. If not provided, a default name will be generated.
-
-        Notes
-        -----
-        Model-backed tokenizers such as ``jieba/default`` and ``lindera/ipadic``
-        require tokenizer models in Lance's language model home. Set
-        ``LANCE_LANGUAGE_MODEL_HOME`` to override the default platform data
-        directory under ``lance/language_models``.
        """
        raise NotImplementedError

@@ -2237,13 +2170,7 @@ class LanceTable(Table):
        index_cache_size: Optional[int] = None,
        num_bits: int = 8,
        index_type: Literal[
-            "IVF_FLAT",
-            "IVF_SQ",
-            "IVF_PQ",
-            "IVF_RQ",
-            "IVF_HNSW_SQ",
-            "IVF_HNSW_PQ",
-            "IVF_HNSW_FLAT",
+            "IVF_FLAT", "IVF_SQ", "IVF_PQ", "IVF_RQ", "IVF_HNSW_SQ", "IVF_HNSW_PQ"
        ] = "IVF_PQ",
        max_iterations: int = 50,
        sample_rate: int = 256,
@@ -2330,16 +2257,6 @@ class LanceTable(Table):
                ef_construction=ef_construction,
                target_partition_size=target_partition_size,
            )
-        elif index_type == "IVF_HNSW_FLAT":
-            config = HnswFlat(
-                distance_type=metric,
-                num_partitions=num_partitions,
-                max_iterations=max_iterations,
-                sample_rate=sample_rate,
-                m=m,
-                ef_construction=ef_construction,
-                target_partition_size=target_partition_size,
-            )
        else:
            raise ValueError(f"Unknown index type {index_type}")

@@ -2545,22 +2462,14 @@ class LanceTable(Table):
            **tokenizer_configs,
        )

-        try:
-            LOOP.run(
-                self._table.create_index(
-                    field_names,
-                    replace=replace,
-                    config=config,
-                    name=name,
-                )
+        LOOP.run(
+            self._table.create_index(
+                field_names,
+                replace=replace,
+                config=config,
+                name=name,
            )
-        except (ValueError, RuntimeError) as e:
-            _maybe_add_fts_error_note(
-                e,
-                base_tokenizer=config.base_tokenizer,
-                language=config.language,
-            )
-            raise e
+        )

    @staticmethod
    def infer_tokenizer_configs(tokenizer_name: str) -> dict:
@@ -3890,18 +3799,7 @@ class AsyncTable:
        *,
        replace: Optional[bool] = None,
        config: Optional[
-            Union[
-                IvfFlat,
-                IvfPq,
-                IvfRq,
-                HnswPq,
-                HnswSq,
-                HnswFlat,
-                BTree,
-                Bitmap,
-                LabelList,
-                FTS,
-            ]
+            Union[IvfFlat, IvfPq, IvfRq, HnswPq, HnswSq, BTree, Bitmap, LabelList, FTS]
        ] = None,
        wait_timeout: Optional[timedelta] = None,
        name: Optional[str] = None,
@@ -3948,7 +3846,6 @@ class AsyncTable:
                    IvfRq,
                    HnswPq,
                    HnswSq,
-                    HnswFlat,
                    BTree,
                    Bitmap,
                    LabelList,
@@ -3968,13 +3865,11 @@ class AsyncTable:
                name=name,
                train=train,
            )
-        except (ValueError, RuntimeError) as e:
-            if isinstance(config, FTS):
-                _maybe_add_fts_error_note(
-                    e,
-                    base_tokenizer=config.base_tokenizer,
-                    language=config.language,
-                )
+        except ValueError as e:
+            if "not support the requested language" in str(e):
+                supported_langs = ", ".join(lang_mapping.values())
+                help_msg = f"Supported languages: {supported_langs}"
+                add_note(e, help_msg)
            raise e

    async def drop_index(self, name: str) -> None:
@@ -5119,7 +5014,6 @@ class IndexStatistics:
        "IVF_RQ",
        "IVF_HNSW_SQ",
        "IVF_HNSW_PQ",
-        "IVF_HNSW_FLAT",
        "FTS",
        "BTREE",
        "BITMAP",
@@ -24,7 +24,6 @@ VectorIndexType = Literal[
    "IVF_PQ",
    "IVF_HNSW_SQ",
    "IVF_HNSW_PQ",
-    "IVF_HNSW_FLAT",
    "IVF_RQ",
 ]
 ScalarIndexType = Literal["BTREE", "BITMAP", "LABEL_LIST"]
@@ -32,7 +31,6 @@ IndexType = Literal[
    "IVF_PQ",
    "IVF_HNSW_PQ",
    "IVF_HNSW_SQ",
-    "IVF_HNSW_FLAT",
    "IVF_SQ",
    "FTS",
    "BTREE",
@@ -42,5 +40,4 @@ IndexType = Literal[
 ]

 # Tokenizer literals
-BuiltinTokenizerType = Literal["simple", "raw", "whitespace", "ngram"]
-BaseTokenizerType = BuiltinTokenizerType | str
+BaseTokenizerType = Literal["simple", "raw", "whitespace", "ngram"]
@@ -1,8 +0,0 @@
-我们 98740 r
-都 202780 d
-有 423765 v
-光明 1219 n
-的 318825 uj
-前途 1263 n
-前 62779 f
-途 857 n
@@ -1,4 +0,0 @@
-segmenter:
-  mode: "normal"
-  dictionary:
-    path: "./python/tests/models/lindera/ipadic/main"
@@ -15,10 +15,7 @@
 #  limitations under the License.
 import os
 import random
-import shutil
 from unittest import mock
-from pathlib import Path
-import zipfile

 import lancedb as ldb
 from lancedb.db import DBConnection
@@ -39,8 +36,6 @@ import pytest
 import pytest_asyncio
 from utils import exception_output

-TEST_LANGUAGE_MODEL_HOME = Path(__file__).parent / "models"
-

@pytest.fixture
 def table(tmp_path) -> ldb.table.LanceTable:
@@ -94,40 +89,6 @@ def table(tmp_path) -> ldb.table.LanceTable:
    return table


-@pytest.fixture
-def language_model_home(monkeypatch, tmp_path):
-    model_home = tmp_path / "language-models"
-    shutil.copytree(TEST_LANGUAGE_MODEL_HOME, model_home)
-    monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(model_home))
-    return model_home
-
-
-@pytest.fixture
-def lindera_ipadic(language_model_home):
-    model_path = language_model_home / "lindera" / "ipadic"
-    extracted_model = model_path / "main"
-    config_path = model_path / "config.yml"
-
-    if extracted_model.exists():
-        shutil.rmtree(extracted_model)
-
-    with zipfile.ZipFile(model_path / "main.zip", "r") as zip_ref:
-        zip_ref.extractall(model_path)
-    config_path.write_text(
-        "segmenter:\n"
-        '  mode: "normal"\n'
-        "  dictionary:\n"
-        f'    path: "{extracted_model.resolve().as_posix()}"\n',
-        encoding="utf-8",
-    )
-
-    try:
-        yield
-    finally:
-        if extracted_model.exists():
-            shutil.rmtree(extracted_model)
-
-
@pytest_asyncio.fixture
 async def async_table(tmp_path) -> ldb.table.AsyncTable:
    # Use local random state to avoid affecting other tests
@@ -723,90 +684,6 @@ def test_fts_ngram(mem_db: DBConnection):
    assert set(r["text"] for r in results) == {"lance database", "lance is cool"}


-def test_fts_jieba_tokenizer(mem_db: DBConnection, language_model_home):
-    data = pa.table({"text": ["我们都有光明的前途", "光明的前途"]})
-    table = mem_db.create_table("test_jieba", data=data)
-    table.create_fts_index(
-        "text",
-        base_tokenizer="jieba/default",
-        stem=False,
-        remove_stop_words=False,
-        ascii_folding=False,
-    )
-
-    results = table.search("我们", query_type="fts").limit(10).to_list()
-    assert [row["text"] for row in results] == ["我们都有光明的前途"]
-
-
-def test_fts_jieba_missing_language_model_note(
-    mem_db: DBConnection, monkeypatch, tmp_path
-):
-    missing_root = tmp_path / "missing-language-models"
-    monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(missing_root))
-    table = mem_db.create_table(
-        "test_missing_jieba_model",
-        data=pa.table({"text": ["我们都有光明的前途"]}),
-    )
-
-    with pytest.raises((ValueError, RuntimeError)) as e:
-        table.create_fts_index(
-            "text",
-            base_tokenizer="jieba/default",
-            stem=False,
-            remove_stop_words=False,
-            ascii_folding=False,
-        )
-
-    output = exception_output(e)
-    assert "Invalid directory path:" in output
-    assert "LANCE_LANGUAGE_MODEL_HOME" in output
-    assert "jieba/default" in output
-
-
-@pytest.mark.asyncio
-async def test_fts_jieba_missing_language_model_note_async(monkeypatch, tmp_path):
-    missing_root = tmp_path / "missing-language-models"
-    monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(missing_root))
-    db = await ldb.connect_async(tmp_path / "async-db")
-    table = await db.create_table(
-        "test_missing_jieba_model_async",
-        data=pa.table({"text": ["我们都有光明的前途"]}),
-    )
-
-    with pytest.raises((ValueError, RuntimeError)) as e:
-        await table.create_index(
-            "text",
-            config=FTS(
-                base_tokenizer="jieba/default",
-                stem=False,
-                remove_stop_words=False,
-                ascii_folding=False,
-            ),
-        )
-
-    output = exception_output(e)
-    assert "Invalid directory path:" in output
-    assert "LANCE_LANGUAGE_MODEL_HOME" in output
-    assert "jieba/default" in output
-
-
-def test_fts_lindera_tokenizer(
-    mem_db: DBConnection, language_model_home, lindera_ipadic
-):
-    data = pa.table({"text": ["成田国際空港", "東京国際空港", "羽田空港"]})
-    table = mem_db.create_table("test_lindera", data=data)
-    table.create_fts_index(
-        "text",
-        base_tokenizer="lindera/ipadic",
-        stem=False,
-        remove_stop_words=False,
-        ascii_folding=False,
-    )
-
-    results = table.search("成田", query_type="fts").limit(10).to_list()
-    assert [row["text"] for row in results] == ["成田国際空港"]
-
-
 def test_fts_query_to_json():
    """Test that FTS query to_json() produces valid JSON strings with exact format."""

@@ -16,13 +16,11 @@ from lancedb.index import (
    IvfSq,
    IvfHnswPq,
    IvfHnswSq,
-    IvfHnswFlat,
    IvfRq,
    Bitmap,
    LabelList,
    HnswPq,
    HnswSq,
-    HnswFlat,
    FTS,
 )
 from lancedb.table import IndexStatistics
@@ -252,21 +250,6 @@ async def test_create_hnswpq_alias_index(some_table: AsyncTable):
    assert indices[0].index_type in {"HnswPq", "IvfHnswPq"}


-@pytest.mark.asyncio
-async def test_create_hnswflat_index(some_table: AsyncTable):
-    await some_table.create_index("vector", config=HnswFlat(num_partitions=10))
-    indices = await some_table.list_indices()
-    assert len(indices) == 1
-
-
-@pytest.mark.asyncio
-async def test_create_hnswflat_alias_index(some_table: AsyncTable):
-    await some_table.create_index("vector", config=IvfHnswFlat(num_partitions=5))
-    indices = await some_table.list_indices()
-    assert len(indices) == 1
-    assert indices[0].index_type in {"HnswFlat", "IvfHnswFlat"}
-
-
@pytest.mark.asyncio
 async def test_create_ivfsq_index(some_table: AsyncTable):
    await some_table.create_index("vector", config=IvfSq(num_partitions=10))
@@ -312,7 +295,6 @@ def test_index_statistics_index_type_lists_all_supported_values():
        "IVF_RQ",
        "IVF_HNSW_SQ",
        "IVF_HNSW_PQ",
-        "IVF_HNSW_FLAT",
        "FTS",
        "BTREE",
        "BITMAP",
@@ -9,6 +9,21 @@ from lancedb import DBConnection, Table, connect
 from lancedb.permutation import Permutation, Permutations, permutation_builder


+def test_permutation_persistence(tmp_path):
+    db = connect(tmp_path)
+    tbl = db.create_table("test_table", pa.table({"x": range(100), "y": range(100)}))
+
+    permutation_tbl = (
+        permutation_builder(tbl).shuffle().persist(db, "test_permutation").execute()
+    )
+    assert permutation_tbl.count_rows() == 100
+
+    re_open = db.open_table("test_permutation")
+    assert re_open.count_rows() == 100
+
+    assert permutation_tbl.to_arrow() == re_open.to_arrow()
+
+
 def test_split_random_ratios(mem_db):
    """Test random splitting with ratios."""
    tbl = mem_db.create_table(
@@ -6,8 +6,6 @@ import contextlib
 from datetime import timedelta
 import http.server
 import json
-import multiprocessing as mp
-import sys
 import threading
 import time
 from unittest.mock import MagicMock, patch
@@ -1232,82 +1230,3 @@ def test_background_loop_cancellation(exception):
        with pytest.raises(exception):
            loop.run(None)
        mock_future.cancel.assert_called_once()
-
-
-def _remote_fork_child(port: int, queue) -> None:
-    # Build a fresh Connection in the child so we exercise the at-fork-child
-    # tokio runtime reset rather than relying on an inherited reqwest client.
-    db = lancedb.connect(
-        "db://dev",
-        api_key="fake",
-        host_override=f"http://localhost:{port}",
-        client_config={
-            "retry_config": {"retries": 0},
-            "timeout_config": {"connect_timeout": 2, "read_timeout": 2},
-        },
-    )
-    queue.put(db.table_names())
-
-
-@pytest.mark.skipif(
-    sys.platform != "linux",
-    reason=(
-        "fork() is unavailable on Windows and unsafe on macOS "
-        "(Apple frameworks/TLS are not fork-safe)"
-    ),
-)
-def test_remote_connection_after_fork():
-    """A freshly-built remote Connection in a forked child should not hang.
-
-    The pyo3-async-runtimes tokio runtime would otherwise be inherited from
-    the parent with dead worker threads; the at-fork-child handler in our
-    runtime module rebuilds it on first use in the child.
-    """
-
-    def handler(request):
-        request.send_response(200)
-        request.send_header("Content-Type", "application/json")
-        request.end_headers()
-        request.wfile.write(b'{"tables": []}')
-
-    server = http.server.HTTPServer(("localhost", 0), make_mock_http_handler(handler))
-    port = server.server_address[1]
-    server_thread = threading.Thread(target=server.serve_forever)
-    server_thread.start()
-    try:
-        # Hit the server in the parent first so the runtime + LOOP are warm
-        # before fork; a fresh child must still succeed.
-        parent_db = lancedb.connect(
-            "db://dev",
-            api_key="fake",
-            host_override=f"http://localhost:{port}",
-            client_config={
-                "retry_config": {"retries": 0},
-                "timeout_config": {"connect_timeout": 2, "read_timeout": 2},
-            },
-        )
-        assert parent_db.table_names() == []
-
-        ctx = mp.get_context("fork")
-        queue = ctx.Queue()
-        proc = ctx.Process(target=_remote_fork_child, args=(port, queue))
-        proc.start()
-        proc.join(timeout=15)
-
-        if proc.is_alive():
-            proc.terminate()
-            proc.join(timeout=5)
-            if proc.is_alive():
-                proc.kill()
-                proc.join()
-            pytest.fail("Remote connection hung after fork")
-
-        assert proc.exitcode == 0, f"child exited with code {proc.exitcode}"
-        assert not queue.empty(), "child produced no result"
-        assert queue.get() == []
-
-        # Parent connection must still be usable after the child returned.
-        assert parent_db.table_names() == []
-    finally:
-        server.shutdown()
-        server_thread.join()
@@ -11,7 +11,7 @@ from unittest.mock import patch

 import lancedb
 from lancedb.dependencies import _PANDAS_AVAILABLE
-from lancedb.index import HnswFlat, HnswPq, HnswSq, IvfPq
+from lancedb.index import HnswPq, HnswSq, IvfPq
 import numpy as np
 import polars as pl
 import pyarrow as pa
@@ -917,21 +917,6 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
        "my_vector", replace=True, config=expected_config, name=None, train=True
    )

-    table.create_index(
-        vector_column_name="my_vector",
-        metric="cosine",
-        index_type="IVF_HNSW_FLAT",
-        sample_rate=0.1,
-        m=29,
-        ef_construction=10,
-    )
-    expected_config = HnswFlat(
-        distance_type="cosine", sample_rate=0.1, m=29, ef_construction=10
-    )
-    mock_create_index.assert_called_with(
-        "my_vector", replace=True, config=expected_config, name=None, train=True
-    )
-

@patch("lancedb.table.AsyncTable.create_index")
 def test_create_index_name_and_train_parameters(
@@ -1,29 +1,14 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors

-import functools
-import multiprocessing as mp
-import pickle
-import sys
-
-import lancedb
 import pyarrow as pa
 import pytest
-from lancedb.permutation import Permutation, Permutations, permutation_builder
 from lancedb.util import tbl_to_tensor
+from lancedb.permutation import Permutation

 torch = pytest.importorskip("torch")


-def _open_native_table(uri: str, table_name: str):
-    """Top-level connection factory used by the explicit-factory pickle test.
-
-    Defined at module scope so that pickle can resolve it by name in the
-    worker / unpickling process.
-    """
-    return lancedb.connect(uri).open_table(table_name)
-
-
 def test_table_dataloader(mem_db):
    table = mem_db.create_table("test_table", pa.table({"a": range(1000)}))
    dataloader = torch.utils.data.DataLoader(
@@ -55,156 +40,3 @@ def test_permutation_dataloader(mem_db):
    for batch in dataloader:
        assert batch.size(0) == 1
        assert batch.size(1) == 10
-
-
-def test_permutation_is_picklable(tmp_db):
-    """A Permutation must be picklable so it can be used with PyTorch's
-    DataLoader when num_workers > 0 (which uses multiprocessing and pickles
-    the dataset to pass it to worker processes)."""
-    table = tmp_db.create_table("test_table", pa.table({"a": range(1000)}))
-    permutation = Permutation.identity(table)
-
-    pickled = pickle.dumps(permutation)
-    restored = pickle.loads(pickled)
-
-    assert len(restored) == 1000
-    rows = restored.__getitems__([0, 1, 2])
-    assert rows == [{"a": 0}, {"a": 1}, {"a": 2}]
-
-
-def test_permutation_with_memory_base_is_picklable(mem_db):
-    """An in-memory base table is inlined into the pickle as Arrow IPC bytes
-    and rebuilt on the other side as an in-memory LanceTable, so the
-    Permutation round-trips even though the original database can't be
-    reopened across processes."""
-    table = mem_db.create_table("test_table", pa.table({"a": range(50)}))
-    permutation = Permutation.identity(table)
-
-    restored = pickle.loads(pickle.dumps(permutation))
-
-    assert len(restored) == 50
-    assert restored.__getitems__([0, 10, 49]) == [{"a": 0}, {"a": 10}, {"a": 49}]
-
-
-def test_permutation_dataloader_multiprocessing(tmp_db):
-    """Using a Permutation with a PyTorch DataLoader that has num_workers > 0
-    must work end-to-end. Each worker process gets a pickled copy of the
-    dataset and reads batches from it."""
-    table = tmp_db.create_table("test_table", pa.table({"a": range(1000)}))
-    permutation = Permutation.identity(table)
-
-    dataloader = torch.utils.data.DataLoader(
-        permutation,
-        batch_size=10,
-        shuffle=True,
-        num_workers=2,
-        multiprocessing_context="spawn",
-    )
-    seen = 0
-    for batch in dataloader:
-        assert batch["a"].size(0) == 10
-        seen += batch["a"].size(0)
-    assert seen == 1000
-
-
-def test_permutation_pickle_with_connection_factory(tmp_path):
-    """When the user provides a connection_factory, pickling should round-trip
-    through that factory rather than introspecting the connection URI. Useful
-    for remote / cloud connections where the URI alone isn't reopenable."""
-    db = lancedb.connect(tmp_path)
-    db.create_table("test_table", pa.table({"a": range(50)}))
-
-    factory = functools.partial(_open_native_table, str(tmp_path))
-    permutation = Permutation.identity(factory("test_table")).with_connection_factory(
-        factory
-    )
-
-    restored = pickle.loads(pickle.dumps(permutation))
-
-    assert len(restored) == 50
-    # The factory survives pickling and is what powered base-table reopen.
-    assert restored.connection_factory is not None
-    assert restored.connection_factory.func is _open_native_table
-    assert restored.__getitems__([0, 1, 2]) == [{"a": 0}, {"a": 1}, {"a": 2}]
-
-
-def test_permutation_with_builder_is_picklable(tmp_db):
-    """A Permutation built from a non-identity permutation table must round-trip
-    through pickle while preserving the row order defined by the permutation."""
-    table = tmp_db.create_table("test_table", pa.table({"a": range(100)}))
-    perm_tbl = (
-        permutation_builder(table)
-        .split_random(ratios=[0.8, 0.2], seed=42, split_names=["train", "test"])
-        .shuffle(seed=42)
-        .execute()
-    )
-    permutations = Permutations(table, perm_tbl)
-    permutation = permutations["train"]
-
-    indices = list(range(len(permutation)))
-    expected = permutation.__getitems__(indices)
-
-    restored = pickle.loads(pickle.dumps(permutation))
-
-    assert len(restored) == len(permutation)
-    assert restored.__getitems__(indices) == expected
-
-
-def _multiworker_dataloader_target(db_uri: str, result_queue):
-    import lancedb
-    from lancedb.permutation import Permutation
-
-    db = lancedb.connect(db_uri)
-    table = db.open_table("test_table")
-    permutation = Permutation.identity(table)
-
-    dataloader = torch.utils.data.DataLoader(
-        permutation,
-        batch_size=10,
-        num_workers=2,
-        multiprocessing_context="fork",
-    )
-    count = 0
-    for batch in dataloader:
-        assert batch["a"].size(0) == 10
-        count += 1
-    result_queue.put(count)
-
-
-@pytest.mark.skipif(
-    sys.platform != "linux",
-    reason=(
-        "fork() is unavailable on Windows and unsafe on macOS "
-        "(Apple frameworks/TLS are not fork-safe)"
-    ),
-)
-def test_permutation_dataloader_fork_workers(tmp_path):
-    """A Permutation used by a fork-based DataLoader should not hang.
-
-    PyTorch's DataLoader uses fork-based multiprocessing by default on Linux.
-    LanceDB drives async work through a background asyncio thread that does
-    not survive a fork, so any LOOP.run() in a worker blocks forever.
-    """
-    import lancedb
-
-    db_uri = str(tmp_path / "db")
-    db = lancedb.connect(db_uri)
-    db.create_table("test_table", pa.table({"a": list(range(1000))}))
-
-    ctx = mp.get_context("spawn")
-    queue = ctx.Queue()
-    proc = ctx.Process(target=_multiworker_dataloader_target, args=(db_uri, queue))
-    proc.start()
-    proc.join(timeout=30)
-
-    if proc.is_alive():
-        proc.terminate()
-        proc.join(timeout=5)
-        if proc.is_alive():
-            proc.kill()
-            proc.join()
-        pytest.fail("Permutation hung when iterated in a fork-based DataLoader worker")
-
-    assert proc.exitcode == 0, f"child exited with code {proc.exitcode}"
-    assert not queue.empty(), "child produced no batches"
-    assert queue.get() == 100
@@ -3,8 +3,6 @@

 use std::sync::Arc;

-use crate::error::PythonErrorExt;
-use crate::runtime::future_into_py;
 use arrow::{
    datatypes::SchemaRef,
    pyarrow::{IntoPyArrow, ToPyArrow},
@@ -14,6 +12,9 @@ use lancedb::arrow::SendableRecordBatchStream;
 use pyo3::{
    Bound, Py, PyAny, PyRef, PyResult, Python, exceptions::PyStopAsyncIteration, pyclass, pymethods,
 };
+use pyo3_async_runtimes::tokio::future_into_py;
+
+use crate::error::PythonErrorExt;

 #[pyclass]
 pub struct RecordBatchStream {
@@ -7,12 +7,6 @@ use std::{
    time::Duration,
 };

-use crate::{
-    error::PythonErrorExt,
-    namespace::{create_namespace_storage_options_provider, extract_namespace_arc},
-    runtime::future_into_py,
-    table::Table,
-};
 use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::FromPyArrow};
 use lancedb::{
    connection::Connection as LanceConnection,
@@ -26,6 +20,13 @@ use pyo3::{
    pyclass, pyfunction, pymethods,
    types::{PyDict, PyDictMethods},
 };
+use pyo3_async_runtimes::tokio::future_into_py;
+
+use crate::{
+    error::PythonErrorExt,
+    namespace::{create_namespace_storage_options_provider, extract_namespace_arc},
+    table::Table,
+};

 #[pyclass]
 pub struct Connection {
@@ -1,13 +1,11 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors

-use lancedb::index::vector::{
-    IvfFlatIndexBuilder, IvfHnswFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder,
-    IvfPqIndexBuilder, IvfRqIndexBuilder, IvfSqIndexBuilder,
-};
+use lancedb::index::vector::{IvfFlatIndexBuilder, IvfRqIndexBuilder, IvfSqIndexBuilder};
 use lancedb::index::{
    Index as LanceDbIndex,
    scalar::{BTreeIndexBuilder, FtsIndexBuilder},
+    vector::{IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, IvfPqIndexBuilder},
 };
 use pyo3::IntoPyObject;
 use pyo3::types::PyStringMethods;
@@ -164,26 +162,8 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                }
                Ok(LanceDbIndex::IvfHnswSq(hnsw_sq_builder))
            }
-            "HnswFlat" => {
-                let params = source.extract::<IvfHnswFlatParams>()?;
-                let distance_type = parse_distance_type(params.distance_type)?;
-                let mut hnsw_flat_builder = IvfHnswFlatIndexBuilder::default()
-                    .distance_type(distance_type)
-                    .max_iterations(params.max_iterations)
-                    .sample_rate(params.sample_rate)
-                    .num_edges(params.m)
-                    .ef_construction(params.ef_construction);
-                if let Some(num_partitions) = params.num_partitions {
-                    hnsw_flat_builder = hnsw_flat_builder.num_partitions(num_partitions);
-                }
-                if let Some(target_partition_size) = params.target_partition_size {
-                    hnsw_flat_builder =
-                        hnsw_flat_builder.target_partition_size(target_partition_size);
-                }
-                Ok(LanceDbIndex::IvfHnswFlat(hnsw_flat_builder))
-            }
            not_supported => Err(PyValueError::new_err(format!(
-                "Invalid index type '{}'.  Must be one of BTree, Bitmap, LabelList, FTS, IvfPq, IvfSq, IvfHnswPq, IvfHnswSq, or IvfHnswFlat",
+                "Invalid index type '{}'.  Must be one of BTree, Bitmap, LabelList, FTS, IvfPq, IvfSq, IvfHnswPq, or IvfHnswSq",
                not_supported
            ))),
        }
@@ -270,17 +250,6 @@ struct IvfHnswSqParams {
    target_partition_size: Option<u32>,
 }

-#[derive(FromPyObject)]
-struct IvfHnswFlatParams {
-    distance_type: String,
-    num_partitions: Option<u32>,
-    max_iterations: u32,
-    sample_rate: u32,
-    m: u32,
-    ef_construction: u32,
-    target_partition_size: Option<u32>,
-}
-
 #[pyclass(get_all)]
 /// A description of an index currently configured on a column
 pub struct IndexConfig {
@@ -28,7 +28,6 @@ pub mod index;
 pub mod namespace;
 pub mod permutation;
 pub mod query;
-pub mod runtime;
 pub mod session;
 pub mod table;
 pub mod util;
@@ -4,7 +4,7 @@
 use std::sync::{Arc, Mutex};

 use crate::{
-    arrow::RecordBatchStream, error::PythonErrorExt, runtime::future_into_py, table::Table,
+    arrow::RecordBatchStream, connection::Connection, error::PythonErrorExt, table::Table,
 };
 use arrow::pyarrow::{PyArrowType, ToPyArrow};
 use lancedb::{
@@ -21,6 +21,7 @@ use pyo3::{
    pyclass, pymethods,
    types::{PyAnyMethods, PyDict, PyDictMethods, PyType},
 };
+use pyo3_async_runtimes::tokio::future_into_py;

 fn table_from_py<'a>(table: Bound<'a, PyAny>) -> PyResult<Bound<'a, Table>> {
    if table.hasattr("_inner")? {
@@ -79,6 +80,24 @@ impl PyAsyncPermutationBuilder {

 #[pymethods]
 impl PyAsyncPermutationBuilder {
+    #[pyo3(signature = (database, table_name))]
+    pub fn persist(
+        slf: PyRefMut<'_, Self>,
+        database: Bound<'_, PyAny>,
+        table_name: String,
+    ) -> PyResult<Self> {
+        let conn = if database.hasattr("_conn")? {
+            database
+                .getattr("_conn")?
+                .getattr("_inner")?
+                .cast_into::<Connection>()?
+        } else {
+            database.getattr("_inner")?.cast_into::<Connection>()?
+        };
+        let database = conn.borrow().database()?;
+        slf.modify(|builder| builder.persist(database, table_name))
+    }
+
    #[pyo3(signature = (*, ratios=None, counts=None, fixed=None, seed=None, split_names=None))]
    pub fn split_random(
        slf: PyRefMut<'_, Self>,
@@ -4,11 +4,6 @@
 use std::sync::Arc;
 use std::time::Duration;

-use crate::expr::PyExpr;
-use crate::runtime::future_into_py;
-use crate::util::parse_distance_type;
-use crate::{arrow::RecordBatchStream, util::PyLanceDB};
-use crate::{error::PythonErrorExt, index::class_name};
 use arrow::array::Array;
 use arrow::array::ArrayData;
 use arrow::array::make_array;
@@ -41,6 +36,12 @@ use pyo3::types::{PyDict, PyString};
 use pyo3::{Borrowed, FromPyObject, exceptions::PyRuntimeError};
 use pyo3::{PyErr, pyclass};
 use pyo3::{exceptions::PyValueError, intern};
+use pyo3_async_runtimes::tokio::future_into_py;
+
+use crate::expr::PyExpr;
+use crate::util::parse_distance_type;
+use crate::{arrow::RecordBatchStream, util::PyLanceDB};
+use crate::{error::PythonErrorExt, index::class_name};

 impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB<FtsQuery> {
    type Error = PyErr;
@@ -1,142 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-//! Fork-safe wrapper around tokio + pyo3-async-runtimes.
-//!
-//! `pyo3_async_runtimes::tokio` keeps its multi-threaded runtime in a
-//! `OnceLock` that can never be replaced.  Tokio's worker threads do not
-//! survive `fork()`, so once a child inherits a "frozen" runtime, every
-//! `future_into_py` call hangs forever.
-//!
-//! We sidestep the global by routing every future through our own
-//! [`LanceRuntime`] (a [`pyo3_async_runtimes::generic::Runtime`] impl) backed
-//! by an [`AtomicPtr`] to a tokio runtime that we own.  A `pthread_atfork`
-//! child handler nulls the pointer; the next `spawn` rebuilds the runtime in
-//! the child.  This mirrors the pattern used in the Lance Python bindings.
-
-use std::future::Future;
-use std::pin::Pin;
-use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering};
-
-use pyo3::{Bound, PyAny, PyResult, Python, conversion::IntoPyObject};
-use pyo3_async_runtimes::{
-    TaskLocals,
-    generic::{ContextExt, JoinError, Runtime},
-};
-use tokio::{runtime, task};
-
-static RUNTIME: AtomicPtr<runtime::Runtime> = AtomicPtr::new(std::ptr::null_mut());
-static RUNTIME_INSTALLING: AtomicBool = AtomicBool::new(false);
-static ATFORK_INSTALLED: AtomicBool = AtomicBool::new(false);
-
-fn create_runtime() -> runtime::Runtime {
-    runtime::Builder::new_multi_thread()
-        .enable_all()
-        .thread_name("lancedb-tokio-worker")
-        .build()
-        .expect("Failed to build tokio runtime")
-}
-
-fn get_runtime() -> &'static runtime::Runtime {
-    loop {
-        let ptr = RUNTIME.load(Ordering::SeqCst);
-        if !ptr.is_null() {
-            return unsafe { &*ptr };
-        }
-        if !RUNTIME_INSTALLING.fetch_or(true, Ordering::SeqCst) {
-            break;
-        }
-        std::thread::yield_now();
-    }
-    if !ATFORK_INSTALLED.fetch_or(true, Ordering::SeqCst) {
-        install_atfork();
-    }
-    let new_ptr = Box::into_raw(Box::new(create_runtime()));
-    RUNTIME.store(new_ptr, Ordering::SeqCst);
-    unsafe { &*new_ptr }
-}
-
-/// Runs in async-signal context after `fork()` in the child.  We can only
-/// touch atomics here; we deliberately leak the previous runtime because
-/// dropping a tokio `Runtime` would try to join its (now-dead) worker
-/// threads and hang.
-extern "C" fn atfork_child() {
-    RUNTIME.store(std::ptr::null_mut(), Ordering::SeqCst);
-    RUNTIME_INSTALLING.store(false, Ordering::SeqCst);
-}
-
-#[cfg(not(windows))]
-fn install_atfork() {
-    unsafe { libc::pthread_atfork(None, None, Some(atfork_child)) };
-}
-
-#[cfg(windows)]
-fn install_atfork() {}
-
-/// Marker type implementing [`Runtime`] over our fork-safe runtime slot.
-pub struct LanceRuntime;
-
-/// Newtype wrapper around `tokio::task::JoinError` so we can implement the
-/// foreign [`JoinError`] trait without violating orphan rules.
-pub struct LanceJoinError(task::JoinError);
-
-impl JoinError for LanceJoinError {
-    fn is_panic(&self) -> bool {
-        self.0.is_panic()
-    }
-    fn into_panic(self) -> Box<dyn std::any::Any + Send + 'static> {
-        self.0.into_panic()
-    }
-}
-
-impl Runtime for LanceRuntime {
-    type JoinError = LanceJoinError;
-    type JoinHandle = Pin<Box<dyn Future<Output = Result<(), Self::JoinError>> + Send>>;
-
-    fn spawn<F>(fut: F) -> Self::JoinHandle
-    where
-        F: Future<Output = ()> + Send + 'static,
-    {
-        let handle = get_runtime().spawn(fut);
-        Box::pin(async move { handle.await.map_err(LanceJoinError) })
-    }
-
-    fn spawn_blocking<F>(f: F) -> Self::JoinHandle
-    where
-        F: FnOnce() + Send + 'static,
-    {
-        let handle = get_runtime().spawn_blocking(f);
-        Box::pin(async move { handle.await.map_err(LanceJoinError) })
-    }
-}
-
-tokio::task_local! {
-    static TASK_LOCALS: std::cell::OnceCell<TaskLocals>;
-}
-
-impl ContextExt for LanceRuntime {
-    fn scope<F, R>(locals: TaskLocals, fut: F) -> Pin<Box<dyn Future<Output = R> + Send>>
-    where
-        F: Future<Output = R> + Send + 'static,
-    {
-        let cell = std::cell::OnceCell::new();
-        cell.set(locals).unwrap();
-        Box::pin(TASK_LOCALS.scope(cell, fut))
-    }
-
-    fn get_task_locals() -> Option<TaskLocals> {
-        TASK_LOCALS
-            .try_with(|c| c.get().cloned())
-            .unwrap_or_default()
-    }
-}
-
-/// Drop-in replacement for `pyo3_async_runtimes::tokio::future_into_py` that
-/// uses our fork-safe runtime.
-pub fn future_into_py<F, T>(py: Python<'_>, fut: F) -> PyResult<Bound<'_, PyAny>>
-where
-    F: Future<Output = PyResult<T>> + Send + 'static,
-    T: for<'py> IntoPyObject<'py> + Send + 'static,
-{
-    pyo3_async_runtimes::generic::future_into_py::<LanceRuntime, _, T>(py, fut)
-}
@@ -2,7 +2,6 @@
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 use std::{collections::HashMap, sync::Arc};

-use crate::runtime::future_into_py;
 use crate::{
    connection::Connection,
    error::PythonErrorExt,
@@ -25,6 +24,7 @@ use pyo3::{
    pyclass, pymethods,
    types::{IntoPyDict, PyAnyMethods, PyDict, PyDictMethods},
 };
+use pyo3_async_runtimes::tokio::future_into_py;

 mod scannable;

@@ -40,7 +40,7 @@ lance-datafusion.workspace = true
 lance-datagen = { workspace = true }
 lance-file = { workspace = true }
 lance-io = { workspace = true }
-lance-index = { workspace = true, features = ["tokenizer-jieba", "tokenizer-lindera"] }
+lance-index = { workspace = true }
 lance-table = { workspace = true }
 lance-linalg = { workspace = true }
 lance-testing = { workspace = true }
@@ -505,15 +505,8 @@ impl ListingDatabase {
                // Filter out the commit store query param -- it's a lancedb param
                url.query_pairs_mut().clear();
                url.query_pairs_mut().extend_pairs(filtered_querys);
-                // Take a copy of the query string so we can propagate it to lance.
-                // `query_pairs_mut()` leaves the URL with `Some("")` even when no
-                // pairs survive (or none existed in the first place), so an empty
-                // string here must be treated the same as "no query" — otherwise
-                // every table URI ends up with a trailing `?`, which makes downstream
-                // sub-paths (e.g. MemWAL gen paths) re-parse as path=<base table> +
-                // query=<sub-path>, causing Lance to find the base table dataset
-                // when looking up the sub-path.
-                let query_string = url.query().filter(|q| !q.is_empty()).map(|s| s.to_string());
+                // Take a copy of the query string so we can propagate it to lance
+                let query_string = url.query().map(|s| s.to_string());
                // clear the query string so we can use the url as the base uri
                // use .set_query(None) instead of .set_query("") because the latter
                // will add a trailing '?' to the url
@@ -2220,133 +2213,6 @@ mod tests {
        assert_eq!(uri, expected);
    }

-    /// Regression: connecting via a URL-style URI (which goes through
-    /// `url::Url::parse` and the `query_pairs_mut()` path) must not
-    /// append a trailing `?` to per-table URIs when the input URI has
-    /// no query string.
-    ///
-    /// Earlier, `query_pairs_mut().clear()` left the URL with
-    /// `query=Some("")`, which then propagated as a trailing `?` on
-    /// every table URI. Sub-path lookups against that URI (e.g. MemWAL
-    /// `<table_uri>/_mem_wal/<shard>/<rand>_gen_<n>`) re-parsed as
-    /// `path=<base table>` + `query=/_mem_wal/...`, causing
-    /// `Dataset::write` to find the base table dataset and falsely
-    /// report `Dataset already exists`.
-    /// Mirrors the URL-mutation step from
-    /// [`ListingDatabase::connect_with_options`] so we can assert the
-    /// fix without going through filesystem setup (which is awkward
-    /// across platforms — see the `file://` test below).
-    fn capture_query_like_connect(input_uri: &str) -> Option<String> {
-        let mut url = url::Url::parse(input_uri).unwrap();
-        let mut filtered_querys = Vec::new();
-        for (key, value) in url.query_pairs() {
-            if key == ENGINE || key == MIRRORED_STORE {
-                continue;
-            }
-            filtered_querys.push((key.to_string(), value.to_string()));
-        }
-        url.query_pairs_mut().clear();
-        url.query_pairs_mut().extend_pairs(filtered_querys);
-        url.query().filter(|q| !q.is_empty()).map(|s| s.to_string())
-    }
-
-    #[test]
-    fn test_capture_query_treats_empty_as_none() {
-        // No query at all. With the bug, `query_pairs_mut()` left the
-        // URL with `query=Some("")` and we used to propagate that.
-        assert_eq!(
-            capture_query_like_connect("s3://bucket/prefix/"),
-            None,
-            "empty query after mutation must be treated as no query"
-        );
-
-        // Real query is propagated.
-        assert_eq!(
-            capture_query_like_connect("s3://bucket/prefix/?foo=bar"),
-            Some("foo=bar".to_string())
-        );
-
-        // lancedb-internal `engine=` is stripped; nothing remains, so
-        // query_string is None — not Some("").
-        assert_eq!(
-            capture_query_like_connect(&format!("s3://bucket/prefix/?{}=mem", ENGINE)),
-            None
-        );
-
-        // Mixed: drop `engine=`, keep the rest.
-        let captured =
-            capture_query_like_connect(&format!("s3://bucket/prefix/?{}=mem&foo=bar", ENGINE));
-        assert_eq!(captured.as_deref(), Some("foo=bar"));
-    }
-
-    /// Regression: connecting via a URL-style URI (which goes through
-    /// `url::Url::parse` and the `query_pairs_mut()` path) must not
-    /// append a trailing `?` to per-table URIs when the input URI has
-    /// no query string. Sub-path lookups against such a URI (e.g.
-    /// MemWAL `<table_uri>/_mem_wal/<shard>/<rand>_gen_<n>`) re-parse
-    /// as `path=<base table>` + `query=/_mem_wal/...`, causing
-    /// `Dataset::write` to find the base table dataset and falsely
-    /// report `Dataset already exists`.
-    ///
-    /// Skipped on Windows: `try_create_dir` does not understand
-    /// `file:///C:/…` paths so `connect_with_options` fails before
-    /// even reaching the URL-mutation logic. The pure URL-mutation
-    /// invariant is covered by
-    /// `test_capture_query_treats_empty_as_none` above, which runs
-    /// on all platforms.
-    #[cfg(not(windows))]
-    #[tokio::test]
-    async fn test_table_uri_url_path_has_no_trailing_question_mark() {
-        let tempdir = tempdir().unwrap();
-        let uri = format!("file://{}", tempdir.path().to_str().unwrap());
-
-        let request = ConnectRequest {
-            uri: uri.clone(),
-            #[cfg(feature = "remote")]
-            client_config: Default::default(),
-            options: Default::default(),
-            namespace_client_properties: Default::default(),
-            manifest_enabled: false,
-            read_consistency_interval: None,
-            session: None,
-        };
-        let db = ListingDatabase::connect_with_options(&request)
-            .await
-            .unwrap();
-
-        assert_eq!(
-            db.query_string, None,
-            "no input query → no captured query_string"
-        );
-
-        let table_uri = db.table_uri("test").unwrap();
-        assert!(
-            !table_uri.ends_with('?'),
-            "table_uri must not have a trailing `?`: {}",
-            table_uri
-        );
-        assert_eq!(table_uri, format!("{}/test.lance", uri));
-
-        // A real query string should still be propagated.
-        let with_query = format!("{}?foo=bar", uri);
-        let request_with_query = ConnectRequest {
-            uri: with_query,
-            #[cfg(feature = "remote")]
-            client_config: Default::default(),
-            options: Default::default(),
-            namespace_client_properties: Default::default(),
-            manifest_enabled: false,
-            read_consistency_interval: None,
-            session: None,
-        };
-        let db_with_query = ListingDatabase::connect_with_options(&request_with_query)
-            .await
-            .unwrap();
-        assert_eq!(db_with_query.query_string.as_deref(), Some("foo=bar"));
-        let table_uri = db_with_query.table_uri("test").unwrap();
-        assert_eq!(table_uri, format!("{}/test.lance?foo=bar", uri));
-    }
-
    #[tokio::test]
    async fn test_namespace_client() {
        let (_tempdir, db) = setup_database().await;
@@ -13,10 +13,7 @@ use crate::{DistanceType, Error, Result, table::BaseTable};

 use self::{
    scalar::{BTreeIndexBuilder, BitmapIndexBuilder, LabelListIndexBuilder},
-    vector::{
-        IvfHnswFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, IvfPqIndexBuilder,
-        IvfSqIndexBuilder,
-    },
+    vector::{IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, IvfPqIndexBuilder, IvfSqIndexBuilder},
 };

 pub mod scalar;
@@ -70,10 +67,6 @@ pub enum Index {
    /// IVF-HNSW index with Scalar Quantization
    /// It is a variant of the HNSW algorithm that uses scalar quantization to compress the vectors.
    IvfHnswSq(IvfHnswSqIndexBuilder),
-
-    /// IVF-HNSW index without quantization.
-    /// Stores raw vectors, providing the highest recall at the cost of more memory and disk space.
-    IvfHnswFlat(IvfHnswFlatIndexBuilder),
 }

 /// Builder for the create_index operation
@@ -297,8 +290,6 @@ pub enum IndexType {
    IvfHnswPq,
    #[serde(alias = "IVF_HNSW_SQ")]
    IvfHnswSq,
-    #[serde(alias = "IVF_HNSW_FLAT")]
-    IvfHnswFlat,
    // Scalar
    #[serde(alias = "BTREE")]
    BTree,
@@ -320,7 +311,6 @@ impl std::fmt::Display for IndexType {
            Self::IvfRq => write!(f, "IVF_RQ"),
            Self::IvfHnswPq => write!(f, "IVF_HNSW_PQ"),
            Self::IvfHnswSq => write!(f, "IVF_HNSW_SQ"),
-            Self::IvfHnswFlat => write!(f, "IVF_HNSW_FLAT"),
            Self::BTree => write!(f, "BTREE"),
            Self::Bitmap => write!(f, "BITMAP"),
            Self::LabelList => write!(f, "LABEL_LIST"),
@@ -344,7 +334,6 @@ impl std::str::FromStr for IndexType {
            "IVF_RQ" => Ok(Self::IvfRq),
            "IVF_HNSW_PQ" => Ok(Self::IvfHnswPq),
            "IVF_HNSW_SQ" => Ok(Self::IvfHnswSq),
-            "IVF_HNSW_FLAT" => Ok(Self::IvfHnswFlat),
            _ => Err(Error::InvalidInput {
                message: format!("the input value {} is not a valid IndexType", value),
            }),
@@ -474,46 +474,3 @@ impl IvfHnswSqIndexBuilder {
    impl_ivf_params_setter!();
    impl_hnsw_params_setter!();
 }
-
-/// Builder for an IVF_HNSW_FLAT index.
-///
-/// This index combines IVF partitioning with an HNSW graph per partition,
-/// storing raw (unquantized) vectors. It offers the highest recall among
-/// the IVF_HNSW family at the cost of more memory and disk space compared
-/// to [`IvfHnswSqIndexBuilder`] or [`IvfHnswPqIndexBuilder`].
-#[derive(Debug, Clone, Serialize)]
-pub struct IvfHnswFlatIndexBuilder {
-    // IVF
-    #[serde(rename = "metric_type")]
-    pub(crate) distance_type: DistanceType,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub(crate) num_partitions: Option<u32>,
-    pub(crate) sample_rate: u32,
-    pub(crate) max_iterations: u32,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub(crate) target_partition_size: Option<u32>,
-
-    // HNSW
-    pub(crate) m: u32,
-    pub(crate) ef_construction: u32,
-}
-
-impl Default for IvfHnswFlatIndexBuilder {
-    fn default() -> Self {
-        Self {
-            distance_type: DistanceType::L2,
-            num_partitions: None,
-            sample_rate: 256,
-            max_iterations: 50,
-            m: 20,
-            ef_construction: 300,
-            target_partition_size: None,
-        }
-    }
-}
-
-impl IvfHnswFlatIndexBuilder {
-    impl_distance_type_setter!();
-    impl_ivf_params_setter!();
-    impl_hnsw_params_setter!();
-}
@@ -1540,7 +1540,6 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
            Index::IvfPq(p) => ("IVF_PQ", Some(to_json(p)?)),
            Index::IvfSq(p) => ("IVF_SQ", Some(to_json(p)?)),
            Index::IvfHnswSq(p) => ("IVF_HNSW_SQ", Some(to_json(p)?)),
-            Index::IvfHnswFlat(p) => ("IVF_HNSW_FLAT", Some(to_json(p)?)),
            Index::IvfRq(p) => ("IVF_RQ", Some(to_json(p)?)),
            Index::BTree(p) => ("BTREE", Some(to_json(p)?)),
            Index::Bitmap(p) => ("BITMAP", Some(to_json(p)?)),
@@ -2069,8 +2068,7 @@ mod tests {
    use serde_json::json;

    use crate::index::vector::{
-        IvfFlatIndexBuilder, IvfHnswFlatIndexBuilder, IvfHnswSqIndexBuilder, IvfRqIndexBuilder,
-        IvfSqIndexBuilder,
+        IvfFlatIndexBuilder, IvfHnswSqIndexBuilder, IvfRqIndexBuilder, IvfSqIndexBuilder,
    };
    use crate::remote::JSON_CONTENT_TYPE;
    use crate::remote::db::DEFAULT_SERVER_VERSION;
@@ -3323,35 +3321,6 @@ mod tests {
                        .ef_construction(500),
                ),
            ),
-            (
-                "IVF_HNSW_FLAT",
-                json!({
-                    "metric_type": "l2",
-                    "sample_rate": 256,
-                    "max_iterations": 50,
-                    "m": 20,
-                    "ef_construction": 300,
-                }),
-                Index::IvfHnswFlat(Default::default()),
-            ),
-            (
-                "IVF_HNSW_FLAT",
-                json!({
-                    "metric_type": "cosine",
-                    "num_partitions": 64,
-                    "sample_rate": 256,
-                    "max_iterations": 50,
-                    "m": 40,
-                    "ef_construction": 500,
-                }),
-                Index::IvfHnswFlat(
-                    IvfHnswFlatIndexBuilder::default()
-                        .distance_type(DistanceType::Cosine)
-                        .num_partitions(64)
-                        .num_edges(40)
-                        .ef_construction(500),
-                ),
-            ),
            (
                "IVF_SQ",
                json!({
@@ -2033,24 +2033,6 @@ impl NativeTable {
                );
                Ok(Box::new(lance_idx_params))
            }
-            Index::IvfHnswFlat(index) => {
-                Self::validate_index_type(field, "IVF HNSW FLAT", supported_vector_data_type)?;
-                let ivf_params = Self::build_ivf_params(
-                    index.num_partitions,
-                    index.target_partition_size,
-                    index.sample_rate,
-                    index.max_iterations,
-                );
-                let hnsw_params = HnswBuildParams::default()
-                    .num_edges(index.m as usize)
-                    .ef_construction(index.ef_construction as usize);
-                let lance_idx_params = VectorIndexParams::ivf_hnsw(
-                    index.distance_type.into(),
-                    ivf_params,
-                    hnsw_params,
-                );
-                Ok(Box::new(lance_idx_params))
-            }
        }
    }

@@ -2076,8 +2058,7 @@ impl NativeTable {
            | Index::IvfPq(_)
            | Index::IvfRq(_)
            | Index::IvfHnswPq(_)
-            | Index::IvfHnswSq(_)
-            | Index::IvfHnswFlat(_) => IndexType::Vector,
+            | Index::IvfHnswSq(_) => IndexType::Vector,
        }
    }

@@ -3195,56 +3176,6 @@ mod tests {
        assert_eq!(stats.num_unindexed_rows, 0);
    }

-    #[tokio::test]
-    async fn test_create_index_ivf_hnsw_flat() {
-        use arrow_array::RecordBatch;
-        use arrow_schema::{DataType, Field, Schema as ArrowSchema};
-        use rand;
-        use std::iter::repeat_with;
-
-        use crate::index::vector::IvfHnswFlatIndexBuilder;
-        use arrow_array::Float32Array;
-
-        let tmp_dir = tempdir().unwrap();
-        let uri = tmp_dir.path().to_str().unwrap();
-        let conn = connect(uri).execute().await.unwrap();
-
-        let dimension = 16;
-        let schema = Arc::new(ArrowSchema::new(vec![Field::new(
-            "embeddings",
-            DataType::FixedSizeList(
-                Arc::new(Field::new("item", DataType::Float32, true)),
-                dimension,
-            ),
-            false,
-        )]));
-
-        let float_arr = Float32Array::from(
-            repeat_with(rand::random::<f32>)
-                .take(512 * dimension as usize)
-                .collect::<Vec<f32>>(),
-        );
-
-        let vectors = Arc::new(create_fixed_size_list(float_arr, dimension).unwrap());
-        let batch = RecordBatch::try_new(schema.clone(), vec![vectors.clone()]).unwrap();
-
-        let table = conn.create_table("test", batch).execute().await.unwrap();
-
-        let index = IvfHnswFlatIndexBuilder::default();
-        table
-            .create_index(&["embeddings"], Index::IvfHnswFlat(index))
-            .execute()
-            .await
-            .unwrap();
-
-        let index_configs = table.list_indices().await.unwrap();
-        assert_eq!(index_configs.len(), 1);
-        let index = index_configs.into_iter().next().unwrap();
-        assert_eq!(index.index_type, crate::index::IndexType::IvfHnswFlat);
-        assert_eq!(index.columns, vec!["embeddings".to_string()]);
-        assert_eq!(table.count_rows(None).await.unwrap(), 512);
-    }
-
    fn create_fixed_size_list<T: Array>(values: T, list_size: i32) -> Result<FixedSizeListArray> {
        let list_type = DataType::FixedSizeList(
            Arc::new(Field::new("item", values.data_type().clone(), true)),