fix: support nested field paths in native index creation

lance v7.0.0-beta.9 bumps lindera 0.44 -> 3.0.7, which changed the
tokenizer config schema (dictionary is now a string path, not a
{ path: ... } map) and the dictionary binary format (now requires
metadata.json). The old fixtures broke test_fts_lindera_tokenizer on
all platforms.

Lift the regenerated config.yml and main.zip from the lance
v7.0.0-beta.9 tag (lance-format/lance#6719) and update the
lindera_ipadic fixture's config writer to the 3.x schema.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

.github/workflows/codex-fix-ci.yml

@@ -45,7 +45,9 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: 20
+          # pnpm 11 (used by the nodejs install step below) requires
+          # Node >= 22.13; use 24 since 22 hits EOL in October.
+          node-version: 24
 
       - name: Install Codex CLI
         run: npm install -g @openai/codex
@@ -79,10 +81,14 @@ jobs:
           java-version: '11'
           cache: maven
 
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 11.1.1
       - name: Install Node.js dependencies for TypeScript bindings
         run: |
           cd nodejs
-          npm ci
+          pnpm install --frozen-lockfile
 
       - name: Configure git user
         run: |
@@ -137,7 +143,7 @@ jobs:
                - For Rust test failures: Run the specific test with "cargo test -p <crate> <test_name>"
                - For Python test failures: Build with "cd python && maturin develop" then run "pytest <specific_test_file>::<test_name>"
                - For Java test failures: Run "cd java && mvn test -Dtest=<TestClass>#<testMethod>"
-               - For TypeScript test failures: Run "cd nodejs && npm run build && npm test -- --testNamePattern='<test_name>'"
+               - For TypeScript test failures: Run "cd nodejs && pnpm build && pnpm test -- --testNamePattern='<test_name>'"
                - Do NOT run the full test suite - only run the tests that were failing
 
           7. If the additional guidelines are provided, follow them as well.

.github/workflows/java-publish.yml

@@ -43,7 +43,7 @@ jobs:
           server-username: SONATYPE_USER
           server-password: SONATYPE_TOKEN
           gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
-          gpg-passphrase: ${{ secrets.GPG_PASSPHRASE }}
+          gpg-passphrase: MAVEN_GPG_PASSPHRASE
       - name: Set git config
         run: |
           git config --global user.email "dev+gha@lancedb.com"
@@ -58,10 +58,11 @@ jobs:
           echo "use-agent" >> ~/.gnupg/gpg.conf
           echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
           export GPG_TTY=$(tty)
-          ./mvnw --batch-mode -DskipTests -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -pl lancedb-core -am -P deploy-to-ossrh
+          ./mvnw --batch-mode -DskipTests -DpushChanges=false deploy -pl lancedb-core -am -P deploy-to-ossrh
         env:
           SONATYPE_USER: ${{ secrets.SONATYPE_USER }}
           SONATYPE_TOKEN: ${{ secrets.SONATYPE_TOKEN }}
+          MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
 
   report-failure:
     name: Report Workflow Failure

.github/workflows/nodejs.yml

@@ -42,11 +42,17 @@ jobs:
       with:
         fetch-depth: 0
         lfs: true
+    - uses: pnpm/action-setup@v4
+      with:
+        version: 11.1.1
     - uses: actions/setup-node@v4
       with:
-        node-version: 20
-        cache: 'npm'
-        cache-dependency-path: nodejs/package-lock.json
+        # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+        # in October. The library itself still supports Node >= 18
+        # (see test matrix below).
+        node-version: 24
+        cache: 'pnpm'
+        cache-dependency-path: nodejs/pnpm-lock.yaml
     - uses: actions-rust-lang/setup-rust-toolchain@v1
       with:
         components: rustfmt, clippy
@@ -61,11 +67,13 @@ jobs:
       run: cargo clippy --profile ci --all --all-features -- -D warnings
     - name: Lint Typescript
       run: |
-        npm ci
-        npm run lint-ci
+        pnpm install --frozen-lockfile
+        pnpm lint-ci
     - name: Lint examples
       working-directory: nodejs/examples
-      run: npm ci && npm run lint-ci
+      # The `@lancedb/lancedb` dep points at file:../dist; pnpm errors if
+      # that dir is missing, so create an empty one for lint-only runs.
+      run: mkdir -p ../dist && pnpm install --frozen-lockfile && pnpm lint-ci
   linux:
     name: Linux (NodeJS ${{ matrix.node-version }})
     timeout-minutes: 30
@@ -82,14 +90,18 @@ jobs:
       with:
         fetch-depth: 0
         lfs: true
-    - uses: actions/setup-node@v4
-      name: Setup Node.js 20 for build
+    - uses: pnpm/action-setup@v4
       with:
-        # @napi-rs/cli v3 requires Node >= 20.12 (via @inquirer/prompts@8).
-        # Build always on Node 20; tests run on the matrix version below.
-        node-version: 20
-        cache: 'npm'
-        cache-dependency-path: nodejs/package-lock.json
+        version: 11.1.1
+    - uses: actions/setup-node@v4
+      name: Setup Node.js 24 for build
+      with:
+        # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+        # in October. Build/install runs on Node 24; tests run on the
+        # matrix version below using direct jest invocation.
+        node-version: 24
+        cache: 'pnpm'
+        cache-dependency-path: nodejs/pnpm-lock.yaml
     - uses: Swatinem/rust-cache@v2
     - name: Install dependencies
       run: |
@@ -97,45 +109,52 @@ jobs:
         sudo apt install -y protobuf-compiler libssl-dev
     - name: Build
       run: |
-        npm ci --include=optional
-        npm run build:debug -- --profile ci
+        pnpm install --frozen-lockfile
+        # No `--` separator: pnpm forwards it literally, which would
+        # make napi-rs treat `--profile ci` as a cargo passthrough arg.
+        pnpm build:debug --profile ci
+        pnpm tsc
+    - name: Setup examples
+      working-directory: nodejs/examples
+      run: pnpm install --frozen-lockfile
+    - name: Check docs
+      run: |
+        # We run this as part of the job because the binary needs to be built
+        # first to export the types of the native code.
+        set -e
+        # `pnpm docs` would invoke pnpm's built-in `docs` command, not
+        # the script — use `pnpm run docs`.
+        pnpm run docs
+        if ! git diff --exit-code -- ../ ':(exclude)Cargo.lock'; then
+          echo "Docs need to be updated"
+          echo "Run 'pnpm run docs', fix any warnings, and commit the changes."
+          exit 1
+        fi
     - uses: actions/setup-node@v4
       name: Setup Node.js ${{ matrix.node-version }} for test
       with:
         node-version: ${{ matrix.node-version }}
-    - name: Compile TypeScript
-      run: npm run tsc
     - name: Setup localstack
       working-directory: .
       run: docker compose up --detach --wait
     - name: Test
       env:
         S3_TEST: "1"
-      run: npm run test
-    - name: Setup examples
-      working-directory: nodejs/examples
-      run: npm ci
+        # Newer @smithy/core uses dynamic ESM imports.
+        NODE_OPTIONS: "--experimental-vm-modules"
+      # Invoke jest directly because pnpm 11 itself requires Node 22+
+      # while the matrix tests on older Node versions.
+      run: npx jest --verbose
     - name: Test examples
       working-directory: ./
       env:
         OPENAI_API_KEY: test
         OPENAI_BASE_URL: http://0.0.0.0:8000
+        NODE_OPTIONS: "--experimental-vm-modules"
       run: |
         python ci/mock_openai.py &
         cd nodejs/examples
-        npm test
-    - name: Check docs
-      run: |
-        # We run this as part of the job because the binary needs to be built
-        # first to export the types of the native code.
-        set -e
-        npm ci
-        npm run docs
-        if ! git diff --exit-code -- ../ ':(exclude)Cargo.lock'; then
-          echo "Docs need to be updated"
-          echo "Run 'npm run docs', fix any warnings, and commit the changes."
-          exit 1
-        fi
+        npx jest --testEnvironment jest-environment-node-single-context --verbose
   macos:
     timeout-minutes: 30
     runs-on: "macos-14"
@@ -148,20 +167,28 @@ jobs:
       with:
         fetch-depth: 0
         lfs: true
+    - uses: pnpm/action-setup@v4
+      with:
+        version: 11.1.1
     - uses: actions/setup-node@v4
       with:
-        node-version: 20
-        cache: 'npm'
-        cache-dependency-path: nodejs/package-lock.json
+        # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+        # in October.
+        node-version: 24
+        cache: 'pnpm'
+        cache-dependency-path: nodejs/pnpm-lock.yaml
+    - uses: dtolnay/rust-toolchain@stable
     - uses: Swatinem/rust-cache@v2
     - name: Install dependencies
       run: |
         brew install protobuf
     - name: Build
       run: |
-        npm ci --include=optional
-        npm run build:debug -- --profile ci
-        npm run tsc
+        pnpm install --frozen-lockfile
+        # No `--` separator: pnpm forwards it literally, which would
+        # make napi-rs treat `--profile ci` as a cargo passthrough arg.
+        pnpm build:debug --profile ci
+        pnpm tsc
     - name: Test
       run: |
-        npm run test
+        pnpm test

.github/workflows/npm-publish.yml

@@ -171,13 +171,18 @@ jobs:
         working-directory: nodejs
     steps:
       - uses: actions/checkout@v4
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 11.1.1
       - name: Setup node
         uses: actions/setup-node@v4
-        if: ${{ !matrix.settings.docker }}
         with:
-          node-version: 20
-          cache: npm
-          cache-dependency-path: nodejs/package-lock.json
+          # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+          # in October.
+          node-version: 24
+          cache: pnpm
+          cache-dependency-path: nodejs/pnpm-lock.yaml
       - name: Install
         uses: dtolnay/rust-toolchain@stable
         if: ${{ !matrix.settings.docker }}
@@ -195,7 +200,7 @@ jobs:
             target/
           key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}
       - name: Install dependencies
-        run: npm ci
+        run: pnpm install --frozen-lockfile
       - name: Install Zig
         uses: mlugg/setup-zig@v2
         if: ${{ contains(matrix.settings.target, 'musl') }}
@@ -248,7 +253,7 @@ jobs:
       # one to do the upload.
       - name: Make generic artifacts
         if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
-        run: npm run tsc
+        run: pnpm tsc
       - name: Upload Generic Artifacts
         if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
         uses: actions/upload-artifact@v4
@@ -283,14 +288,24 @@ jobs:
         working-directory: nodejs
     steps:
       - uses: actions/checkout@v4
-      - name: Setup node
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 11.1.1
+      - name: Setup Node.js 24 for install
+        uses: actions/setup-node@v4
+        with:
+          # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
+          # in October.
+          node-version: 24
+          cache: pnpm
+          cache-dependency-path: nodejs/pnpm-lock.yaml
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+      - name: Setup Node.js ${{ matrix.node }} for test
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node }}
-          cache: npm
-          cache-dependency-path: nodejs/package-lock.json
-      - name: Install dependencies
-        run: npm ci
       - name: Download artifacts
         uses: actions/download-artifact@v4
         with:
@@ -311,7 +326,9 @@ jobs:
       - name: Move built files
         run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
       - name: Test bindings
-        run: npm test
+        # Invoke jest directly because pnpm 11 itself requires Node 22+
+        # while the matrix tests on older Node versions.
+        run: npx jest --verbose
   publish:
     name: Publish
     runs-on: ubuntu-latest
@@ -323,15 +340,19 @@ jobs:
       - test-lancedb
     steps:
       - uses: actions/checkout@v4
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 11.1.1
       - name: Setup node
         uses: actions/setup-node@v4
         with:
           node-version: 24
-          cache: npm
-          cache-dependency-path: nodejs/package-lock.json
+          cache: pnpm
+          cache-dependency-path: nodejs/pnpm-lock.yaml
           registry-url: "https://registry.npmjs.org"
       - name: Install dependencies
-        run: npm ci
+        run: pnpm install --frozen-lockfile
       - uses: actions/download-artifact@v4
         with:
           name: nodejs-dist
@@ -351,7 +372,7 @@ jobs:
       - name: Display structure of downloaded files
         run: find dist && find nodejs-artifacts
       - name: Move artifacts
-        run: npx napi artifacts -d nodejs-artifacts
+        run: pnpm exec napi artifacts -d nodejs-artifacts
       - name: List packages
         run: find npm
       - name: Publish

Cargo.lock

@@ -146,6 +146,15 @@ dependencies = [
  "object",
 ]
 
+[[package]]
+name = "arc-swap"
+version = "1.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207"
+dependencies = [
+ "rustversion",
+]
+
 [[package]]
 name = "argminmax"
 version = "0.6.3"
@@ -374,7 +383,7 @@ version = "58.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "serde_core",
  "serde_json",
 ]
@@ -1210,6 +1219,12 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
 
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
 [[package]]
 name = "bitflags"
 version = "2.11.0"
@@ -1337,6 +1352,29 @@ version = "3.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
 
+[[package]]
+name = "bytecheck"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0caa33a2c0edca0419d15ac723dff03f1956f7978329b1e3b5fdaaaed9d3ca8b"
+dependencies = [
+ "bytecheck_derive",
+ "ptr_meta",
+ "rancor",
+ "simdutf8",
+]
+
+[[package]]
+name = "bytecheck_derive"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
 [[package]]
 name = "bytemuck"
 version = "1.25.0"
@@ -1458,6 +1496,15 @@ dependencies = [
  "shlex",
 ]
 
+[[package]]
+name = "cedarwood"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d910bedd62c24733263d0bed247460853c9d22e8956bd4cd964302095e04e90"
+dependencies = [
+ "smallvec",
+]
+
 [[package]]
 name = "cfg-if"
 version = "0.1.10"
@@ -1530,7 +1577,7 @@ checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f"
 dependencies = [
  "parse-zoneinfo",
  "phf 0.11.3",
- "phf_codegen",
+ "phf_codegen 0.11.3",
 ]
 
 [[package]]
@@ -1880,7 +1927,7 @@ version = "0.29.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "crossterm_winapi",
  "document-features",
  "parking_lot",
@@ -1972,6 +2019,12 @@ version = "0.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1"
 
+[[package]]
+name = "daachorse"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db756b5eb7d81d31f31f660f4132f8cf5698de52fca144c143d0ae0cbb5f2e06"
+
 [[package]]
 name = "darling"
 version = "0.20.11"
@@ -2910,6 +2963,15 @@ dependencies = [
  "cfg-if 1.0.4",
 ]
 
+[[package]]
+name = "encoding_rs_io"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1cc3c5651fb62ab8aa3103998dade57efdd028544bd300516baa31840c252a83"
+dependencies = [
+ "encoding_rs",
+]
+
 [[package]]
 name = "enum-as-inner"
 version = "0.6.1"
@@ -3083,7 +3145,7 @@ version = "25.12.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "rustc_version",
 ]
 
@@ -3150,8 +3212,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "7.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.5#1538ec33cf9eb6d06ae324b1b2a2f413a2aca05c"
+version = "7.0.0-beta.9"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.9#109ebf71fcfddef8faae1af519df339a330debfc"
 dependencies = [
  "arrow-array",
  "rand 0.9.4",
@@ -3605,6 +3667,12 @@ dependencies = [
  "serde_core",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+
 [[package]]
 name = "heapify"
 version = "0.2.0"
@@ -4084,7 +4152,7 @@ version = "0.7.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fdd7bddefd0a8833b88a4b68f90dae22c7450d11b354198baee3874fd811b344"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "cfg-if 1.0.4",
  "libc",
 ]
@@ -4159,6 +4227,28 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9028f49264629065d057f340a86acb84867925865f73bbf8d47b4d149a7e88b8"
 
+[[package]]
+name = "jieba-macros"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a29cfc5dcd898604c6f80363411fa6b6b08e27d1d253d6225b9cb6702ea02fc0"
+dependencies = [
+ "phf_codegen 0.13.1",
+]
+
+[[package]]
+name = "jieba-rs"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3245d6e9d1d5facbd6a23848d6b67e3439738ccbb4fa5a3d65da315ba1a910a2"
+dependencies = [
+ "cedarwood",
+ "jieba-macros",
+ "phf 0.13.1",
+ "regex",
+ "rustc-hash",
+]
+
 [[package]]
 name = "jiff"
 version = "0.2.23"
@@ -4308,6 +4398,15 @@ dependencies = [
  "simple_asn1",
 ]
 
+[[package]]
+name = "kanaria"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0f9d9652540055ac4fded998a73aca97d965899077ab1212587437da44196ff"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
 [[package]]
 name = "konst"
 version = "0.4.3"
@@ -4327,8 +4426,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 
 [[package]]
 name = "lance"
-version = "7.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.5#1538ec33cf9eb6d06ae324b1b2a2f413a2aca05c"
+version = "7.0.0-beta.9"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.9#109ebf71fcfddef8faae1af519df339a330debfc"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -4373,7 +4472,6 @@ dependencies = [
  "lance-table",
  "lance-tokenizer",
  "log",
- "moka",
  "object_store",
  "permutation",
  "pin-project",
@@ -4396,12 +4494,11 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "7.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.5#1538ec33cf9eb6d06ae324b1b2a2f413a2aca05c"
+version = "7.0.0-beta.9"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.9#109ebf71fcfddef8faae1af519df339a330debfc"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
- "arrow-cast",
  "arrow-data",
  "arrow-ipc",
  "arrow-ord",
@@ -4418,8 +4515,8 @@ dependencies = [
 
 [[package]]
 name = "lance-bitpacking"
-version = "7.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.5#1538ec33cf9eb6d06ae324b1b2a2f413a2aca05c"
+version = "7.0.0-beta.9"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.9#109ebf71fcfddef8faae1af519df339a330debfc"
 dependencies = [
  "arrayref",
  "paste",
@@ -4428,8 +4525,8 @@ dependencies = [
 
 [[package]]
 name = "lance-core"
-version = "7.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.5#1538ec33cf9eb6d06ae324b1b2a2f413a2aca05c"
+version = "7.0.0-beta.9"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.9#109ebf71fcfddef8faae1af519df339a330debfc"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4437,7 +4534,6 @@ dependencies = [
  "async-trait",
  "byteorder",
  "bytes",
- "chrono",
  "datafusion-common",
  "datafusion-sql",
  "deepsize",
@@ -4446,7 +4542,6 @@ dependencies = [
  "lance-arrow",
  "libc",
  "log",
- "mock_instant",
  "moka",
  "num_cpus",
  "object_store",
@@ -4466,8 +4561,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datafusion"
-version = "7.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.5#1538ec33cf9eb6d06ae324b1b2a2f413a2aca05c"
+version = "7.0.0-beta.9"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.9#109ebf71fcfddef8faae1af519df339a330debfc"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4491,15 +4586,14 @@ dependencies = [
  "pin-project",
  "prost",
  "prost-build",
- "snafu 0.9.0",
  "tokio",
  "tracing",
 ]
 
 [[package]]
 name = "lance-datagen"
-version = "7.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.5#1538ec33cf9eb6d06ae324b1b2a2f413a2aca05c"
+version = "7.0.0-beta.9"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.9#109ebf71fcfddef8faae1af519df339a330debfc"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4517,8 +4611,8 @@ dependencies = [
 
 [[package]]
 name = "lance-encoding"
-version = "7.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.5#1538ec33cf9eb6d06ae324b1b2a2f413a2aca05c"
+version = "7.0.0-beta.9"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.9#109ebf71fcfddef8faae1af519df339a330debfc"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -4543,10 +4637,8 @@ dependencies = [
  "num-traits",
  "prost",
  "prost-build",
- "prost-types",
  "rand 0.9.4",
- "snafu 0.9.0",
- "strum",
+ "strum 0.26.3",
  "tokio",
  "tracing",
  "xxhash-rust",
@@ -4555,8 +4647,8 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "7.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.5#1538ec33cf9eb6d06ae324b1b2a2f413a2aca05c"
+version = "7.0.0-beta.9"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.9#109ebf71fcfddef8faae1af519df339a330debfc"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -4581,16 +4673,16 @@ dependencies = [
  "prost",
  "prost-build",
  "prost-types",
- "snafu 0.9.0",
  "tokio",
  "tracing",
 ]
 
 [[package]]
 name = "lance-index"
-version = "7.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.5#1538ec33cf9eb6d06ae324b1b2a2f413a2aca05c"
+version = "7.0.0-beta.9"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.9#109ebf71fcfddef8faae1af519df339a330debfc"
 dependencies = [
+ "arc-swap",
  "arrow",
  "arrow-arith",
  "arrow-array",
@@ -4609,13 +4701,13 @@ dependencies = [
  "datafusion-common",
  "datafusion-expr",
  "datafusion-physical-expr",
- "datafusion-sql",
  "deepsize",
  "dirs",
  "fst",
  "futures",
  "half",
  "itertools 0.13.0",
+ "jieba-rs",
  "jsonb",
  "lance-arrow",
  "lance-core",
@@ -4643,7 +4735,6 @@ dependencies = [
  "serde",
  "serde_json",
  "smallvec",
- "snafu 0.9.0",
  "tempfile",
  "tokio",
  "tracing",
@@ -4653,8 +4744,8 @@ dependencies = [
 
 [[package]]
 name = "lance-io"
-version = "7.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.5#1538ec33cf9eb6d06ae324b1b2a2f413a2aca05c"
+version = "7.0.0-beta.9"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.9#109ebf71fcfddef8faae1af519df339a330debfc"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -4678,7 +4769,6 @@ dependencies = [
  "lance-arrow",
  "lance-core",
  "lance-namespace",
- "libc",
  "log",
  "moka",
  "object_store",
@@ -4689,7 +4779,6 @@ dependencies = [
  "prost",
  "rand 0.9.4",
  "serde",
- "snafu 0.9.0",
  "tempfile",
  "tokio",
  "tracing",
@@ -4698,8 +4787,8 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "7.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.5#1538ec33cf9eb6d06ae324b1b2a2f413a2aca05c"
+version = "7.0.0-beta.9"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.9#109ebf71fcfddef8faae1af519df339a330debfc"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4715,22 +4804,21 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace"
-version = "7.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.5#1538ec33cf9eb6d06ae324b1b2a2f413a2aca05c"
+version = "7.0.0-beta.9"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.9#109ebf71fcfddef8faae1af519df339a330debfc"
 dependencies = [
  "arrow",
  "async-trait",
  "bytes",
  "lance-core",
  "lance-namespace-reqwest-client",
- "serde",
  "snafu 0.9.0",
 ]
 
 [[package]]
 name = "lance-namespace-impls"
-version = "7.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.5#1538ec33cf9eb6d06ae324b1b2a2f413a2aca05c"
+version = "7.0.0-beta.9"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.9#109ebf71fcfddef8faae1af519df339a330debfc"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -4757,7 +4845,6 @@ dependencies = [
  "serde",
  "serde_json",
  "sha2",
- "snafu 0.9.0",
  "tokio",
  "tower",
  "tower-http 0.5.2",
@@ -4780,8 +4867,8 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "7.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.5#1538ec33cf9eb6d06ae324b1b2a2f413a2aca05c"
+version = "7.0.0-beta.9"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.9#109ebf71fcfddef8faae1af519df339a330debfc"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4820,8 +4907,8 @@ dependencies = [
 
 [[package]]
 name = "lance-testing"
-version = "7.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.5#1538ec33cf9eb6d06ae324b1b2a2f413a2aca05c"
+version = "7.0.0-beta.9"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.9#109ebf71fcfddef8faae1af519df339a330debfc"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -4832,9 +4919,11 @@ dependencies = [
 
 [[package]]
 name = "lance-tokenizer"
-version = "7.0.0-beta.5"
-source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.5#1538ec33cf9eb6d06ae324b1b2a2f413a2aca05c"
+version = "7.0.0-beta.9"
+source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-beta.9#109ebf71fcfddef8faae1af519df339a330debfc"
 dependencies = [
+ "jieba-rs",
+ "lindera",
  "rust-stemmers",
  "serde",
  "unicode-normalization",
@@ -4936,6 +5025,7 @@ dependencies = [
  "env_logger",
  "futures",
  "half",
+ "lance-namespace",
  "lancedb",
  "log",
  "lzma-sys",
@@ -4951,6 +5041,7 @@ dependencies = [
  "arrow",
  "async-trait",
  "bytes",
+ "datafusion-common",
  "env_logger",
  "futures",
  "lance-core",
@@ -5073,6 +5164,60 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "lindera"
+version = "3.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74cda79d7161e99b414e4d292ff673cc3f8d22f070d8be3b6185c033363a9216"
+dependencies = [
+ "anyhow",
+ "byteorder",
+ "csv",
+ "daachorse",
+ "kanaria",
+ "lindera-dictionary",
+ "log",
+ "once_cell",
+ "percent-encoding",
+ "regex",
+ "serde",
+ "serde_json",
+ "serde_yaml_ng",
+ "strum 0.28.0",
+ "strum_macros 0.28.0",
+ "unicode-blocks",
+ "unicode-normalization",
+ "unicode-segmentation",
+ "url",
+]
+
+[[package]]
+name = "lindera-dictionary"
+version = "3.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2385456ca9fe87c29072c5f156b52fdd5e28d5b5738ddfb3979501dbd736530"
+dependencies = [
+ "anyhow",
+ "byteorder",
+ "csv",
+ "daachorse",
+ "derive_builder",
+ "encoding_rs",
+ "encoding_rs_io",
+ "glob",
+ "log",
+ "memmap2 0.9.10",
+ "num_cpus",
+ "once_cell",
+ "regex",
+ "rkyv",
+ "serde",
+ "serde_json",
+ "strum 0.28.0",
+ "strum_macros 0.28.0",
+ "thiserror 2.0.18",
+]
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.12.1"
@@ -5304,12 +5449,6 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
-[[package]]
-name = "mock_instant"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dce6dd36094cac388f119d2e9dc82dc730ef91c32a6222170d630e5414b956e6"
-
 [[package]]
 name = "moka"
 version = "0.12.15"
@@ -5386,13 +5525,33 @@ dependencies = [
  "target-features",
 ]
 
+[[package]]
+name = "munge"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c"
+dependencies = [
+ "munge_macro",
+]
+
+[[package]]
+name = "munge_macro"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
 [[package]]
 name = "napi"
 version = "3.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6944d0bf100571cd6e1a98a316cdca262deb6fccf8d93f5ae1502ca3fc88bd3"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "ctor",
  "futures",
  "napi-build",
@@ -5605,7 +5764,7 @@ version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
 ]
 
 [[package]]
@@ -5717,7 +5876,7 @@ version = "6.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "libc",
  "once_cell",
  "onig_sys",
@@ -6140,16 +6299,36 @@ dependencies = [
  "phf_shared 0.12.1",
 ]
 
+[[package]]
+name = "phf"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
+dependencies = [
+ "phf_shared 0.13.1",
+ "serde",
+]
+
 [[package]]
 name = "phf_codegen"
 version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"
 dependencies = [
- "phf_generator",
+ "phf_generator 0.11.3",
  "phf_shared 0.11.3",
 ]
 
+[[package]]
+name = "phf_codegen"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1"
+dependencies = [
+ "phf_generator 0.13.1",
+ "phf_shared 0.13.1",
+]
+
 [[package]]
 name = "phf_generator"
 version = "0.11.3"
@@ -6160,6 +6339,16 @@ dependencies = [
  "rand 0.8.5",
 ]
 
+[[package]]
+name = "phf_generator"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737"
+dependencies = [
+ "fastrand",
+ "phf_shared 0.13.1",
+]
+
 [[package]]
 name = "phf_shared"
 version = "0.11.3"
@@ -6178,6 +6367,15 @@ dependencies = [
  "siphasher",
 ]
 
+[[package]]
+name = "phf_shared"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266"
+dependencies = [
+ "siphasher",
+]
+
 [[package]]
 name = "pin-project"
 version = "1.1.11"
@@ -6361,7 +6559,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "465f70d3e96b6d0b1a43c358ba451286b8c8bd56696feff020d65702aa33e35c"
 dependencies = [
  "ahash",
- "bitflags",
+ "bitflags 2.11.0",
  "bytemuck",
  "chrono",
  "chrono-tz 0.8.6",
@@ -6435,7 +6633,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "89b2632b1af668e2058d5f8f916d8fbde3cac63d03ae29a705f598e41dcfeb7f"
 dependencies = [
  "ahash",
- "bitflags",
+ "bitflags 2.11.0",
  "glob",
  "once_cell",
  "polars-arrow",
@@ -6750,6 +6948,26 @@ dependencies = [
  "cc",
 ]
 
+[[package]]
+name = "ptr_meta"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79"
+dependencies = [
+ "ptr_meta_derive",
+]
+
+[[package]]
+name = "ptr_meta_derive"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
 [[package]]
 name = "pulp"
 version = "0.22.2"
@@ -6960,6 +7178,15 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
 
+[[package]]
+name = "rancor"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee"
+dependencies = [
+ "ptr_meta",
+]
+
 [[package]]
 name = "rand"
 version = "0.8.5"
@@ -7104,7 +7331,7 @@ version = "11.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
 ]
 
 [[package]]
@@ -7185,7 +7412,7 @@ version = "0.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
 ]
 
 [[package]]
@@ -7260,6 +7487,15 @@ version = "1.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2"
 
+[[package]]
+name = "rend"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6"
+dependencies = [
+ "bytecheck",
+]
+
 [[package]]
 name = "reqsign-aliyun-oss"
 version = "3.0.0"
@@ -7514,6 +7750,36 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "rkyv"
+version = "0.8.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3"
+dependencies = [
+ "bytecheck",
+ "bytes",
+ "hashbrown 0.17.1",
+ "indexmap 2.13.0",
+ "munge",
+ "ptr_meta",
+ "rancor",
+ "rend",
+ "rkyv_derive",
+ "tinyvec",
+ "uuid",
+]
+
+[[package]]
+name = "rkyv_derive"
+version = "0.8.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
 [[package]]
 name = "roaring"
 version = "0.11.4"
@@ -7616,7 +7882,7 @@ version = "1.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "errno",
  "libc",
  "linux-raw-sys",
@@ -7865,7 +8131,7 @@ version = "3.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "core-foundation 0.10.1",
  "core-foundation-sys",
  "libc",
@@ -8011,6 +8277,19 @@ dependencies = [
  "syn 2.0.117",
 ]
 
+[[package]]
+name = "serde_yaml_ng"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f"
+dependencies = [
+ "indexmap 2.13.0",
+ "itoa",
+ "ryu",
+ "serde",
+ "unsafe-libyaml",
+]
+
 [[package]]
 name = "sha1"
 version = "0.10.6"
@@ -8392,6 +8671,15 @@ dependencies = [
  "strum_macros 0.26.4",
 ]
 
+[[package]]
+name = "strum"
+version = "0.28.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd"
+dependencies = [
+ "strum_macros 0.28.0",
+]
+
 [[package]]
 name = "strum_macros"
 version = "0.25.3"
@@ -8418,6 +8706,18 @@ dependencies = [
  "syn 2.0.117",
 ]
 
+[[package]]
+name = "strum_macros"
+version = "0.28.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664"
+dependencies = [
+ "heck 0.5.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
 [[package]]
 name = "subtle"
 version = "2.6.1"
@@ -8478,7 +8778,7 @@ version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "byteorder",
  "enum-as-inner",
  "libc",
@@ -8520,7 +8820,7 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "core-foundation 0.9.4",
  "system-configuration-sys",
 ]
@@ -8885,7 +9185,7 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "bytes",
  "http 1.4.0",
  "http-body 1.0.1",
@@ -8903,7 +9203,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8"
 dependencies = [
  "async-compression",
- "bitflags",
+ "bitflags 2.11.0",
  "bytes",
  "futures-core",
  "futures-util",
@@ -9058,6 +9358,12 @@ version = "2.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
 
+[[package]]
+name = "unicode-blocks"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b12e05d9e06373163a9bb6bb8c263c261b396643a99445fe6b9811fd376581b"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.24"
@@ -9115,6 +9421,12 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
 
+[[package]]
+name = "unsafe-libyaml"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
+
 [[package]]
 name = "untrusted"
 version = "0.7.1"
@@ -9386,7 +9698,7 @@ version = "0.244.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
 dependencies = [
- "bitflags",
+ "bitflags 2.11.0",
  "hashbrown 0.15.5",
  "indexmap 2.13.0",
  "semver",
@@ -9855,7 +10167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
 dependencies = [
  "anyhow",
- "bitflags",
+ "bitflags 2.11.0",
  "indexmap 2.13.0",
  "log",
  "serde",

Cargo.toml

@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"
 
 [workspace.dependencies]
-lance = { "version" = "=7.0.0-beta.5", default-features = false, "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=7.0.0-beta.5", default-features = false, "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=7.0.0-beta.5", default-features = false, "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=7.0.0-beta.9", default-features = false, "tag" = "v7.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=7.0.0-beta.9", "tag" = "v7.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=7.0.0-beta.9", "tag" = "v7.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=7.0.0-beta.9", "tag" = "v7.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=7.0.0-beta.9", default-features = false, "tag" = "v7.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=7.0.0-beta.9", "tag" = "v7.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=7.0.0-beta.9", "tag" = "v7.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=7.0.0-beta.9", "tag" = "v7.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=7.0.0-beta.9", default-features = false, "tag" = "v7.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=7.0.0-beta.9", "tag" = "v7.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=7.0.0-beta.9", "tag" = "v7.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=7.0.0-beta.9", "tag" = "v7.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=7.0.0-beta.9", "tag" = "v7.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=7.0.0-beta.9", "tag" = "v7.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
 arrow = { version = "58.0.0", optional = false }

deny.toml

@@ -51,6 +51,18 @@ ignore = [
     # https://rustsec.org/advisories/RUSTSEC-2024-0436
     { id = "RUSTSEC-2024-0436", reason = "transitive via datafusion; awaiting ecosystem migration" },
 
+    # encoding: unmaintained. Reached through lindera-dictionary, which is
+    # required by the native Lindera tokenizer path. Lindera has not migrated
+    # off this crate yet.
+    # https://rustsec.org/advisories/RUSTSEC-2021-0153
+    { id = "RUSTSEC-2021-0153", reason = "transitive via lindera-dictionary for native Lindera tokenizer" },
+
+    # fast-float: unsound and unmaintained. Reached only through polars-arrow
+    # from the optional Polars integration; replacement requires a Polars
+    # dependency upgrade.
+    # https://rustsec.org/advisories/RUSTSEC-2024-0379
+    { id = "RUSTSEC-2024-0379", reason = "transitive via polars-arrow; waiting on Polars migration" },
+
     # tantivy: segfault on malformed input due to missing bounds check.
     # Pulled in via lance for full-text search. We only feed tantivy
     # documents we construct ourselves, not attacker-controlled bytes.
@@ -68,11 +80,17 @@ ignore = [
     # https://rustsec.org/advisories/RUSTSEC-2025-0119
     { id = "RUSTSEC-2025-0119", reason = "transitive via hf-hub/indicatif; cosmetic formatting crate" },
 
-    # rustls-pemfile: unmaintained. Reached from two separate chains:
-    # rustls-native-certs 0.6 (via hyper-rustls 0.24) and object_store 0.12.
-    # Both upstream dependencies need to move before we can drop it.
-    # https://rustsec.org/advisories/RUSTSEC-2025-0134
-    { id = "RUSTSEC-2025-0134", reason = "transitive via rustls-native-certs/object_store; waiting on upstream migration" },
+    # bincode: unmaintained. Reached through lindera and lindera-dictionary,
+    # which are required by the native Lindera tokenizer path. Lindera has not
+    # migrated to another serialization format yet.
+    # https://rustsec.org/advisories/RUSTSEC-2025-0141
+    { id = "RUSTSEC-2025-0141", reason = "transitive via lindera/lindera-dictionary for native Lindera tokenizer" },
+
+    # lru: soundness issue in IterMut. Reached only through aws-sdk-s3 in
+    # LanceDB's dev-dependency graph; LanceDB does not use that iterator
+    # directly. Clearing this requires the AWS SDK chain to update lru.
+    # https://rustsec.org/advisories/RUSTSEC-2026-0002
+    { id = "RUSTSEC-2026-0002", reason = "transitive via aws-sdk-s3 dev-dependency; waiting on AWS SDK lru upgrade" },
 
     # rustls-webpki 0.101.7 (old major line): name-constraint checks for
     # URI / wildcard names. Pulled in only via the legacy rustls 0.21 chain
@@ -89,6 +107,12 @@ ignore = [
     # we actively use is upgraded to 0.103.13 which contains the fix.
     # https://rustsec.org/advisories/RUSTSEC-2026-0104
     { id = "RUSTSEC-2026-0104", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
+
+    # rand 0.8.5: soundness issue only when ThreadRng reseeds inside a custom
+    # logger. Reached through several transitive chains. LanceDB does not use
+    # rand from a custom logger; upgrade once all pinned chains accept 0.8.6+.
+    # https://rustsec.org/advisories/RUSTSEC-2026-0097
+    { id = "RUSTSEC-2026-0097", reason = "transitive rand 0.8.5; LanceDB does not call ThreadRng from custom logging" },
 ]
 
 # ---------------------------------------------------------------------------

docs/src/js/_media/CONTRIBUTING.md

@@ -12,20 +12,22 @@ Typescript.
 * `src/`: Rust bindings source code
 * `lancedb/`: Typescript package source code
 * `__test__/`: Unit tests
-* `examples/`: An npm package with the examples shown in the documentation
+* `examples/`: A pnpm package with the examples shown in the documentation
 
 ## Development environment
 
 To set up your development environment, you will need to install the following:
 
-1. Node.js 14 or later
-2. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
-3. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
+1. Node.js 22 or later (required by pnpm 11)
+2. [pnpm](https://pnpm.io/installation) 11 or later (or run via `corepack enable`,
+   which uses the `packageManager` field in `package.json`)
+3. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
+4. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
 
 Initial setup:
 
 ```shell
-npm install
+pnpm install
 ```
 
 ### Commit Hooks
@@ -39,38 +41,38 @@ pre-commit install
 
 ## Development
 
-Most common development commands can be run using the npm scripts.
+Most common development commands can be run using the pnpm scripts.
 
 Build the package
 
 ```shell
-npm install
-npm run build
+pnpm install
+pnpm build
 ```
 
 Lint:
 
 ```shell
-npm run lint
+pnpm lint
 ```
 
 Format and fix lints:
 
 ```shell
-npm run lint-fix
+pnpm lint-fix
 ```
 
 Run tests:
 
 ```shell
-npm test
+pnpm test
 ```
 
 To run a single test:
 
 ```shell
 # Single file: table.test.ts
-npm test -- table.test.ts
+pnpm test -- table.test.ts
 # Single test: 'merge insert' in table.test.ts
-npm test -- table.test.ts --testNamePattern=merge\ insert
+pnpm test -- table.test.ts --testNamePattern=merge\ insert
 ```

docs/src/js/classes/Connection.md

@@ -148,6 +148,33 @@ Creates a new empty Table
 
 ***
 
+### createNamespace()
+
+```ts
+abstract createNamespace(namespacePath, options?): Promise<CreateNamespaceResponse>
+```
+
+Create a new namespace at the given path.
+
+#### Parameters
+
+* **namespacePath**: `string`[]
+    The namespace path to create.
+
+* **options?**: `Partial`&lt;[`CreateNamespaceOptions`](../interfaces/CreateNamespaceOptions.md)&gt;
+    Creation `mode`
+    ("create" | "exist_ok" | "overwrite") and optional `properties`
+    to attach to the namespace.
+
+#### Returns
+
+`Promise`&lt;[`CreateNamespaceResponse`](../interfaces/CreateNamespaceResponse.md)&gt;
+
+The properties of the
+  created namespace and an optional transaction id.
+
+***
+
 ### createTable()
 
 #### createTable(options, namespacePath)
@@ -230,6 +257,29 @@ Creates a new Table and initialize it with new data.
 
 ***
 
+### describeNamespace()
+
+```ts
+abstract describeNamespace(namespacePath): Promise<DescribeNamespaceResponse>
+```
+
+Describe a namespace, returning its properties.
+
+#### Parameters
+
+* **namespacePath**: `string`[]
+    The namespace path to describe, in
+    parent → child order, e.g. `["analytics", "sales"]`.
+
+#### Returns
+
+`Promise`&lt;[`DescribeNamespaceResponse`](../interfaces/DescribeNamespaceResponse.md)&gt;
+
+The namespace's properties
+  (may be undefined if the namespace has none).
+
+***
+
 ### display()
 
 ```ts
@@ -263,6 +313,36 @@ Drop all tables in the database.
 
 ***
 
+### dropNamespace()
+
+```ts
+abstract dropNamespace(namespacePath, options?): Promise<DropNamespaceResponse>
+```
+
+Drop a namespace.
+
+Use `behavior: "cascade"` to also drop everything contained in the
+namespace (sub-namespaces and tables). The default `"restrict"`
+behavior refuses to drop a non-empty namespace.
+
+#### Parameters
+
+* **namespacePath**: `string`[]
+    The namespace path to drop.
+
+* **options?**: `Partial`&lt;[`DropNamespaceOptions`](../interfaces/DropNamespaceOptions.md)&gt;
+    `mode` ("skip" | "fail"
+    for missing-namespace handling) and `behavior` ("restrict" | "cascade").
+
+#### Returns
+
+`Promise`&lt;[`DropNamespaceResponse`](../interfaces/DropNamespaceResponse.md)&gt;
+
+Any properties returned by
+  the server and an optional transaction id.
+
+***
+
 ### dropTable()
 
 ```ts
@@ -299,6 +379,36 @@ Return true if the connection has not been closed
 
 ***
 
+### listNamespaces()
+
+```ts
+abstract listNamespaces(namespacePath?, options?): Promise<ListNamespacesResponse>
+```
+
+List the immediate child namespaces under the given parent.
+
+Results may be paginated. To retrieve subsequent pages, pass the
+`pageToken` returned by a previous call.
+
+#### Parameters
+
+* **namespacePath?**: `string`[]
+    The parent namespace path. Defaults
+    to the root namespace if omitted.
+
+* **options?**: `Partial`&lt;[`ListNamespacesOptions`](../interfaces/ListNamespacesOptions.md)&gt;
+    Pagination options
+    (`pageToken`, `limit`).
+
+#### Returns
+
+`Promise`&lt;[`ListNamespacesResponse`](../interfaces/ListNamespacesResponse.md)&gt;
+
+Child namespace names and
+  an optional token for fetching the next page.
+
+***
+
 ### openTable()
 
 ```ts
@@ -327,6 +437,29 @@ Open a table in the database.
 
 ***
 
+### renameTable()
+
+```ts
+abstract renameTable(
+   oldName,
+   newName,
+   namespacePath?): Promise<void>
+```
+
+#### Parameters
+
+* **oldName**: `string`
+
+* **newName**: `string`
+
+* **namespacePath?**: `string`[]
+
+#### Returns
+
+`Promise`&lt;`void`&gt;
+
+***
+
 ### tableNames()
 
 #### tableNames(options)

docs/src/js/classes/Scannable.md

@@ -0,0 +1,173 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / Scannable
+
+# Class: Scannable
+
+A data source that can be scanned as a stream of Arrow `RecordBatch`es.
+
+`Scannable` wraps the schema + optional row count + rescannable flag and
+a callback that yields batches one at a time. It is passed to consumers
+(e.g. `Table.add`, `createTable`, `mergeInsert` — follow-up work) that
+need to pull data without materializing the full dataset in JS memory.
+
+Batches cross the JS↔Rust boundary as Arrow IPC Stream messages; a fresh
+writer serializes each batch, and the Rust side decodes it with
+`arrow_ipc::reader::StreamReader`. One batch is in flight at a time.
+
+## Properties
+
+### numRows
+
+```ts
+readonly numRows: null | number;
+```
+
+***
+
+### rescannable
+
+```ts
+readonly rescannable: boolean;
+```
+
+***
+
+### schema
+
+```ts
+readonly schema: Schema<any>;
+```
+
+## Methods
+
+### fromFactory()
+
+```ts
+static fromFactory(
+   schema,
+   factory,
+   opts): Promise<Scannable>
+```
+
+Build a Scannable from an explicit schema and a factory that returns a
+fresh batch iterator on each call.
+
+The factory is invoked once per scan. Each iterator yields
+`RecordBatch`es matching the declared schema. Use this when you need
+direct control over the pull loop — for example, to wrap a streaming
+source whose batches are produced lazily.
+
+#### Parameters
+
+* **schema**: `Schema`&lt;`any`&gt;
+    The Arrow schema of the produced batches.
+
+* **factory**
+    Called at the start of each scan to produce a batch
+    iterator. Must be idempotent when `rescannable` is true.
+
+* **opts**: [`ScannableOptions`](../interfaces/ScannableOptions.md) = `{}`
+    Optional hints. `rescannable` defaults to `true`; set to
+    `false` if calling `factory()` twice would not reproduce the same data.
+
+#### Returns
+
+`Promise`&lt;[`Scannable`](Scannable.md)&gt;
+
+***
+
+### fromIterable()
+
+```ts
+static fromIterable(
+   schema,
+   iter,
+   opts): Promise<Scannable>
+```
+
+Build a Scannable from an iterable of `RecordBatch`es. `rescannable`
+defaults to `false`. Pass an explicit schema so the consumer can
+validate before any batch is pulled.
+
+`opts.rescannable: true` is honest for replayable iterables (Arrays,
+Sets, or custom iterables whose `[Symbol.iterator]()` returns a fresh
+iterator each call). It is rejected for one-shot iterables (generators,
+async generators, or already-an-iterator inputs) because their
+`[Symbol.iterator]()` returns the same exhausted object on the second
+scan. For replayable sources outside this shape, use
+`fromFactory(schema, () => createIter(), { rescannable: true })`.
+
+Note: when `opts.rescannable` is `true`, the constructor calls
+`[Symbol.iterator]()` once on the input to perform the structural check.
+
+#### Parameters
+
+* **schema**: `Schema`&lt;`any`&gt;
+
+* **iter**: `Iterable`&lt;`RecordBatch`&lt;`any`&gt;&gt; \| `AsyncIterable`&lt;`RecordBatch`&lt;`any`&gt;&gt;
+
+* **opts**: [`ScannableOptions`](../interfaces/ScannableOptions.md) = `{}`
+
+#### Returns
+
+`Promise`&lt;[`Scannable`](Scannable.md)&gt;
+
+***
+
+### fromRecordBatchReader()
+
+```ts
+static fromRecordBatchReader(reader, opts): Promise<Scannable>
+```
+
+Build a Scannable from an Arrow `RecordBatchReader`. A reader can only
+be consumed once; `rescannable` defaults to `false`.
+
+The reader must already be opened (via `.open()`) so its `.schema` is
+populated. `RecordBatchReader.from(...)` returns an unopened reader.
+
+`opts.rescannable: true` is rejected because `RecordBatchReader` is a
+self-iterator (its `[Symbol.iterator]()` returns itself), and this
+constructor does not call `reader.reset()` between scans, so a second
+scan would always see an exhausted reader. For genuinely replayable
+sources, use
+`fromFactory(schema, () => openReader(), { rescannable: true })`,
+which mints a fresh reader on each scan.
+
+#### Parameters
+
+* **reader**: `RecordBatchReader`&lt;`any`&gt;
+
+* **opts**: [`ScannableOptions`](../interfaces/ScannableOptions.md) = `{}`
+
+#### Returns
+
+`Promise`&lt;[`Scannable`](Scannable.md)&gt;
+
+***
+
+### fromTable()
+
+```ts
+static fromTable(table, opts): Promise<Scannable>
+```
+
+Build a Scannable from an in-memory Arrow `Table`. Always rescannable;
+the table's batches are replayed on each scan.
+
+The table's row count is authoritative: `opts.numRows` must either be
+omitted or equal to `table.numRows`. `opts.rescannable` of `false` is
+rejected because in-memory Tables are always rescannable.
+
+#### Parameters
+
+* **table**: `Table`&lt;`any`&gt;
+
+* **opts**: [`ScannableOptions`](../interfaces/ScannableOptions.md) = `{}`
+
+#### Returns
+
+`Promise`&lt;[`Scannable`](Scannable.md)&gt;

docs/src/js/classes/Table.md

@@ -501,6 +501,34 @@ Modeled after ``VACUUM`` in PostgreSQL.
 
 ***
 
+### prewarmData()
+
+```ts
+abstract prewarmData(columns?): Promise<void>
+```
+
+Prewarm one or more columns of data in the table.
+
+#### Parameters
+
+* **columns?**: `string`[]
+    The columns to prewarm. If undefined, all columns are prewarmed.
+    This will load the column data into the page cache so that future queries that
+    read those columns avoid the initial cold-start latency.  This call initiates
+    prewarming and returns once the request is accepted; the warming itself may
+    continue in the background.  Calling it on already-prewarmed columns is a
+    no-op on the server.
+    Prewarming is generally useful for columns used in filters or projections.
+    Large columns (e.g. high-dimensional vectors or binary data) may not be
+    practical to prewarm.
+    This feature is currently only supported on remote tables.
+
+#### Returns
+
+`Promise`&lt;`void`&gt;
+
+***
+
 ### prewarmIndex()
 
 ```ts

docs/src/js/functions/connectNamespace.md

@@ -0,0 +1,131 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / connectNamespace
+
+# Function: connectNamespace()
+
+## connectNamespace(implName, config, options)
+
+```ts
+function connectNamespace(
+   implName,
+   config,
+   options?): Promise<Connection>
+```
+
+Connect to a LanceDB database through a namespace.
+
+Unlike [connect](connect.md), which routes by URI scheme (local path vs.
+`db://` cloud), `connectNamespace` always returns a namespace-backed
+connection. The `implName` selects the namespace implementation:
+
+- `"dir"` — directory namespace, configured with [DirNamespaceConfig](../interfaces/DirNamespaceConfig.md).
+- `"rest"` — remote REST catalog, configured with [RestNamespaceConfig](../interfaces/RestNamespaceConfig.md).
+- Any other string — full module path for a custom implementation,
+  configured with a free-form string-keyed `properties` map.
+
+### Parameters
+
+* **implName**: `"dir"`
+
+* **config**: [`DirNamespaceConfig`](../interfaces/DirNamespaceConfig.md)
+
+* **options?**: `Partial`&lt;[`ConnectNamespaceOptions`](../interfaces/ConnectNamespaceOptions.md)&gt;
+
+### Returns
+
+`Promise`&lt;[`Connection`](../classes/Connection.md)&gt;
+
+### Examples
+
+```ts
+const db = await connectNamespace("dir", { root: "/path/to/db" });
+await db.createTable("users", [{ id: 1 }]);
+```
+
+```ts
+const db = await connectNamespace("rest", {
+  uri: "https://catalog.example.com",
+  headers: { "x-api-key": process.env.CATALOG_KEY ?? "" },
+});
+```
+
+```ts
+const db = await connectNamespace("my.custom.Namespace", {
+  endpoint: "...",
+});
+```
+
+## connectNamespace(implName, config, options)
+
+```ts
+function connectNamespace(
+   implName,
+   config,
+   options?): Promise<Connection>
+```
+
+Connect through the built-in REST namespace.
+
+Configured with [RestNamespaceConfig](../interfaces/RestNamespaceConfig.md). See the function-level
+documentation above for the full surface, examples, and how this
+relates to [connect](connect.md).
+
+### Parameters
+
+* **implName**: `"rest"`
+
+* **config**: [`RestNamespaceConfig`](../interfaces/RestNamespaceConfig.md)
+
+* **options?**: `Partial`&lt;[`ConnectNamespaceOptions`](../interfaces/ConnectNamespaceOptions.md)&gt;
+
+### Returns
+
+`Promise`&lt;[`Connection`](../classes/Connection.md)&gt;
+
+### Example
+
+```ts
+const db = await connectNamespace("rest", {
+  uri: "https://catalog.example.com",
+  headers: { "x-api-key": process.env.CATALOG_KEY ?? "" },
+});
+```
+
+## connectNamespace(implName, properties, options)
+
+```ts
+function connectNamespace(
+   implName,
+   properties,
+   options?): Promise<Connection>
+```
+
+Connect through a custom namespace implementation by full module path,
+configured with a free-form string-keyed `properties` map. Use the
+typed overloads above for the built-in `"dir"` and `"rest"` impls.
+
+See the function-level documentation above for examples and how this
+relates to [connect](connect.md).
+
+### Parameters
+
+* **implName**: `string`
+
+* **properties**: `Record`&lt;`string`, `string`&gt;
+
+* **options?**: `Partial`&lt;[`ConnectNamespaceOptions`](../interfaces/ConnectNamespaceOptions.md)&gt;
+
+### Returns
+
+`Promise`&lt;[`Connection`](../classes/Connection.md)&gt;
+
+### Example
+
+```ts
+const db = await connectNamespace("my.custom.Namespace", {
+  endpoint: "...",
+});
+```

docs/src/js/globals.md

@@ -32,6 +32,7 @@
 - [PhraseQuery](classes/PhraseQuery.md)
 - [Query](classes/Query.md)
 - [QueryBase](classes/QueryBase.md)
+- [Scannable](classes/Scannable.md)
 - [Session](classes/Session.md)
 - [StaticHeaderProvider](classes/StaticHeaderProvider.md)
 - [Table](classes/Table.md)
@@ -51,10 +52,17 @@
 - [ClientConfig](interfaces/ClientConfig.md)
 - [ColumnAlteration](interfaces/ColumnAlteration.md)
 - [CompactionStats](interfaces/CompactionStats.md)
+- [ConnectNamespaceOptions](interfaces/ConnectNamespaceOptions.md)
 - [ConnectionOptions](interfaces/ConnectionOptions.md)
+- [CreateNamespaceOptions](interfaces/CreateNamespaceOptions.md)
+- [CreateNamespaceResponse](interfaces/CreateNamespaceResponse.md)
 - [CreateTableOptions](interfaces/CreateTableOptions.md)
 - [DeleteResult](interfaces/DeleteResult.md)
+- [DescribeNamespaceResponse](interfaces/DescribeNamespaceResponse.md)
+- [DirNamespaceConfig](interfaces/DirNamespaceConfig.md)
 - [DropColumnsResult](interfaces/DropColumnsResult.md)
+- [DropNamespaceOptions](interfaces/DropNamespaceOptions.md)
+- [DropNamespaceResponse](interfaces/DropNamespaceResponse.md)
 - [ExecutableQuery](interfaces/ExecutableQuery.md)
 - [FragmentStatistics](interfaces/FragmentStatistics.md)
 - [FragmentSummaryStats](interfaces/FragmentSummaryStats.md)
@@ -69,13 +77,17 @@
 - [IvfFlatOptions](interfaces/IvfFlatOptions.md)
 - [IvfPqOptions](interfaces/IvfPqOptions.md)
 - [IvfRqOptions](interfaces/IvfRqOptions.md)
+- [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
+- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
 - [MergeResult](interfaces/MergeResult.md)
 - [OpenTableOptions](interfaces/OpenTableOptions.md)
 - [OptimizeOptions](interfaces/OptimizeOptions.md)
 - [OptimizeStats](interfaces/OptimizeStats.md)
 - [QueryExecutionOptions](interfaces/QueryExecutionOptions.md)
 - [RemovalStats](interfaces/RemovalStats.md)
+- [RestNamespaceConfig](interfaces/RestNamespaceConfig.md)
 - [RetryConfig](interfaces/RetryConfig.md)
+- [ScannableOptions](interfaces/ScannableOptions.md)
 - [ShuffleOptions](interfaces/ShuffleOptions.md)
 - [SplitCalculatedOptions](interfaces/SplitCalculatedOptions.md)
 - [SplitHashOptions](interfaces/SplitHashOptions.md)
@@ -107,6 +119,7 @@
 
 - [RecordBatchIterator](functions/RecordBatchIterator.md)
 - [connect](functions/connect.md)
+- [connectNamespace](functions/connectNamespace.md)
 - [makeArrowTable](functions/makeArrowTable.md)
 - [packBits](functions/packBits.md)
 - [permutationBuilder](functions/permutationBuilder.md)

docs/src/js/interfaces/ConnectNamespaceOptions.md

@@ -0,0 +1,54 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ConnectNamespaceOptions
+
+# Interface: ConnectNamespaceOptions
+
+## Properties
+
+### namespaceClientProperties?
+
+```ts
+optional namespaceClientProperties: Record<string, string>;
+```
+
+Extra properties for the backing namespace client.
+
+***
+
+### readConsistencyInterval?
+
+```ts
+optional readConsistencyInterval: number;
+```
+
+The interval, in seconds, at which to check for updates to the table
+from other processes. If None, then consistency is not checked. For
+performance reasons, this is the default. For strong consistency, set
+this to zero seconds. Then every read will check for updates from other
+processes. As a compromise, you can set this to a non-zero value for
+eventual consistency.
+
+***
+
+### session?
+
+```ts
+optional session: Session;
+```
+
+The session to use for this connection. Holds shared caches and other
+session-specific state.
+
+***
+
+### storageOptions?
+
+```ts
+optional storageOptions: Record<string, string>;
+```
+
+Configuration for object storage. The available options are described
+at https://docs.lancedb.com/storage/

docs/src/js/interfaces/CreateNamespaceOptions.md

@@ -0,0 +1,27 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / CreateNamespaceOptions
+
+# Interface: CreateNamespaceOptions
+
+## Properties
+
+### mode?
+
+```ts
+optional mode: "overwrite" | "create" | "exist_ok";
+```
+
+Creation mode.
+
+***
+
+### properties?
+
+```ts
+optional properties: Record<string, string>;
+```
+
+Properties to set on the new namespace.

docs/src/js/interfaces/CreateNamespaceResponse.md

@@ -0,0 +1,23 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / CreateNamespaceResponse
+
+# Interface: CreateNamespaceResponse
+
+## Properties
+
+### properties?
+
+```ts
+optional properties: Record<string, string>;
+```
+
+***
+
+### transactionId?
+
+```ts
+optional transactionId: string;
+```

docs/src/js/interfaces/DescribeNamespaceResponse.md

@@ -0,0 +1,15 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / DescribeNamespaceResponse
+
+# Interface: DescribeNamespaceResponse
+
+## Properties
+
+### properties?
+
+```ts
+optional properties: Record<string, string>;
+```

docs/src/js/interfaces/DirNamespaceConfig.md

@@ -0,0 +1,47 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / DirNamespaceConfig
+
+# Interface: DirNamespaceConfig
+
+Configuration for the built-in directory namespace (`"dir"`).
+
+The directory namespace stores tables under a single root path (local
+filesystem or object storage URI). See
+[https://docs.lancedb.com/namespaces](https://docs.lancedb.com/namespaces) for the documented surface;
+less-common knobs live under [DirNamespaceConfig.extraProperties](DirNamespaceConfig.md#extraproperties).
+
+## Properties
+
+### extraProperties?
+
+```ts
+optional extraProperties: Record<string, string>;
+```
+
+Additional raw properties passed verbatim to the namespace
+implementation (e.g. `storage.*`, `credential_vendor.*`). Typed
+fields above take precedence on key collision.
+
+***
+
+### manifestEnabled?
+
+```ts
+optional manifestEnabled: boolean;
+```
+
+Whether to maintain a namespace manifest at the root. Required for
+child namespaces. Defaults to true on the impl side.
+
+***
+
+### root
+
+```ts
+root: string;
+```
+
+Root path or URI containing the LanceDB tables.

docs/src/js/interfaces/DropNamespaceOptions.md

@@ -0,0 +1,27 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / DropNamespaceOptions
+
+# Interface: DropNamespaceOptions
+
+## Properties
+
+### behavior?
+
+```ts
+optional behavior: "restrict" | "cascade";
+```
+
+Refuse to drop if non-empty (restrict) or drop recursively (cascade).
+
+***
+
+### mode?
+
+```ts
+optional mode: "fail" | "skip";
+```
+
+Whether to skip if the namespace doesn't exist, or fail.

docs/src/js/interfaces/DropNamespaceResponse.md

@@ -0,0 +1,23 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / DropNamespaceResponse
+
+# Interface: DropNamespaceResponse
+
+## Properties
+
+### properties?
+
+```ts
+optional properties: Record<string, string>;
+```
+
+***
+
+### transactionId?
+
+```ts
+optional transactionId: string[];
+```

docs/src/js/interfaces/ListNamespacesOptions.md

@@ -0,0 +1,27 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ListNamespacesOptions
+
+# Interface: ListNamespacesOptions
+
+## Properties
+
+### limit?
+
+```ts
+optional limit: number;
+```
+
+An optional limit to the number of results to return.
+
+***
+
+### pageToken?
+
+```ts
+optional pageToken: string;
+```
+
+Token from a previous response for pagination.

docs/src/js/interfaces/ListNamespacesResponse.md

@@ -0,0 +1,23 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ListNamespacesResponse
+
+# Interface: ListNamespacesResponse
+
+## Properties
+
+### namespaces
+
+```ts
+namespaces: string[];
+```
+
+***
+
+### pageToken?
+
+```ts
+optional pageToken: string;
+```

docs/src/js/interfaces/RestNamespaceConfig.md

@@ -0,0 +1,47 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / RestNamespaceConfig
+
+# Interface: RestNamespaceConfig
+
+Configuration for the built-in REST namespace (`"rest"`).
+
+The REST namespace talks to a remote catalog server over HTTP. See
+[https://docs.lancedb.com/namespaces](https://docs.lancedb.com/namespaces) for the documented surface;
+less-common knobs (TLS, metrics) live under
+[RestNamespaceConfig.extraProperties](RestNamespaceConfig.md#extraproperties).
+
+## Properties
+
+### extraProperties?
+
+```ts
+optional extraProperties: Record<string, string>;
+```
+
+Additional raw properties passed verbatim to the namespace
+implementation (e.g. `tls.*`, `ops_metrics_enabled`, `delimiter`).
+Typed fields above take precedence on key collision.
+
+***
+
+### headers?
+
+```ts
+optional headers: Record<string, string>;
+```
+
+HTTP headers forwarded with each request. Keys are passed through
+as-is (e.g. `"x-api-key"`, `"Authorization"`).
+
+***
+
+### uri
+
+```ts
+uri: string;
+```
+
+Catalog endpoint URL.

docs/src/js/interfaces/ScannableOptions.md

@@ -0,0 +1,29 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ScannableOptions
+
+# Interface: ScannableOptions
+
+## Properties
+
+### numRows?
+
+```ts
+optional numRows: number;
+```
+
+Hint about the number of rows. Not validated against the stream.
+
+***
+
+### rescannable?
+
+```ts
+optional rescannable: boolean;
+```
+
+Whether the source can be scanned more than once. Defaults to `true` for
+`fromTable` / `fromFactory` and `false` for `fromIterable` /
+`fromRecordBatchReader`.

docs/src/python/python.md

@@ -94,11 +94,11 @@ of raw SQL strings with [where][lancedb.query.LanceQueryBuilder.where] and
 
 ## Full text search
 
-::: lancedb.fts.create_index
+Use [lancedb.table.Table.create_fts_index][] for the synchronous API or
+[lancedb.table.AsyncTable.create_index][] with [lancedb.index.FTS][] for the
+asynchronous API.
 
-::: lancedb.fts.populate_index
-
-::: lancedb.fts.search_index
+::: lancedb.index.FTS
 
 ## Utilities
 

java/pom.xml

@@ -28,7 +28,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>7.0.0-beta.5</lance-core.version>
+        <lance-core.version>7.0.0-beta.9</lance-core.version>
         <spotless.skip>false</spotless.skip>
         <spotless.version>2.30.0</spotless.version>
         <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>

nodejs/AGENTS.md

@@ -3,11 +3,11 @@ The core Rust library is in the `../rust/lancedb` directory, the rust binding
 code is in the `src/` directory and the typescript bindings are in
 the `lancedb/` directory.
 
-Whenever you change the Rust code, you will need to recompile: `npm run build`.
+Whenever you change the Rust code, you will need to recompile: `pnpm build`.
 
 Common commands:
-* Build: `npm run build`
-* Lint: `npm run lint`
-* Fix lints: `npm run lint-fix`
-* Test: `npm test`
-* Run single test file: `npm test __test__/arrow.test.ts`
+* Build: `pnpm build`
+* Lint: `pnpm lint`
+* Fix lints: `pnpm lint-fix`
+* Test: `pnpm test`
+* Run single test file: `pnpm test __test__/arrow.test.ts`

nodejs/CONTRIBUTING.md

@@ -12,20 +12,22 @@ Typescript.
 * `src/`: Rust bindings source code
 * `lancedb/`: Typescript package source code
 * `__test__/`: Unit tests
-* `examples/`: An npm package with the examples shown in the documentation
+* `examples/`: A pnpm package with the examples shown in the documentation
 
 ## Development environment
 
 To set up your development environment, you will need to install the following:
 
-1. Node.js 14 or later
-2. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
-3. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
+1. Node.js 22 or later (required by pnpm 11)
+2. [pnpm](https://pnpm.io/installation) 11 or later (or run via `corepack enable`,
+   which uses the `packageManager` field in `package.json`)
+3. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
+4. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
 
 Initial setup:
 
 ```shell
-npm install
+pnpm install
 ```
 
 ### Commit Hooks
@@ -39,38 +41,38 @@ pre-commit install
 
 ## Development
 
-Most common development commands can be run using the npm scripts.
+Most common development commands can be run using the pnpm scripts.
 
 Build the package
 
 ```shell
-npm install
-npm run build
+pnpm install
+pnpm build
 ```
 
 Lint:
 
 ```shell
-npm run lint
+pnpm lint
 ```
 
 Format and fix lints:
 
 ```shell
-npm run lint-fix
+pnpm lint-fix
 ```
 
 Run tests:
 
 ```shell
-npm test
+pnpm test
 ```
 
 To run a single test:
 
 ```shell
 # Single file: table.test.ts
-npm test -- table.test.ts
+pnpm test -- table.test.ts
 # Single test: 'merge insert' in table.test.ts
-npm test -- table.test.ts --testNamePattern=merge\ insert
+pnpm test -- table.test.ts --testNamePattern=merge\ insert
 ```

nodejs/Cargo.toml

@@ -22,6 +22,7 @@ arrow-schema.workspace = true
 env_logger.workspace = true
 futures.workspace = true
 lancedb = { path = "../rust/lancedb", default-features = false }
+lance-namespace.workspace = true
 napi = { version = "3.8.3", default-features = false, features = [
     "napi9",
     "async"

nodejs/__test__/connection.test.ts

@@ -4,7 +4,7 @@
 import { readdirSync } from "fs";
 import { Field, Float64, Schema } from "apache-arrow";
 import * as tmp from "tmp";
-import { Connection, Table, connect } from "../lancedb";
+import { Connection, Table, connect, connectNamespace } from "../lancedb";
 import { LocalTable } from "../lancedb/table";
 
 describe("when connecting", () => {
@@ -81,6 +81,16 @@ describe("given a connection", () => {
     await db.createTable("test4", [{ id: 1 }, { id: 2 }]);
   });
 
+  it("should expose renameTable and reject on OSS listing DB", async () => {
+    await db.createTable("old_name", [{ id: 1 }]);
+
+    await expect(db.renameTable("old_name", "new_name")).rejects.toThrow(
+      "rename_table is not supported in LanceDB OSS",
+    );
+
+    await expect(db.tableNames()).resolves.toEqual(["old_name"]);
+  });
+
   it("should fail if creating table twice, unless overwrite is true", async () => {
     let tbl = await db.createTable("test", [{ id: 1 }, { id: 2 }]);
     await expect(tbl.countRows()).resolves.toBe(2);
@@ -306,3 +316,186 @@ describe("clone table functionality", () => {
     ).rejects.toThrow("Deep clone is not yet implemented");
   });
 });
+
+describe("namespaces", () => {
+  let tmpDir: tmp.DirResult;
+  let db: Connection;
+
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    // The local DirectoryNamespace backend only supports child namespaces
+    // when manifest mode is enabled (see lance-namespace-impls/src/dir.rs).
+    db = await connect(tmpDir.name, {
+      // biome-ignore lint/style/useNamingConvention: opaque backend property key, must match Rust
+      namespaceClientProperties: { manifest_enabled: "true" },
+    });
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  it("should create and describe a namespace", async () => {
+    await db.createNamespace(["myns"]);
+    const desc = await db.describeNamespace(["myns"]);
+    expect(desc).toBeDefined();
+  });
+
+  it("should list namespaces created at the root", async () => {
+    await db.createNamespace(["alpha"]);
+    await db.createNamespace(["beta"]);
+    const list = await db.listNamespaces();
+    expect(list.namespaces).toEqual(expect.arrayContaining(["alpha", "beta"]));
+  });
+
+  it("should list child namespaces under a parent", async () => {
+    await db.createNamespace(["parent"]);
+    await db.createNamespace(["parent", "child"]);
+    const list = await db.listNamespaces(["parent"]);
+    expect(list.namespaces).toContain("child");
+  });
+
+  it("should drop a namespace", async () => {
+    await db.createNamespace(["ephemeral"]);
+    await db.dropNamespace(["ephemeral"]);
+    const list = await db.listNamespaces();
+    expect(list.namespaces).not.toContain("ephemeral");
+  });
+
+  it("should raise an error on any namespace op after close", async () => {
+    await db.close();
+    await expect(db.describeNamespace(["foo"])).rejects.toThrow(
+      "Connection is closed",
+    );
+    await expect(db.listNamespaces()).rejects.toThrow("Connection is closed");
+    await expect(db.createNamespace(["foo"])).rejects.toThrow(
+      "Connection is closed",
+    );
+    await expect(db.dropNamespace(["foo"])).rejects.toThrow(
+      "Connection is closed",
+    );
+  });
+
+  it("should raise an understandable error when describing a non-existent namespace", async () => {
+    await expect(db.describeNamespace(["does-not-exist"])).rejects.toThrow(
+      /not found/i,
+    );
+  });
+
+  it("should raise an error when creating a namespace that already exists", async () => {
+    await db.createNamespace(["dup"]);
+    await expect(db.createNamespace(["dup"])).rejects.toThrow();
+  });
+
+  it("should reject an unrecognized createNamespace mode with a clear error", async () => {
+    await expect(
+      // biome-ignore lint/suspicious/noExplicitAny: deliberately bypass TS to test runtime validation
+      db.createNamespace(["x"], { mode: "frobnicate" as any }),
+    ).rejects.toThrow(/Invalid mode 'frobnicate'/);
+  });
+
+  it("should reject an unrecognized dropNamespace mode with a clear error", async () => {
+    await db.createNamespace(["x"]);
+    await expect(
+      // biome-ignore lint/suspicious/noExplicitAny: deliberately bypass TS to test runtime validation
+      db.dropNamespace(["x"], { mode: "frobnicate" as any }),
+    ).rejects.toThrow(/Invalid mode 'frobnicate'/);
+  });
+
+  it("should reject an unrecognized dropNamespace behavior with a clear error", async () => {
+    await db.createNamespace(["x"]);
+    await expect(
+      // biome-ignore lint/suspicious/noExplicitAny: deliberately bypass TS to test runtime validation
+      db.dropNamespace(["x"], { behavior: "frobnicate" as any }),
+    ).rejects.toThrow(/Invalid behavior 'frobnicate'/);
+  });
+});
+
+describe("connectNamespace", () => {
+  let tmpDir: tmp.DirResult;
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  it("connects via the dir implementation and supports table ops", async () => {
+    const db = await connectNamespace("dir", { root: tmpDir.name });
+    await db.createTable("users", [{ id: 1 }, { id: 2 }]);
+    await expect(db.tableNames()).resolves.toContain("users");
+  });
+
+  it("throws a clear error when implName is empty", async () => {
+    await expect(connectNamespace("", {})).rejects.toThrow(
+      "implName must be a non-empty string",
+    );
+  });
+
+  it("throws when the namespace implementation is unknown", async () => {
+    await expect(connectNamespace("not-a-real-impl", {})).rejects.toThrow();
+  });
+
+  it("passes storage options through to the namespace", async () => {
+    const db = await connectNamespace(
+      "dir",
+      { root: tmpDir.name },
+      { storageOptions: { newTableDataStorageVersion: "stable" } },
+    );
+    await db.createTable("plumbing", [{ id: 1 }]);
+    await expect(db.tableNames()).resolves.toContain("plumbing");
+  });
+
+  it("supports child namespaces when manifestEnabled is true on the dir config", async () => {
+    const writer = await connectNamespace("dir", {
+      root: tmpDir.name,
+      manifestEnabled: true,
+    });
+    await writer.createNamespace(["analytics"]);
+    await writer.createTable("orders", [{ id: 1 }, { id: 2 }], ["analytics"]);
+    await writer.close();
+
+    const reader = await connectNamespace("dir", {
+      root: tmpDir.name,
+      manifestEnabled: true,
+    });
+    await expect(reader.tableNames(["analytics"])).resolves.toContain("orders");
+    const orders = await reader.openTable("orders", ["analytics"]);
+    await expect(orders.countRows()).resolves.toBe(2);
+  });
+
+  it("merges extraProperties into the dir config and is overridden by typed fields", async () => {
+    // Two observable assertions:
+    // - Typed `root` overrides extraProperties.root: createTable would fail
+    //   under the bogus path if the override didn't happen.
+    // - extraProperties.manifest_enabled="false" is honored end-to-end. Child
+    //   namespaces require manifest mode (default true), so explicitly
+    //   disabling it via extraProperties must make createNamespace reject. If
+    //   extraProperties pass-through were silently broken, the default would
+    //   let createNamespace succeed.
+    const db = await connectNamespace("dir", {
+      root: tmpDir.name,
+      extraProperties: {
+        root: "/should/be/overridden",
+        // biome-ignore lint/style/useNamingConvention: backend property key
+        manifest_enabled: "false",
+      },
+    });
+    await db.createTable("base", [{ id: 1 }]);
+    await expect(db.tableNames()).resolves.toContain("base");
+    await expect(db.createNamespace(["analytics"])).rejects.toThrow();
+  });
+
+  it("flows unknown top-level keys through when implName is dynamic (no silent drop)", async () => {
+    // Routes via the third overload because `impl` is `string`, not the
+    // literal `"dir"`. The dispatcher still notices the runtime value is
+    // "dir", but unknown keys like `manifest_enabled` must not be silently
+    // dropped during the conversion.
+    //
+    // Asserting a *negative* outcome (manifest disabled -> createNamespace
+    // rejects) is required for observability, since the backend default for
+    // `manifest_enabled` is true.
+    const impl: string = "dir";
+    const db = await connectNamespace(impl, {
+      root: tmpDir.name,
+      // biome-ignore lint/style/useNamingConvention: backend property key
+      manifest_enabled: "false",
+    });
+    await expect(db.createNamespace(["mixed"])).rejects.toThrow();
+  });
+});

nodejs/__test__/scannable.test.ts

@@ -0,0 +1,438 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import {
+  Field,
+  Float16,
+  Int32,
+  type RecordBatch,
+  RecordBatchReader,
+  Schema,
+  tableToIPC,
+} from "apache-arrow";
+import { makeArrowTable, makeEmptyTable } from "../lancedb/arrow";
+import { Scannable } from "../lancedb/scannable";
+
+function makeTable() {
+  return makeArrowTable(
+    [
+      { id: 1, name: "a" },
+      { id: 2, name: "b" },
+      { id: 3, name: "c" },
+    ],
+    { vectorColumns: {} },
+  );
+}
+
+async function makeReader(): Promise<RecordBatchReader> {
+  // `RecordBatchReader.from()` returns an unopened reader; `.schema` is only
+  // populated after `.open()`. Opening sync readers is synchronous.
+  const reader = RecordBatchReader.from(tableToIPC(makeTable()));
+  return reader.open() as RecordBatchReader;
+}
+
+describe("Scannable", () => {
+  describe("fromTable", () => {
+    test("reflects schema, numRows, and defaults rescannable=true", async () => {
+      const table = makeTable();
+      const scannable = await Scannable.fromTable(table);
+
+      expect(scannable.schema).toBe(table.schema);
+      expect(scannable.numRows).toBe(table.numRows);
+      expect(scannable.rescannable).toBe(true);
+    });
+
+    test("throws when opts.numRows does not match table.numRows", async () => {
+      await expect(
+        Scannable.fromTable(makeTable(), { numRows: 42 }),
+      ).rejects.toThrow(/does not match table\.numRows/);
+    });
+
+    test("throws when opts.rescannable is false", async () => {
+      await expect(
+        Scannable.fromTable(makeTable(), { rescannable: false }),
+      ).rejects.toThrow(/always rescannable/);
+    });
+  });
+
+  describe("fromRecordBatchReader", () => {
+    test("reflects schema and defaults numRows=null, rescannable=false", async () => {
+      const reader = await makeReader();
+      const scannable = await Scannable.fromRecordBatchReader(reader);
+
+      expect(scannable.schema).toBe(reader.schema);
+      expect(scannable.numRows).toBeNull();
+      expect(scannable.rescannable).toBe(false);
+    });
+
+    test("honors numRows override", async () => {
+      const scannable = await Scannable.fromRecordBatchReader(
+        await makeReader(),
+        { numRows: 3 },
+      );
+
+      expect(scannable.numRows).toBe(3);
+      expect(scannable.rescannable).toBe(false);
+    });
+
+    test("rescannable: false explicit does not throw", async () => {
+      const reader = await makeReader();
+      const scannable = await Scannable.fromRecordBatchReader(reader, {
+        rescannable: false,
+      });
+      expect(scannable.rescannable).toBe(false);
+    });
+
+    test("throws when opts.rescannable is true", async () => {
+      const reader = await makeReader();
+      await expect(
+        Scannable.fromRecordBatchReader(reader, { rescannable: true }),
+      ).rejects.toThrow(/does not accept rescannable/);
+    });
+
+    test("throws when opts.rescannable is true even alongside numRows", async () => {
+      const reader = await makeReader();
+      await expect(
+        Scannable.fromRecordBatchReader(reader, {
+          numRows: 3,
+          rescannable: true,
+        }),
+      ).rejects.toThrow(/does not accept rescannable/);
+    });
+  });
+
+  describe("fromIterable", () => {
+    test("accepts a sync iterable of batches", async () => {
+      const table = makeTable();
+      const scannable = await Scannable.fromIterable(
+        table.schema,
+        table.batches,
+      );
+
+      expect(scannable.schema).toBe(table.schema);
+      expect(scannable.numRows).toBeNull();
+      expect(scannable.rescannable).toBe(false);
+    });
+
+    test("accepts an async iterable of batches", async () => {
+      const table = makeTable();
+      async function* generator(): AsyncGenerator<RecordBatch> {
+        for (const batch of table.batches) {
+          yield batch;
+        }
+      }
+
+      const scannable = await Scannable.fromIterable(table.schema, generator());
+      expect(scannable.schema).toBe(table.schema);
+      expect(scannable.rescannable).toBe(false);
+    });
+
+    describe("rescannable: true detection", () => {
+      // Replayable inputs: [Symbol.iterator]() / [Symbol.asyncIterator]()
+      // returns a fresh iterator each call. Must NOT throw.
+
+      test("Array passes (fresh ArrayIterator each call)", async () => {
+        const table = makeTable();
+        const scannable = await Scannable.fromIterable(
+          table.schema,
+          table.batches,
+          { rescannable: true },
+        );
+        expect(scannable.rescannable).toBe(true);
+      });
+
+      test("Set passes (fresh SetIterator each call)", async () => {
+        const table = makeTable();
+        const set = new Set<RecordBatch>(table.batches);
+        const scannable = await Scannable.fromIterable(table.schema, set, {
+          rescannable: true,
+        });
+        expect(scannable.rescannable).toBe(true);
+      });
+
+      test("custom Iterable returning a fresh iterator passes", async () => {
+        const table = makeTable();
+        const replayable: Iterable<RecordBatch> = {
+          [Symbol.iterator]() {
+            return table.batches[Symbol.iterator]();
+          },
+        };
+        const scannable = await Scannable.fromIterable(
+          table.schema,
+          replayable,
+          { rescannable: true },
+        );
+        expect(scannable.rescannable).toBe(true);
+      });
+
+      test("object with generator method passes (fresh generator each call)", async () => {
+        const table = makeTable();
+        const replayable: Iterable<RecordBatch> = {
+          *[Symbol.iterator]() {
+            for (const batch of table.batches) yield batch;
+          },
+        };
+        const scannable = await Scannable.fromIterable(
+          table.schema,
+          replayable,
+          { rescannable: true },
+        );
+        expect(scannable.rescannable).toBe(true);
+      });
+
+      test("empty Array passes (replayable degenerate case)", async () => {
+        const schema = makeTable().schema;
+        const scannable = await Scannable.fromIterable(
+          schema,
+          [] as RecordBatch[],
+          { rescannable: true },
+        );
+        expect(scannable.rescannable).toBe(true);
+      });
+
+      // One-shot inputs: [Symbol.iterator]() / [Symbol.asyncIterator]()
+      // returns the same object, or the input is already-an-iterator.
+      // Must throw with a /one-shot/ message.
+
+      test("sync generator throws", async () => {
+        const table = makeTable();
+        function* generator(): Generator<RecordBatch> {
+          for (const batch of table.batches) yield batch;
+        }
+        await expect(
+          Scannable.fromIterable(table.schema, generator(), {
+            rescannable: true,
+          }),
+        ).rejects.toThrow(/one-shot/);
+      });
+
+      test("async generator throws", async () => {
+        const table = makeTable();
+        async function* generator(): AsyncGenerator<RecordBatch> {
+          for (const batch of table.batches) yield batch;
+        }
+        await expect(
+          Scannable.fromIterable(table.schema, generator(), {
+            rescannable: true,
+          }),
+        ).rejects.toThrow(/one-shot/);
+      });
+
+      test("empty generator throws (one-shot degenerate case)", async () => {
+        const schema = makeTable().schema;
+        function* generator(): Generator<RecordBatch> {
+          // intentionally empty; yields nothing.
+        }
+        await expect(
+          Scannable.fromIterable(schema, generator(), { rescannable: true }),
+        ).rejects.toThrow(/one-shot/);
+      });
+
+      test("custom self-iterator throws", async () => {
+        const table = makeTable();
+        const batches = table.batches;
+        let i = 0;
+        const oneShot: Iterable<RecordBatch> & Iterator<RecordBatch> = {
+          [Symbol.iterator]() {
+            return this;
+          },
+          next() {
+            if (i >= batches.length) {
+              return { done: true, value: undefined };
+            }
+            return { done: false, value: batches[i++] };
+          },
+        };
+        await expect(
+          Scannable.fromIterable(table.schema, oneShot, { rescannable: true }),
+        ).rejects.toThrow(/one-shot/);
+      });
+
+      test("Array.values() (IterableIterator) throws", async () => {
+        const table = makeTable();
+        const iter = table.batches.values();
+        await expect(
+          Scannable.fromIterable(table.schema, iter, { rescannable: true }),
+        ).rejects.toThrow(/one-shot/);
+      });
+
+      test("raw iterator (only `.next`) throws", async () => {
+        const table = makeTable();
+        const batches = table.batches;
+        let i = 0;
+        const rawIter = {
+          next(): IteratorResult<RecordBatch> {
+            if (i >= batches.length) {
+              return { done: true, value: undefined };
+            }
+            return { done: false, value: batches[i++] };
+          },
+        };
+        await expect(
+          Scannable.fromIterable(
+            table.schema,
+            rawIter as unknown as Iterable<RecordBatch>,
+            { rescannable: true },
+          ),
+        ).rejects.toThrow(/one-shot/);
+      });
+
+      // Edge: null/undefined must not crash the detection helper. The
+      // null check belongs to `normalizeIterator` and only fires when a
+      // scan starts.
+
+      test("null input does not crash detection at construction", async () => {
+        const schema = makeTable().schema;
+        await expect(
+          Scannable.fromIterable(
+            schema,
+            null as unknown as Iterable<RecordBatch>,
+            {
+              rescannable: true,
+            },
+          ),
+        ).resolves.toBeDefined();
+      });
+
+      test("undefined input does not crash detection at construction", async () => {
+        const schema = makeTable().schema;
+        await expect(
+          Scannable.fromIterable(
+            schema,
+            undefined as unknown as Iterable<RecordBatch>,
+            { rescannable: true },
+          ),
+        ).resolves.toBeDefined();
+      });
+
+      // Default (rescannable omitted) skips the check entirely, so even
+      // pathological inputs construct without throwing here.
+
+      test("rescannable omitted skips detection entirely (generator passes)", async () => {
+        const table = makeTable();
+        function* generator(): Generator<RecordBatch> {
+          for (const batch of table.batches) yield batch;
+        }
+        const scannable = await Scannable.fromIterable(
+          table.schema,
+          generator(),
+        );
+        expect(scannable.rescannable).toBe(false);
+      });
+
+      test("rescannable: false explicit skips detection entirely (generator passes)", async () => {
+        const table = makeTable();
+        function* generator(): Generator<RecordBatch> {
+          for (const batch of table.batches) yield batch;
+        }
+        const scannable = await Scannable.fromIterable(
+          table.schema,
+          generator(),
+          { rescannable: false },
+        );
+        expect(scannable.rescannable).toBe(false);
+      });
+    });
+  });
+
+  describe("fromFactory", () => {
+    test("defaults rescannable=true and does not invoke the factory eagerly", async () => {
+      const table = makeTable();
+      const factory = jest.fn(() => table.batches);
+
+      const scannable = await Scannable.fromFactory(table.schema, factory);
+
+      expect(scannable.schema).toBe(table.schema);
+      expect(scannable.rescannable).toBe(true);
+      expect(factory).not.toHaveBeenCalled();
+    });
+
+    test("honors rescannable and numRows overrides", async () => {
+      const table = makeTable();
+      const scannable = await Scannable.fromFactory(
+        table.schema,
+        () => table.batches,
+        { numRows: 7, rescannable: false },
+      );
+
+      expect(scannable.numRows).toBe(7);
+      expect(scannable.rescannable).toBe(false);
+    });
+  });
+
+  describe("validation", () => {
+    test("throws when numRows is negative", async () => {
+      await expect(
+        Scannable.fromFactory(makeTable().schema, () => [], { numRows: -1 }),
+      ).rejects.toThrow(/non-negative/);
+    });
+
+    test("throws when numRows is not an integer", async () => {
+      await expect(
+        Scannable.fromFactory(makeTable().schema, () => [], { numRows: 3.5 }),
+      ).rejects.toThrow(/integer/);
+    });
+  });
+
+  describe("native handle", () => {
+    test("exposes a native handle via inner", async () => {
+      const scannable = await Scannable.fromTable(makeTable());
+      expect(scannable.inner).toBeDefined();
+      expect(typeof scannable.inner).toBe("object");
+      expect(scannable.inner).not.toBeNull();
+    });
+  });
+
+  // Schema-variety construction tests. Each asserts that construction
+  // succeeds against a richer Arrow schema, which transitively exercises
+  // schema serialization and the Rust-side `ipc_file_to_schema` for types
+  // beyond flat primitives.
+  describe("schema variety", () => {
+    test("accepts an empty table", async () => {
+      const schema = new Schema([new Field("id", new Int32(), true)]);
+      const table = makeEmptyTable(schema);
+      const scannable = await Scannable.fromTable(table);
+
+      expect(scannable.numRows).toBe(0);
+      expect(scannable.schema).toBe(table.schema);
+    });
+
+    test("accepts nested struct and list columns", async () => {
+      const table = makeArrowTable(
+        [
+          { id: 1, point: { x: 0, y: 0 }, tags: ["a", "b"] },
+          { id: 2, point: { x: 1, y: 2 }, tags: ["c"] },
+        ],
+        { vectorColumns: {} },
+      );
+      const scannable = await Scannable.fromTable(table);
+
+      expect(scannable.schema).toBe(table.schema);
+      expect(scannable.numRows).toBe(2);
+    });
+
+    test("accepts a FixedSizeList (vector) column", async () => {
+      const table = makeArrowTable(
+        [
+          { id: 1, vec: [1, 2, 3] },
+          { id: 2, vec: [4, 5, 6] },
+        ],
+        { vectorColumns: { vec: { type: new Float16() } } },
+      );
+      const scannable = await Scannable.fromTable(table);
+
+      expect(scannable.schema).toBe(table.schema);
+      expect(scannable.numRows).toBe(2);
+    });
+
+    test("accepts a table with many columns", async () => {
+      const row: Record<string, number> = {};
+      for (let i = 0; i < 50; i++) row[`c${i}`] = i;
+      const table = makeArrowTable([row, row], { vectorColumns: {} });
+      const scannable = await Scannable.fromTable(table);
+
+      expect(scannable.schema.fields.length).toBe(50);
+      expect(scannable.numRows).toBe(2);
+    });
+  });
+});

nodejs/__test__/table.test.ts

@@ -1870,6 +1870,25 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       expect(results.length).toBe(3);
     });
 
+    test("prewarmData errors on local tables", async () => {
+      const db = await connect(tmpDir.name);
+      const data = [
+        { text: "alpha", vector: [0.1, 0.2, 0.3] },
+        { text: "beta", vector: [0.4, 0.5, 0.6] },
+      ];
+      const table = await db.createTable("prewarm_data_test", data);
+
+      // prewarmData is only supported on remote tables. We verify the call
+      // is wired through napi and surfaces the expected error for both
+      // arg shapes (undefined and string[]).
+      await expect(table.prewarmData()).rejects.toThrow(
+        "prewarm_data is currently only supported on remote tables",
+      );
+      await expect(table.prewarmData(["text"])).rejects.toThrow(
+        "prewarm_data is currently only supported on remote tables",
+      );
+    });
+
     test("full text index on list", async () => {
       const db = await connect(tmpDir.name);
       const data = [

nodejs/examples/package.json

@@ -11,16 +11,17 @@
     "test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
     "lint": "biome check *.ts && biome format *.ts",
     "lint-ci": "biome ci .",
-    "lint-fix": "biome check --write *.ts && npm run format",
+    "lint-fix": "biome check --write *.ts && pnpm format",
     "format": "biome format --write *.ts"
   },
   "author": "Lance Devs",
   "license": "Apache-2.0",
+  "packageManager": "pnpm@11.1.1",
   "dependencies": {
-    "@huggingface/transformers": "^3.0.2",
+    "@huggingface/transformers": "3.0.2",
     "@lancedb/lancedb": "file:../dist",
-    "openai": "^4.29.2",
-    "sharp": "^0.33.5"
+    "openai": "4.29.2",
+    "sharp": "0.33.5"
   },
   "devDependencies": {
     "@biomejs/biome": "^1.7.3",

nodejs/examples/pnpm-workspace.yaml

@@ -0,0 +1,13 @@
+# Block resolution of versions less than 24h old (Shai-Hulud window).
+# This is the pnpm 11 default but pinned here so it's visible to
+# reviewers and survives a future pnpm major flipping the default.
+minimumReleaseAge: 1440
+
+# Fail install if a transitive dep tries to run an unapproved script.
+strictDepBuilds: true
+
+allowBuilds:
+  '@biomejs/biome': true
+  onnxruntime-node: true
+  protobufjs: true
+  sharp: true

nodejs/lancedb/arrow.ts

@@ -1291,6 +1291,18 @@ export async function fromRecordBatchToBuffer(
   return Buffer.from(await writer.toUint8Array());
 }
 
+/**
+ * Create a buffer containing a single record batch using the Arrow IPC Stream
+ * serialization. Each call produces a self-contained Stream message (schema +
+ * batch + EOS) suitable for incremental decode by `arrow_ipc::reader::StreamReader`.
+ */
+export async function fromRecordBatchToStreamBuffer(
+  batch: RecordBatch,
+): Promise<Buffer> {
+  const writer = RecordBatchStreamWriter.writeAll([batch]);
+  return Buffer.from(await writer.toUint8Array());
+}
+
 /**
  * Serialize an Arrow Table into a buffer using the Arrow IPC Stream serialization
  *

nodejs/lancedb/connection.ts

@@ -16,6 +16,18 @@ import {
 } from "./arrow";
 import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
 import { Connection as LanceDbConnection } from "./native";
+import type {
+  CreateNamespaceResponse,
+  DescribeNamespaceResponse,
+  DropNamespaceResponse,
+  ListNamespacesResponse,
+} from "./native";
+export type {
+  CreateNamespaceResponse,
+  DescribeNamespaceResponse,
+  DropNamespaceResponse,
+  ListNamespacesResponse,
+};
 import { sanitizeTable } from "./sanitize";
 import { LocalTable, Table } from "./table";
 
@@ -110,6 +122,28 @@ export interface TableNamesOptions {
   /** An optional limit to the number of results to return. */
   limit?: number;
 }
+
+export interface ListNamespacesOptions {
+  /** Token from a previous response for pagination. */
+  pageToken?: string;
+  /** An optional limit to the number of results to return. */
+  limit?: number;
+}
+
+export interface CreateNamespaceOptions {
+  /** Creation mode. */
+  mode?: "create" | "exist_ok" | "overwrite";
+  /** Properties to set on the new namespace. */
+  properties?: Record<string, string>;
+}
+
+export interface DropNamespaceOptions {
+  /** Whether to skip if the namespace doesn't exist, or fail. */
+  mode?: "skip" | "fail";
+  /** Refuse to drop if non-empty (restrict) or drop recursively (cascade). */
+  behavior?: "restrict" | "cascade";
+}
+
 /**
  * A LanceDB Connection that allows you to open tables and create new ones.
  *
@@ -262,12 +296,81 @@ export abstract class Connection {
    */
   abstract dropTable(name: string, namespacePath?: string[]): Promise<void>;
 
+  abstract renameTable(
+    oldName: string,
+    newName: string,
+    namespacePath?: string[],
+  ): Promise<void>;
+
   /**
    * Drop all tables in the database.
    * @param {string[]} namespacePath The namespace path to drop tables from (defaults to root namespace).
    */
   abstract dropAllTables(namespacePath?: string[]): Promise<void>;
 
+  /**
+   * Describe a namespace, returning its properties.
+   *
+   * @param {string[]} namespacePath - The namespace path to describe, in
+   *   parent → child order, e.g. `["analytics", "sales"]`.
+   * @returns {Promise<DescribeNamespaceResponse>} The namespace's properties
+   *   (may be undefined if the namespace has none).
+   */
+  abstract describeNamespace(
+    namespacePath: string[],
+  ): Promise<DescribeNamespaceResponse>;
+
+  /**
+   * List the immediate child namespaces under the given parent.
+   *
+   * Results may be paginated. To retrieve subsequent pages, pass the
+   * `pageToken` returned by a previous call.
+   *
+   * @param {string[]} namespacePath - The parent namespace path. Defaults
+   *   to the root namespace if omitted.
+   * @param {Partial<ListNamespacesOptions>} options - Pagination options
+   *   (`pageToken`, `limit`).
+   * @returns {Promise<ListNamespacesResponse>} Child namespace names and
+   *   an optional token for fetching the next page.
+   */
+  abstract listNamespaces(
+    namespacePath?: string[],
+    options?: Partial<ListNamespacesOptions>,
+  ): Promise<ListNamespacesResponse>;
+
+  /**
+   * Create a new namespace at the given path.
+   *
+   * @param {string[]} namespacePath - The namespace path to create.
+   * @param {Partial<CreateNamespaceOptions>} options - Creation `mode`
+   *   ("create" | "exist_ok" | "overwrite") and optional `properties`
+   *   to attach to the namespace.
+   * @returns {Promise<CreateNamespaceResponse>} The properties of the
+   *   created namespace and an optional transaction id.
+   */
+  abstract createNamespace(
+    namespacePath: string[],
+    options?: Partial<CreateNamespaceOptions>,
+  ): Promise<CreateNamespaceResponse>;
+
+  /**
+   * Drop a namespace.
+   *
+   * Use `behavior: "cascade"` to also drop everything contained in the
+   * namespace (sub-namespaces and tables). The default `"restrict"`
+   * behavior refuses to drop a non-empty namespace.
+   *
+   * @param {string[]} namespacePath - The namespace path to drop.
+   * @param {Partial<DropNamespaceOptions>} options - `mode` ("skip" | "fail"
+   *   for missing-namespace handling) and `behavior` ("restrict" | "cascade").
+   * @returns {Promise<DropNamespaceResponse>} Any properties returned by
+   *   the server and an optional transaction id.
+   */
+  abstract dropNamespace(
+    namespacePath: string[],
+    options?: Partial<DropNamespaceOptions>,
+  ): Promise<DropNamespaceResponse>;
+
   /**
    * Clone a table from a source table.
    *
@@ -512,9 +615,56 @@ export class LocalConnection extends Connection {
     return this.inner.dropTable(name, namespacePath ?? []);
   }
 
+  async renameTable(
+    oldName: string,
+    newName: string,
+    namespacePath?: string[],
+  ): Promise<void> {
+    return this.inner.renameTable(oldName, newName, namespacePath ?? []);
+  }
+
   async dropAllTables(namespacePath?: string[]): Promise<void> {
     return this.inner.dropAllTables(namespacePath ?? []);
   }
+
+  describeNamespace(
+    namespacePath: string[],
+  ): Promise<DescribeNamespaceResponse> {
+    return this.inner.describeNamespace(namespacePath);
+  }
+
+  listNamespaces(
+    namespacePath?: string[],
+    options?: Partial<ListNamespacesOptions>,
+  ): Promise<ListNamespacesResponse> {
+    return this.inner.listNamespaces(
+      namespacePath ?? [],
+      options?.pageToken,
+      options?.limit,
+    );
+  }
+
+  createNamespace(
+    namespacePath: string[],
+    options?: Partial<CreateNamespaceOptions>,
+  ): Promise<CreateNamespaceResponse> {
+    return this.inner.createNamespace(
+      namespacePath,
+      options?.mode,
+      options?.properties,
+    );
+  }
+
+  dropNamespace(
+    namespacePath: string[],
+    options?: Partial<DropNamespaceOptions>,
+  ): Promise<DropNamespaceResponse> {
+    return this.inner.dropNamespace(
+      namespacePath,
+      options?.mode,
+      options?.behavior,
+    );
+  }
 }
 
 /**

nodejs/lancedb/index.ts

@@ -8,6 +8,7 @@ import {
 } from "./connection";
 
 import {
+  ConnectNamespaceOptions,
   ConnectionOptions,
   Connection as LanceDbConnection,
   JsHeaderProvider as NativeJsHeaderProvider,
@@ -22,6 +23,7 @@ export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
 export {
   AddColumnsSql,
   ConnectionOptions,
+  ConnectNamespaceOptions,
   IndexStatistics,
   IndexConfig,
   ClientConfig,
@@ -62,6 +64,13 @@ export {
   CreateTableOptions,
   TableNamesOptions,
   OpenTableOptions,
+  ListNamespacesOptions,
+  CreateNamespaceOptions,
+  DropNamespaceOptions,
+  ListNamespacesResponse,
+  CreateNamespaceResponse,
+  DropNamespaceResponse,
+  DescribeNamespaceResponse,
 } from "./connection";
 
 export { Session } from "./native.js";
@@ -117,6 +126,7 @@ export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
 
 export * as embedding from "./embedding";
 export { permutationBuilder, PermutationBuilder } from "./permutation";
+export { Scannable, ScannableOptions } from "./scannable";
 export * as rerankers from "./rerankers";
 export {
   SchemaLike,
@@ -293,3 +303,197 @@ export async function connect(
   );
   return new LocalConnection(nativeConn);
 }
+
+/**
+ * Configuration for the built-in directory namespace (`"dir"`).
+ *
+ * The directory namespace stores tables under a single root path (local
+ * filesystem or object storage URI). See
+ * {@link https://docs.lancedb.com/namespaces} for the documented surface;
+ * less-common knobs live under {@link DirNamespaceConfig.extraProperties}.
+ */
+export interface DirNamespaceConfig {
+  /** Root path or URI containing the LanceDB tables. */
+  root: string;
+  /**
+   * Whether to maintain a namespace manifest at the root. Required for
+   * child namespaces. Defaults to true on the impl side.
+   */
+  manifestEnabled?: boolean;
+  /**
+   * Additional raw properties passed verbatim to the namespace
+   * implementation (e.g. `storage.*`, `credential_vendor.*`). Typed
+   * fields above take precedence on key collision.
+   */
+  extraProperties?: Record<string, string>;
+}
+
+/**
+ * Configuration for the built-in REST namespace (`"rest"`).
+ *
+ * The REST namespace talks to a remote catalog server over HTTP. See
+ * {@link https://docs.lancedb.com/namespaces} for the documented surface;
+ * less-common knobs (TLS, metrics) live under
+ * {@link RestNamespaceConfig.extraProperties}.
+ */
+export interface RestNamespaceConfig {
+  /** Catalog endpoint URL. */
+  uri: string;
+  /**
+   * HTTP headers forwarded with each request. Keys are passed through
+   * as-is (e.g. `"x-api-key"`, `"Authorization"`).
+   */
+  headers?: Record<string, string>;
+  /**
+   * Additional raw properties passed verbatim to the namespace
+   * implementation (e.g. `tls.*`, `ops_metrics_enabled`, `delimiter`).
+   * Typed fields above take precedence on key collision.
+   */
+  extraProperties?: Record<string, string>;
+}
+
+function dirConfigToProperties(
+  config: DirNamespaceConfig,
+): Record<string, string> {
+  // Spread the whole input so that unknown keys (e.g. a raw `manifest_enabled`
+  // passed via the dynamic-impl path) flow through instead of being dropped.
+  // Typed transformations layer on top.
+  const { manifestEnabled, extraProperties, ...rest } = config;
+  const properties: Record<string, string> = {
+    ...(extraProperties ?? {}),
+    ...(rest as Record<string, string>),
+  };
+  if (manifestEnabled !== undefined) {
+    properties.manifest_enabled = String(manifestEnabled);
+  }
+  return properties;
+}
+
+function restConfigToProperties(
+  config: RestNamespaceConfig,
+): Record<string, string> {
+  const { headers, extraProperties, ...rest } = config;
+  const properties: Record<string, string> = {
+    ...(extraProperties ?? {}),
+    ...(rest as Record<string, string>),
+  };
+  if (headers) {
+    for (const [name, value] of Object.entries(headers)) {
+      properties[`headers.${name}`] = value;
+    }
+  }
+  return properties;
+}
+
+/**
+ * Connect to a LanceDB database through a namespace.
+ *
+ * Unlike {@link connect}, which routes by URI scheme (local path vs.
+ * `db://` cloud), `connectNamespace` always returns a namespace-backed
+ * connection. The `implName` selects the namespace implementation:
+ *
+ * - `"dir"` — directory namespace, configured with {@link DirNamespaceConfig}.
+ * - `"rest"` — remote REST catalog, configured with {@link RestNamespaceConfig}.
+ * - Any other string — full module path for a custom implementation,
+ *   configured with a free-form string-keyed `properties` map.
+ *
+ * @example Typed dir namespace
+ * ```ts
+ * const db = await connectNamespace("dir", { root: "/path/to/db" });
+ * await db.createTable("users", [{ id: 1 }]);
+ * ```
+ *
+ * @example Typed REST namespace with auth headers
+ * ```ts
+ * const db = await connectNamespace("rest", {
+ *   uri: "https://catalog.example.com",
+ *   headers: { "x-api-key": process.env.CATALOG_KEY ?? "" },
+ * });
+ * ```
+ *
+ * @example Custom implementation with raw properties
+ * ```ts
+ * const db = await connectNamespace("my.custom.Namespace", {
+ *   endpoint: "...",
+ * });
+ * ```
+ */
+export function connectNamespace(
+  implName: "dir",
+  config: DirNamespaceConfig,
+  options?: Partial<ConnectNamespaceOptions>,
+): Promise<Connection>;
+/**
+ * Connect through the built-in REST namespace.
+ *
+ * Configured with {@link RestNamespaceConfig}. See the function-level
+ * documentation above for the full surface, examples, and how this
+ * relates to {@link connect}.
+ *
+ * @example
+ * ```ts
+ * const db = await connectNamespace("rest", {
+ *   uri: "https://catalog.example.com",
+ *   headers: { "x-api-key": process.env.CATALOG_KEY ?? "" },
+ * });
+ * ```
+ */
+export function connectNamespace(
+  implName: "rest",
+  config: RestNamespaceConfig,
+  options?: Partial<ConnectNamespaceOptions>,
+): Promise<Connection>;
+/**
+ * Connect through a custom namespace implementation by full module path,
+ * configured with a free-form string-keyed `properties` map. Use the
+ * typed overloads above for the built-in `"dir"` and `"rest"` impls.
+ *
+ * See the function-level documentation above for examples and how this
+ * relates to {@link connect}.
+ *
+ * @example
+ * ```ts
+ * const db = await connectNamespace("my.custom.Namespace", {
+ *   endpoint: "...",
+ * });
+ * ```
+ */
+export function connectNamespace(
+  implName: string,
+  properties: Record<string, string>,
+  options?: Partial<ConnectNamespaceOptions>,
+): Promise<Connection>;
+export async function connectNamespace(
+  implName: string,
+  configOrProperties:
+    | DirNamespaceConfig
+    | RestNamespaceConfig
+    | Record<string, string>,
+  options?: Partial<ConnectNamespaceOptions>,
+): Promise<Connection> {
+  let properties: Record<string, string>;
+  if (implName === "dir") {
+    properties = dirConfigToProperties(
+      configOrProperties as DirNamespaceConfig,
+    );
+  } else if (implName === "rest") {
+    properties = restConfigToProperties(
+      configOrProperties as RestNamespaceConfig,
+    );
+  } else {
+    properties = configOrProperties as Record<string, string>;
+  }
+
+  const finalOptions: ConnectNamespaceOptions = (options ??
+    {}) as ConnectNamespaceOptions;
+  finalOptions.storageOptions = cleanseStorageOptions(
+    finalOptions.storageOptions,
+  );
+
+  const nativeConn = await LanceDbConnection.newWithNamespace(
+    implName,
+    properties,
+    finalOptions,
+  );
+  return new LocalConnection(nativeConn);
+}

nodejs/lancedb/scannable.ts

@@ -0,0 +1,274 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import {
+  Table as ArrowTable,
+  RecordBatch,
+  RecordBatchReader,
+  Schema,
+} from "apache-arrow";
+import {
+  fromRecordBatchToStreamBuffer,
+  fromTableToBuffer,
+  makeEmptyTable,
+} from "./arrow";
+import { NapiScannable } from "./native.js";
+
+export interface ScannableOptions {
+  /** Hint about the number of rows. Not validated against the stream. */
+  numRows?: number;
+  /**
+   * Whether the source can be scanned more than once. Defaults to `true` for
+   * `fromTable` / `fromFactory` and `false` for `fromIterable` /
+   * `fromRecordBatchReader`.
+   */
+  rescannable?: boolean;
+}
+
+/**
+ * A data source that can be scanned as a stream of Arrow `RecordBatch`es.
+ *
+ * `Scannable` wraps the schema + optional row count + rescannable flag and
+ * a callback that yields batches one at a time. It is passed to consumers
+ * (e.g. `Table.add`, `createTable`, `mergeInsert` — follow-up work) that
+ * need to pull data without materializing the full dataset in JS memory.
+ *
+ * Batches cross the JS↔Rust boundary as Arrow IPC Stream messages; a fresh
+ * writer serializes each batch, and the Rust side decodes it with
+ * `arrow_ipc::reader::StreamReader`. One batch is in flight at a time.
+ */
+export class Scannable {
+  readonly schema: Schema;
+  readonly numRows: number | null;
+  readonly rescannable: boolean;
+
+  /** @hidden */
+  private readonly native: NapiScannable;
+
+  private constructor(
+    native: NapiScannable,
+    schema: Schema,
+    numRows: number | null,
+    rescannable: boolean,
+  ) {
+    this.native = native;
+    this.schema = schema;
+    this.numRows = numRows;
+    this.rescannable = rescannable;
+  }
+
+  /** @hidden Access the native handle for passing through to Rust consumers. */
+  get inner(): NapiScannable {
+    return this.native;
+  }
+
+  /**
+   * Build a Scannable from an explicit schema and a factory that returns a
+   * fresh batch iterator on each call.
+   *
+   * The factory is invoked once per scan. Each iterator yields
+   * `RecordBatch`es matching the declared schema. Use this when you need
+   * direct control over the pull loop — for example, to wrap a streaming
+   * source whose batches are produced lazily.
+   *
+   * @param schema - The Arrow schema of the produced batches.
+   * @param factory - Called at the start of each scan to produce a batch
+   *   iterator. Must be idempotent when `rescannable` is true.
+   * @param opts - Optional hints. `rescannable` defaults to `true`; set to
+   *   `false` if calling `factory()` twice would not reproduce the same data.
+   */
+  static async fromFactory(
+    schema: Schema,
+    factory: () =>
+      | AsyncIterable<RecordBatch>
+      | Iterable<RecordBatch>
+      | AsyncIterator<RecordBatch>
+      | Iterator<RecordBatch>,
+    opts: ScannableOptions = {},
+  ): Promise<Scannable> {
+    const numRows = opts.numRows ?? null;
+    if (numRows != null && !Number.isInteger(numRows)) {
+      throw new TypeError("numRows must be an integer");
+    }
+    const rescannable = opts.rescannable ?? true;
+
+    let iter: AsyncIterator<RecordBatch> | Iterator<RecordBatch> | null = null;
+    const getNextBatch = async (isStart: boolean): Promise<Buffer | null> => {
+      // `isStart` is true on the first pull of every new scan_as_stream.
+      // Drop any cached iterator so factory() is re-invoked for the next scan
+      if (isStart) {
+        iter = null;
+      }
+      if (iter === null) {
+        iter = normalizeIterator(factory());
+      }
+      const result = await iter.next();
+      if (result.done) {
+        iter = null;
+        return null;
+      }
+      return fromRecordBatchToStreamBuffer(result.value);
+    };
+
+    const schemaBuf = await fromTableToBuffer(makeEmptyTable(schema));
+    const native = new NapiScannable(
+      schemaBuf,
+      numRows,
+      rescannable,
+      getNextBatch,
+    );
+    return new Scannable(native, schema, numRows, rescannable);
+  }
+
+  /**
+   * Build a Scannable from an in-memory Arrow `Table`. Always rescannable;
+   * the table's batches are replayed on each scan.
+   *
+   * The table's row count is authoritative: `opts.numRows` must either be
+   * omitted or equal to `table.numRows`. `opts.rescannable` of `false` is
+   * rejected because in-memory Tables are always rescannable.
+   */
+  static async fromTable(
+    table: ArrowTable,
+    opts: ScannableOptions = {},
+  ): Promise<Scannable> {
+    if (opts.numRows != null && opts.numRows !== table.numRows) {
+      throw new TypeError(
+        `opts.numRows (${opts.numRows}) does not match table.numRows (${table.numRows}). ` +
+          `The table's row count is authoritative; omit numRows or pass the matching value.`,
+      );
+    }
+    if (opts.rescannable === false) {
+      throw new TypeError(
+        `fromTable does not accept rescannable: false. ` +
+          `In-memory Arrow Tables are always rescannable; omit the option or pass true.`,
+      );
+    }
+    return Scannable.fromFactory(table.schema, () => table.batches, {
+      numRows: table.numRows,
+      rescannable: true,
+    });
+  }
+
+  /**
+   * Build a Scannable from an iterable of `RecordBatch`es. `rescannable`
+   * defaults to `false`. Pass an explicit schema so the consumer can
+   * validate before any batch is pulled.
+   *
+   * `opts.rescannable: true` is honest for replayable iterables (Arrays,
+   * Sets, or custom iterables whose `[Symbol.iterator]()` returns a fresh
+   * iterator each call). It is rejected for one-shot iterables (generators,
+   * async generators, or already-an-iterator inputs) because their
+   * `[Symbol.iterator]()` returns the same exhausted object on the second
+   * scan. For replayable sources outside this shape, use
+   * `fromFactory(schema, () => createIter(), { rescannable: true })`.
+   *
+   * Note: when `opts.rescannable` is `true`, the constructor calls
+   * `[Symbol.iterator]()` once on the input to perform the structural check.
+   */
+  static async fromIterable(
+    schema: Schema,
+    iter: AsyncIterable<RecordBatch> | Iterable<RecordBatch>,
+    opts: ScannableOptions = {},
+  ): Promise<Scannable> {
+    if (opts.rescannable === true && isOneShotIterable(iter)) {
+      throw new TypeError(
+        `fromIterable: rescannable: true is not honest for one-shot iterables ` +
+          `(generators, async generators, or iterators where [Symbol.iterator]() ` +
+          `returns the same object). The source would be exhausted after the first scan. ` +
+          `Use fromFactory(schema, () => createIter(), { rescannable: true }) for sources ` +
+          `where each call mints a fresh iterator.`,
+      );
+    }
+    return Scannable.fromFactory(schema, () => iter, {
+      numRows: opts.numRows,
+      rescannable: opts.rescannable ?? false,
+    });
+  }
+
+  /**
+   * Build a Scannable from an Arrow `RecordBatchReader`. A reader can only
+   * be consumed once; `rescannable` defaults to `false`.
+   *
+   * The reader must already be opened (via `.open()`) so its `.schema` is
+   * populated. `RecordBatchReader.from(...)` returns an unopened reader.
+   *
+   * `opts.rescannable: true` is rejected because `RecordBatchReader` is a
+   * self-iterator (its `[Symbol.iterator]()` returns itself), and this
+   * constructor does not call `reader.reset()` between scans, so a second
+   * scan would always see an exhausted reader. For genuinely replayable
+   * sources, use
+   * `fromFactory(schema, () => openReader(), { rescannable: true })`,
+   * which mints a fresh reader on each scan.
+   */
+  static async fromRecordBatchReader(
+    reader: RecordBatchReader,
+    opts: ScannableOptions = {},
+  ): Promise<Scannable> {
+    if (opts.rescannable === true) {
+      throw new TypeError(
+        `fromRecordBatchReader does not accept rescannable: true. ` +
+          `RecordBatchReader is a self-iterator (its [Symbol.iterator]() ` +
+          `returns itself) and would be exhausted after the first scan. ` +
+          `Use fromFactory(schema, () => openReader(), { rescannable: true }) ` +
+          `for sources where each call mints a fresh reader.`,
+      );
+    }
+    return Scannable.fromFactory(reader.schema, () => reader, {
+      numRows: opts.numRows,
+      rescannable: false,
+    });
+  }
+}
+
+function normalizeIterator<T>(
+  source: AsyncIterable<T> | Iterable<T> | AsyncIterator<T> | Iterator<T>,
+): AsyncIterator<T> | Iterator<T> {
+  if (source == null) {
+    throw new TypeError("Scannable factory returned null/undefined");
+  }
+  if (
+    typeof (source as AsyncIterable<T>)[Symbol.asyncIterator] === "function"
+  ) {
+    return (source as AsyncIterable<T>)[Symbol.asyncIterator]();
+  }
+  if (typeof (source as Iterable<T>)[Symbol.iterator] === "function") {
+    return (source as Iterable<T>)[Symbol.iterator]();
+  }
+  // Already an iterator (has `.next`).
+  if (typeof (source as Iterator<T>).next === "function") {
+    return source as Iterator<T>;
+  }
+  throw new TypeError("Scannable factory returned a non-iterable value");
+}
+
+// A "self-iterator" returns the same object from `[Symbol.iterator]()` /
+// `[Symbol.asyncIterator]()`. Generators behave this way, so they exhaust
+// after one pass. Replayable iterables (Array, Set, custom) return a fresh
+// iterator each call. Detection mirrors `normalizeIterator`'s ordering so
+// classification matches scan-time behavior.
+function isOneShotIterable(
+  source: AsyncIterable<unknown> | Iterable<unknown>,
+): boolean {
+  // null/undefined are not one-shot in any meaningful sense; let
+  // `normalizeIterator` raise the actual error at scan time.
+  if (source == null) return false;
+  const ref = source as unknown;
+  if (
+    typeof (source as AsyncIterable<unknown>)[Symbol.asyncIterator] ===
+    "function"
+  ) {
+    const it = (source as AsyncIterable<unknown>)[
+      Symbol.asyncIterator
+    ]() as unknown;
+    return it === ref;
+  }
+  if (typeof (source as Iterable<unknown>)[Symbol.iterator] === "function") {
+    const it = (source as Iterable<unknown>)[Symbol.iterator]() as unknown;
+    return it === ref;
+  }
+  // Already-an-iterator (has `.next` but no `Symbol.iterator`) is by
+  // definition one-shot.
+  if (typeof (source as { next?: unknown }).next === "function") return true;
+  return false;
+}

nodejs/lancedb/table.ts

@@ -285,6 +285,25 @@ export abstract class Table {
    */
   abstract prewarmIndex(name: string): Promise<void>;
 
+  /**
+   * Prewarm one or more columns of data in the table.
+   *
+   * @param columns The columns to prewarm. If undefined, all columns are prewarmed.
+   *
+   * This will load the column data into the page cache so that future queries that
+   * read those columns avoid the initial cold-start latency.  This call initiates
+   * prewarming and returns once the request is accepted; the warming itself may
+   * continue in the background.  Calling it on already-prewarmed columns is a
+   * no-op on the server.
+   *
+   * Prewarming is generally useful for columns used in filters or projections.
+   * Large columns (e.g. high-dimensional vectors or binary data) may not be
+   * practical to prewarm.
+   *
+   * This feature is currently only supported on remote tables.
+   */
+  abstract prewarmData(columns?: string[]): Promise<void>;
+
   /**
    * Waits for asynchronous indexing to complete on the table.
    *
@@ -710,6 +729,10 @@ export class LocalTable extends Table {
     await this.inner.prewarmIndex(name);
   }
 
+  async prewarmData(columns?: string[]): Promise<void> {
+    await this.inner.prewarmData(columns);
+  }
+
   async waitForIndex(
     indexNames: string[],
     timeoutSeconds: number,

nodejs/package.json

@@ -38,15 +38,15 @@
     "url": "https://github.com/lancedb/lancedb"
   },
   "devDependencies": {
-    "@aws-sdk/client-dynamodb": "^3.33.0",
-    "@aws-sdk/client-kms": "^3.33.0",
-    "@aws-sdk/client-s3": "^3.33.0",
+    "@aws-sdk/client-dynamodb": "3.1003.0",
+    "@aws-sdk/client-kms": "3.1003.0",
+    "@aws-sdk/client-s3": "3.1003.0",
     "@biomejs/biome": "^1.7.3",
     "@jest/globals": "^29.7.0",
-    "@napi-rs/cli": "^3.5.1",
+    "@napi-rs/cli": "3.5.1",
     "@types/axios": "^0.14.0",
     "@types/jest": "^29.1.2",
-    "@types/node": "^22.7.4",
+    "@types/node": "22.7.4",
     "@types/tmp": "^0.2.6",
     "apache-arrow-15": "npm:apache-arrow@15.0.0",
     "apache-arrow-16": "npm:apache-arrow@16.0.0",
@@ -57,9 +57,9 @@
     "shx": "^0.3.4",
     "tmp": "^0.2.3",
     "ts-jest": "^29.1.2",
-    "typedoc": "^0.26.4",
-    "typedoc-plugin-markdown": "^4.2.1",
-    "typescript": "^5.5.4",
+    "typedoc": "0.26.4",
+    "typedoc-plugin-markdown": "4.2.1",
+    "typescript": "5.5.4",
     "typescript-eslint": "^7.1.0"
   },
   "ava": {
@@ -68,15 +68,16 @@
   "engines": {
     "node": ">= 18"
   },
+  "packageManager": "pnpm@11.1.1",
   "cpu": ["x64", "arm64"],
   "os": ["darwin", "linux", "win32"],
   "scripts": {
     "artifacts": "napi artifacts",
     "build:debug": "napi build --platform --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir lancedb",
-    "postbuild:debug": "shx mkdir -p dist && shx cp lancedb/*.node dist/",
+    "postbuild:debug": "shx mkdir -p dist && shx cp lancedb/*.node dist/ && node -e \"require('fs').writeFileSync('dist/package.json', JSON.stringify({name:'@lancedb/lancedb',type:'commonjs'}))\"",
     "build:release": "napi build --platform --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir dist",
-    "build": "npm run build:debug && npm run tsc",
-    "build-release": "npm run build:release && npm run tsc",
+    "build": "pnpm build:debug && pnpm tsc",
+    "build-release": "pnpm build:release && pnpm tsc",
     "tsc": "tsc -b",
     "posttsc": "shx cp lancedb/native.d.ts dist/native.d.ts",
     "lint-ci": "biome ci .",
@@ -86,7 +87,7 @@
     "lint-fix": "biome check --write . && biome format --write .",
     "prepublishOnly": "napi prepublish -t npm",
     "test": "jest --verbose",
-    "integration": "S3_TEST=1 npm run test",
+    "integration": "S3_TEST=1 pnpm test",
     "universal": "napi universalize",
     "version": "napi version"
   },
@@ -94,8 +95,8 @@
     "reflect-metadata": "^0.2.2"
   },
   "optionalDependencies": {
-    "@huggingface/transformers": "^3.0.2",
-    "openai": "^4.29.2"
+    "@huggingface/transformers": "3.0.2",
+    "openai": "4.29.2"
   },
   "peerDependencies": {
     "apache-arrow": ">=15.0.0 <=18.1.0"

nodejs/pnpm-workspace.yaml

@@ -0,0 +1,18 @@
+# Flat node_modules layout. The @napi-rs/cli build step fails to locate
+# the cdylib artifact under pnpm's isolated layout; the hoisted linker
+# mirrors npm's structure and unblocks the native build.
+nodeLinker: hoisted
+
+# Block resolution of versions less than 24h old (Shai-Hulud window).
+# This is the pnpm 11 default but pinned here so it's visible to
+# reviewers and survives a future pnpm major flipping the default.
+minimumReleaseAge: 1440
+
+# Fail install if a transitive dep tries to run an unapproved script.
+strictDepBuilds: true
+
+allowBuilds:
+  '@biomejs/biome': true
+  onnxruntime-node: true
+  protobufjs: true
+  sharp: true

nodejs/src/connection.rs

@@ -8,12 +8,16 @@ use lancedb::database::{CreateTableMode, Database};
 use napi::bindgen_prelude::*;
 use napi_derive::*;
 
+use crate::ConnectNamespaceOptions;
 use crate::ConnectionOptions;
 use crate::error::NapiErrorExt;
 use crate::header::JsHeaderProvider;
 use crate::table::Table;
-use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection};
+use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection, connect_namespace};
 
+use lance_namespace::models::{
+    CreateNamespaceRequest, DescribeNamespaceRequest, DropNamespaceRequest, ListNamespacesRequest,
+};
 use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
 
 #[napi]
@@ -21,6 +25,29 @@ pub struct Connection {
     inner: Option<LanceDBConnection>,
 }
 
+#[napi(object)]
+pub struct DescribeNamespaceResponse {
+    pub properties: Option<HashMap<String, String>>,
+}
+
+#[napi(object)]
+pub struct ListNamespacesResponse {
+    pub namespaces: Vec<String>,
+    pub page_token: Option<String>,
+}
+
+#[napi(object)]
+pub struct CreateNamespaceResponse {
+    pub properties: Option<HashMap<String, String>>,
+    pub transaction_id: Option<String>,
+}
+
+#[napi(object)]
+pub struct DropNamespaceResponse {
+    pub properties: Option<HashMap<String, String>>,
+    pub transaction_id: Option<Vec<String>>,
+}
+
 impl Connection {
     pub(crate) fn inner_new(inner: LanceDBConnection) -> Self {
         Self { inner: Some(inner) }
@@ -106,6 +133,39 @@ impl Connection {
         Ok(Self::inner_new(builder.execute().await.default_error()?))
     }
 
+    /// Create a new Connection instance backed by a namespace implementation.
+    #[napi(factory)]
+    pub async fn new_with_namespace(
+        impl_name: String,
+        properties: HashMap<String, String>,
+        options: ConnectNamespaceOptions,
+    ) -> napi::Result<Self> {
+        if impl_name.is_empty() {
+            return Err(napi::Error::from_reason(
+                "implName must be a non-empty string",
+            ));
+        }
+
+        let mut builder = connect_namespace(&impl_name, properties);
+        if let Some(interval) = options.read_consistency_interval {
+            builder =
+                builder.read_consistency_interval(std::time::Duration::from_secs_f64(interval));
+        }
+        if let Some(storage_options) = options.storage_options {
+            for (key, value) in storage_options {
+                builder = builder.storage_option(key, value);
+            }
+        }
+        if let Some(namespace_client_properties) = options.namespace_client_properties {
+            builder = builder.namespace_client_properties(namespace_client_properties);
+        }
+        if let Some(session) = options.session {
+            builder = builder.session(session.inner.clone());
+        }
+
+        Ok(Self::inner_new(builder.execute().await.default_error()?))
+    }
+
     #[napi]
     pub fn display(&self) -> napi::Result<String> {
         Ok(self.get_inner()?.to_string())
@@ -268,9 +328,149 @@ impl Connection {
             .default_error()
     }
 
+    #[napi(catch_unwind)]
+    pub async fn rename_table(
+        &self,
+        old_name: String,
+        new_name: String,
+        namespace_path: Option<Vec<String>>,
+    ) -> napi::Result<()> {
+        let ns = namespace_path.unwrap_or_default();
+        self.get_inner()?
+            .rename_table(&old_name, &new_name, &ns, &ns)
+            .await
+            .default_error()
+    }
+
     #[napi(catch_unwind)]
     pub async fn drop_all_tables(&self, namespace_path: Option<Vec<String>>) -> napi::Result<()> {
         let ns = namespace_path.unwrap_or_default();
         self.get_inner()?.drop_all_tables(&ns).await.default_error()
     }
+
+    #[napi(catch_unwind)]
+    /// Describe a namespace and return its properties.
+    pub async fn describe_namespace(
+        &self,
+        namespace_path: Vec<String>,
+    ) -> napi::Result<DescribeNamespaceResponse> {
+        let req = DescribeNamespaceRequest {
+            id: Some(namespace_path),
+            ..Default::default()
+        };
+        let resp = self
+            .get_inner()?
+            .describe_namespace(req)
+            .await
+            .default_error()?;
+        Ok(DescribeNamespaceResponse {
+            properties: resp.properties,
+        })
+    }
+
+    #[napi(catch_unwind)]
+    /// List child namespaces under the given namespace path
+    pub async fn list_namespaces(
+        &self,
+        namespace_path: Option<Vec<String>>,
+        page_token: Option<String>,
+        limit: Option<u32>,
+    ) -> napi::Result<ListNamespacesResponse> {
+        let req = ListNamespacesRequest {
+            id: namespace_path,
+            page_token,
+            limit: limit.map(|l| l as i32),
+            ..Default::default()
+        };
+        let resp = self
+            .get_inner()?
+            .list_namespaces(req)
+            .await
+            .default_error()?;
+        Ok(ListNamespacesResponse {
+            namespaces: resp.namespaces,
+            page_token: resp.page_token,
+        })
+    }
+
+    #[napi(catch_unwind)]
+    /// Create a new namespace with optional properties.
+    pub async fn create_namespace(
+        &self,
+        namespace_path: Vec<String>,
+        mode: Option<String>,
+        properties: Option<HashMap<String, String>>,
+    ) -> napi::Result<CreateNamespaceResponse> {
+        let mode_str = mode
+            .map(|m| match m.to_lowercase().as_str() {
+                "create" => Ok("Create".to_string()),
+                "exist_ok" => Ok("ExistOk".to_string()),
+                "overwrite" => Ok("Overwrite".to_string()),
+                _ => Err(napi::Error::from_reason(format!(
+                    "Invalid mode '{}': expected one of 'create', 'exist_ok', 'overwrite'",
+                    m
+                ))),
+            })
+            .transpose()?;
+        let req = CreateNamespaceRequest {
+            id: Some(namespace_path),
+            mode: mode_str,
+            properties,
+            ..Default::default()
+        };
+        let resp = self
+            .get_inner()?
+            .create_namespace(req)
+            .await
+            .default_error()?;
+        Ok(CreateNamespaceResponse {
+            properties: resp.properties,
+            transaction_id: resp.transaction_id,
+        })
+    }
+
+    #[napi(catch_unwind)]
+    /// Drop a namespace.
+    pub async fn drop_namespace(
+        &self,
+        namespace_path: Vec<String>,
+        mode: Option<String>,
+        behavior: Option<String>,
+    ) -> napi::Result<DropNamespaceResponse> {
+        let mode_str = mode
+            .map(|m| match m.to_lowercase().as_str() {
+                "skip" => Ok("Skip".to_string()),
+                "fail" => Ok("Fail".to_string()),
+                _ => Err(napi::Error::from_reason(format!(
+                    "Invalid mode '{}': expected one of 'skip', 'fail'",
+                    m
+                ))),
+            })
+            .transpose()?;
+        let behavior_str = behavior
+            .map(|b| match b.to_lowercase().as_str() {
+                "restrict" => Ok("Restrict".to_string()),
+                "cascade" => Ok("Cascade".to_string()),
+                _ => Err(napi::Error::from_reason(format!(
+                    "Invalid behavior '{}': expected one of 'restrict', 'cascade'",
+                    b
+                ))),
+            })
+            .transpose()?;
+        let req = DropNamespaceRequest {
+            id: Some(namespace_path),
+            mode: mode_str,
+            behavior: behavior_str,
+            ..Default::default()
+        };
+        let resp = self
+            .get_inner()?
+            .drop_namespace(req)
+            .await
+            .default_error()?;
+        Ok(DropNamespaceResponse {
+            properties: resp.properties,
+            transaction_id: resp.transaction_id,
+        })
+    }
 }

nodejs/src/lib.rs

@@ -16,6 +16,7 @@ pub mod permutation;
 mod query;
 pub mod remote;
 mod rerankers;
+mod scannable;
 mod session;
 mod table;
 mod util;
@@ -67,6 +68,26 @@ pub struct OpenTableOptions {
     pub storage_options: Option<HashMap<String, String>>,
 }
 
+#[napi(object)]
+#[derive(Debug)]
+pub struct ConnectNamespaceOptions {
+    /// The interval, in seconds, at which to check for updates to the table
+    /// from other processes. If None, then consistency is not checked. For
+    /// performance reasons, this is the default. For strong consistency, set
+    /// this to zero seconds. Then every read will check for updates from other
+    /// processes. As a compromise, you can set this to a non-zero value for
+    /// eventual consistency.
+    pub read_consistency_interval: Option<f64>,
+    /// Configuration for object storage. The available options are described
+    /// at https://docs.lancedb.com/storage/
+    pub storage_options: Option<HashMap<String, String>>,
+    /// Extra properties for the backing namespace client.
+    pub namespace_client_properties: Option<HashMap<String, String>>,
+    /// The session to use for this connection. Holds shared caches and other
+    /// session-specific state.
+    pub session: Option<session::Session>,
+}
+
 #[napi_derive::module_init]
 fn init() {
     let env = Env::new()

nodejs/src/scannable.rs

@@ -0,0 +1,253 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+//! NodeJS binding for the [`lancedb::data::scannable::Scannable`] trait.
+//!
+//! The JS side supplies a `getNextBatch(isStart)` callback that returns the
+//! next Arrow `RecordBatch` encoded as a self-contained Arrow IPC Stream
+//! message (schema message + record batch message + EOS marker) wrapped in a
+//! `Buffer`, or `null` when the stream is exhausted. The Rust side parses
+//! each buffer with `arrow_ipc::reader::StreamReader`, validates every
+//! standalone batch stream against the declared schema, and yields decoded
+//! `RecordBatch`es as a [`SendableRecordBatchStream`].
+//!
+//! `isStart` is `true` on the first `getNextBatch` call of each new
+//! `scan_as_stream` and `false` thereafter. JS uses it to drop any cached
+//! iterator and re-invoke its factory at scan boundaries, so retries
+//! triggered by mid-stream failures restart at batch 0.
+
+use std::io::Cursor;
+use std::sync::Arc;
+
+use arrow_array::RecordBatch;
+use arrow_ipc::reader::StreamReader;
+use arrow_schema::SchemaRef;
+use futures::stream::once;
+use lancedb::arrow::{SendableRecordBatchStream, SimpleRecordBatchStream};
+use lancedb::data::scannable::Scannable as LanceScannable;
+use lancedb::ipc::ipc_file_to_schema;
+use lancedb::{Error, Result as LanceResult};
+use napi::bindgen_prelude::*;
+use napi::threadsafe_function::ThreadsafeFunction;
+use napi_derive::napi;
+
+/// Threadsafe handle to the JS `getNextBatch` callback. The callback takes a
+/// single boolean `isStart` (`true` on the first call of each new scan) and
+/// returns a Promise that resolves to a `Buffer` containing one IPC Stream
+/// message, or `null` at end-of-stream.
+type GetNextBatchFn = ThreadsafeFunction<bool, Promise<Option<Buffer>>, bool, Status, false>;
+
+/// A Rust-side view of a JS-constructed `Scannable`.
+///
+/// Held in JS as the return value of the `Scannable` class constructor. When
+/// passed to a consumer that accepts `impl lancedb::data::scannable::Scannable`,
+/// the consumer invokes `scan_as_stream()` to pull batches through the JS
+/// callback.
+#[napi]
+pub struct NapiScannable {
+    schema: SchemaRef,
+    num_rows: Option<usize>,
+    rescannable: bool,
+    // `ThreadsafeFunction` is not `Clone`; wrap in `Arc` so the stream
+    // returned by `scan_as_stream` can own a handle independent of `self`.
+    get_next_batch: Arc<GetNextBatchFn>,
+    // Tracks whether a scan has already started; used to enforce one-shot
+    // semantics on non-rescannable sources.
+    scanned: bool,
+}
+
+#[napi]
+impl NapiScannable {
+    /// Construct a new `NapiScannable`.
+    ///
+    /// - `schema_buf` — Arrow IPC File buffer carrying only the schema (no batches).
+    /// - `num_rows` — optional row count hint; not validated against the stream.
+    /// - `rescannable` — whether `get_next_batch` may be re-driven after the
+    ///   scan completes.
+    /// - `get_next_batch` -- JS callback that yields the next batch as an Arrow
+    ///   IPC Stream message wrapped in a `Buffer`, or `null` at EOF. The
+    ///   `isStart` argument is `true` on the first call of each new scan;
+    ///   JS uses it to discard any cached iterator before pulling.
+    #[napi(constructor)]
+    pub fn new(
+        schema_buf: Buffer,
+        num_rows: Option<i64>,
+        rescannable: bool,
+        get_next_batch: Function<bool, Promise<Option<Buffer>>>,
+    ) -> napi::Result<Self> {
+        let schema = ipc_file_to_schema(schema_buf.to_vec())
+            .map_err(|e| napi::Error::from_reason(format!("Invalid schema buffer: {}", e)))?;
+        let num_rows = num_rows
+            .map(|n| {
+                usize::try_from(n)
+                    .map_err(|_| napi::Error::from_reason("num_rows must be non-negative"))
+            })
+            .transpose()?;
+        let get_next_batch = Arc::new(get_next_batch.build_threadsafe_function().build()?);
+        Ok(Self {
+            schema,
+            num_rows,
+            rescannable,
+            get_next_batch,
+            scanned: false,
+        })
+    }
+}
+
+impl std::fmt::Debug for NapiScannable {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("NapiScannable")
+            .field("schema", &self.schema)
+            .field("num_rows", &self.num_rows)
+            .field("rescannable", &self.rescannable)
+            .finish()
+    }
+}
+
+impl LanceScannable for NapiScannable {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+
+    fn scan_as_stream(&mut self) -> SendableRecordBatchStream {
+        let schema = self.schema.clone();
+
+        // One-shot enforcement for non-rescannable sources: return a stream
+        // whose first item is an error.
+        if self.scanned && !self.rescannable {
+            let err_stream = once(async {
+                Err(Error::InvalidInput {
+                    message: "Scannable has already been consumed (non-rescannable source)"
+                        .to_string(),
+                })
+            });
+            return Box::pin(SimpleRecordBatchStream::new(err_stream, schema));
+        }
+        self.scanned = true;
+
+        let tsfn = Arc::clone(&self.get_next_batch);
+        let declared_schema = schema.clone();
+
+        // State threaded through the unfold. `is_first_pull` starts true so
+        // the first call into JS signals a new-scan boundary; JS uses it to
+        // reset any cached iterator before factory()-ing a fresh one.
+        let initial = State {
+            tsfn,
+            batch_index: 0,
+            declared_schema,
+            errored: false,
+            is_first_pull: true,
+        };
+
+        let stream = futures::stream::unfold(initial, |mut state| async move {
+            if state.errored {
+                return None;
+            }
+
+            // Pull the next IPC Stream buffer from JS. `is_first_pull` is
+            // consumed here and cleared so subsequent pulls continue the
+            // same scan rather than restarting it.
+            let is_start = state.is_first_pull;
+            state.is_first_pull = false;
+            let buf = match pull_next(&state.tsfn, is_start).await {
+                Ok(Some(buf)) => buf,
+                Ok(None) => return None,
+                Err(e) => {
+                    state.errored = true;
+                    return Some((Err(e), state));
+                }
+            };
+
+            match decode_one_batch(buf.as_ref(), &state.declared_schema) {
+                Ok(batch) => {
+                    state.batch_index += 1;
+                    Some((Ok(batch), state))
+                }
+                Err(e) => {
+                    let tagged = Error::Runtime {
+                        message: format!(
+                            "[scannable/rust-bridge] failure at batch index {}: {}",
+                            state.batch_index, e
+                        ),
+                    };
+                    state.errored = true;
+                    Some((Err(tagged), state))
+                }
+            }
+        });
+
+        Box::pin(SimpleRecordBatchStream::new(stream, schema))
+    }
+
+    fn num_rows(&self) -> Option<usize> {
+        self.num_rows
+    }
+
+    fn rescannable(&self) -> bool {
+        self.rescannable
+    }
+}
+
+struct State {
+    tsfn: Arc<GetNextBatchFn>,
+    batch_index: usize,
+    declared_schema: SchemaRef,
+    errored: bool,
+    /// True for the very first pull of a new scan. Forwarded to JS so the
+    /// callback can drop any cached iterator and call its factory fresh,
+    /// which makes rescannable sources restart at batch 0 even when the
+    /// previous scan ended mid-stream.
+    is_first_pull: bool,
+}
+
+/// Invoke the JS callback and await its Promise. `is_start` is forwarded to
+/// the JS side as the `isStart` argument so it can reset its iterator at the
+/// scan boundary. Errors on the JS side surface here as rejected promises
+/// and are tunneled back as `lancedb::Error::Runtime`.
+async fn pull_next(tsfn: &GetNextBatchFn, is_start: bool) -> LanceResult<Option<Buffer>> {
+    let promise = tsfn
+        .call_async(is_start)
+        .await
+        .map_err(|e| Error::Runtime {
+            message: format!(
+                "[scannable/js-factory] napi error status={}, reason={}",
+                e.status, e.reason
+            ),
+        })?;
+    promise.await.map_err(|e| Error::Runtime {
+        message: format!(
+            "[scannable/js-iterator] napi error status={}, reason={}",
+            e.status, e.reason
+        ),
+    })
+}
+
+/// Decode one IPC Stream buffer (schema + batch + EOS) into a `RecordBatch`.
+/// Each buffer is a standalone IPC stream, so every decoded stream schema must
+/// match the one declared at construction.
+fn decode_one_batch(buf: &[u8], declared: &SchemaRef) -> LanceResult<RecordBatch> {
+    let reader = StreamReader::try_new(Cursor::new(buf), None).map_err(|e| Error::Runtime {
+        message: format!("failed to open IPC stream reader: {}", e),
+    })?;
+
+    let actual = reader.schema();
+    if actual.as_ref() != declared.as_ref() {
+        return Err(Error::InvalidInput {
+            message: format!(
+                "declared schema does not match stream schema: declared={:?} actual={:?}",
+                declared, actual
+            ),
+        });
+    }
+
+    let mut iter = reader;
+    let batch = iter
+        .next()
+        .ok_or_else(|| Error::Runtime {
+            message: "IPC stream contained schema but no record batch".to_string(),
+        })?
+        .map_err(|e| Error::Runtime {
+            message: format!("failed to decode record batch: {}", e),
+        })?;
+    Ok(batch)
+}

nodejs/src/table.rs

@@ -159,6 +159,14 @@ impl Table {
             .default_error()
     }
 
+    #[napi(catch_unwind)]
+    pub async fn prewarm_data(&self, columns: Option<Vec<String>>) -> napi::Result<()> {
+        self.inner_ref()?
+            .prewarm_data(columns)
+            .await
+            .default_error()
+    }
+
     #[napi(catch_unwind)]
     pub async fn wait_for_index(&self, index_names: Vec<String>, timeout_s: i64) -> Result<()> {
         let timeout = std::time::Duration::from_secs(timeout_s.try_into().unwrap());

python/Cargo.toml

@@ -19,6 +19,7 @@ arrow = { version = "58.0.0", features = ["pyarrow"] }
 async-trait = "0.1"
 bytes = "1"
 lancedb = { path = "../rust/lancedb", default-features = false }
+datafusion-common.workspace = true
 lance-core.workspace = true
 lance-namespace.workspace = true
 lance-namespace-impls.workspace = true

python/python/lancedb/_lancedb.pyi

@@ -12,6 +12,7 @@ from .index import (
     LabelList,
     HnswPq,
     HnswSq,
+    HnswFlat,
     FTS,
 )
 from lance_namespace import (
@@ -25,6 +26,7 @@ from .remote import ClientConfig
 
 IvfHnswPq: type[HnswPq] = HnswPq
 IvfHnswSq: type[HnswSq] = HnswSq
+IvfHnswFlat: type[HnswFlat] = HnswFlat
 
 class PyExpr:
     """A type-safe DataFusion expression node (Rust-side handle)."""
@@ -49,7 +51,7 @@ class PyExpr:
     def to_sql(self) -> str: ...
 
 def expr_col(name: str) -> PyExpr: ...
-def expr_lit(value: Union[bool, int, float, str]) -> PyExpr: ...
+def expr_lit(value: Union[bool, int, float, str, bytes]) -> PyExpr: ...
 def expr_func(name: str, args: List[PyExpr]) -> PyExpr: ...
 
 class Session:
@@ -180,6 +182,7 @@ class Table:
             IvfPq,
             HnswPq,
             HnswSq,
+            HnswFlat,
             BTree,
             Bitmap,
             LabelList,

python/python/lancedb/expr.py

@@ -63,7 +63,7 @@ def _coerce(value: "ExprLike") -> "Expr":
 
 
 # Type alias used in annotations.
-ExprLike = Union["Expr", bool, int, float, str]
+ExprLike = Union["Expr", bool, int, float, str, bytes]
 
 
 class Expr:
@@ -261,13 +261,13 @@ def col(name: str) -> Expr:
     return Expr(expr_col(name))
 
 
-def lit(value: Union[bool, int, float, str]) -> Expr:
+def lit(value: Union[bool, int, float, str, bytes]) -> Expr:
     """Create a literal (constant) value expression.
 
     Parameters
     ----------
     value:
-        A Python ``bool``, ``int``, ``float``, or ``str``.
+        A Python ``bool``, ``int``, ``float``, ``str``, or ``bytes``.
 
     Examples
     --------

python/python/lancedb/index.py

@@ -7,6 +7,7 @@ from typing import Literal, Optional
 from ._lancedb import (
     IndexConfig,
 )
+from .types import BaseTokenizerType
 
 lang_mapping = {
     "ar": "Arabic",
@@ -111,8 +112,12 @@ class FTS:
         - "simple": Splits text by whitespace and punctuation.
         - "whitespace": Split text by whitespace, but not punctuation.
         - "raw": No tokenization. The entire text is treated as a single token.
+        - "ngram": N-gram tokenizer for substring-style matching.
+        - "jieba/*": Jieba tokenizer loaded from Lance's language model home.
+        - "lindera/*": Lindera tokenizer loaded from Lance's language model home.
     language : str, default "English"
-        The language to use for tokenization.
+        The language to use for stemming and stop-word removal. This is not the
+        primary way to enable CJK tokenization.
     max_token_length : int, default 40
         The maximum token length to index. Tokens longer than this length will be
         ignored.
@@ -127,10 +132,17 @@ class FTS:
     ascii_folding : bool, default True
         Whether to fold ASCII characters. This converts accented characters to
         their ASCII equivalent. For example, "café" would be converted to "cafe".
+
+    Notes
+    -----
+    Model-backed tokenizers such as ``jieba/default`` and ``lindera/ipadic``
+    require tokenizer models in Lance's language model home. Set
+    ``LANCE_LANGUAGE_MODEL_HOME`` to override the default platform data
+    directory under ``lance/language_models``.
     """
 
     with_position: bool = False
-    base_tokenizer: Literal["simple", "raw", "whitespace"] = "simple"
+    base_tokenizer: BaseTokenizerType = "simple"
     language: str = "English"
     max_token_length: Optional[int] = 40
     lower_case: bool = True
@@ -376,9 +388,98 @@ class HnswSq:
     target_partition_size: Optional[int] = None
 
 
+@dataclass
+class HnswFlat:
+    """Describe a HNSW-FLAT index configuration.
+
+    HNSW-FLAT stands for Hierarchical Navigable Small World without quantization.
+    It stores raw vectors in the HNSW graph, providing the highest recall among
+    the IVF_HNSW family at the cost of more memory and disk space compared to
+    :class:`HnswSq` or :class:`HnswPq`.
+
+    Parameters
+    ----------
+
+    distance_type: str, default "l2"
+
+        The distance metric used to train the index.
+
+        The following distance types are available:
+
+        "l2" - Euclidean distance. This is a very common distance metric that
+        accounts for both magnitude and direction when determining the distance
+        between vectors. l2 distance has a range of [0, ∞).
+
+        "cosine" - Cosine distance.  Cosine distance is a distance metric
+        calculated from the cosine similarity between two vectors. Cosine
+        similarity is a measure of similarity between two non-zero vectors of an
+        inner product space. It is defined to equal the cosine of the angle
+        between them.  Unlike l2, the cosine distance is not affected by the
+        magnitude of the vectors.  Cosine distance has a range of [0, 2].
+
+        "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
+        distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
+        l2 norm is 1), then dot distance is equivalent to the cosine distance.
+
+    num_partitions, default sqrt(num_rows)
+
+        The number of IVF partitions to create.
+
+        For HNSW, we recommend a small number of partitions. Setting this to 1
+        works well for most tables. For very large tables, training just one HNSW
+        graph will require too much memory. Each partition becomes its own HNSW
+        graph, so setting this value higher reduces the peak memory use of
+        training.
+
+    max_iterations, default 50
+
+        Max iterations to train kmeans.
+
+        When training an IVF index we use kmeans to calculate the partitions.
+        This parameter controls how many iterations of kmeans to run.
+
+    sample_rate, default 256
+
+        The rate used to calculate the number of training vectors for kmeans.
+
+    m, default 20
+
+        The number of neighbors to select for each vector in the HNSW graph.
+
+        This value controls the tradeoff between search speed and accuracy.
+        The higher the value the more accurate the search but the slower it
+        will be.
+
+    ef_construction, default 300
+
+        The number of candidates to evaluate during the construction of the HNSW
+        graph.
+
+        This value controls the tradeoff between build speed and accuracy.
+        The higher the value the more accurate the build but the slower it will
+        be.  150 to 300 is the typical range. 100 is a minimum for good quality
+        search results. In most cases, there is no benefit to setting this higher
+        than 500.  This value should be set to a value that is not less than `ef`
+        in the search phase.
+
+    target_partition_size, default is 1,048,576
+
+        The target size of each partition.
+    """
+
+    distance_type: Literal["l2", "cosine", "dot"] = "l2"
+    num_partitions: Optional[int] = None
+    max_iterations: int = 50
+    sample_rate: int = 256
+    m: int = 20
+    ef_construction: int = 300
+    target_partition_size: Optional[int] = None
+
+
 # Backwards-compatible aliases
 IvfHnswPq = HnswPq
 IvfHnswSq = HnswSq
+IvfHnswFlat = HnswFlat
 
 
 @dataclass
@@ -698,11 +799,13 @@ __all__ = [
     "IvfPq",
     "IvfHnswPq",
     "IvfHnswSq",
+    "IvfHnswFlat",
     "IvfSq",
     "IvfRq",
     "IvfFlat",
     "HnswPq",
     "HnswSq",
+    "HnswFlat",
     "IndexConfig",
     "FTS",
     "Bitmap",

python/python/lancedb/namespace_utils.py

@@ -6,22 +6,44 @@
 from typing import Optional
 
 
+_CREATE_NAMESPACE_MODES = frozenset({"create", "exist_ok", "overwrite"})
+_DROP_NAMESPACE_MODES = frozenset({"SKIP", "FAIL"})
+_DROP_NAMESPACE_BEHAVIORS = frozenset({"RESTRICT", "CASCADE"})
+
+
 def _normalize_create_namespace_mode(mode: Optional[str]) -> Optional[str]:
     """Normalize create namespace mode to lowercase (API expects lowercase)."""
     if mode is None:
         return None
-    return mode.lower()
+    normalized = mode.lower()
+    if normalized not in _CREATE_NAMESPACE_MODES:
+        raise ValueError(
+            f"Invalid create namespace mode {mode!r}: "
+            f"expected one of 'create', 'exist_ok', 'overwrite'"
+        )
+    return normalized
 
 
 def _normalize_drop_namespace_mode(mode: Optional[str]) -> Optional[str]:
     """Normalize drop namespace mode to uppercase (API expects uppercase)."""
     if mode is None:
         return None
-    return mode.upper()
+    normalized = mode.upper()
+    if normalized not in _DROP_NAMESPACE_MODES:
+        raise ValueError(
+            f"Invalid drop namespace mode {mode!r}: expected one of 'skip', 'fail'"
+        )
+    return normalized
 
 
 def _normalize_drop_namespace_behavior(behavior: Optional[str]) -> Optional[str]:
     """Normalize drop namespace behavior to uppercase (API expects uppercase)."""
     if behavior is None:
         return None
-    return behavior.upper()
+    normalized = behavior.upper()
+    if normalized not in _DROP_NAMESPACE_BEHAVIORS:
+        raise ValueError(
+            f"Invalid drop namespace behavior {behavior!r}: "
+            f"expected one of 'restrict', 'cascade'"
+        )
+    return normalized

python/python/lancedb/remote/table.py

@@ -22,6 +22,7 @@ from lancedb.index import (
     FTS,
     BTree,
     Bitmap,
+    HnswFlat,
     HnswSq,
     IvfFlat,
     IvfPq,
@@ -39,6 +40,7 @@ from lancedb.table import _normalize_progress
 
 from ..query import LanceVectorQueryBuilder, LanceQueryBuilder, LanceTakeQueryBuilder
 from ..table import AsyncTable, IndexStatistics, Query, Table, Tags
+from ..types import BaseTokenizerType
 
 
 class RemoteTable(Table):
@@ -167,7 +169,7 @@ class RemoteTable(Table):
         wait_timeout: Optional[timedelta] = None,
         with_position: bool = False,
         # tokenizer configs:
-        base_tokenizer: str = "simple",
+        base_tokenizer: BaseTokenizerType = "simple",
         language: str = "English",
         max_token_length: Optional[int] = 40,
         lower_case: bool = True,
@@ -284,13 +286,15 @@ class RemoteTable(Table):
             )
         elif index_type == "IVF_HNSW_SQ":
             config = HnswSq(distance_type=metric, num_partitions=num_partitions)
+        elif index_type == "IVF_HNSW_FLAT":
+            config = HnswFlat(distance_type=metric, num_partitions=num_partitions)
         elif index_type == "IVF_FLAT":
             config = IvfFlat(distance_type=metric, num_partitions=num_partitions)
         else:
             raise ValueError(
                 f"Unknown vector index type: {index_type}. Valid options are"
                 " 'IVF_FLAT', 'IVF_PQ', 'IVF_RQ', 'IVF_SQ',"
-                " 'IVF_HNSW_PQ', 'IVF_HNSW_SQ'"
+                " 'IVF_HNSW_PQ', 'IVF_HNSW_SQ', 'IVF_HNSW_FLAT'"
             )
 
         LOOP.run(

python/python/lancedb/table.py

@@ -57,6 +57,7 @@ from .index import (
     LabelList,
     HnswPq,
     HnswSq,
+    HnswFlat,
     FTS,
 )
 from .merge import LanceMergeInsertBuilder
@@ -86,6 +87,59 @@ from .util import (
 )
 from .index import lang_mapping
 
+_MODEL_BACKED_TOKENIZER_PREFIXES = ("jieba", "lindera")
+_MODEL_BACKED_TOKENIZER_ERRORS = (
+    "unknown base tokenizer",
+    "Invalid directory path:",
+    "Failed to load Jieba",
+    "Failed to load tokenizer config",
+    "Failed to initialize default tokenizer",
+)
+
+
+def _add_unique_note(exception: BaseException, note: str) -> None:
+    existing_notes = getattr(exception, "__notes__", ()) or ()
+    message = (
+        exception.args[0]
+        if exception.args and isinstance(exception.args[0], str)
+        else ""
+    )
+    if note not in existing_notes and note not in message:
+        add_note(exception, note)
+
+
+def _is_model_backed_tokenizer(base_tokenizer: str) -> bool:
+    return any(
+        base_tokenizer == prefix or base_tokenizer.startswith(f"{prefix}/")
+        for prefix in _MODEL_BACKED_TOKENIZER_PREFIXES
+    )
+
+
+def _maybe_add_fts_error_note(
+    exception: BaseException, *, base_tokenizer: str, language: Optional[str] = None
+) -> None:
+    message = str(exception)
+    if language is not None and "not support the requested language" in message:
+        supported_langs = ", ".join(lang_mapping.values())
+        _add_unique_note(exception, f"Supported languages: {supported_langs}")
+        return
+
+    if not _is_model_backed_tokenizer(base_tokenizer):
+        return
+
+    if not any(marker in message for marker in _MODEL_BACKED_TOKENIZER_ERRORS):
+        return
+
+    _add_unique_note(
+        exception,
+        "Model-backed tokenizers such as 'jieba/default' and 'lindera/ipadic' "
+        "require tokenizer models in Lance's language model home. Set "
+        "LANCE_LANGUAGE_MODEL_HOME to override the default platform data "
+        "directory under 'lance/language_models'. Expected layouts include "
+        "'<model-home>/jieba/default/...' and "
+        "'<model-home>/lindera/ipadic/...'.",
+    )
+
 
 if TYPE_CHECKING:
     from .db import LanceDBConnection
@@ -958,7 +1012,10 @@ class Table(ABC):
         tokenizer_name: str, default "default"
             A compatibility alias for native tokenizer configs. Can be "raw",
             "default" or the 2 letter language code followed by "_stem". So
-            for english it would be "en_stem".
+            for english it would be "en_stem". For new native FTS indexes, use
+            ``base_tokenizer`` directly; ``tokenizer_name`` is a legacy
+            compatibility alias and does not expose model-backed tokenizer names
+            such as ``jieba/default`` or ``lindera/ipadic``.
         use_tantivy: bool, default False
             Deprecated legacy Tantivy parameter. Setting this to True raises an
             error.
@@ -972,8 +1029,11 @@ class Table(ABC):
             - "whitespace": Split text by whitespace, but not punctuation.
             - "raw": No tokenization. The entire text is treated as a single token.
             - "ngram": N-Gram tokenizer.
+            - "jieba/*": Jieba tokenizer loaded from Lance's language model home.
+            - "lindera/*": Lindera tokenizer loaded from Lance's language model home.
         language : str, default "English"
-            The language to use for tokenization.
+            The language to use for stemming and stop-word removal. This is not
+            the primary way to enable CJK tokenization.
         max_token_length : int, default 40
             The maximum token length to index. Tokens longer than this length will be
             ignored.
@@ -999,6 +1059,13 @@ class Table(ABC):
             The timeout to wait if indexing is asynchronous.
         name: str, optional
             The name of the index. If not provided, a default name will be generated.
+
+        Notes
+        -----
+        Model-backed tokenizers such as ``jieba/default`` and ``lindera/ipadic``
+        require tokenizer models in Lance's language model home. Set
+        ``LANCE_LANGUAGE_MODEL_HOME`` to override the default platform data
+        directory under ``lance/language_models``.
         """
         raise NotImplementedError
 
@@ -2170,7 +2237,13 @@ class LanceTable(Table):
         index_cache_size: Optional[int] = None,
         num_bits: int = 8,
         index_type: Literal[
-            "IVF_FLAT", "IVF_SQ", "IVF_PQ", "IVF_RQ", "IVF_HNSW_SQ", "IVF_HNSW_PQ"
+            "IVF_FLAT",
+            "IVF_SQ",
+            "IVF_PQ",
+            "IVF_RQ",
+            "IVF_HNSW_SQ",
+            "IVF_HNSW_PQ",
+            "IVF_HNSW_FLAT",
         ] = "IVF_PQ",
         max_iterations: int = 50,
         sample_rate: int = 256,
@@ -2257,6 +2330,16 @@ class LanceTable(Table):
                 ef_construction=ef_construction,
                 target_partition_size=target_partition_size,
             )
+        elif index_type == "IVF_HNSW_FLAT":
+            config = HnswFlat(
+                distance_type=metric,
+                num_partitions=num_partitions,
+                max_iterations=max_iterations,
+                sample_rate=sample_rate,
+                m=m,
+                ef_construction=ef_construction,
+                target_partition_size=target_partition_size,
+            )
         else:
             raise ValueError(f"Unknown index type {index_type}")
 
@@ -2462,14 +2545,22 @@ class LanceTable(Table):
             **tokenizer_configs,
         )
 
-        LOOP.run(
-            self._table.create_index(
-                field_names,
-                replace=replace,
-                config=config,
-                name=name,
+        try:
+            LOOP.run(
+                self._table.create_index(
+                    field_names,
+                    replace=replace,
+                    config=config,
+                    name=name,
+                )
             )
-        )
+        except (ValueError, RuntimeError) as e:
+            _maybe_add_fts_error_note(
+                e,
+                base_tokenizer=config.base_tokenizer,
+                language=config.language,
+            )
+            raise e
 
     @staticmethod
     def infer_tokenizer_configs(tokenizer_name: str) -> dict:
@@ -3799,7 +3890,18 @@ class AsyncTable:
         *,
         replace: Optional[bool] = None,
         config: Optional[
-            Union[IvfFlat, IvfPq, IvfRq, HnswPq, HnswSq, BTree, Bitmap, LabelList, FTS]
+            Union[
+                IvfFlat,
+                IvfPq,
+                IvfRq,
+                HnswPq,
+                HnswSq,
+                HnswFlat,
+                BTree,
+                Bitmap,
+                LabelList,
+                FTS,
+            ]
         ] = None,
         wait_timeout: Optional[timedelta] = None,
         name: Optional[str] = None,
@@ -3846,6 +3948,7 @@ class AsyncTable:
                     IvfRq,
                     HnswPq,
                     HnswSq,
+                    HnswFlat,
                     BTree,
                     Bitmap,
                     LabelList,
@@ -3865,11 +3968,13 @@ class AsyncTable:
                 name=name,
                 train=train,
             )
-        except ValueError as e:
-            if "not support the requested language" in str(e):
-                supported_langs = ", ".join(lang_mapping.values())
-                help_msg = f"Supported languages: {supported_langs}"
-                add_note(e, help_msg)
+        except (ValueError, RuntimeError) as e:
+            if isinstance(config, FTS):
+                _maybe_add_fts_error_note(
+                    e,
+                    base_tokenizer=config.base_tokenizer,
+                    language=config.language,
+                )
             raise e
 
     async def drop_index(self, name: str) -> None:
@@ -5014,6 +5119,7 @@ class IndexStatistics:
         "IVF_RQ",
         "IVF_HNSW_SQ",
         "IVF_HNSW_PQ",
+        "IVF_HNSW_FLAT",
         "FTS",
         "BTREE",
         "BITMAP",

python/python/lancedb/types.py

@@ -24,6 +24,7 @@ VectorIndexType = Literal[
     "IVF_PQ",
     "IVF_HNSW_SQ",
     "IVF_HNSW_PQ",
+    "IVF_HNSW_FLAT",
     "IVF_RQ",
 ]
 ScalarIndexType = Literal["BTREE", "BITMAP", "LABEL_LIST"]
@@ -31,6 +32,7 @@ IndexType = Literal[
     "IVF_PQ",
     "IVF_HNSW_PQ",
     "IVF_HNSW_SQ",
+    "IVF_HNSW_FLAT",
     "IVF_SQ",
     "FTS",
     "BTREE",
@@ -40,4 +42,5 @@ IndexType = Literal[
 ]
 
 # Tokenizer literals
-BaseTokenizerType = Literal["simple", "raw", "whitespace", "ngram"]
+BuiltinTokenizerType = Literal["simple", "raw", "whitespace", "ngram"]
+BaseTokenizerType = BuiltinTokenizerType | str

python/python/tests/models/jieba/default/dict.txt

@@ -0,0 +1,8 @@
+我们 98740 r
+都 202780 d
+有 423765 v
+光明 1219 n
+的 318825 uj
+前途 1263 n
+前 62779 f
+途 857 n

python/python/tests/models/lindera/ipadic/config.yml

@@ -0,0 +1,3 @@
+segmenter:
+  mode: "normal"
+  dictionary: "./python/tests/models/lindera/ipadic/main"

python/python/tests/test_db.py

@@ -914,6 +914,29 @@ def test_local_namespace_operations(tmp_path):
     assert db.list_namespaces().namespaces == []
 
 
+def test_create_namespace_invalid_mode_raises(tmp_path):
+    """Unrecognized create namespace modes raise a clear error."""
+    db = lancedb.connect(tmp_path)
+    with pytest.raises(ValueError, match="Invalid create namespace mode"):
+        db.create_namespace(["child"], mode="frobnicate")
+
+
+def test_drop_namespace_invalid_mode_raises(tmp_path):
+    """Unrecognized drop namespace modes raise a clear error."""
+    db = lancedb.connect(tmp_path)
+    db.create_namespace(["child"])
+    with pytest.raises(ValueError, match="Invalid drop namespace mode"):
+        db.drop_namespace(["child"], mode="frobnicate")
+
+
+def test_drop_namespace_invalid_behavior_raises(tmp_path):
+    """Unrecognized drop namespace behaviors raise a clear error."""
+    db = lancedb.connect(tmp_path)
+    db.create_namespace(["child"])
+    with pytest.raises(ValueError, match="Invalid drop namespace behavior"):
+        db.drop_namespace(["child"], behavior="frobnicate")
+
+
 def test_clone_table_latest_version(tmp_path):
     """Test cloning a table with the latest version (default behavior)"""
     import os

python/python/tests/test_fts.py

@@ -15,7 +15,10 @@
 #  limitations under the License.
 import os
 import random
+import shutil
 from unittest import mock
+from pathlib import Path
+import zipfile
 
 import lancedb as ldb
 from lancedb.db import DBConnection
@@ -36,6 +39,8 @@ import pytest
 import pytest_asyncio
 from utils import exception_output
 
+TEST_LANGUAGE_MODEL_HOME = Path(__file__).parent / "models"
+
 
 @pytest.fixture
 def table(tmp_path) -> ldb.table.LanceTable:
@@ -89,6 +94,39 @@ def table(tmp_path) -> ldb.table.LanceTable:
     return table
 
 
+@pytest.fixture
+def language_model_home(monkeypatch, tmp_path):
+    model_home = tmp_path / "language-models"
+    shutil.copytree(TEST_LANGUAGE_MODEL_HOME, model_home)
+    monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(model_home))
+    return model_home
+
+
+@pytest.fixture
+def lindera_ipadic(language_model_home):
+    model_path = language_model_home / "lindera" / "ipadic"
+    extracted_model = model_path / "main"
+    config_path = model_path / "config.yml"
+
+    if extracted_model.exists():
+        shutil.rmtree(extracted_model)
+
+    with zipfile.ZipFile(model_path / "main.zip", "r") as zip_ref:
+        zip_ref.extractall(model_path)
+    config_path.write_text(
+        "segmenter:\n"
+        '  mode: "normal"\n'
+        f'  dictionary: "{extracted_model.resolve().as_posix()}"\n',
+        encoding="utf-8",
+    )
+
+    try:
+        yield
+    finally:
+        if extracted_model.exists():
+            shutil.rmtree(extracted_model)
+
+
 @pytest_asyncio.fixture
 async def async_table(tmp_path) -> ldb.table.AsyncTable:
     # Use local random state to avoid affecting other tests
@@ -684,6 +722,90 @@ def test_fts_ngram(mem_db: DBConnection):
     assert set(r["text"] for r in results) == {"lance database", "lance is cool"}
 
 
+def test_fts_jieba_tokenizer(mem_db: DBConnection, language_model_home):
+    data = pa.table({"text": ["我们都有光明的前途", "光明的前途"]})
+    table = mem_db.create_table("test_jieba", data=data)
+    table.create_fts_index(
+        "text",
+        base_tokenizer="jieba/default",
+        stem=False,
+        remove_stop_words=False,
+        ascii_folding=False,
+    )
+
+    results = table.search("我们", query_type="fts").limit(10).to_list()
+    assert [row["text"] for row in results] == ["我们都有光明的前途"]
+
+
+def test_fts_jieba_missing_language_model_note(
+    mem_db: DBConnection, monkeypatch, tmp_path
+):
+    missing_root = tmp_path / "missing-language-models"
+    monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(missing_root))
+    table = mem_db.create_table(
+        "test_missing_jieba_model",
+        data=pa.table({"text": ["我们都有光明的前途"]}),
+    )
+
+    with pytest.raises((ValueError, RuntimeError)) as e:
+        table.create_fts_index(
+            "text",
+            base_tokenizer="jieba/default",
+            stem=False,
+            remove_stop_words=False,
+            ascii_folding=False,
+        )
+
+    output = exception_output(e)
+    assert "Invalid directory path:" in output
+    assert "LANCE_LANGUAGE_MODEL_HOME" in output
+    assert "jieba/default" in output
+
+
+@pytest.mark.asyncio
+async def test_fts_jieba_missing_language_model_note_async(monkeypatch, tmp_path):
+    missing_root = tmp_path / "missing-language-models"
+    monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(missing_root))
+    db = await ldb.connect_async(tmp_path / "async-db")
+    table = await db.create_table(
+        "test_missing_jieba_model_async",
+        data=pa.table({"text": ["我们都有光明的前途"]}),
+    )
+
+    with pytest.raises((ValueError, RuntimeError)) as e:
+        await table.create_index(
+            "text",
+            config=FTS(
+                base_tokenizer="jieba/default",
+                stem=False,
+                remove_stop_words=False,
+                ascii_folding=False,
+            ),
+        )
+
+    output = exception_output(e)
+    assert "Invalid directory path:" in output
+    assert "LANCE_LANGUAGE_MODEL_HOME" in output
+    assert "jieba/default" in output
+
+
+def test_fts_lindera_tokenizer(
+    mem_db: DBConnection, language_model_home, lindera_ipadic
+):
+    data = pa.table({"text": ["成田国際空港", "東京国際空港", "羽田空港"]})
+    table = mem_db.create_table("test_lindera", data=data)
+    table.create_fts_index(
+        "text",
+        base_tokenizer="lindera/ipadic",
+        stem=False,
+        remove_stop_words=False,
+        ascii_folding=False,
+    )
+
+    results = table.search("成田", query_type="fts").limit(10).to_list()
+    assert [row["text"] for row in results] == ["成田国際空港"]
+
+
 def test_fts_query_to_json():
     """Test that FTS query to_json() produces valid JSON strings with exact format."""
 

python/python/tests/test_index.py

@@ -16,11 +16,13 @@ from lancedb.index import (
     IvfSq,
     IvfHnswPq,
     IvfHnswSq,
+    IvfHnswFlat,
     IvfRq,
     Bitmap,
     LabelList,
     HnswPq,
     HnswSq,
+    HnswFlat,
     FTS,
 )
 from lancedb.table import IndexStatistics
@@ -250,6 +252,21 @@ async def test_create_hnswpq_alias_index(some_table: AsyncTable):
     assert indices[0].index_type in {"HnswPq", "IvfHnswPq"}
 
 
+@pytest.mark.asyncio
+async def test_create_hnswflat_index(some_table: AsyncTable):
+    await some_table.create_index("vector", config=HnswFlat(num_partitions=10))
+    indices = await some_table.list_indices()
+    assert len(indices) == 1
+
+
+@pytest.mark.asyncio
+async def test_create_hnswflat_alias_index(some_table: AsyncTable):
+    await some_table.create_index("vector", config=IvfHnswFlat(num_partitions=5))
+    indices = await some_table.list_indices()
+    assert len(indices) == 1
+    assert indices[0].index_type in {"HnswFlat", "IvfHnswFlat"}
+
+
 @pytest.mark.asyncio
 async def test_create_ivfsq_index(some_table: AsyncTable):
     await some_table.create_index("vector", config=IvfSq(num_partitions=10))
@@ -295,6 +312,7 @@ def test_index_statistics_index_type_lists_all_supported_values():
         "IVF_RQ",
         "IVF_HNSW_SQ",
         "IVF_HNSW_PQ",
+        "IVF_HNSW_FLAT",
         "FTS",
         "BTREE",
         "BITMAP",

python/python/tests/test_table.py

@@ -11,7 +11,7 @@ from unittest.mock import patch
 
 import lancedb
 from lancedb.dependencies import _PANDAS_AVAILABLE
-from lancedb.index import HnswPq, HnswSq, IvfPq
+from lancedb.index import HnswFlat, HnswPq, HnswSq, IvfPq
 import numpy as np
 import polars as pl
 import pyarrow as pa
@@ -917,6 +917,21 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
         "my_vector", replace=True, config=expected_config, name=None, train=True
     )
 
+    table.create_index(
+        vector_column_name="my_vector",
+        metric="cosine",
+        index_type="IVF_HNSW_FLAT",
+        sample_rate=0.1,
+        m=29,
+        ef_construction=10,
+    )
+    expected_config = HnswFlat(
+        distance_type="cosine", sample_rate=0.1, m=29, ef_construction=10
+    )
+    mock_create_index.assert_called_with(
+        "my_vector", replace=True, config=expected_config, name=None, train=True
+    )
+
 
 @patch("lancedb.table.AsyncTable.create_index")
 def test_create_index_name_and_train_parameters(

python/src/connection.rs

@@ -395,12 +395,17 @@ impl Connection {
         future_into_py(py, async move {
             use lance_namespace::models::CreateNamespaceRequest;
             // Mode is now a string field
-            let mode_str = mode.and_then(|m| match m.to_lowercase().as_str() {
-                "create" => Some("Create".to_string()),
-                "exist_ok" => Some("ExistOk".to_string()),
-                "overwrite" => Some("Overwrite".to_string()),
-                _ => None,
-            });
+            let mode_str = mode
+                .map(|m| match m.to_lowercase().as_str() {
+                    "create" => Ok("Create".to_string()),
+                    "exist_ok" => Ok("ExistOk".to_string()),
+                    "overwrite" => Ok("Overwrite".to_string()),
+                    _ => Err(PyValueError::new_err(format!(
+                        "Invalid mode {:?}: expected one of 'create', 'exist_ok', 'overwrite'",
+                        m
+                    ))),
+                })
+                .transpose()?;
             let request = CreateNamespaceRequest {
                 id: Some(namespace_path),
                 mode: mode_str,
@@ -428,16 +433,26 @@ impl Connection {
         future_into_py(py, async move {
             use lance_namespace::models::DropNamespaceRequest;
             // Mode and Behavior are now string fields
-            let mode_str = mode.and_then(|m| match m.to_uppercase().as_str() {
-                "SKIP" => Some("Skip".to_string()),
-                "FAIL" => Some("Fail".to_string()),
-                _ => None,
-            });
-            let behavior_str = behavior.and_then(|b| match b.to_uppercase().as_str() {
-                "RESTRICT" => Some("Restrict".to_string()),
-                "CASCADE" => Some("Cascade".to_string()),
-                _ => None,
-            });
+            let mode_str = mode
+                .map(|m| match m.to_uppercase().as_str() {
+                    "SKIP" => Ok("Skip".to_string()),
+                    "FAIL" => Ok("Fail".to_string()),
+                    _ => Err(PyValueError::new_err(format!(
+                        "Invalid mode {:?}: expected one of 'skip', 'fail'",
+                        m
+                    ))),
+                })
+                .transpose()?;
+            let behavior_str = behavior
+                .map(|b| match b.to_uppercase().as_str() {
+                    "RESTRICT" => Ok("Restrict".to_string()),
+                    "CASCADE" => Ok("Cascade".to_string()),
+                    _ => Err(PyValueError::new_err(format!(
+                        "Invalid behavior {:?}: expected one of 'restrict', 'cascade'",
+                        b
+                    ))),
+                })
+                .transpose()?;
             let request = DropNamespaceRequest {
                 id: Some(namespace_path),
                 mode: mode_str,

python/src/expr.rs

@@ -8,7 +8,9 @@
 //! DataFusion [`Expr`] nodes, bypassing SQL string parsing.
 
 use arrow::{datatypes::DataType, pyarrow::PyArrowType};
+use datafusion_common::ScalarValue;
 use lancedb::expr::{DfExpr, col as ldb_col, contains, expr_cast, lit as df_lit, lower, upper};
+use pyo3::types::PyBytes;
 use pyo3::{Bound, PyAny, PyResult, exceptions::PyValueError, prelude::*, pyfunction};
 
 /// A type-safe DataFusion expression.
@@ -141,7 +143,7 @@ pub fn expr_col(name: &str) -> PyExpr {
 
 /// Create a literal value expression.
 ///
-/// Supported Python types: `bool`, `int`, `float`, `str`.
+/// Supported Python types: `bool`, `int`, `float`, `str`, `bytes`.
 #[pyfunction]
 pub fn expr_lit(value: Bound<'_, PyAny>) -> PyResult<PyExpr> {
     // bool must be checked before int because bool is a subclass of int in Python
@@ -157,8 +159,12 @@ pub fn expr_lit(value: Bound<'_, PyAny>) -> PyResult<PyExpr> {
     if let Ok(s) = value.extract::<String>() {
         return Ok(PyExpr(df_lit(s)));
     }
+    if value.is_instance_of::<PyBytes>() {
+        let bytes = value.extract::<Vec<u8>>()?;
+        return Ok(PyExpr(df_lit(ScalarValue::Binary(Some(bytes)))));
+    }
     Err(PyValueError::new_err(format!(
-        "unsupported literal type: {}. Supported: bool, int, float, str",
+        "unsupported literal type: {}. Supported: bool, int, float, str, bytes",
         value.get_type().name()?
     )))
 }

python/src/index.rs

@@ -1,11 +1,13 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-use lancedb::index::vector::{IvfFlatIndexBuilder, IvfRqIndexBuilder, IvfSqIndexBuilder};
+use lancedb::index::vector::{
+    IvfFlatIndexBuilder, IvfHnswFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder,
+    IvfPqIndexBuilder, IvfRqIndexBuilder, IvfSqIndexBuilder,
+};
 use lancedb::index::{
     Index as LanceDbIndex,
     scalar::{BTreeIndexBuilder, FtsIndexBuilder},
-    vector::{IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, IvfPqIndexBuilder},
 };
 use pyo3::IntoPyObject;
 use pyo3::types::PyStringMethods;
@@ -162,8 +164,26 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                 }
                 Ok(LanceDbIndex::IvfHnswSq(hnsw_sq_builder))
             }
+            "HnswFlat" => {
+                let params = source.extract::<IvfHnswFlatParams>()?;
+                let distance_type = parse_distance_type(params.distance_type)?;
+                let mut hnsw_flat_builder = IvfHnswFlatIndexBuilder::default()
+                    .distance_type(distance_type)
+                    .max_iterations(params.max_iterations)
+                    .sample_rate(params.sample_rate)
+                    .num_edges(params.m)
+                    .ef_construction(params.ef_construction);
+                if let Some(num_partitions) = params.num_partitions {
+                    hnsw_flat_builder = hnsw_flat_builder.num_partitions(num_partitions);
+                }
+                if let Some(target_partition_size) = params.target_partition_size {
+                    hnsw_flat_builder =
+                        hnsw_flat_builder.target_partition_size(target_partition_size);
+                }
+                Ok(LanceDbIndex::IvfHnswFlat(hnsw_flat_builder))
+            }
             not_supported => Err(PyValueError::new_err(format!(
-                "Invalid index type '{}'.  Must be one of BTree, Bitmap, LabelList, FTS, IvfPq, IvfSq, IvfHnswPq, or IvfHnswSq",
+                "Invalid index type '{}'.  Must be one of BTree, Bitmap, LabelList, FTS, IvfPq, IvfSq, IvfHnswPq, IvfHnswSq, or IvfHnswFlat",
                 not_supported
             ))),
         }
@@ -250,6 +270,17 @@ struct IvfHnswSqParams {
     target_partition_size: Option<u32>,
 }
 
+#[derive(FromPyObject)]
+struct IvfHnswFlatParams {
+    distance_type: String,
+    num_partitions: Option<u32>,
+    max_iterations: u32,
+    sample_rate: u32,
+    m: u32,
+    ef_construction: u32,
+    target_partition_size: Option<u32>,
+}
+
 #[pyclass(get_all)]
 /// A description of an index currently configured on a column
 pub struct IndexConfig {

python/tests/test_expr.py

@@ -33,6 +33,14 @@ class TestExprConstruction:
         e = lit(True)
         assert isinstance(e, Expr)
 
+    def test_lit_bytes(self):
+        e = lit(b"\xde\xad\xbe\xef")
+        assert isinstance(e, Expr)
+
+    def test_lit_bytes_empty(self):
+        e = lit(b"")
+        assert isinstance(e, Expr)
+
     def test_lit_unsupported_type_raises(self):
         with pytest.raises(Exception):
             lit([1, 2, 3])
@@ -135,6 +143,43 @@ class TestExprOperators:
         assert e.to_sql() == "(name = 'alice')"
 
 
+class TestExprBytesLiteral:
+    def test_bytes_to_sql(self):
+        e = lit(b"\xde\xad\xbe\xef")
+        assert e.to_sql() == "X'DEADBEEF'"
+
+    def test_empty_bytes_to_sql(self):
+        e = lit(b"")
+        assert e.to_sql() == "X''"
+
+    def test_bytes_repr(self):
+        e = lit(b"\x01\x02")
+        assert repr(e) == "Expr(X'0102')"
+
+    def test_bytes_equality_expr_sql(self):
+        e = col("data") == lit(b"\xca\xfe")
+        assert e.to_sql() == "(data = X'CAFE')"
+
+    def test_bytes_ne_expr_sql(self):
+        e = col("data") != lit(b"\xff")
+        assert e.to_sql() == "(data <> X'FF')"
+
+    def test_bytes_compound_expr_sql(self):
+        e = (col("data") == lit(b"\x01")) & (col("id") > lit(5))
+        assert e.to_sql() == "((data = X'01') AND (id > 5))"
+
+    def test_bytes_in_function_call(self):
+        # Regression test: binary literals inside scalar function calls
+        # used to fail because DataFusion's unparser does not support Binary
+        # scalars.  Now handled via a placeholder-substitution rewrite.
+        e = func("contains", col("data"), lit(b"\xff"))
+        assert e.to_sql() == "contains(data, X'FF')"
+
+    def test_bytes_in_not(self):
+        e = ~(col("data") == lit(b"\xff"))
+        assert e.to_sql() == "NOT (data = X'FF')"
+
+
 class TestExprStringMethods:
     def test_lower(self):
         e = col("name").lower()
@@ -385,3 +430,44 @@ class TestColNamingIntegration:
         )
         assert "upper_name" in result.schema.names
         assert sorted(result["upper_name"].to_pylist()) == ["ALICE", "BOB", "CHARLIE"]
+
+
+# ── bytes / binary column integration tests ───────────────────────────────────
+
+
+@pytest.fixture
+def binary_table(tmp_path):
+    db = lancedb.connect(str(tmp_path))
+    data = pa.table(
+        {
+            "id": [1, 2, 3],
+            "payload": pa.array(
+                [b"\x01\x02", b"\xca\xfe", b"\xff\x00"],
+                type=pa.binary(),
+            ),
+        }
+    )
+    return db.create_table("binary_test", data)
+
+
+class TestExprBytesIntegration:
+    def test_binary_equality_filter(self, binary_table):
+        result = (
+            binary_table.search().where(col("payload") == lit(b"\xca\xfe")).to_arrow()
+        )
+        assert result.num_rows == 1
+        assert result["id"][0].as_py() == 2
+
+    def test_binary_ne_filter(self, binary_table):
+        result = (
+            binary_table.search().where(col("payload") != lit(b"\x01\x02")).to_arrow()
+        )
+        assert result.num_rows == 2
+
+    def test_binary_compound_filter(self, binary_table):
+        result = (
+            binary_table.search()
+            .where((col("payload") == lit(b"\x01\x02")) | (col("id") == lit(3)))
+            .to_arrow()
+        )
+        assert result.num_rows == 2

rust/lancedb/Cargo.toml

@@ -40,7 +40,7 @@ lance-datafusion.workspace = true
 lance-datagen = { workspace = true }
 lance-file = { workspace = true }
 lance-io = { workspace = true }
-lance-index = { workspace = true }
+lance-index = { workspace = true, features = ["tokenizer-jieba", "tokenizer-lindera"] }
 lance-table = { workspace = true }
 lance-linalg = { workspace = true }
 lance-testing = { workspace = true }

rust/lancedb/src/database/listing.rs

@@ -505,8 +505,15 @@ impl ListingDatabase {
                 // Filter out the commit store query param -- it's a lancedb param
                 url.query_pairs_mut().clear();
                 url.query_pairs_mut().extend_pairs(filtered_querys);
-                // Take a copy of the query string so we can propagate it to lance
-                let query_string = url.query().map(|s| s.to_string());
+                // Take a copy of the query string so we can propagate it to lance.
+                // `query_pairs_mut()` leaves the URL with `Some("")` even when no
+                // pairs survive (or none existed in the first place), so an empty
+                // string here must be treated the same as "no query" — otherwise
+                // every table URI ends up with a trailing `?`, which makes downstream
+                // sub-paths (e.g. MemWAL gen paths) re-parse as path=<base table> +
+                // query=<sub-path>, causing Lance to find the base table dataset
+                // when looking up the sub-path.
+                let query_string = url.query().filter(|q| !q.is_empty()).map(|s| s.to_string());
                 // clear the query string so we can use the url as the base uri
                 // use .set_query(None) instead of .set_query("") because the latter
                 // will add a trailing '?' to the url
@@ -2213,6 +2220,133 @@ mod tests {
         assert_eq!(uri, expected);
     }
 
+    /// Regression: connecting via a URL-style URI (which goes through
+    /// `url::Url::parse` and the `query_pairs_mut()` path) must not
+    /// append a trailing `?` to per-table URIs when the input URI has
+    /// no query string.
+    ///
+    /// Earlier, `query_pairs_mut().clear()` left the URL with
+    /// `query=Some("")`, which then propagated as a trailing `?` on
+    /// every table URI. Sub-path lookups against that URI (e.g. MemWAL
+    /// `<table_uri>/_mem_wal/<shard>/<rand>_gen_<n>`) re-parsed as
+    /// `path=<base table>` + `query=/_mem_wal/...`, causing
+    /// `Dataset::write` to find the base table dataset and falsely
+    /// report `Dataset already exists`.
+    /// Mirrors the URL-mutation step from
+    /// [`ListingDatabase::connect_with_options`] so we can assert the
+    /// fix without going through filesystem setup (which is awkward
+    /// across platforms — see the `file://` test below).
+    fn capture_query_like_connect(input_uri: &str) -> Option<String> {
+        let mut url = url::Url::parse(input_uri).unwrap();
+        let mut filtered_querys = Vec::new();
+        for (key, value) in url.query_pairs() {
+            if key == ENGINE || key == MIRRORED_STORE {
+                continue;
+            }
+            filtered_querys.push((key.to_string(), value.to_string()));
+        }
+        url.query_pairs_mut().clear();
+        url.query_pairs_mut().extend_pairs(filtered_querys);
+        url.query().filter(|q| !q.is_empty()).map(|s| s.to_string())
+    }
+
+    #[test]
+    fn test_capture_query_treats_empty_as_none() {
+        // No query at all. With the bug, `query_pairs_mut()` left the
+        // URL with `query=Some("")` and we used to propagate that.
+        assert_eq!(
+            capture_query_like_connect("s3://bucket/prefix/"),
+            None,
+            "empty query after mutation must be treated as no query"
+        );
+
+        // Real query is propagated.
+        assert_eq!(
+            capture_query_like_connect("s3://bucket/prefix/?foo=bar"),
+            Some("foo=bar".to_string())
+        );
+
+        // lancedb-internal `engine=` is stripped; nothing remains, so
+        // query_string is None — not Some("").
+        assert_eq!(
+            capture_query_like_connect(&format!("s3://bucket/prefix/?{}=mem", ENGINE)),
+            None
+        );
+
+        // Mixed: drop `engine=`, keep the rest.
+        let captured =
+            capture_query_like_connect(&format!("s3://bucket/prefix/?{}=mem&foo=bar", ENGINE));
+        assert_eq!(captured.as_deref(), Some("foo=bar"));
+    }
+
+    /// Regression: connecting via a URL-style URI (which goes through
+    /// `url::Url::parse` and the `query_pairs_mut()` path) must not
+    /// append a trailing `?` to per-table URIs when the input URI has
+    /// no query string. Sub-path lookups against such a URI (e.g.
+    /// MemWAL `<table_uri>/_mem_wal/<shard>/<rand>_gen_<n>`) re-parse
+    /// as `path=<base table>` + `query=/_mem_wal/...`, causing
+    /// `Dataset::write` to find the base table dataset and falsely
+    /// report `Dataset already exists`.
+    ///
+    /// Skipped on Windows: `try_create_dir` does not understand
+    /// `file:///C:/…` paths so `connect_with_options` fails before
+    /// even reaching the URL-mutation logic. The pure URL-mutation
+    /// invariant is covered by
+    /// `test_capture_query_treats_empty_as_none` above, which runs
+    /// on all platforms.
+    #[cfg(not(windows))]
+    #[tokio::test]
+    async fn test_table_uri_url_path_has_no_trailing_question_mark() {
+        let tempdir = tempdir().unwrap();
+        let uri = format!("file://{}", tempdir.path().to_str().unwrap());
+
+        let request = ConnectRequest {
+            uri: uri.clone(),
+            #[cfg(feature = "remote")]
+            client_config: Default::default(),
+            options: Default::default(),
+            namespace_client_properties: Default::default(),
+            manifest_enabled: false,
+            read_consistency_interval: None,
+            session: None,
+        };
+        let db = ListingDatabase::connect_with_options(&request)
+            .await
+            .unwrap();
+
+        assert_eq!(
+            db.query_string, None,
+            "no input query → no captured query_string"
+        );
+
+        let table_uri = db.table_uri("test").unwrap();
+        assert!(
+            !table_uri.ends_with('?'),
+            "table_uri must not have a trailing `?`: {}",
+            table_uri
+        );
+        assert_eq!(table_uri, format!("{}/test.lance", uri));
+
+        // A real query string should still be propagated.
+        let with_query = format!("{}?foo=bar", uri);
+        let request_with_query = ConnectRequest {
+            uri: with_query,
+            #[cfg(feature = "remote")]
+            client_config: Default::default(),
+            options: Default::default(),
+            namespace_client_properties: Default::default(),
+            manifest_enabled: false,
+            read_consistency_interval: None,
+            session: None,
+        };
+        let db_with_query = ListingDatabase::connect_with_options(&request_with_query)
+            .await
+            .unwrap();
+        assert_eq!(db_with_query.query_string.as_deref(), Some("foo=bar"));
+        let table_uri = db_with_query.table_uri("test").unwrap();
+        assert_eq!(table_uri, format!("{}/test.lance?foo=bar", uri));
+    }
+
     #[tokio::test]
     async fn test_namespace_client() {
         let (_tempdir, db) = setup_database().await;

rust/lancedb/src/database/namespace.rs

@@ -11,6 +11,7 @@ use lance::io::commit::namespace_manifest::LanceNamespaceExternalManifestStore;
 use lance_io::object_store::{ObjectStoreParams, StorageOptionsAccessor};
 use lance_namespace::{
     LanceNamespace,
+    error::{ErrorCode, NamespaceError},
     models::{
         CreateNamespaceRequest, CreateNamespaceResponse, DeclareTableRequest,
         DescribeNamespaceRequest, DescribeNamespaceResponse, DescribeTableRequest,
@@ -29,7 +30,7 @@ use crate::database::listing::{
     OPT_NEW_TABLE_V2_MANIFEST_PATHS,
 };
 use crate::error::{Error, Result};
-use crate::table::NativeTable;
+use crate::table::{NativeTable, map_namespace_lance_error};
 use lance::dataset::WriteMode;
 
 use super::{
@@ -37,6 +38,19 @@ use super::{
     Database, OpenTableRequest, TableNamesRequest,
 };
 
+/// Returns true if the given `lance::Error` (anywhere in its source chain) is a
+/// `NamespaceError::TableAlreadyExists`.
+fn is_table_already_exists_namespace_error(err: &lance::Error) -> bool {
+    let mut current: Option<&(dyn std::error::Error + 'static)> = Some(err);
+    while let Some(e) = current {
+        if let Some(ns_err) = e.downcast_ref::<NamespaceError>() {
+            return ns_err.code() == ErrorCode::TableAlreadyExists;
+        }
+        current = e.source();
+    }
+    false
+}
+
 /// A database implementation that uses lance-namespace for table management
 pub struct LanceNamespaceDatabase {
     namespace: Arc<dyn LanceNamespace>,
@@ -356,13 +370,15 @@ impl Database for LanceNamespaceDatabase {
                         (loc, opts, response.managed_versioning)
                     }
                     Err(e)
-                        if matches!(request.mode, CreateTableMode::Create) && {
-                            let err_str = e.to_string();
-                            err_str.contains("already exists")
-                                || err_str.contains("TableAlreadyExists")
-                                || err_str.contains("table already exists")
-                        } =>
+                        if matches!(request.mode, CreateTableMode::Create)
+                            && is_table_already_exists_namespace_error(&e) =>
                     {
+                        // A declare conflict can either mean (a) the table was previously
+                        // *declared* but never written (in which case we should proceed and
+                        // create it), or (b) the table is fully realized (in which case the
+                        // user is creating something that already exists and we should
+                        // surface TableAlreadyExists). Disambiguate by describing the table
+                        // and checking whether it has both a version and a schema.
                         let response = self
                             .namespace
                             .describe_table(DescribeTableRequest {
@@ -370,11 +386,8 @@ impl Database for LanceNamespaceDatabase {
                                 ..Default::default()
                             })
                             .await
-                            .map_err(|describe_err| Error::Runtime {
-                                message: format!(
-                                    "Failed to describe existing declared table after declare conflict: {}",
-                                    describe_err
-                                ),
+                            .map_err(|describe_err| {
+                                map_namespace_lance_error(describe_err, &request.name)
                             })?;
 
                         if response.version.is_some() && response.schema.is_some() {
@@ -394,9 +407,7 @@ impl Database for LanceNamespaceDatabase {
                         (loc, opts, response.managed_versioning)
                     }
                     Err(e) => {
-                        return Err(Error::Runtime {
-                            message: format!("Failed to declare table: {}", e),
-                        });
+                        return Err(map_namespace_lance_error(e, &request.name));
                     }
                 }
             }
@@ -1086,8 +1097,120 @@ mod tests {
             .execute()
             .await;
 
-        // Verify: Should return an error
-        assert!(result.is_err());
+        // Verify: Should return TableNotFound — not a generic Runtime/internal error
+        // (regression test for ENT-1235: open_table on missing table previously surfaced as
+        // a generic 500/Runtime error rather than TableNotFound).
+        match result {
+            Err(Error::TableNotFound { name, .. }) => {
+                assert_eq!(name, "non_existent_table");
+            }
+            Err(other) => panic!("Expected TableNotFound, got: {:?}", other),
+            Ok(_) => panic!("Expected open_table to fail, but it succeeded"),
+        }
+    }
+
+    #[tokio::test]
+    async fn test_namespace_open_table_not_found_at_root() {
+        // Same as above, but at the root namespace (no parent namespace creation).
+        // Covers the common code path used by `db.open_table("foo")` without a namespace.
+        let tmp_dir = tempdir().unwrap();
+        let root_path = tmp_dir.path().to_str().unwrap().to_string();
+
+        let mut properties = HashMap::new();
+        properties.insert("root".to_string(), root_path);
+
+        let conn = connect_namespace("dir", properties)
+            .execute()
+            .await
+            .expect("Failed to connect to namespace");
+
+        let result = conn.open_table("missing_at_root").execute().await;
+
+        match result {
+            Err(Error::TableNotFound { name, .. }) => {
+                assert_eq!(name, "missing_at_root");
+            }
+            Err(other) => panic!("Expected TableNotFound, got: {:?}", other),
+            Ok(_) => panic!("Expected open_table to fail, but it succeeded"),
+        }
+    }
+
+    #[tokio::test]
+    async fn test_namespace_create_table_already_exists() {
+        // Regression test for ENT-1235: create_table on an existing table (in default
+        // Create mode) should return TableAlreadyExists, not a generic Runtime/500 error.
+        let tmp_dir = tempdir().unwrap();
+        let root_path = tmp_dir.path().to_str().unwrap().to_string();
+
+        let mut properties = HashMap::new();
+        properties.insert("root".to_string(), root_path);
+
+        let conn = connect_namespace("dir", properties)
+            .execute()
+            .await
+            .expect("Failed to connect to namespace");
+
+        conn.create_namespace(CreateNamespaceRequest {
+            id: Some(vec!["test_ns".into()]),
+            ..Default::default()
+        })
+        .await
+        .expect("Failed to create namespace");
+
+        // Create the table once.
+        conn.create_table("dup_table", create_test_data())
+            .namespace(vec!["test_ns".into()])
+            .execute()
+            .await
+            .expect("Failed to create table the first time");
+
+        // Try to create it again with the default Create mode.
+        let result = conn
+            .create_table("dup_table", create_test_data())
+            .namespace(vec!["test_ns".into()])
+            .execute()
+            .await;
+
+        match result {
+            Err(Error::TableAlreadyExists { name }) => {
+                assert_eq!(name, "dup_table");
+            }
+            Err(other) => panic!("Expected TableAlreadyExists, got: {:?}", other),
+            Ok(_) => panic!("Expected create_table to fail, but it succeeded"),
+        }
+    }
+
+    #[tokio::test]
+    async fn test_namespace_create_table_already_exists_at_root() {
+        // Same as above, but at the root namespace.
+        let tmp_dir = tempdir().unwrap();
+        let root_path = tmp_dir.path().to_str().unwrap().to_string();
+
+        let mut properties = HashMap::new();
+        properties.insert("root".to_string(), root_path);
+
+        let conn = connect_namespace("dir", properties)
+            .execute()
+            .await
+            .expect("Failed to connect to namespace");
+
+        conn.create_table("dup_root", create_test_data())
+            .execute()
+            .await
+            .expect("Failed to create table the first time");
+
+        let result = conn
+            .create_table("dup_root", create_test_data())
+            .execute()
+            .await;
+
+        match result {
+            Err(Error::TableAlreadyExists { name }) => {
+                assert_eq!(name, "dup_root");
+            }
+            Err(other) => panic!("Expected TableAlreadyExists, got: {:?}", other),
+            Ok(_) => panic!("Expected create_table to fail, but it succeeded"),
+        }
     }
 
     #[tokio::test]

rust/lancedb/src/expr.rs

@@ -138,4 +138,69 @@ mod tests {
         let sql = expr_to_sql_string(&expr).unwrap();
         assert!(sql.contains("price"));
     }
+
+    #[test]
+    fn test_binary_literal() {
+        use datafusion_common::ScalarValue;
+        let expr = lit(ScalarValue::Binary(Some(vec![0xde, 0xad, 0xbe, 0xef])));
+        let sql = expr_to_sql_string(&expr).unwrap();
+        assert_eq!(sql, "X'DEADBEEF'");
+    }
+
+    #[test]
+    fn test_binary_literal_in_filter() {
+        use datafusion_common::ScalarValue;
+        let expr = col("data").eq(lit(ScalarValue::Binary(Some(vec![0xca, 0xfe]))));
+        let sql = expr_to_sql_string(&expr).unwrap();
+        assert_eq!(sql, "(data = X'CAFE')");
+    }
+
+    #[test]
+    fn test_binary_literal_compound() {
+        use datafusion_common::ScalarValue;
+        let bin_expr = col("data").eq(lit(ScalarValue::Binary(Some(vec![0x01]))));
+        let int_expr = col("id").gt(lit(5i64));
+        let combined = bin_expr.and(int_expr);
+        let sql = expr_to_sql_string(&combined).unwrap();
+        assert_eq!(sql, "((data = X'01') AND (id > 5))");
+    }
+
+    #[test]
+    fn test_null_binary_literal() {
+        use datafusion_common::ScalarValue;
+        let expr = lit(ScalarValue::Binary(None));
+        let sql = expr_to_sql_string(&expr).unwrap();
+        assert_eq!(sql, "NULL");
+    }
+
+    #[test]
+    fn test_binary_literal_in_function_call() {
+        use datafusion_common::ScalarValue;
+        // Binary literals inside scalar function arguments must also be
+        // serialized correctly (regression test for placeholder rewrite path).
+        let expr = contains(col("data"), lit(ScalarValue::Binary(Some(vec![0xff]))));
+        let sql = expr_to_sql_string(&expr).unwrap();
+        assert_eq!(sql, "contains(data, X'FF')");
+    }
+
+    #[test]
+    fn test_binary_literal_in_negation() {
+        use datafusion_common::ScalarValue;
+        use std::ops::Not;
+        let expr = col("data")
+            .eq(lit(ScalarValue::Binary(Some(vec![0xab, 0xcd]))))
+            .not();
+        let sql = expr_to_sql_string(&expr).unwrap();
+        assert_eq!(sql, "NOT (data = X'ABCD')");
+    }
+
+    #[test]
+    fn test_multiple_binary_literals() {
+        use datafusion_common::ScalarValue;
+        let lhs = col("a").eq(lit(ScalarValue::Binary(Some(vec![0x01]))));
+        let rhs = col("b").eq(lit(ScalarValue::Binary(Some(vec![0x02, 0x03]))));
+        let expr = lhs.and(rhs);
+        let sql = expr_to_sql_string(&expr).unwrap();
+        assert_eq!(sql, "((a = X'01') AND (b = X'0203'))");
+    }
 }

rust/lancedb/src/expr/sql.rs

@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
+use datafusion_common::ScalarValue;
+use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
 use datafusion_expr::Expr;
 use datafusion_sql::unparser::{self, dialect::Dialect};
 
@@ -28,7 +30,36 @@ impl Dialect for LanceSqlDialect {
     }
 }
 
-pub fn expr_to_sql_string(expr: &Expr) -> crate::Result<String> {
+/// Prefix for placeholder strings inserted in place of binary literals.  Chosen
+/// to be extremely unlikely to occur in user data.
+const BINARY_PLACEHOLDER_PREFIX: &str = "__lancedb_binary_placeholder_";
+
+fn bytes_to_hex_sql(bytes: &[u8]) -> String {
+    let hex: String = bytes.iter().map(|b| format!("{b:02X}")).collect();
+    format!("X'{hex}'")
+}
+
+/// Returns true if *expr* contains a `Binary` or `LargeBinary` scalar literal
+/// anywhere in its subtree.  DataFusion's SQL unparser cannot serialize those
+/// variants, so we route such expressions through a placeholder-substitution
+/// path that emits SQL `X'...'` byte-string literals.
+fn has_binary_literal(expr: &Expr) -> bool {
+    let mut found = false;
+    let _ = expr.apply(&mut |e: &Expr| {
+        if matches!(
+            e,
+            Expr::Literal(ScalarValue::Binary(_) | ScalarValue::LargeBinary(_), _)
+        ) {
+            found = true;
+            Ok(TreeNodeRecursion::Stop)
+        } else {
+            Ok(TreeNodeRecursion::Continue)
+        }
+    });
+    found
+}
+
+fn run_unparser(expr: &Expr) -> crate::Result<String> {
     let ast = unparser::Unparser::new(&LanceSqlDialect)
         .expr_to_sql(expr)
         .map_err(|e| crate::Error::InvalidInput {
@@ -36,3 +67,49 @@ pub fn expr_to_sql_string(expr: &Expr) -> crate::Result<String> {
         })?;
     Ok(ast.to_string())
 }
+
+pub fn expr_to_sql_string(expr: &Expr) -> crate::Result<String> {
+    // Fast path: no binary literals — DataFusion's unparser handles everything.
+    if !has_binary_literal(expr) {
+        return run_unparser(expr);
+    }
+
+    // Slow path: DataFusion's unparser cannot serialize `Binary`/`LargeBinary`
+    // scalars, so we rewrite each one to a unique string-literal placeholder,
+    // let the unparser do the rest of the work, then substitute the SQL
+    // `X'...'` byte-string literal back in.  This keeps the operator/function
+    // serialization logic centralized in DataFusion and works for every
+    // expression node type the unparser supports.
+    let mut bindings: Vec<Vec<u8>> = Vec::new();
+    let rewritten = expr
+        .clone()
+        .transform(|e: Expr| match e {
+            Expr::Literal(ScalarValue::Binary(Some(bytes)), m)
+            | Expr::Literal(ScalarValue::LargeBinary(Some(bytes)), m) => {
+                let placeholder = format!("{}{}__", BINARY_PLACEHOLDER_PREFIX, bindings.len());
+                bindings.push(bytes);
+                Ok(Transformed::yes(Expr::Literal(
+                    ScalarValue::Utf8(Some(placeholder)),
+                    m,
+                )))
+            }
+            Expr::Literal(ScalarValue::Binary(None), m)
+            | Expr::Literal(ScalarValue::LargeBinary(None), m) => {
+                Ok(Transformed::yes(Expr::Literal(ScalarValue::Null, m)))
+            }
+            other => Ok(Transformed::no(other)),
+        })
+        .map_err(|e| crate::Error::InvalidInput {
+            message: format!("failed to rewrite expression: {}", e),
+        })?
+        .data;
+
+    let mut sql = run_unparser(&rewritten)?;
+    for (i, bytes) in bindings.iter().enumerate() {
+        // The unparser quotes string literals with single quotes, so the
+        // placeholder appears as `'__lancedb_binary_placeholder_<i>__'`.
+        let quoted = format!("'{}{}__'", BINARY_PLACEHOLDER_PREFIX, i);
+        sql = sql.replace(&quoted, &bytes_to_hex_sql(bytes));
+    }
+    Ok(sql)
+}

rust/lancedb/src/index.rs

@@ -13,7 +13,10 @@ use crate::{DistanceType, Error, Result, table::BaseTable};
 
 use self::{
     scalar::{BTreeIndexBuilder, BitmapIndexBuilder, LabelListIndexBuilder},
-    vector::{IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, IvfPqIndexBuilder, IvfSqIndexBuilder},
+    vector::{
+        IvfHnswFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, IvfPqIndexBuilder,
+        IvfSqIndexBuilder,
+    },
 };
 
 pub mod scalar;
@@ -67,6 +70,10 @@ pub enum Index {
     /// IVF-HNSW index with Scalar Quantization
     /// It is a variant of the HNSW algorithm that uses scalar quantization to compress the vectors.
     IvfHnswSq(IvfHnswSqIndexBuilder),
+
+    /// IVF-HNSW index without quantization.
+    /// Stores raw vectors, providing the highest recall at the cost of more memory and disk space.
+    IvfHnswFlat(IvfHnswFlatIndexBuilder),
 }
 
 /// Builder for the create_index operation
@@ -290,6 +297,8 @@ pub enum IndexType {
     IvfHnswPq,
     #[serde(alias = "IVF_HNSW_SQ")]
     IvfHnswSq,
+    #[serde(alias = "IVF_HNSW_FLAT")]
+    IvfHnswFlat,
     // Scalar
     #[serde(alias = "BTREE")]
     BTree,
@@ -311,6 +320,7 @@ impl std::fmt::Display for IndexType {
             Self::IvfRq => write!(f, "IVF_RQ"),
             Self::IvfHnswPq => write!(f, "IVF_HNSW_PQ"),
             Self::IvfHnswSq => write!(f, "IVF_HNSW_SQ"),
+            Self::IvfHnswFlat => write!(f, "IVF_HNSW_FLAT"),
             Self::BTree => write!(f, "BTREE"),
             Self::Bitmap => write!(f, "BITMAP"),
             Self::LabelList => write!(f, "LABEL_LIST"),
@@ -334,6 +344,7 @@ impl std::str::FromStr for IndexType {
             "IVF_RQ" => Ok(Self::IvfRq),
             "IVF_HNSW_PQ" => Ok(Self::IvfHnswPq),
             "IVF_HNSW_SQ" => Ok(Self::IvfHnswSq),
+            "IVF_HNSW_FLAT" => Ok(Self::IvfHnswFlat),
             _ => Err(Error::InvalidInput {
                 message: format!("the input value {} is not a valid IndexType", value),
             }),

rust/lancedb/src/index/vector.rs

@@ -474,3 +474,46 @@ impl IvfHnswSqIndexBuilder {
     impl_ivf_params_setter!();
     impl_hnsw_params_setter!();
 }
+
+/// Builder for an IVF_HNSW_FLAT index.
+///
+/// This index combines IVF partitioning with an HNSW graph per partition,
+/// storing raw (unquantized) vectors. It offers the highest recall among
+/// the IVF_HNSW family at the cost of more memory and disk space compared
+/// to [`IvfHnswSqIndexBuilder`] or [`IvfHnswPqIndexBuilder`].
+#[derive(Debug, Clone, Serialize)]
+pub struct IvfHnswFlatIndexBuilder {
+    // IVF
+    #[serde(rename = "metric_type")]
+    pub(crate) distance_type: DistanceType,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub(crate) num_partitions: Option<u32>,
+    pub(crate) sample_rate: u32,
+    pub(crate) max_iterations: u32,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub(crate) target_partition_size: Option<u32>,
+
+    // HNSW
+    pub(crate) m: u32,
+    pub(crate) ef_construction: u32,
+}
+
+impl Default for IvfHnswFlatIndexBuilder {
+    fn default() -> Self {
+        Self {
+            distance_type: DistanceType::L2,
+            num_partitions: None,
+            sample_rate: 256,
+            max_iterations: 50,
+            m: 20,
+            ef_construction: 300,
+            target_partition_size: None,
+        }
+    }
+}
+
+impl IvfHnswFlatIndexBuilder {
+    impl_distance_type_setter!();
+    impl_ivf_params_setter!();
+    impl_hnsw_params_setter!();
+}

rust/lancedb/src/remote/table.rs

@@ -1540,6 +1540,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
             Index::IvfPq(p) => ("IVF_PQ", Some(to_json(p)?)),
             Index::IvfSq(p) => ("IVF_SQ", Some(to_json(p)?)),
             Index::IvfHnswSq(p) => ("IVF_HNSW_SQ", Some(to_json(p)?)),
+            Index::IvfHnswFlat(p) => ("IVF_HNSW_FLAT", Some(to_json(p)?)),
             Index::IvfRq(p) => ("IVF_RQ", Some(to_json(p)?)),
             Index::BTree(p) => ("BTREE", Some(to_json(p)?)),
             Index::Bitmap(p) => ("BITMAP", Some(to_json(p)?)),
@@ -2068,7 +2069,8 @@ mod tests {
     use serde_json::json;
 
     use crate::index::vector::{
-        IvfFlatIndexBuilder, IvfHnswSqIndexBuilder, IvfRqIndexBuilder, IvfSqIndexBuilder,
+        IvfFlatIndexBuilder, IvfHnswFlatIndexBuilder, IvfHnswSqIndexBuilder, IvfRqIndexBuilder,
+        IvfSqIndexBuilder,
     };
     use crate::remote::JSON_CONTENT_TYPE;
     use crate::remote::db::DEFAULT_SERVER_VERSION;
@@ -3321,6 +3323,35 @@ mod tests {
                         .ef_construction(500),
                 ),
             ),
+            (
+                "IVF_HNSW_FLAT",
+                json!({
+                    "metric_type": "l2",
+                    "sample_rate": 256,
+                    "max_iterations": 50,
+                    "m": 20,
+                    "ef_construction": 300,
+                }),
+                Index::IvfHnswFlat(Default::default()),
+            ),
+            (
+                "IVF_HNSW_FLAT",
+                json!({
+                    "metric_type": "cosine",
+                    "num_partitions": 64,
+                    "sample_rate": 256,
+                    "max_iterations": 50,
+                    "m": 40,
+                    "ef_construction": 500,
+                }),
+                Index::IvfHnswFlat(
+                    IvfHnswFlatIndexBuilder::default()
+                        .distance_type(DistanceType::Cosine)
+                        .num_partitions(64)
+                        .num_edges(40)
+                        .ef_construction(500),
+                ),
+            ),
             (
                 "IVF_SQ",
                 json!({

rust/lancedb/src/table.rs

@@ -36,6 +36,7 @@ pub use query::AnyQuery;
 
 use lance::io::commit::namespace_manifest::LanceNamespaceExternalManifestStore;
 use lance_namespace::LanceNamespace;
+use lance_namespace::error::NamespaceError;
 use lance_namespace::models::DescribeTableRequest;
 use lance_table::format::Manifest;
 use lance_table::io::commit::CommitHandler;
@@ -94,6 +95,53 @@ pub use schema_evolution::{AddColumnsResult, AlterColumnsResult, DropColumnsResu
 use serde_with::skip_serializing_none;
 pub use update::{UpdateBuilder, UpdateResult};
 
+/// Walk a boxed error chain to find the innermost `NamespaceError`.
+///
+/// Callers like `DatasetBuilder::from_namespace` re-wrap their inner namespace error
+/// inside a fresh `lance::Error::Namespace`, so a single downcast at the top level
+/// won't find it. This walks `.source()` to unwrap arbitrarily nested layers.
+fn find_namespace_error<'a>(
+    err: &'a (dyn std::error::Error + 'static),
+) -> Option<&'a NamespaceError> {
+    let mut current: Option<&(dyn std::error::Error + 'static)> = Some(err);
+    while let Some(e) = current {
+        if let Some(ns_err) = e.downcast_ref::<NamespaceError>() {
+            return Some(ns_err);
+        }
+        current = e.source();
+    }
+    None
+}
+
+/// Map a `lance::Error` coming from a `lance-namespace` call into a `lancedb::Error`,
+/// preserving the fine-grained namespace error code (e.g. `TableNotFound`,
+/// `TableAlreadyExists`). Errors that aren't recognized namespace error variants fall
+/// through to a generic runtime error rather than `TableNotFound`/`TableAlreadyExists`.
+pub(crate) fn map_namespace_lance_error(err: lance::Error, table_name: &str) -> Error {
+    if let Some(code) = find_namespace_error(&err).map(NamespaceError::code) {
+        match code {
+            lance_namespace::error::ErrorCode::TableNotFound => {
+                return Error::TableNotFound {
+                    name: table_name.to_string(),
+                    source: Box::new(err),
+                };
+            }
+            lance_namespace::error::ErrorCode::TableAlreadyExists => {
+                return Error::TableAlreadyExists {
+                    name: table_name.to_string(),
+                };
+            }
+            _ => {}
+        }
+    }
+    match err {
+        lance::Error::Namespace { source, .. } => Error::Runtime {
+            message: format!("Namespace error: {}", source),
+        },
+        other => other.into(),
+    }
+}
+
 /// Defines the type of column
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum ColumnKind {
@@ -1494,12 +1542,7 @@ impl NativeTable {
         // and storage options from the namespace
         let builder = DatasetBuilder::from_namespace(namespace_client.clone(), table_id)
             .await
-            .map_err(|e| match e {
-                lance::Error::Namespace { source, .. } => Error::Runtime {
-                    message: format!("Failed to get table info from namespace: {:?}", source),
-                },
-                e => e.into(),
-            })?;
+            .map_err(|e| map_namespace_lance_error(e, name))?;
 
         let dataset = builder
             .with_read_params(params)
@@ -2033,6 +2076,24 @@ impl NativeTable {
                 );
                 Ok(Box::new(lance_idx_params))
             }
+            Index::IvfHnswFlat(index) => {
+                Self::validate_index_type(field, "IVF HNSW FLAT", supported_vector_data_type)?;
+                let ivf_params = Self::build_ivf_params(
+                    index.num_partitions,
+                    index.target_partition_size,
+                    index.sample_rate,
+                    index.max_iterations,
+                );
+                let hnsw_params = HnswBuildParams::default()
+                    .num_edges(index.m as usize)
+                    .ef_construction(index.ef_construction as usize);
+                let lance_idx_params = VectorIndexParams::ivf_hnsw(
+                    index.distance_type.into(),
+                    ivf_params,
+                    hnsw_params,
+                );
+                Ok(Box::new(lance_idx_params))
+            }
         }
     }
 
@@ -2058,7 +2119,8 @@ impl NativeTable {
             | Index::IvfPq(_)
             | Index::IvfRq(_)
             | Index::IvfHnswPq(_)
-            | Index::IvfHnswSq(_) => IndexType::Vector,
+            | Index::IvfHnswSq(_)
+            | Index::IvfHnswFlat(_) => IndexType::Vector,
         }
     }
 
@@ -2329,13 +2391,28 @@ impl BaseTable for NativeTable {
                 message: "Multi-column (composite) indices are not yet supported".to_string(),
             });
         }
-        let schema = self.schema().await?;
+        let dataset = self.dataset.get().await?;
+        let Some(field_path) = dataset.schema().resolve_case_insensitive(&opts.columns[0]) else {
+            return Err(Error::Schema {
+                message: format!(
+                    "Unable to get field named {:?}. Valid fields: {:?}",
+                    opts.columns[0],
+                    dataset.schema().field_paths()
+                ),
+            });
+        };
+        let field = (*field_path.last().expect("resolved field path is non-empty")).clone();
+        let names = field_path
+            .iter()
+            .map(|f| f.name.as_str())
+            .collect::<Vec<_>>();
+        let column = lance_core::datatypes::format_field_path(&names);
+        drop(dataset);
 
-        let field = schema.field_with_name(&opts.columns[0])?;
-
-        let lance_idx_params = self.make_index_params(field, opts.index.clone()).await?;
-        let index_type = self.get_index_type_for_field(field, &opts.index);
-        let columns = [field.name().as_str()];
+        let field = Field::from(&field);
+        let lance_idx_params = self.make_index_params(&field, opts.index.clone()).await?;
+        let index_type = self.get_index_type_for_field(&field, &opts.index);
+        let columns = [column.as_str()];
         self.dataset.ensure_mutable()?;
         let mut dataset = (*self.dataset.get().await?).clone();
         let mut builder = dataset
@@ -2481,11 +2558,11 @@ impl BaseTable for NativeTable {
 
             let mut columns = Vec::with_capacity(idx.fields.len());
             for field_id in &idx.fields {
-                let Some(field) = dataset.schema().field_by_id(*field_id) else {
+                let Ok(field_path) = dataset.schema().field_path(*field_id) else {
                     log::warn!("The index {} ({}) referenced a field with id {} which does not exist in the schema", idx.name, idx.uuid, field_id);
                     return None;
                 };
-                columns.push(field.name.clone());
+                columns.push(field_path);
             }
 
             let name = idx.name.clone();
@@ -2699,7 +2776,7 @@ mod tests {
 
     use arrow_array::{
         Array, BooleanArray, FixedSizeListArray, Int32Array, LargeStringArray, RecordBatch,
-        RecordBatchIterator, RecordBatchReader, StringArray,
+        RecordBatchIterator, RecordBatchReader, StringArray, StructArray,
         builder::{ListBuilder, StringBuilder},
     };
     use arrow_array::{BinaryArray, LargeBinaryArray};
@@ -3176,6 +3253,56 @@ mod tests {
         assert_eq!(stats.num_unindexed_rows, 0);
     }
 
+    #[tokio::test]
+    async fn test_create_index_ivf_hnsw_flat() {
+        use arrow_array::RecordBatch;
+        use arrow_schema::{DataType, Field, Schema as ArrowSchema};
+        use rand;
+        use std::iter::repeat_with;
+
+        use crate::index::vector::IvfHnswFlatIndexBuilder;
+        use arrow_array::Float32Array;
+
+        let tmp_dir = tempdir().unwrap();
+        let uri = tmp_dir.path().to_str().unwrap();
+        let conn = connect(uri).execute().await.unwrap();
+
+        let dimension = 16;
+        let schema = Arc::new(ArrowSchema::new(vec![Field::new(
+            "embeddings",
+            DataType::FixedSizeList(
+                Arc::new(Field::new("item", DataType::Float32, true)),
+                dimension,
+            ),
+            false,
+        )]));
+
+        let float_arr = Float32Array::from(
+            repeat_with(rand::random::<f32>)
+                .take(512 * dimension as usize)
+                .collect::<Vec<f32>>(),
+        );
+
+        let vectors = Arc::new(create_fixed_size_list(float_arr, dimension).unwrap());
+        let batch = RecordBatch::try_new(schema.clone(), vec![vectors.clone()]).unwrap();
+
+        let table = conn.create_table("test", batch).execute().await.unwrap();
+
+        let index = IvfHnswFlatIndexBuilder::default();
+        table
+            .create_index(&["embeddings"], Index::IvfHnswFlat(index))
+            .execute()
+            .await
+            .unwrap();
+
+        let index_configs = table.list_indices().await.unwrap();
+        assert_eq!(index_configs.len(), 1);
+        let index = index_configs.into_iter().next().unwrap();
+        assert_eq!(index.index_type, crate::index::IndexType::IvfHnswFlat);
+        assert_eq!(index.columns, vec!["embeddings".to_string()]);
+        assert_eq!(table.count_rows(None).await.unwrap(), 512);
+    }
+
     fn create_fixed_size_list<T: Array>(values: T, list_size: i32) -> Result<FixedSizeListArray> {
         let list_type = DataType::FixedSizeList(
             Arc::new(Field::new("item", values.data_type().clone(), true)),
@@ -3255,6 +3382,57 @@ mod tests {
         assert_eq!(stats.num_unindexed_rows, 0);
     }
 
+    #[tokio::test]
+    async fn test_create_scalar_index_on_nested_field() {
+        let tmp_dir = tempdir().unwrap();
+        let uri = tmp_dir.path().to_str().unwrap();
+
+        let metadata_fields = vec![Arc::new(Field::new("user_id", DataType::Int32, false))];
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new(
+                "metadata",
+                DataType::Struct(metadata_fields.clone().into()),
+                false,
+            ),
+        ]));
+        let metadata = StructArray::new(
+            metadata_fields.into(),
+            vec![Arc::new(Int32Array::from_iter_values(0..10))],
+            None,
+        );
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..10)),
+                Arc::new(metadata),
+            ],
+        )
+        .unwrap();
+        let conn = ConnectBuilder::new(uri).execute().await.unwrap();
+        let table = conn
+            .create_table("my_table", batch)
+            .execute()
+            .await
+            .unwrap();
+
+        table
+            .create_index(
+                &["metadata.user_id"],
+                Index::BTree(BTreeIndexBuilder::default()),
+            )
+            .execute()
+            .await
+            .unwrap();
+
+        let index_configs = table.list_indices().await.unwrap();
+        assert_eq!(index_configs.len(), 1);
+        assert_eq!(
+            index_configs[0].columns,
+            vec!["metadata.user_id".to_string()]
+        );
+    }
+
     #[tokio::test]
     async fn test_create_bitmap_index() {
         let tmp_dir = tempdir().unwrap();