Bump version: 0.4.13 → 0.4.14

This will start publishing `@lancedb/lancedb` with the new nodejs
package on our releases.

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.4
+current_version = 0.4.14
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True
@@ -9,4 +9,4 @@ tag_name = v{new_version}
 
 [bumpversion:file:rust/ffi/node/Cargo.toml]
 
-[bumpversion:file:rust/vectordb/Cargo.toml]
+[bumpversion:file:rust/lancedb/Cargo.toml]

.cargo/config.toml

@@ -0,0 +1,40 @@
+[profile.release]
+lto = "fat"
+codegen-units = 1
+
+[profile.release-with-debug]
+inherits = "release"
+debug = true
+# Prioritize compile time over runtime performance
+codegen-units = 16
+lto = "thin"
+
+[target.'cfg(all())']
+rustflags = [
+    "-Wclippy::all",
+    "-Wclippy::style",
+    "-Wclippy::fallible_impl_from",
+    "-Wclippy::manual_let_else",
+    "-Wclippy::redundant_pub_crate",
+    "-Wclippy::string_add_assign",
+    "-Wclippy::string_add",
+    "-Wclippy::string_lit_as_bytes",
+    "-Wclippy::string_to_string",
+    "-Wclippy::use_self",
+    "-Dclippy::cargo",
+    "-Dclippy::dbg_macro",
+    # not too much we can do to avoid multiple crate versions
+    "-Aclippy::multiple-crate-versions",
+    "-Aclippy::wildcard_dependencies",
+]
+
+[target.x86_64-unknown-linux-gnu]
+rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"]
+
+[target.aarch64-apple-darwin]
+rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm,+dotprod"]
+
+# Not all Windows systems have the C runtime installed, so this avoids library
+# not found errors on systems that are missing it.
+[target.x86_64-pc-windows-msvc]
+rustflags = ["-Ctarget-feature=+crt-static"]

.github/workflows/build_linux_wheel/action.yml

@@ -0,0 +1,58 @@
+# We create a composite action to be re-used both for testing and for releasing
+name: build-linux-wheel
+description: "Build a manylinux wheel for lance"
+inputs:
+  python-minor-version:
+    description: "8, 9, 10, 11, 12"
+    required: true
+  args:
+    description: "--release"
+    required: false
+    default: ""
+  arm-build:
+    description: "Build for arm64 instead of x86_64"
+    # Note: this does *not* mean the host is arm64, since we might be cross-compiling.
+    required: false
+    default: "false"
+runs:
+  using: "composite"
+  steps:
+    - name: CONFIRM ARM BUILD
+      shell: bash
+      run: |
+        echo "ARM BUILD: ${{ inputs.arm-build }}"
+    - name: Build x86_64 Manylinux wheel
+      if: ${{ inputs.arm-build == 'false' }}
+      uses: PyO3/maturin-action@v1
+      with:
+        command: build
+        working-directory: python
+        target: x86_64-unknown-linux-gnu
+        manylinux: "2_17"
+        args: ${{ inputs.args }}
+        before-script-linux: |
+          set -e
+          yum install -y openssl-devel \
+            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
+            && unzip /tmp/protoc.zip -d /usr/local \
+            && rm /tmp/protoc.zip
+    - name: Build Arm Manylinux Wheel
+      if: ${{ inputs.arm-build == 'true' }}
+      uses: PyO3/maturin-action@v1
+      with:
+        command: build
+        working-directory: python
+        target: aarch64-unknown-linux-gnu
+        manylinux: "2_24"
+        args: ${{ inputs.args }}
+        before-script-linux: |
+          set -e
+          apt install -y unzip
+          if [ $(uname -m) = "x86_64" ]; then
+            PROTOC_ARCH="x86_64"
+          else
+            PROTOC_ARCH="aarch_64"
+          fi
+          curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$PROTOC_ARCH.zip > /tmp/protoc.zip \
+            && unzip /tmp/protoc.zip -d /usr/local \
+            && rm /tmp/protoc.zip

.github/workflows/build_mac_wheel/action.yml

@@ -0,0 +1,25 @@
+# We create a composite action to be re-used both for testing and for releasing
+name: build_wheel
+description: "Build a lance wheel"
+inputs:
+  python-minor-version:
+    description: "8, 9, 10, 11"
+    required: true
+  args:
+    description: "--release"
+    required: false
+    default: ""
+runs:
+  using: "composite"
+  steps:
+    - name: Install macos dependency
+      shell: bash
+      run: |
+        brew install protobuf
+    - name: Build wheel
+      uses: PyO3/maturin-action@v1
+      with:
+        command: build
+        args: ${{ inputs.args }}
+        working-directory: python
+        interpreter: 3.${{ inputs.python-minor-version }}

.github/workflows/build_windows_wheel/action.yml

@@ -0,0 +1,33 @@
+# We create a composite action to be re-used both for testing and for releasing
+name: build_wheel
+description: "Build a lance wheel"
+inputs:
+  python-minor-version:
+    description: "8, 9, 10, 11"
+    required: true
+  args:
+    description: "--release"
+    required: false
+    default: ""
+runs:
+  using: "composite"
+  steps:
+    - name: Install Protoc v21.12
+      working-directory: C:\
+      run: |
+        New-Item -Path 'C:\protoc' -ItemType Directory
+        Set-Location C:\protoc
+        Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+        7z x protoc.zip
+        Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+      shell: powershell
+    - name: Build wheel
+      uses: PyO3/maturin-action@v1
+      with:
+        command: build
+        args: ${{ inputs.args }}
+        working-directory: python
+    - uses: actions/upload-artifact@v3
+      with:
+        name: windows-wheels
+        path: python\target\wheels

.github/workflows/cargo-publish.yml

@@ -16,7 +16,7 @@ jobs:
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: rust
@@ -26,4 +26,4 @@ jobs:
           sudo apt install -y protobuf-compiler libssl-dev
       - name: Publish the package
         run: |
-          cargo publish -p vectordb --all-features --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          cargo publish -p lancedb --all-features --token ${{ secrets.CARGO_REGISTRY_TOKEN }}

.github/workflows/docs.yml

@@ -24,12 +24,16 @@ jobs:
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-22.04
+    runs-on: buildjet-8vcpu-ubuntu-2204
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+      - name: Install dependecies needed for ubuntu
+        run: |
+          sudo apt install -y protobuf-compiler libssl-dev
+          rustup update && rustup default        
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.10"
           cache: "pip"
@@ -42,7 +46,7 @@ jobs:
       - name: Set up node
         uses: actions/setup-node@v3
         with:
-          node-version: ${{ matrix.node-version }}
+          node-version: 20
           cache: 'npm'
           cache-dependency-path: node/package-lock.json
       - uses: Swatinem/rust-cache@v2
@@ -62,8 +66,9 @@ jobs:
         run: |
           npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
       - name: Build docs
+        working-directory: docs
         run: |
-          PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
+          PYTHONPATH=. mkdocs build
       - name: Setup Pages
         uses: actions/configure-pages@v2
       - name: Upload artifact
@@ -72,4 +77,4 @@ jobs:
           path: "docs/site"
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v1
+        uses: actions/deploy-pages@v1

.github/workflows/docs_test.yml

@@ -18,26 +18,28 @@ on:
 env:
   # Disable full debug symbol generation to speed up CI build and keep memory down
   # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1"
+  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=native -C target-feature=+f16c,+avx2,+fma"
   RUST_BACKTRACE: "1"
 
 jobs:
   test-python:
     name: Test doc python code
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        python-minor-version: [ "11" ]
-        os: ["ubuntu-22.04"]
+    runs-on: "buildjet-8vcpu-ubuntu-2204"
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
+    - name: Install dependecies needed for ubuntu
+      run: |
+        sudo apt install -y protobuf-compiler libssl-dev
+        rustup update && rustup default
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
-        python-version: 3.${{ matrix.python-minor-version }}
+        python-version: 3.11
         cache: "pip"
         cache-dependency-path: "docs/test/requirements.txt"
+    - name: Rust cache
+      uses: swatinem/rust-cache@v2        
     - name: Build Python
       working-directory: docs/test
       run:
@@ -52,45 +54,43 @@ jobs:
         for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
   test-node:
     name: Test doc nodejs code
-    runs-on: ${{ matrix.os }}
+    runs-on: "buildjet-8vcpu-ubuntu-2204"
+    timeout-minutes: 60
     strategy:
-      matrix:
-        node-version: [ "18" ]
-        os: ["ubuntu-22.04"]
+      fail-fast: false
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
         lfs: true
     - name: Set up Node
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
-        node-version: ${{ matrix.node-version }}
+        node-version: 20
     - name: Install dependecies needed for ubuntu
-      if: ${{ matrix.os == 'ubuntu-22.04' }}
       run: |
         sudo apt install -y protobuf-compiler libssl-dev
-    - name: Install node dependencies
-      run: |
-        cd docs/test
-        npm install 
+        rustup update && rustup default
     - name: Rust cache
       uses: swatinem/rust-cache@v2
-    - name: Install LanceDB
+    - name: Install node dependencies
       run: |
-        cd docs/test/node_modules/vectordb 
+        sudo swapoff -a
+        sudo fallocate -l 8G /swapfile
+        sudo chmod 600 /swapfile
+        sudo mkswap /swapfile
+        sudo swapon /swapfile
+        sudo swapon --show
+        cd node
         npm ci
         npm run build-release
-        npm run tsc
-    - name: Create test files
-      run: |
-        cd docs/test
-        node md_testing.js
+        cd ../docs
+        npm install
     - name: Test
       env:
         LANCEDB_URI: ${{ secrets.LANCEDB_URI }}
         LANCEDB_DEV_API_KEY: ${{ secrets.LANCEDB_DEV_API_KEY }}
       run: |
-        cd docs/test/node
-        for d in *; do cd "$d"; echo "$d".js; node "$d".js; cd ..; done
+        cd docs
+        npm t

.github/workflows/make-release-commit.yml

@@ -26,7 +26,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out main
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: main
           persist-credentials: false
@@ -37,10 +37,10 @@ jobs:
         run: |
           git config user.name 'Lance Release'
           git config user.email 'lance-dev@lancedb.com'
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v4
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - name: Bump version, create tag and commit
         run: |
           pip install bump2version

.github/workflows/node.yml

@@ -24,27 +24,6 @@ env:
   RUST_BACKTRACE: "1"
 
 jobs:
-  lint:
-    name: Lint
-    runs-on: ubuntu-22.04
-    defaults:
-      run:
-        shell: bash
-        working-directory: node
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        lfs: true
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 20
-        cache: 'npm'
-        cache-dependency-path: node/package-lock.json
-    - name: Lint
-      run: |
-        npm ci
-        npm run lint
   linux:
     name: Linux (Node ${{ matrix.node-version }})
     timeout-minutes: 30
@@ -57,7 +36,7 @@ jobs:
         shell: bash
         working-directory: node
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
         lfs: true
@@ -89,7 +68,7 @@ jobs:
         shell: bash
         working-directory: node
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
         lfs: true
@@ -128,7 +107,7 @@ jobs:
       # this one is for dynamodb
       DYNAMODB_ENDPOINT: http://localhost:4566
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
         lfs: true

.github/workflows/nodejs.yml

@@ -29,7 +29,7 @@ jobs:
         shell: bash
         working-directory: nodejs
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
         lfs: true
@@ -49,6 +49,7 @@ jobs:
         cargo clippy --all --all-features -- -D warnings
         npm ci
         npm run lint
+        npm run chkformat
   linux:
     name: Linux (NodeJS ${{ matrix.node-version }})
     timeout-minutes: 30
@@ -61,7 +62,7 @@ jobs:
         shell: bash
         working-directory: nodejs
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
         lfs: true
@@ -84,13 +85,13 @@ jobs:
       run: npm run test
   macos:
     timeout-minutes: 30
-    runs-on: "macos-13"
+    runs-on: "macos-14"
     defaults:
       run:
         shell: bash
         working-directory: nodejs
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
         lfs: true
@@ -111,4 +112,3 @@ jobs:
     - name: Test
       run: |
         npm run test
-

.github/workflows/npm-publish.yml

@@ -2,7 +2,7 @@ name: NPM Publish
 
 on:
   release:
-    types: [ published ]
+    types: [published]
 
 jobs:
   node:
@@ -15,11 +15,11 @@ jobs:
         working-directory: node
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - uses: actions/setup-node@v3
         with:
           node-version: 20
-          cache: 'npm'
+          cache: "npm"
           cache-dependency-path: node/package-lock.json
       - name: Install dependencies
         run: |
@@ -31,7 +31,7 @@ jobs:
           npm run tsc
           npm pack
       - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: node-package
           path: |
@@ -45,13 +45,13 @@ jobs:
             runner: macos-13
           - arch: aarch64-apple-darwin
             # xlarge is implicitly arm64.
-            runner: macos-13-xlarge
+            runner: macos-14
     runs-on: ${{ matrix.config.runner }}
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install system dependencies
         run: brew install protobuf
       - name: Install npm dependencies
@@ -61,12 +61,41 @@ jobs:
       - name: Build MacOS native node modules
         run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }}
       - name: Upload Darwin Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: native-darwin
+          name: node-native-darwin
           path: |
             node/dist/lancedb-vectordb-darwin*.tgz
-         
+
+  nodejs-macos:
+    strategy:
+      matrix:
+        config:
+          - arch: x86_64-apple-darwin
+            runner: macos-13
+          - arch: aarch64-apple-darwin
+            # xlarge is implicitly arm64.
+            runner: macos-14
+    runs-on: ${{ matrix.config.runner }}
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install system dependencies
+        run: brew install protobuf
+      - name: Install npm dependencies
+        run: |
+          cd nodejs
+          npm ci
+      - name: Build MacOS native nodejs modules
+        run: bash ci/build_macos_artifacts_nodejs.sh ${{ matrix.config.arch }}
+      - name: Upload Darwin Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-native-darwin-${{ matrix.config.arch }}
+          path: |
+            nodejs/dist/*.node
 
   node-linux:
     name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu
@@ -80,20 +109,86 @@ jobs:
           - arch: x86_64
             runner: ubuntu-latest
           - arch: aarch64
-            runner: buildjet-4vcpu-ubuntu-2204-arm
+            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
+            runner: buildjet-16vcpu-ubuntu-2204-arm
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+      # Buildjet aarch64 runners have only 1.5 GB RAM per core, vs 3.5 GB per core for
+      # x86_64 runners. To avoid OOM errors on ARM, we create a swap file.
+      - name: Configure aarch64 build
+        if: ${{ matrix.config.arch == 'aarch64' }}
+        run: |
+          free -h
+          sudo fallocate -l 16G /swapfile
+          sudo chmod 600 /swapfile
+          sudo mkswap /swapfile
+          sudo swapon /swapfile
+          echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab
+          # print info
+          swapon --show
+          free -h
       - name: Build Linux Artifacts
         run: |
           bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}
       - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: native-linux
+          name: node-native-linux
           path: |
             node/dist/lancedb-vectordb-linux*.tgz
 
+  nodejs-linux:
+    name: nodejs-linux (${{ matrix.config.arch}}-unknown-linux-gnu
+    runs-on: ${{ matrix.config.runner }}
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - arch: x86_64
+            runner: ubuntu-latest
+          - arch: aarch64
+            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
+            runner: buildjet-16vcpu-ubuntu-2204-arm
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      # Buildjet aarch64 runners have only 1.5 GB RAM per core, vs 3.5 GB per core for
+      # x86_64 runners. To avoid OOM errors on ARM, we create a swap file.
+      - name: Configure aarch64 build
+        if: ${{ matrix.config.arch == 'aarch64' }}
+        run: |
+          free -h
+          sudo fallocate -l 16G /swapfile
+          sudo chmod 600 /swapfile
+          sudo mkswap /swapfile
+          sudo swapon /swapfile
+          echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab
+          # print info
+          swapon --show
+          free -h
+      - name: Build Linux Artifacts
+        run: |
+          bash ci/build_linux_artifacts_nodejs.sh ${{ matrix.config.arch }}
+      - name: Upload Linux Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-native-linux-${{ matrix.config.arch }}
+          path: |
+            nodejs/dist/*.node
+      # The generic files are the same in all distros so we just pick
+      # one to do the upload.
+      - name: Upload Generic Artifacts
+        if: ${{ matrix.config.arch == 'x86_64' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-dist
+          path: |
+            nodejs/dist/*
+            !nodejs/dist/*.node
+
   node-windows:
     runs-on: windows-2022
     # Only runs on tags that matches the make-release action
@@ -104,7 +199,7 @@ jobs:
         target: [x86_64-pc-windows-msvc]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install Protoc v21.12
         working-directory: C:\
         run: |
@@ -121,25 +216,60 @@ jobs:
       - name: Build Windows native node modules
         run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }}
       - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: native-windows
+          name: node-native-windows
           path: |
             node/dist/lancedb-vectordb-win32*.tgz
 
+  nodejs-windows:
+    runs-on: windows-2022
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64-pc-windows-msvc]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install Protoc v21.12
+        working-directory: C:\
+        run: |
+          New-Item -Path 'C:\protoc' -ItemType Directory
+          Set-Location C:\protoc
+          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+          7z x protoc.zip
+          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+        shell: powershell
+      - name: Install npm dependencies
+        run: |
+          cd nodejs
+          npm ci
+      - name: Build Windows native node modules
+        run: .\ci\build_windows_artifacts_nodejs.ps1 ${{ matrix.target }}
+      - name: Upload Windows Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-native-windows
+          path: |
+            nodejs/dist/*.node
+
   release:
     needs: [node, node-macos, node-linux, node-windows]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: node-*
       - name: Display structure of downloaded files
         run: ls -R
       - uses: actions/setup-node@v3
         with:
           node-version: 20
-          registry-url: 'https://registry.npmjs.org'
+          registry-url: "https://registry.npmjs.org"
       - name: Publish to NPM
         env:
           NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
@@ -149,12 +279,51 @@ jobs:
             npm publish $filename
           done
 
+  release-nodejs:
+    needs: [nodejs-macos, nodejs-linux, nodejs-windows]
+    runs-on: ubuntu-latest
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    defaults:
+      run:
+        shell: bash
+        working-directory: nodejs
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: nodejs-dist
+          path: nodejs/dist
+      - uses: actions/download-artifact@v4
+        name: Download arch-specific binaries
+        with:
+          pattern: nodejs-*
+          path: nodejs/nodejs-artifacts
+          merge-multiple: true
+      - name: Display structure of downloaded files
+        run: find .
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          registry-url: "https://registry.npmjs.org"
+      - name: Install napi-rs
+        run: npm install -g @napi-rs/cli
+      - name: Prepare artifacts
+        run: npx napi artifacts -d nodejs-artifacts
+      - name: Display structure of staged files
+        run: find npm
+      - name: Publish to NPM
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
+        run: npm publish --access public
+
   update-package-lock:
     needs: [release]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: main
           persist-credentials: false
@@ -163,3 +332,18 @@ jobs:
       - uses: ./.github/workflows/update_package_lock
         with:
           github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+
+  update-package-lock-nodejs:
+    needs: [release-nodejs]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          persist-credentials: false
+          fetch-depth: 0
+          lfs: true
+      - uses: ./.github/workflows/update_package_lock_nodejs
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.github/workflows/pypi-publish.yml

@@ -2,30 +2,91 @@ name: PyPI Publish
 
 on:
   release:
-    types: [ published ]
+    types: [published]
 
 jobs:
-  publish:
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the python-make-release action
-    if: startsWith(github.ref, 'refs/tags/python-v')
-    defaults:
-      run:
-        shell: bash
-        working-directory: python
+  linux:
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        python-minor-version: ["8"]
+        platform:
+          - x86_64
+          - aarch64
+    runs-on: "ubuntu-22.04"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
-      - name: Build distribution
-        run: |
-          ls -la
-          pip install wheel setuptools --upgrade
-          python setup.py sdist bdist_wheel
-      - name: Publish
-        uses: pypa/gh-action-pypi-publish@v1.8.5
+          python-version: 3.${{ matrix.python-minor-version }}
+      - uses: ./.github/workflows/build_linux_wheel
         with:
-          password: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
-          packages-dir: python/dist
+          python-minor-version: ${{ matrix.python-minor-version }}
+          args: "--release --strip"
+          arm-build: ${{ matrix.platform == 'aarch64' }}
+      - uses: ./.github/workflows/upload_wheel
+        with:
+          token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
+          repo: "pypi"
+  mac:
+    timeout-minutes: 60
+    runs-on: ${{ matrix.config.runner }}
+    strategy:
+      matrix:
+        python-minor-version: ["8"]
+        config:
+          - target: x86_64-apple-darwin
+            runner: macos-13
+          - target: aarch64-apple-darwin
+            runner: macos-14
+    env:
+      MACOSX_DEPLOYMENT_TARGET: 10.15
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.12
+      - uses: ./.github/workflows/build_mac_wheel
+        with:
+          python-minor-version: ${{ matrix.python-minor-version }}
+          args: "--release --strip --target ${{ matrix.config.target }}"
+      - uses: ./.github/workflows/upload_wheel
+        with:
+          python-minor-version: ${{ matrix.python-minor-version }}
+          token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
+          repo: "pypi"
+  windows:
+    timeout-minutes: 60
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        python-minor-version: ["8"]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.${{ matrix.python-minor-version }}
+      - uses: ./.github/workflows/build_windows_wheel
+        with:
+          python-minor-version: ${{ matrix.python-minor-version }}
+          args: "--release --strip"
+          vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }}
+      - uses: ./.github/workflows/upload_wheel
+        with:
+          python-minor-version: ${{ matrix.python-minor-version }}
+          token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
+          repo: "pypi"

.github/workflows/python-make-release-commit.yml

@@ -26,7 +26,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out main
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         ref: main
         persist-credentials: false
@@ -37,10 +37,10 @@ jobs:
       run: |
         git config user.name 'Lance Release'
         git config user.email 'lance-dev@lancedb.com'
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
       with:
-        python-version: "3.10"
+        python-version: "3.11"
     - name: Bump version, create tag and commit
       working-directory: python
       run: |

.github/workflows/python.yml

@@ -14,49 +14,133 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  linux:
+  lint:
+    name: "Lint"
     timeout-minutes: 30
-    strategy:
-      matrix:
-        python-minor-version: [ "8", "9", "10", "11" ]
     runs-on: "ubuntu-22.04"
     defaults:
       run:
         shell: bash
         working-directory: python
     steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.${{ matrix.python-minor-version }}
-    - name: Install lancedb
-      run: |
-        pip install -e .[tests]
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock ruff
-    - name: Format check
-      run: ruff format --check .
-    - name: Lint
-      run: ruff .
-    - name: Run tests
-      run: pytest -m "not slow" -x -v --durations=30 tests
-    - name: doctest
-      run: pytest --doctest-modules lancedb
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install ruff
+        run: |
+          pip install ruff==0.2.2
+      - name: Format check
+        run: ruff format --check .
+      - name: Lint
+        run: ruff .
+  doctest:
+    name: "Doctest"
+    timeout-minutes: 30
+    runs-on: "ubuntu-22.04"
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: "pip"
+      - name: Install protobuf
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+      - name: Install
+        run: |
+          pip install -e .[tests,dev,embeddings]
+          pip install tantivy
+          pip install mlx
+      - name: Doctest
+        run: pytest --doctest-modules python/lancedb
+  linux:
+    name: "Linux: python-3.${{ matrix.python-minor-version }}"
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        python-minor-version: ["8", "11"]
+    runs-on: "ubuntu-22.04"
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Install protobuf
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.${{ matrix.python-minor-version }}
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+      - uses: ./.github/workflows/build_linux_wheel
+      - uses: ./.github/workflows/run_tests
+      # Make sure wheels are not included in the Rust cache
+      - name: Delete wheels
+        run: rm -rf target/wheels
   platform:
-    name: "Platform: ${{ matrix.config.name }}"
+    name: "Mac: ${{ matrix.config.name }}"
     timeout-minutes: 30
     strategy:
       matrix:
         config:
-          - name: x86 Mac
+          - name: x86
             runner: macos-13
-          - name: Arm Mac
-            runner: macos-13-xlarge
-          - name: x86 Windows
+          - name: Arm
+            runner: macos-14
+    runs-on: "${{ matrix.config.runner }}"
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+      - uses: ./.github/workflows/build_mac_wheel
+      - uses: ./.github/workflows/run_tests
+      # Make sure wheels are not included in the Rust cache
+      - name: Delete wheels
+        run: rm -rf target/wheels
+  windows:
+    name: "Windows: ${{ matrix.config.name }}"
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        config:
+          - name: x86
             runner: windows-latest
     runs-on: "${{ matrix.config.runner }}"
     defaults:
@@ -64,21 +148,22 @@ jobs:
         shell: bash
         working-directory: python
     steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.11"
-    - name: Install lancedb
-      run: |
-        pip install -e .[tests]
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock
-    - name: Run tests
-      run: pytest -m "not slow" -x -v --durations=30 tests
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+      - uses: ./.github/workflows/build_windows_wheel
+      - uses: ./.github/workflows/run_tests
+      # Make sure wheels are not included in the Rust cache
+      - name: Delete wheels
+        run: rm -rf target/wheels
   pydantic1x:
     timeout-minutes: 30
     runs-on: "ubuntu-22.04"
@@ -87,21 +172,22 @@ jobs:
         shell: bash
         working-directory: python
     steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.9
-    - name: Install lancedb
-      run: |
-        pip install "pydantic<2"
-        pip install -e .[tests]
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock
-    - name: Run tests
-      run: pytest -m "not slow" -x -v --durations=30 tests
-    - name: doctest
-      run: pytest --doctest-modules lancedb
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+      - name: Install lancedb
+        run: |
+          pip install "pydantic<2"
+          pip install -e .[tests]
+          pip install tantivy
+      - name: Run tests
+        run: pytest -m "not slow" -x -v --durations=30 python/tests

.github/workflows/run_tests/action.yml

@@ -0,0 +1,17 @@
+name: run-tests
+
+description: "Install lance wheel and run unit tests"
+inputs:
+  python-minor-version:
+    required: true
+    description: "8 9 10 11 12"
+runs:
+  using: "composite"
+  steps:
+    - name: Install lancedb
+      shell: bash
+      run: |
+        pip3 install $(ls target/wheels/lancedb-*.whl)[tests,dev]
+    - name: pytest
+      shell: bash
+      run: pytest -m "not slow" -x -v --durations=30 python/python/tests

.github/workflows/rust.yml

@@ -32,7 +32,7 @@ jobs:
         shell: bash
         working-directory: rust
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
           fetch-depth: 0
           lfs: true
@@ -55,7 +55,7 @@ jobs:
         shell: bash
         working-directory: rust
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
           fetch-depth: 0
           lfs: true
@@ -70,18 +70,20 @@ jobs:
       run: cargo build --all-features
     - name: Run tests
       run: cargo test --all-features
+    - name: Run examples
+      run: cargo run --example simple
   macos:
     timeout-minutes: 30
     strategy:
       matrix:
-        mac-runner: [ "macos-13", "macos-13-xlarge" ]
+        mac-runner: [ "macos-13", "macos-14" ]
     runs-on: "${{ matrix.mac-runner }}"
     defaults:
       run:
         shell: bash
         working-directory: rust
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
             fetch-depth: 0
             lfs: true
@@ -99,7 +101,7 @@ jobs:
   windows:
     runs-on: windows-2022
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: rust
@@ -117,3 +119,4 @@ jobs:
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
           cargo build
           cargo test
+  

.github/workflows/update_package_lock_nodejs/action.yml

@@ -0,0 +1,33 @@
+name: update_package_lock_nodejs
+description: "Update nodejs's package.lock"
+
+inputs:
+  github_token:
+    required: true
+    description: "github token for the repo"
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 20
+    - name: Set git configs
+      shell: bash
+      run: |
+        git config user.name 'Lance Release'
+        git config user.email 'lance-dev@lancedb.com'
+    - name: Update package-lock.json file
+      working-directory: ./nodejs
+      run: |
+        npm install
+        git add package-lock.json
+        git commit -m "Updating package-lock.json"
+      shell: bash
+    - name: Push changes
+      if: ${{ inputs.dry_run }} == "false"
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ inputs.github_token }}
+        branch: main
+        tags: true

.github/workflows/update_package_lock_run.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: main
           persist-credentials: false

.github/workflows/update_package_lock_run_nodejs.yml

@@ -0,0 +1,19 @@
+name: Update NodeJs package-lock.json
+
+on:
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          persist-credentials: false
+          fetch-depth: 0
+          lfs: true
+      - uses: ./.github/workflows/update_package_lock_nodejs
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.github/workflows/upload_wheel/action.yml

@@ -0,0 +1,29 @@
+name: upload-wheel
+
+description: "Upload wheels to Pypi"
+inputs:
+  os:
+    required: true
+    description: "ubuntu-22.04 or macos-13"
+  repo:
+    required: false
+    description: "pypi or testpypi"
+    default: "pypi"
+  token:
+    required: true
+    description: "release token for the repo"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install dependencies
+      shell: bash
+      run: |
+        python -m pip install --upgrade pip
+        pip install twine
+    - name: Publish wheel
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ inputs.token }}
+      shell: bash
+      run: twine upload --repository ${{ inputs.repo }} target/wheels/lancedb-*.whl

.gitignore

@@ -22,6 +22,11 @@ python/dist
 
 **/.hypothesis
 
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
 ## Javascript
 *.node
 **/node_modules
@@ -29,9 +34,12 @@ python/dist
 node/dist
 node/examples/**/package-lock.json
 node/examples/**/dist
+nodejs/lancedb/native*
 dist
 
 ## Rust
 target
 
+**/sccache.log
+
 Cargo.lock

.pre-commit-config.yaml

@@ -5,17 +5,14 @@ repos:
     -   id: check-yaml
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
--   repo: https://github.com/psf/black
-    rev: 22.12.0
-    hooks:
-    -   id: black
 -   repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.0.277
+    rev: v0.2.2
     hooks:
     - id: ruff
--   repo: https://github.com/pycqa/isort
-    rev: 5.12.0
-    hooks:
-    - id: isort
-      name: isort (python)
+- repo: https://github.com/pre-commit/mirrors-prettier
+  rev: v3.1.0
+  hooks:
+    - id: prettier
+      files: "nodejs/.*"
+      exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*

Cargo.toml

@@ -1,37 +1,43 @@
 [workspace]
-members = ["rust/ffi/node", "rust/vectordb", "nodejs"]
+members = ["rust/ffi/node", "rust/lancedb", "nodejs", "python"]
 # Python package needs to be built by maturin.
 exclude = ["python"]
 resolver = "2"
 
 [workspace.package]
 edition = "2021"
-authors = ["Lance Devs <dev@lancedb.com>"]
+authors = ["LanceDB Devs <dev@lancedb.com>"]
 license = "Apache-2.0"
 repository = "https://github.com/lancedb/lancedb"
+description = "Serverless, low-latency vector database for AI applications"
+keywords = ["lancedb", "lance", "database", "vector", "search"]
+categories = ["database-implementations"]
 
 [workspace.dependencies]
-lance = { "version" = "=0.9.9", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.9.9" }
-lance-linalg = { "version" = "=0.9.9" }
-lance-testing = { "version" = "=0.9.9" }
+lance = { "version" = "=0.10.5", "features" = ["dynamodb"] }
+lance-index = { "version" = "=0.10.5" }
+lance-linalg = { "version" = "=0.10.5" }
+lance-testing = { "version" = "=0.10.5" }
 # Note that this one does not include pyarrow
-arrow = { version = "49.0.0", optional = false }
-arrow-array = "49.0"
-arrow-data = "49.0"
-arrow-ipc = "49.0"
-arrow-ord = "49.0"
-arrow-schema = "49.0"
-arrow-arith = "49.0"
-arrow-cast = "49.0"
+arrow = { version = "50.0", optional = false }
+arrow-array = "50.0"
+arrow-data = "50.0"
+arrow-ipc = "50.0"
+arrow-ord = "50.0"
+arrow-schema = "50.0"
+arrow-arith = "50.0"
+arrow-cast = "50.0"
 async-trait = "0"
-chrono = "0.4.23"
+chrono = "0.4.35"
 half = { "version" = "=2.3.1", default-features = false, features = [
     "num-traits",
 ] }
 futures = "0"
 log = "0.4"
 object_store = "0.9.0"
+pin-project = "1.0.7"
 snafu = "0.7.4"
 url = "2"
 num-traits = "0.2"
+regex = "1.10"
+lazy_static = "1"

README.md

@@ -51,12 +51,19 @@ npm install vectordb
 const lancedb = require('vectordb');
 const db = await lancedb.connect('data/sample-lancedb');
 
-const table = await db.createTable('vectors',
-      [{ id: 1, vector: [0.1, 0.2], item: "foo", price: 10 },
-       { id: 2, vector: [1.1, 1.2], item: "bar", price: 50 }])
+const table = await db.createTable({
+  name: 'vectors',
+  data:  [
+    { id: 1, vector: [0.1, 0.2], item: "foo", price: 10 },
+    { id: 2, vector: [1.1, 1.2], item: "bar", price: 50 }
+  ]
+})
 
 const query = table.search([0.1, 0.3]).limit(2);
 const results = await query.execute();
+
+// You can also search for rows by specific criteria without involving a vector search.
+const rowsByCriteria = await table.search(undefined).where("price >= 10").execute();
 ```
 
 **Python**

ci/build_linux_artifacts.sh

@@ -13,7 +13,9 @@ docker build \
     .
 popd
 
+# We turn on memory swap to avoid OOM killer
 docker run \
     -v $(pwd):/io -w /io \
+    --memory-swap=-1 \
     lancedb-node-manylinux \
     bash ci/manylinux_node/build.sh $ARCH

ci/build_linux_artifacts_nodejs.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+ARCH=${1:-x86_64}
+
+# We pass down the current user so that when we later mount the local files
+# into the container, the files are accessible by the current user.
+pushd ci/manylinux_nodejs
+docker build \
+    -t lancedb-nodejs-manylinux \
+    --build-arg="ARCH=$ARCH" \
+    --build-arg="DOCKER_USER=$(id -u)" \
+    --progress=plain \
+    .
+popd
+
+# We turn on memory swap to avoid OOM killer
+docker run \
+    -v $(pwd):/io -w /io \
+    --memory-swap=-1 \
+    lancedb-nodejs-manylinux \
+    bash ci/manylinux_nodejs/build.sh $ARCH

ci/build_macos_artifacts_nodejs.sh

@@ -0,0 +1,34 @@
+# Builds the macOS artifacts (nodejs binaries).
+# Usage: ./ci/build_macos_artifacts_nodejs.sh [target]
+# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
+set -e
+
+prebuild_rust() {
+    # Building here for the sake of easier debugging.
+    pushd rust/lancedb
+    echo "Building rust library for $1"
+    export RUST_BACKTRACE=1
+    cargo build --release --target $1
+    popd
+}
+
+build_node_binaries() {
+    pushd nodejs
+    echo "Building nodejs library for $1"
+    export RUST_TARGET=$1
+    npm run build-release
+    popd
+}
+
+if [ -n "$1" ]; then
+    targets=$1
+else
+    targets="x86_64-apple-darwin aarch64-apple-darwin"
+fi
+
+echo "Building artifacts for targets: $targets"
+for target in $targets
+    do
+    prebuild_rust $target
+    build_node_binaries $target
+done

ci/build_windows_artifacts_nodejs.ps1

@@ -0,0 +1,41 @@
+# Builds the Windows artifacts (nodejs binaries).
+# Usage:  .\ci\build_windows_artifacts_nodejs.ps1 [target]
+# Targets supported:
+# - x86_64-pc-windows-msvc
+# - i686-pc-windows-msvc
+
+function Prebuild-Rust {
+    param (
+        [string]$target
+    )
+
+    # Building here for the sake of easier debugging.
+    Push-Location -Path "rust/lancedb"
+    Write-Host "Building rust library for $target"
+    $env:RUST_BACKTRACE=1
+    cargo build --release --target $target
+    Pop-Location
+}
+
+function Build-NodeBinaries {
+    param (
+        [string]$target
+    )
+
+    Push-Location -Path "nodejs"
+    Write-Host "Building nodejs library for $target"
+    $env:RUST_TARGET=$target
+    npm run build-release
+    Pop-Location
+}
+
+$targets = $args[0]
+if (-not $targets) {
+    $targets = "x86_64-pc-windows-msvc"
+}
+
+Write-Host "Building artifacts for targets: $targets"
+foreach ($target in $targets) {
+    Prebuild-Rust $target
+    Build-NodeBinaries $target
+}

ci/manylinux_nodejs/Dockerfile

@@ -0,0 +1,31 @@
+# Many linux dockerfile with Rust, Node, and Lance dependencies installed.
+# This container allows building the node modules native libraries in an
+# environment with a very old glibc, so that we are compatible with a wide
+# range of linux distributions.
+ARG ARCH=x86_64
+
+FROM quay.io/pypa/manylinux2014_${ARCH}
+
+ARG ARCH=x86_64
+ARG DOCKER_USER=default_user
+
+# Install static openssl
+COPY install_openssl.sh install_openssl.sh
+RUN ./install_openssl.sh ${ARCH} > /dev/null
+
+# Protobuf is also installed as root.
+COPY install_protobuf.sh install_protobuf.sh
+RUN ./install_protobuf.sh ${ARCH}
+
+ENV DOCKER_USER=${DOCKER_USER}
+# Create a group and user
+RUN echo ${ARCH} && adduser --user-group --create-home --uid ${DOCKER_USER} build_user
+
+# We switch to the user to install Rust and Node, since those like to be
+# installed at the user level.
+USER ${DOCKER_USER}
+
+COPY prepare_manylinux_node.sh prepare_manylinux_node.sh
+RUN cp /prepare_manylinux_node.sh $HOME/ && \
+    cd $HOME && \
+    ./prepare_manylinux_node.sh ${ARCH}

ci/manylinux_nodejs/build.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Builds the nodejs module for manylinux. Invoked by ci/build_linux_artifacts_nodejs.sh.
+set -e
+ARCH=${1:-x86_64}
+
+if [ "$ARCH" = "x86_64" ]; then
+    export OPENSSL_LIB_DIR=/usr/local/lib64/
+else
+    export OPENSSL_LIB_DIR=/usr/local/lib/
+fi
+export OPENSSL_STATIC=1
+export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
+
+source $HOME/.bashrc
+
+cd nodejs
+npm ci
+npm run build-release

ci/manylinux_nodejs/install_openssl.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Builds openssl from source so we can statically link to it
+
+# this is to avoid the error we get with the system installation:
+# /usr/bin/ld: <library>: version node not found for symbol SSLeay@@OPENSSL_1.0.1
+# /usr/bin/ld: failed to set dynamic section sizes: Bad value
+set -e
+
+git clone -b OpenSSL_1_1_1u \
+    --single-branch \
+    https://github.com/openssl/openssl.git
+
+pushd openssl
+
+if [[ $1 == x86_64* ]]; then
+    ARCH=linux-x86_64
+else
+    # gnu target
+    ARCH=linux-aarch64
+fi
+
+./Configure no-shared $ARCH
+
+make
+
+make install

ci/manylinux_nodejs/install_protobuf.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Installs protobuf compiler. Should be run as root.
+set -e
+
+if [[ $1 == x86_64* ]]; then
+    ARCH=x86_64
+else
+    # gnu target
+    ARCH=aarch_64
+fi
+
+PB_REL=https://github.com/protocolbuffers/protobuf/releases
+PB_VERSION=23.1
+curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
+unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local

ci/manylinux_nodejs/prepare_manylinux_node.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+
+install_node() {
+    echo "Installing node..."
+
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+
+    source "$HOME"/.bashrc
+
+    nvm install --no-progress 16
+}
+
+install_rust() {
+    echo "Installing rust..."
+    curl https://sh.rustup.rs -sSf | bash -s -- -y
+    export PATH="$PATH:/root/.cargo/bin"
+}
+
+install_node
+install_rust

dockerfiles/Dockerfile

@@ -0,0 +1,27 @@
+#Simple base dockerfile that supports basic dependencies required to run lance with FTS and Hybrid Search
+#Usage docker build -t lancedb:latest -f Dockerfile .
+FROM python:3.10-slim-buster
+
+# Install Rust
+RUN apt-get update && apt-get install -y curl build-essential && \
+  curl https://sh.rustup.rs -sSf | sh -s -- -y
+
+# Set the environment variable for Rust
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install protobuf compiler
+RUN apt-get install -y protobuf-compiler && \
+  apt-get clean && \
+  rm -rf /var/lib/apt/lists/*
+
+RUN apt-get -y update &&\
+  apt-get -y upgrade && \
+  apt-get -y install git
+
+
+# Verify installations
+RUN python --version && \
+  rustc --version && \
+  protoc --version
+
+RUN pip install tantivy lancedb

docs/README.md

@@ -33,3 +33,12 @@ You can run a local server to test the docs prior to deployment by navigating to
 cd docs
 mkdocs serve
 ```
+
+### Run doctest for typescript example
+
+```bash
+cd lancedb/docs
+npm i
+npm run build
+npm run all
+```

docs/mkdocs.yml

@@ -27,7 +27,6 @@ theme:
     - content.tabs.link
     - content.action.edit
     - toc.follow
-    # - toc.integrate
     - navigation.top
     - navigation.tabs
     - navigation.tabs.sticky
@@ -57,6 +56,16 @@ plugins:
           - https://arrow.apache.org/docs/objects.inv
           - https://pandas.pydata.org/docs/objects.inv
 - mkdocs-jupyter
+- ultralytics:
+    verbose: True
+    enabled: True
+    default_image: "assets/lancedb_and_lance.png" # Default image for all pages
+    add_image: True # Automatically add meta image
+    add_keywords: True # Add page keywords in the header tag
+    add_share_buttons: True # Add social share buttons
+    add_authors: False # Display page authors
+    add_desc: False
+    add_dates: False
 
 markdown_extensions:
 - admonition
@@ -67,7 +76,9 @@ markdown_extensions:
     line_spans: __span
     pygments_lang_class: true
 - pymdownx.inlinehilite
-- pymdownx.snippets
+- pymdownx.snippets:
+    base_path: ..
+    dedent_subsections: true
 - pymdownx.superfences
 - pymdownx.tabbed:
     alternate_style: true
@@ -88,15 +99,18 @@ nav:
     - Building an ANN index: ann_indexes.md
     - Vector Search: search.md
     - Full-text search: fts.md
+    - Hybrid search:
+      - Overview: hybrid_search/hybrid_search.md
+      - Comparing Rerankers: hybrid_search/eval.md
+      - Airbnb financial data example: notebooks/hybrid_search.ipynb
     - Filtering: sql.md
     - Versioning & Reproducibility: notebooks/reproducibility.ipynb
     - Configuring Storage: guides/storage.md
   - 🧬 Managing embeddings:
     - Overview: embeddings/index.md
-    - Explicit management: embeddings/embedding_explicit.md
-    - Implicit management: embeddings/embedding_functions.md
-    - Available Functions: embeddings/default_embedding_functions.md
-    - Custom Embedding Functions: embeddings/api.md
+    - Embedding functions: embeddings/embedding_functions.md
+    - Available models: embeddings/default_embedding_functions.md
+    - User-defined embedding functions: embeddings/custom_embedding_function.md
     - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
     - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
   - 🔌 Integrations:
@@ -125,11 +139,14 @@ nav:
       - Serverless Website Chatbot: examples/serverless_website_chatbot.md
       - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
       - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
+    - 🦀 Rust:
+      - Overview: examples/examples_rust.md
   - 🔧 CLI & Config: cli_config.md
   - 💭 FAQs: faq.md
   - ⚙️ API reference:
     - 🐍 Python: python/python.md
     - 👾 JavaScript: javascript/modules.md
+    - 🦀 Rust: https://docs.rs/lancedb/latest/lancedb/
   - ☁️ LanceDB Cloud:
     - Overview: cloud/index.md
     - API reference:
@@ -148,15 +165,18 @@ nav:
     - Building an ANN index: ann_indexes.md
     - Vector Search: search.md
     - Full-text search: fts.md
+    - Hybrid search:
+      - Overview: hybrid_search/hybrid_search.md
+      - Comparing Rerankers: hybrid_search/eval.md
+      - Airbnb financial data example: notebooks/hybrid_search.ipynb
     - Filtering: sql.md
     - Versioning & Reproducibility: notebooks/reproducibility.ipynb
     - Configuring Storage: guides/storage.md
 - Managing Embeddings:
   - Overview: embeddings/index.md
-  - Explicit management: embeddings/embedding_explicit.md
-  - Implicit management: embeddings/embedding_functions.md
-  - Available Functions: embeddings/default_embedding_functions.md
-  - Custom Embedding Functions: embeddings/api.md
+  - Embedding functions: embeddings/embedding_functions.md
+  - Available models: embeddings/default_embedding_functions.md
+  - User-defined embedding functions: embeddings/custom_embedding_function.md
   - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
   - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
 - Integrations:
@@ -170,21 +190,21 @@ nav:
   - Pydantic: python/pydantic.md
   - Voxel51: integrations/voxel51.md
   - PromptTools: integrations/prompttools.md
-- Python examples:
+- Examples:
   - examples/index.md
   - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
   - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
   - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
   - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
   - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
-- Javascript examples:
-  - Overview: examples/examples_js.md
-  - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
+  - YouTube Transcript Search (JS): examples/youtube_transcript_bot_with_nodejs.md
   - Serverless Chatbot from any website: examples/serverless_website_chatbot.md
   - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
 - API reference:
+  - Overview: api_reference.md
   - Python: python/python.md
   - Javascript: javascript/modules.md
+  - Rust: https://docs.rs/lancedb/latest/lancedb/index.html
 - LanceDB Cloud:
     - Overview: cloud/index.md
     - API reference:
@@ -195,6 +215,9 @@ extra_css:
   - styles/global.css
   - styles/extra.css
 
+extra_javascript:
+  - "extra_js/init_ask_ai_widget.js"
+
 extra:
   analytics:
     provider: google

docs/package-lock.json

@@ -0,0 +1,132 @@
+{
+  "name": "lancedb-docs-test",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "lancedb-docs-test",
+      "version": "1.0.0",
+      "license": "Apache 2",
+      "dependencies": {
+        "apache-arrow": "file:../node/node_modules/apache-arrow",
+        "vectordb": "file:../node"
+      },
+      "devDependencies": {
+        "@types/node": "^20.11.8",
+        "typescript": "^5.3.3"
+      }
+    },
+    "../node": {
+      "name": "vectordb",
+      "version": "0.4.6",
+      "cpu": [
+        "x64",
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "os": [
+        "darwin",
+        "linux",
+        "win32"
+      ],
+      "dependencies": {
+        "@apache-arrow/ts": "^14.0.2",
+        "@neon-rs/load": "^0.0.74",
+        "apache-arrow": "^14.0.2",
+        "axios": "^1.4.0"
+      },
+      "devDependencies": {
+        "@neon-rs/cli": "^0.0.160",
+        "@types/chai": "^4.3.4",
+        "@types/chai-as-promised": "^7.1.5",
+        "@types/mocha": "^10.0.1",
+        "@types/node": "^18.16.2",
+        "@types/sinon": "^10.0.15",
+        "@types/temp": "^0.9.1",
+        "@types/uuid": "^9.0.3",
+        "@typescript-eslint/eslint-plugin": "^5.59.1",
+        "cargo-cp-artifact": "^0.1",
+        "chai": "^4.3.7",
+        "chai-as-promised": "^7.1.1",
+        "eslint": "^8.39.0",
+        "eslint-config-standard-with-typescript": "^34.0.1",
+        "eslint-plugin-import": "^2.26.0",
+        "eslint-plugin-n": "^15.7.0",
+        "eslint-plugin-promise": "^6.1.1",
+        "mocha": "^10.2.0",
+        "openai": "^4.24.1",
+        "sinon": "^15.1.0",
+        "temp": "^0.9.4",
+        "ts-node": "^10.9.1",
+        "ts-node-dev": "^2.0.0",
+        "typedoc": "^0.24.7",
+        "typedoc-plugin-markdown": "^3.15.3",
+        "typescript": "*",
+        "uuid": "^9.0.0"
+      },
+      "optionalDependencies": {
+        "@lancedb/vectordb-darwin-arm64": "0.4.6",
+        "@lancedb/vectordb-darwin-x64": "0.4.6",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.4.6",
+        "@lancedb/vectordb-linux-x64-gnu": "0.4.6",
+        "@lancedb/vectordb-win32-x64-msvc": "0.4.6"
+      }
+    },
+    "../node/node_modules/apache-arrow": {
+      "version": "14.0.2",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@types/command-line-args": "5.2.0",
+        "@types/command-line-usage": "5.0.2",
+        "@types/node": "20.3.0",
+        "@types/pad-left": "2.1.1",
+        "command-line-args": "5.2.1",
+        "command-line-usage": "7.0.1",
+        "flatbuffers": "23.5.26",
+        "json-bignum": "^0.0.3",
+        "pad-left": "^2.1.0",
+        "tslib": "^2.5.3"
+      },
+      "bin": {
+        "arrow2csv": "bin/arrow2csv.js"
+      }
+    },
+    "node_modules/@types/node": {
+      "version": "20.11.8",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.8.tgz",
+      "integrity": "sha512-i7omyekpPTNdv4Jb/Rgqg0RU8YqLcNsI12quKSDkRXNfx7Wxdm6HhK1awT3xTgEkgxPn3bvnSpiEAc7a7Lpyow==",
+      "dev": true,
+      "dependencies": {
+        "undici-types": "~5.26.4"
+      }
+    },
+    "node_modules/apache-arrow": {
+      "resolved": "../node/node_modules/apache-arrow",
+      "link": true
+    },
+    "node_modules/typescript": {
+      "version": "5.3.3",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz",
+      "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==",
+      "dev": true,
+      "bin": {
+        "tsc": "bin/tsc",
+        "tsserver": "bin/tsserver"
+      },
+      "engines": {
+        "node": ">=14.17"
+      }
+    },
+    "node_modules/undici-types": {
+      "version": "5.26.5",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
+      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
+      "dev": true
+    },
+    "node_modules/vectordb": {
+      "resolved": "../node",
+      "link": true
+    }
+  }
+}

docs/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "lancedb-docs-test",
+  "version": "1.0.0",
+  "description": "auto-generated tests from doc",
+  "author": "dev@lancedb.com",
+  "license": "Apache 2",
+  "dependencies": {
+    "apache-arrow": "file:../node/node_modules/apache-arrow",
+    "vectordb": "file:../node"
+  },
+  "scripts": {
+    "build": "tsc -b && cd ../node && npm run build-release",
+    "example": "npm run build && node",
+    "test": "npm run build && ls dist/*.js | xargs -n 1 node"
+  },
+  "devDependencies": {
+    "@types/node": "^20.11.8",
+    "typescript": "^5.3.3"
+  }
+}

docs/requirements.txt

@@ -2,4 +2,5 @@ mkdocs==1.5.3
 mkdocs-jupyter==0.24.1
 mkdocs-material==9.5.3
 mkdocstrings[python]==0.20.0
-pydantic
+pydantic
+mkdocs-ultralytics-plugin==0.0.44

docs/src/ann_indexes.md

@@ -7,64 +7,73 @@ for brute-force scanning of the entire vector space.
 A vector index is faster but less accurate than exhaustive search (kNN or flat search).
 LanceDB provides many parameters to fine-tune the index's size, the speed of queries, and the accuracy of results.
 
-Currently, LanceDB does *not* automatically create the ANN index.
-LanceDB has optimized code for kNN as well. For many use-cases, datasets under 100K vectors won't require index creation at all.
-If you can live with <100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
+## Disk-based Index
 
-In the future we will look to automatically create and configure the ANN index as data comes in.
-
-## Types of Index
-
-Lance can support multiple index types, the most widely used one is `IVF_PQ`.
-
-* `IVF_PQ`: use **Inverted File Index (IVF)** to first divide the dataset into `N` partitions,
-   and then use **Product Quantization** to compress vectors in each partition.
-* `DiskANN` (**Experimental**): organize the vector as a on-disk graph, where the vertices approximately
-   represent the nearest neighbors of each vector.
+Lance provides an `IVF_PQ` disk-based index. It uses **Inverted File Index (IVF)** to first divide
+the dataset into `N` partitions, and then applies **Product Quantization** to compress vectors in each partition.
+See the [indexing](concepts/index_ivfpq.md) concepts guide for more information on how this works.
 
 ## Creating an IVF_PQ Index
 
 Lance supports `IVF_PQ` index type by default.
 
 === "Python"
-     Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
 
-     ```python
-     import lancedb
-     import numpy as np
-     uri = "data/sample-lancedb"
-     db = lancedb.connect(uri)
+    Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
 
-     # Create 10,000 sample vectors
-     data = [{"vector": row, "item": f"item {i}"}
+    ```python
+    import lancedb
+    import numpy as np
+    uri = "data/sample-lancedb"
+    db = lancedb.connect(uri)
+
+    # Create 10,000 sample vectors
+    data = [{"vector": row, "item": f"item {i}"}
         for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]
 
-     # Add the vectors to a table
-     tbl = db.create_table("my_vectors", data=data)
+    # Add the vectors to a table
+    tbl = db.create_table("my_vectors", data=data)
 
-     # Create and train the index - you need to have enough data in the table for an effective training step
-     tbl.create_index(num_partitions=256, num_sub_vectors=96)
-     ```
+    # Create and train the index - you need to have enough data in the table for an effective training step
+    tbl.create_index(num_partitions=256, num_sub_vectors=96)
+    ```
 
-=== "Javascript"
-     ```javascript
-     const vectordb = require('vectordb')
-     const db = await vectordb.connect('data/sample-lancedb')
+=== "Typescript"
 
-     let data = []
-     for (let i = 0; i < 10_000; i++) {
-         data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
-     }
-     const table = await db.createTable('my_vectors', data)
-     await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 256, num_sub_vectors: 96 })
-     ```
+    ```typescript
+    --8<--- "docs/src/ann_indexes.ts:import"
 
-- **metric** (default: "L2"): The distance metric to use. By default it uses euclidean distance "`L2`".
-We also support "cosine" and "dot" distance as well.
-- **num_partitions** (default: 256): The number of partitions of the index.
-- **num_sub_vectors** (default: 96): The number of sub-vectors (M) that will be created during Product Quantization (PQ).
-For D dimensional vector, it will be divided into `M` of `D/M` sub-vectors, each of which is presented by
-a single PQ code.
+    --8<-- "docs/src/ann_indexes.ts:ingest"
+    ```
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/ivf_pq.rs:create_index"
+    ```
+
+    IVF_PQ index parameters are more fully defined in the [crate docs](https://docs.rs/lancedb/latest/lancedb/index/vector/struct.IvfPqIndexBuilder.html).
+
+The following IVF_PQ paramters can be specified:
+
+- **distance_type**: The distance metric to use. By default it uses euclidean distance "`L2`".
+  We also support "cosine" and "dot" distance as well.
+- **num_partitions**: The number of partitions in the index. The default is the square root
+  of the number of rows.
+
+!!! note
+
+    In the synchronous python SDK and node's `vectordb` the default is 256. This default has
+    changed in the asynchronous python SDK and node's `lancedb`.
+
+- **num_sub_vectors**: The number of sub-vectors (M) that will be created during Product Quantization (PQ).
+  For D dimensional vector, it will be divided into `M` subvectors with dimension `D/M`, each of which is replaced by
+  a single PQ code. The default is the dimension of the vector divided by 16.
+
+!!! note
+
+    In the synchronous python SDK and node's `vectordb` the default is currently 96. This default has
+    changed in the asynchronous python SDK and node's `lancedb`.
 
 <figure markdown>
   ![IVF PQ](./assets/ivf_pq.png)
@@ -78,7 +87,7 @@ Using GPU for index creation requires [PyTorch>2.0](https://pytorch.org/) being
 
 You can specify the GPU device to train IVF partitions via
 
-- **accelerator**: Specify to ``cuda`` or ``mps`` (on Apple Silicon) to enable GPU training.
+- **accelerator**: Specify to `cuda` or `mps` (on Apple Silicon) to enable GPU training.
 
 === "Linux"
 
@@ -92,7 +101,7 @@ You can specify the GPU device to train IVF partitions via
      )
      ```
 
-=== "Macos"
+=== "MacOS"
 
      <!-- skip-test -->
      ```python
@@ -104,12 +113,11 @@ You can specify the GPU device to train IVF partitions via
      )
      ```
 
-Trouble shootings:
+Troubleshooting:
 
-If you see ``AssertionError: Torch not compiled with CUDA enabled``, you need to [install
+If you see `AssertionError: Torch not compiled with CUDA enabled`, you need to [install
 PyTorch with CUDA support](https://pytorch.org/get-started/locally/).
 
-
 ## Querying an ANN Index
 
 Querying vector indexes is done via the [search](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.search) function.
@@ -127,48 +135,51 @@ There are a couple of parameters that can be used to fine-tune the search:
   Note: refine_factor is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
 
 === "Python"
-     ```python
-     tbl.search(np.random.random((1536))) \
-         .limit(2) \
-         .nprobes(20) \
-         .refine_factor(10) \
-         .to_pandas()
-     ```
-     ```
-                                              vector       item       _distance
-     0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
-     1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
-     ```
 
-=== "Javascript"
-     ```javascript
-     const results_1 = await table
-         .search(Array(1536).fill(1.2))
-         .limit(2)
-         .nprobes(20)
-         .refineFactor(10)
-         .execute()
-     ```
+    ```python
+    tbl.search(np.random.random((1536))) \
+        .limit(2) \
+        .nprobes(20) \
+        .refine_factor(10) \
+        .to_pandas()
+    ```
+
+    ```text
+                                              vector       item       _distance
+    0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
+    1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
+    ```
+
+=== "Typescript"
+
+    ```typescript
+    --8<-- "docs/src/ann_indexes.ts:search1"
+    ```
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/ivf_pq.rs:search1"
+    ```
+
+    Vector search options are more fully defined in the [crate docs](https://docs.rs/lancedb/latest/lancedb/query/struct.Query.html#method.nearest_to).
 
 The search will return the data requested in addition to the distance of each item.
 
-
 ### Filtering (where clause)
 
 You can further filter the elements returned by a search using a where clause.
 
 === "Python"
+
      ```python
      tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
      ```
 
-=== "Javascript"
+=== "Typescript"
+
      ```javascript
-     const results_2 = await table
-         .search(Array(1536).fill(1.2))
-         .where("id != '1141'")
-         .limit(2)
-         .execute()
+     --8<-- "docs/src/ann_indexes.ts:search2"
      ```
 
 ### Projections (select clause)
@@ -176,34 +187,42 @@ You can further filter the elements returned by a search using a where clause.
 You can select the columns returned by the query using a select clause.
 
 === "Python"
+
      ```python
      tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
      ```
-     ```
-        vector                                             _distance
+
+
+     ```text
+                                                   vector _distance
      0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
      1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
      ...
      ```
 
-=== "Javascript"
-     ```javascript
-     const results_3 = await table
-         .search(Array(1536).fill(1.2))
-         .select(["id"])
-         .limit(2)
-         .execute()
+=== "Typescript"
+
+     ```typescript
+     --8<-- "docs/src/ann_indexes.ts:search3"
      ```
 
 ## FAQ
 
+### Why do I need to manually create an index?
+
+Currently, LanceDB does _not_ automatically create the ANN index.
+LanceDB is well-optimized for kNN (exhaustive search) via a disk-based index. For many use-cases,
+datasets of the order of ~100K vectors don't require index creation. If you can live with up to
+100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
+
 ### When is it necessary to create an ANN vector index?
 
-`LanceDB` has manually-tuned SIMD code for computing vector distances.
-In our benchmarks, computing 100K pairs of 1K dimension vectors takes **less than 20ms**.
-For small datasets (< 100K rows) or applications that can accept 100ms latency, vector indices are usually not necessary.
+`LanceDB` comes out-of-the-box with highly optimized SIMD code for computing vector similarity.
+In our benchmarks, computing distances for 100K pairs of 1K dimension vectors takes **less than 20ms**.
+We observe that for small datasets (~100K rows) or for applications that can accept 100ms latency,
+vector indices are usually not necessary.
 
-For large-scale or higher dimension vectors, it is beneficial to create vector index.
+For large-scale or higher dimension vectors, it can beneficial to create vector index for performance.
 
 ### How big is my index, and how many memory will it take?
 
@@ -221,4 +240,4 @@ On `SIFT-1M` dataset, our benchmark shows that keeping each partition 1K-4K rows
 `num_sub_vectors` specifies how many Product Quantization (PQ) short codes to generate on each vector. Because
 PQ is a lossy compression of the original vector, a higher `num_sub_vectors` usually results in
 less space distortion, and thus yields better accuracy. However, a higher `num_sub_vectors` also causes heavier I/O and
-more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.
+more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.

docs/src/ann_indexes.ts

@@ -0,0 +1,53 @@
+// --8<-- [start:import]
+import * as vectordb from "vectordb";
+// --8<-- [end:import]
+
+(async () => {
+  // --8<-- [start:ingest]
+  const db = await vectordb.connect("data/sample-lancedb");
+
+  let data = [];
+  for (let i = 0; i < 10_000; i++) {
+    data.push({
+      vector: Array(1536).fill(i),
+      id: `${i}`,
+      content: "",
+      longId: `${i}`,
+    });
+  }
+  const table = await db.createTable("my_vectors", data);
+  await table.createIndex({
+    type: "ivf_pq",
+    column: "vector",
+    num_partitions: 16,
+    num_sub_vectors: 48,
+  });
+  // --8<-- [end:ingest]
+
+  // --8<-- [start:search1]
+  const results_1 = await table
+    .search(Array(1536).fill(1.2))
+    .limit(2)
+    .nprobes(20)
+    .refineFactor(10)
+    .execute();
+  // --8<-- [end:search1]
+
+  // --8<-- [start:search2]
+  const results_2 = await table
+    .search(Array(1536).fill(1.2))
+    .where("id != '1141'")
+    .limit(2)
+    .execute();
+  // --8<-- [end:search2]
+
+  // --8<-- [start:search3]
+  const results_3 = await table
+    .search(Array(1536).fill(1.2))
+    .select(["id"])
+    .limit(2)
+    .execute();
+  // --8<-- [end:search3]
+
+  console.log("Ann indexes: done");
+})();

docs/src/api_reference.md

@@ -0,0 +1,7 @@
+# API Reference
+
+The API reference for the LanceDB client SDKs are available at the following locations:
+
+- [Python](python/python.md)
+- [JavaScript](javascript/modules.md)
+- [Rust](https://docs.rs/lancedb/latest/lancedb/index.html)

docs/src/basic.md

@@ -3,7 +3,7 @@
 !!! info "LanceDB can be run in a number of ways:"
 
     * Embedded within an existing backend (like your Django, Flask, Node.js or FastAPI application)
-    * Connected to directly from a client application like a Jupyter notebook for analytical workloads
+    * Directly from a client application like a Jupyter notebook for analytical workloads
     * Deployed as a remote serverless database
 
 ![](assets/lancedb_embedded_explanation.png)
@@ -11,43 +11,82 @@
 ## Installation
 
 === "Python"
+
       ```shell
       pip install lancedb
       ```
 
-=== "Javascript"
+=== "Typescript"
+
       ```shell
       npm install vectordb
       ```
 
-## How to connect to a database
+=== "Rust"
+
+    ```shell
+    cargo add lancedb
+    ```
+
+    !!! info "To use the lancedb create, you first need to install protobuf."
+
+    === "macOS"
+
+        ```shell
+        brew install protobuf
+        ```
+
+    === "Ubuntu/Debian"
+
+        ```shell
+        sudo apt install -y protobuf-compiler libssl-dev
+        ```
+
+    !!! info "Please also make sure you're using the same version of Arrow as in the [lancedb crate](https://github.com/lancedb/lancedb/blob/main/Cargo.toml)"
+
+## Connect to a database
 
 === "Python"
+
       ```python
       import lancedb
       uri = "data/sample-lancedb"
       db = lancedb.connect(uri)
       ```
 
-      LanceDB will create the directory if it doesn't exist (including parent directories).
+=== "Typescript"
 
-      If you need a reminder of the uri, use the `db.uri` property.
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:import"
 
-=== "Javascript"
-      ```javascript
-      const lancedb = require("vectordb");
+    --8<-- "docs/src/basic_legacy.ts:open_db"
+    ```
 
-      const uri = "data/sample-lancedb";
-      const db = await lancedb.connect(uri);
-      ```
-      
-      LanceDB will create the directory if it doesn't exist (including parent directories).
+=== "Rust"
 
-      If you need a reminder of the uri, you can call `db.uri()`.
+    ```rust
+    #[tokio::main]
+    async fn main() -> Result<()> {
+        --8<-- "rust/lancedb/examples/simple.rs:connect"
+    }
+    ```
 
-## How to create a table
+    !!! info "See [examples/simple.rs](https://github.com/lancedb/lancedb/tree/main/rust/lancedb/examples/simple.rs) for a full working example."
+
+LanceDB will create the directory if it doesn't exist (including parent directories).
+
+If you need a reminder of the uri, you can call `db.uri()`.
+
+## Create a table
+
+### Create a table from initial data
+
+If you have data to insert into the table at creation time, you can simultaneously create a
+table and insert the data into it.  The schema of the data will be used as the schema of the
+table.
 
 === "Python"
+
     ```python
     tbl = db.create_table("my_table",
                     data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
@@ -59,6 +98,7 @@
     to the `create_table` method.
 
     You can also pass in a pandas DataFrame directly:
+
     ```python
     import pandas as pd
     df = pd.DataFrame([{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
@@ -66,119 +106,224 @@
     tbl = db.create_table("table_from_df", data=df)
     ```
 
-=== "Javascript"
-    ```javascript
-    const tb = await db.createTable(
-        "myTable",
-        [{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-         {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}]
-    )
+=== "Typescript"
+
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:create_table"
     ```
 
     If the table already exists, LanceDB will raise an error by default.
     If you want to overwrite the table, you can pass in `mode="overwrite"`
     to the `createTable` function.
 
+=== "Rust"
 
-!!! info "Under the hood, LanceDB is converting the input data into an Apache Arrow table and persisting it to disk in [Lance format](https://www.github.com/lancedb/lance)."
+    ```rust
+    --8<-- "rust/lancedb/examples/simple.rs:create_table"
+    ```
 
-### Creating an empty table
+    If the table already exists, LanceDB will raise an error by default.  See
+    [the mode option](https://docs.rs/lancedb/latest/lancedb/connection/struct.CreateTableBuilder.html#method.mode)
+    for details on how to overwrite (or open) existing tables instead.
+
+    !!! Providing table records in Rust
+
+        The Rust SDK currently expects data to be provided as an Arrow
+        [RecordBatchReader](https://docs.rs/arrow-array/latest/arrow_array/trait.RecordBatchReader.html)
+        Support for additional formats (such as serde or polars) is on the roadmap.
+
+!!! info "Under the hood, LanceDB reads in the Apache Arrow data and persists it to disk using the [Lance format](https://www.github.com/lancedb/lance)."
+
+### Create an empty table
 
 Sometimes you may not have the data to insert into the table at creation time.
-In this case, you can create an empty table and specify the schema.
+In this case, you can create an empty table and specify the schema, so that you can add
+data to the table at a later time (as long as it conforms to the schema).  This is
+similar to a `CREATE TABLE` statement in SQL.
 
 === "Python"
+
       ```python
       import pyarrow as pa
       schema = pa.schema([pa.field("vector", pa.list_(pa.float32(), list_size=2))])
       tbl = db.create_table("empty_table", schema=schema)
       ```
 
-## How to open an existing table
+=== "Typescript"
 
-Once created, you can open a table using the following code:
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:create_empty_table"
+    ```
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/simple.rs:create_empty_table"
+    ```
+
+## Open an existing table
+
+Once created, you can open a table as follows:
 
 === "Python"
-      ```python
-      tbl = db.open_table("my_table")
-      ```
 
-      If you forget the name of your table, you can always get a listing of all table names:
+    ```python
+    tbl = db.open_table("my_table")
+    ```
 
-      ```python
-      print(db.table_names())
-      ```
+=== "Typescript"
 
-=== "Javascript"
-      ```javascript
-      const tbl = await db.openTable("myTable");
-      ```
+    ```typescript
+    const tbl = await db.openTable("myTable");
+    ```
 
-      If you forget the name of your table, you can always get a listing of all table names:
+=== "Rust"
 
-      ```javascript
-      console.log(await db.tableNames());
-      ```
+    ```rust
+    --8<-- "rust/lancedb/examples/simple.rs:open_existing_tbl"
+    ```
 
-## How to add data to a table
-
-After a table has been created, you can always add more data to it using
+If you forget the name of your table, you can always get a listing of all table names:
 
 === "Python"
-      ```python
 
-      # Option 1: Add a list of dicts to a table
-      data = [{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
-              {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}]
-      tbl.add(data)
-
-      # Option 2: Add a pandas DataFrame to a table
-      df = pd.DataFrame(data)
-      tbl.add(data)
-      ```
+    ```python
+    print(db.table_names())
+    ```
 
 === "Javascript"
-      ```javascript
-      await tbl.add([{vector: [1.3, 1.4], item: "fizz", price: 100.0},
-                     {vector: [9.5, 56.2], item: "buzz", price: 200.0}])
-      ```
 
-## How to search for (approximate) nearest neighbors
+    ```javascript
+    console.log(await db.tableNames());
+    ```
 
-Once you've embedded the query, you can find its nearest neighbors using the following code:
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/simple.rs:list_names"
+    ```
+
+## Add data to a table
+
+After a table has been created, you can always add more data to it as follows:
 
 === "Python"
-      ```python
-      tbl.search([100, 100]).limit(2).to_pandas()
-      ```
 
-      This returns a pandas DataFrame with the results.
+    ```python
 
-=== "Javascript"
-      ```javascript
-      const query = await tbl.search([100, 100]).limit(2).execute();
-      ```
+    # Option 1: Add a list of dicts to a table
+    data = [{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
+            {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}]
+    tbl.add(data)
 
-## How to delete rows from a table
+    # Option 2: Add a pandas DataFrame to a table
+    df = pd.DataFrame(data)
+    tbl.add(data)
+    ```
+
+=== "Typescript"
+
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:add"
+    ```
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/simple.rs:add"
+    ```
+
+## Search for nearest neighbors
+
+Once you've embedded the query, you can find its nearest neighbors as follows:
+
+=== "Python"
+
+    ```python
+    tbl.search([100, 100]).limit(2).to_pandas()
+    ```
+
+    This returns a pandas DataFrame with the results.
+
+=== "Typescript"
+
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:search"
+    ```
+
+=== "Rust"
+
+    ```rust
+    use futures::TryStreamExt;
+
+    --8<-- "rust/lancedb/examples/simple.rs:search"
+    ```
+
+    !!! Query vectors in Rust
+        Rust does not yet support automatic execution of embedding functions.  You will need to
+        calculate embeddings yourself.  Support for this is on the roadmap and can be tracked at
+        https://github.com/lancedb/lancedb/issues/994
+
+        Query vectors can be provided as Arrow arrays or a Vec/slice of Rust floats.
+        Support for additional formats (e.g. `polars::series::Series`) is on the roadmap.
+
+By default, LanceDB runs a brute-force scan over dataset to find the K nearest neighbours (KNN).
+For tables with more than 50K vectors, creating an ANN index is recommended to speed up search performance.
+LanceDB allows you to create an ANN index on a table as follows:
+
+=== "Python"
+
+    ```py
+    tbl.create_index()
+    ```
+
+=== "Typescript"
+
+    ```{.typescript .ignore}
+    --8<-- "docs/src/basic_legacy.ts:create_index"
+    ```
+
+=== "Rust"
+
+    ```rust
+     --8<-- "rust/lancedb/examples/simple.rs:create_index"
+    ```
+
+!!! note "Why do I need to create an index manually?"
+    LanceDB does not automatically create the ANN index for two reasons. The first is that it's optimized
+    for really fast retrievals via a disk-based index, and the second is that data and query workloads can
+    be very diverse, so there's no one-size-fits-all index configuration. LanceDB provides many parameters
+    to fine-tune index size, query latency and accuracy. See the section on
+    [ANN indexes](ann_indexes.md) for more details.
+
+## Delete rows from a table
 
 Use the `delete()` method on tables to delete rows from a table. To choose
 which rows to delete, provide a filter that matches on the metadata columns.
 This can delete any number of rows that match the filter.
 
 === "Python"
-      ```python
-      tbl.delete('item = "fizz"')
-      ```
 
-=== "Javascript"
-      ```javascript
-      await tbl.delete('item = "fizz"')
-      ```
+    ```python
+    tbl.delete('item = "fizz"')
+    ```
+
+=== "Typescript"
+
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:delete"
+    ```
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/simple.rs:delete"
+    ```
 
 The deletion predicate is a SQL expression that supports the same expressions
-as the `where()` clause on a search. They can be as simple or complex as needed.
-To see what expressions are supported, see the [SQL filters](sql.md) section.
-
+as the `where()` clause (`only_if()` in Rust) on a search. They can be as
+simple or complex as needed.  To see what expressions are supported, see the
+[SQL filters](sql.md) section.
 
 === "Python"
 
@@ -188,11 +333,16 @@ To see what expressions are supported, see the [SQL filters](sql.md) section.
 
       Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
 
-## How to remove a table
+=== "Rust"
+
+      Read more: [lancedb::Table::delete](https://docs.rs/lancedb/latest/lancedb/table/struct.Table.html#method.delete)
+
+## Drop a table
 
 Use the `drop_table()` method on the database to remove a table.
 
 === "Python"
+
       ```python
       db.drop_table("my_table")
       ```
@@ -201,13 +351,20 @@ Use the `drop_table()` method on the database to remove a table.
       By default, if the table does not exist an exception is raised. To suppress this,
       you can pass in `ignore_missing=True`.
 
-=== "JavaScript"
-      ```javascript
-      await db.dropTable('myTable')
+=== "Typescript"
+
+      ```typescript
+      --8<-- "docs/src/basic_legacy.ts:drop_table"
       ```
 
       This permanently removes the table and is not recoverable, unlike deleting rows.
-      If the table does not exist an exception is raised. 
+      If the table does not exist an exception is raised.
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/simple.rs:drop_table"
+    ```
 
 !!! note "Bundling `vectordb` apps with Webpack"
 

docs/src/basic_legacy.ts

@@ -0,0 +1,92 @@
+// --8<-- [start:import]
+import * as lancedb from "vectordb";
+import { Schema, Field, Float32, FixedSizeList, Int32, Float16 } from "apache-arrow";
+// --8<-- [end:import]
+import * as fs from "fs";
+import { Table as ArrowTable, Utf8 } from "apache-arrow";
+
+const example = async () => {
+  fs.rmSync("data/sample-lancedb", { recursive: true, force: true });
+  // --8<-- [start:open_db]
+  const lancedb = require("vectordb");
+  const uri = "data/sample-lancedb";
+  const db = await lancedb.connect(uri);
+  // --8<-- [end:open_db]
+
+  // --8<-- [start:create_table]
+  const tbl = await db.createTable(
+    "myTable",
+    [
+      { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+      { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+    ],
+    { writeMode: lancedb.WriteMode.Overwrite }
+  );
+  // --8<-- [end:create_table]
+
+  // --8<-- [start:add]
+  const newData = Array.from({ length: 500 }, (_, i) => ({
+    vector: [i, i + 1],
+    item: "fizz",
+    price: i * 0.1,
+  }));
+  await tbl.add(newData);
+  // --8<-- [end:add]
+
+  // --8<-- [start:create_index]
+  await tbl.createIndex({
+    type: "ivf_pq",
+    num_partitions: 2,
+    num_sub_vectors: 2,
+  });
+  // --8<-- [end:create_index]
+
+  // --8<-- [start:create_empty_table]
+  const schema = new Schema([
+    new Field("id", new Int32()),
+    new Field("name", new Utf8()),
+  ]);
+  const empty_tbl = await db.createTable({ name: "empty_table", schema });
+  // --8<-- [end:create_empty_table]
+
+  // --8<-- [start:create_f16_table]
+  const dim = 16
+  const total = 10
+  const f16_schema = new Schema([
+      new Field('id', new Int32()),
+      new Field(
+        'vector',
+        new FixedSizeList(dim, new Field('item', new Float16(), true)),
+        false
+      )
+    ])
+  const data = lancedb.makeArrowTable(
+      Array.from(Array(total), (_, i) => ({
+        id: i,
+        vector: Array.from(Array(dim), Math.random)
+      })),
+      { f16_schema }
+    )
+  const table = await db.createTable('f16_tbl', data)
+  // --8<-- [end:create_f16_table]
+
+  // --8<-- [start:search]
+  const query = await tbl.search([100, 100]).limit(2).execute();
+  // --8<-- [end:search]
+  console.log(query);
+
+  // --8<-- [start:delete]
+  await tbl.delete('item = "fizz"');
+  // --8<-- [end:delete]
+
+  // --8<-- [start:drop_table]
+  await db.dropTable("myTable");
+  // --8<-- [end:drop_table]
+};
+
+async function main() {
+  await example();
+  console.log("Basic example: done");
+}
+
+main();

docs/src/concepts/index_ivfpq.md

@@ -31,7 +31,7 @@ As an example, consider starting with 128-dimensional vector consisting of 32-bi
 
 While PQ helps with reducing the size of the index, IVF primarily addresses search performance. The primary purpose of an inverted file index is to facilitate rapid and effective nearest neighbor search by narrowing down the search space.
 
-In IVF, the PQ vector space is divided into *Voronoi cells*, which are essentially partitions that consist of all the points in the space that are within a threshold distance of the given region's seed point. These seed points are used to create an inverted index that correlates each centroid with a list of vectors in the space, allowing a search to be restricted to just a subset of vectors in the index.
+In IVF, the PQ vector space is divided into *Voronoi cells*, which are essentially partitions that consist of all the points in the space that are within a threshold distance of the given region's seed point. These seed points are initialized by running K-means over the stored vectors. The centroids of K-means turn into the seed points which then each define a region. These regions are then are used to create an inverted index that correlates each centroid with a list of vectors in the space, allowing a search to be restricted to just a subset of vectors in the index.
 
 ![](../assets/ivfpq_ivf_desc.webp)
 
@@ -81,24 +81,4 @@ The above query will perform a search on the table `tbl` using the given query v
 * `to_pandas()`: Convert the results to a pandas DataFrame
 
 And there you have it! You now understand what an IVF-PQ index is, and how to create and query it in LanceDB.
-
-
-## FAQ
-
-### When is it necessary to create a vector index?
-
-LanceDB has manually-tuned SIMD code for computing vector distances. In our benchmarks, computing 100K pairs of 1K dimension vectors takes **<20ms**. For small datasets (<100K rows) or applications that can accept up to 100ms latency, vector indices are usually not necessary.
-
-For large-scale or higher dimension vectors, it is beneficial to create vector index.
-
-### How big is my index, and how much memory will it take?
-
-In LanceDB, all vector indices are disk-based, meaning that when responding to a vector query, only the relevant pages from the index file are loaded from disk and cached in memory. Additionally, each sub-vector is usually encoded into 1 byte PQ code.
-
-For example, with 1024-dimension vectors, if we choose `num_sub_vectors = 64`, each sub-vector has `1024 / 64 = 16` float32 numbers. Product quantization can lead to approximately `16 * sizeof(float32) / 1 = 64` times of space reduction.
-
-### How to choose `num_partitions` and `num_sub_vectors` for IVF_PQ index?
-
-`num_partitions` is used to decide how many partitions the first level IVF index uses. Higher number of partitions could lead to more efficient I/O during queries and better accuracy, but it takes much more time to train. On SIFT-1M dataset, our benchmark shows that keeping each partition 1K-4K rows lead to a good latency/recall.
-
-`num_sub_vectors` specifies how many PQ short codes to generate on each vector. Because PQ is a lossy compression of the original vector, a higher `num_sub_vectors` usually results in less space distortion, and thus yields better accuracy. However, a higher `num_sub_vectors` also causes heavier I/O and more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.
+To see how to create an IVF-PQ index in LanceDB, take a look at the [ANN indexes](../ann_indexes.md) section.

docs/src/embeddings/custom_embedding_function.md

@@ -17,6 +17,7 @@ Let's implement `SentenceTransformerEmbeddings` class. All you need to do is imp
 
 ```python
 from lancedb.embeddings import register
+from lancedb.util import attempt_import_or_raise
 
 @register("sentence-transformers")
 class SentenceTransformerEmbeddings(TextEmbeddingFunction):
@@ -81,7 +82,7 @@ class OpenClipEmbeddings(EmbeddingFunction):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        open_clip = self.safe_import("open_clip", "open-clip") # EmbeddingFunction util to import external libs and raise if not found
+        open_clip = attempt_import_or_raise("open_clip", "open-clip") # EmbeddingFunction util to import external libs and raise if not found
         model, _, preprocess = open_clip.create_model_and_transforms(
             self.name, pretrained=self.pretrained
         )
@@ -109,14 +110,14 @@ class OpenClipEmbeddings(EmbeddingFunction):
         if isinstance(query, str):
             return [self.generate_text_embeddings(query)]
         else:
-            PIL = self.safe_import("PIL", "pillow")
+            PIL = attempt_import_or_raise("PIL", "pillow")
             if isinstance(query, PIL.Image.Image):
                 return [self.generate_image_embedding(query)]
             else:
                 raise TypeError("OpenClip supports str or PIL Image as query")
 
     def generate_text_embeddings(self, text: str) -> np.ndarray:
-        torch = self.safe_import("torch")
+        torch = attempt_import_or_raise("torch")
         text = self.sanitize_input(text)
         text = self._tokenizer(text)
         text.to(self.device)
@@ -175,7 +176,7 @@ class OpenClipEmbeddings(EmbeddingFunction):
             The image to embed. If the image is a str, it is treated as a uri.
             If the image is bytes, it is treated as the raw image bytes.
         """
-        torch = self.safe_import("torch")
+        torch = attempt_import_or_raise("torch")
         # TODO handle retry and errors for https
         image = self._to_pil(image)
         image = self._preprocess(image).unsqueeze(0)
@@ -183,7 +184,7 @@ class OpenClipEmbeddings(EmbeddingFunction):
             return self._encode_and_normalize_image(image)
 
     def _to_pil(self, image: Union[str, bytes]):
-        PIL = self.safe_import("PIL", "pillow")
+        PIL = attempt_import_or_raise("PIL", "pillow")
         if isinstance(image, bytes):
             return PIL.Image.open(io.BytesIO(image))
         if isinstance(image, PIL.Image.Image):

docs/src/embeddings/default_embedding_functions.md

@@ -9,6 +9,9 @@ Contains the text embedding functions registered by default.
 ### Sentence transformers
 Allows you to set parameters when registering a `sentence-transformers` object.
 
+!!! info
+    Sentence transformer embeddings are normalized by default. It is recommended to use normalized embeddings for similarity search.
+
 | Parameter | Type | Default Value | Description |
 |---|---|---|---|
 | `name` | `str` | `all-MiniLM-L6-v2` | The name of the model |
@@ -44,6 +47,7 @@ LanceDB registers the OpenAI embeddings function in the registry by default, as
 | Parameter | Type | Default Value | Description |
 |---|---|---|---|
 | `name` | `str` | `"text-embedding-ada-002"` | The name of the model. |
+| `dim` | `int` |  Model default   | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it |
 
 
 ```python
@@ -119,7 +123,7 @@ texts = [{"text": "Capitalism has been dominant in the Western world since the e
 tbl.add(texts)
 ```
 
-## Gemini Embedding Function
+### Gemini Embeddings
 With Google's Gemini, you can represent text (words, sentences, and blocks of text) in a vectorized form, making it easier to compare and contrast embeddings. For example, two texts that share a similar subject matter or sentiment should have similar embeddings, which can be identified through mathematical comparison techniques such as cosine similarity. For more on how and why you should use embeddings, refer to the Embeddings guide.
 The Gemini Embedding Model API supports various task types:
 
@@ -155,6 +159,52 @@ tbl.add(df)
 rs = tbl.search("hello").limit(1).to_pandas()
 ```
 
+### AWS Bedrock Text Embedding Functions
+AWS Bedrock supports multiple base models for generating text embeddings. You need to setup the AWS credentials to use this embedding function.
+You can do so by using `awscli` and also add your session_token:
+```shell
+aws configure
+aws configure set aws_session_token "<your_session_token>"
+```
+to ensure that the credentials are set up correctly, you can run the following command:
+```shell
+aws sts get-caller-identity
+```
+
+Supported Embedding modelIDs are:
+* `amazon.titan-embed-text-v1`
+* `cohere.embed-english-v3`
+* `cohere.embed-multilingual-v3`
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| **name** | str | "amazon.titan-embed-text-v1" | The model ID of the bedrock model to use. Supported base models for Text Embeddings: amazon.titan-embed-text-v1, cohere.embed-english-v3, cohere.embed-multilingual-v3 |
+| **region** | str | "us-east-1" | Optional name of the AWS Region in which the service should be called (e.g., "us-east-1"). |
+| **profile_name** | str | None | Optional name of the AWS profile to use for calling the Bedrock service. If not specified, the default profile will be used. |
+| **assumed_role** | str | None | Optional ARN of an AWS IAM role to assume for calling the Bedrock service. If not specified, the current active credentials will be used. |
+| **role_session_name** | str | "lancedb-embeddings" | Optional name of the AWS IAM role session to use for calling the Bedrock service. If not specified, a "lancedb-embeddings" name will be used. |
+| **runtime** | bool | True | Optional choice of getting different client to perform operations with the Amazon Bedrock service. |
+| **max_retries** | int | 7 | Optional number of retries to perform when a request fails. |
+
+Usage Example:
+
+```python
+model = get_registry().get("bedrock-text").create()
+
+class TextModel(LanceModel):
+    text: str = model.SourceField()
+    vector: Vector(model.ndims()) = model.VectorField()
+
+df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
+db = lancedb.connect("tmp_path")
+tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+tbl.add(df)
+rs = tbl.search("hello").limit(1).to_pandas()
+```
+
 ## Multi-modal embedding functions
 Multi-modal embedding functions allow you to query your table using both images and text.
 
@@ -174,7 +224,6 @@ This embedding function supports ingesting images as both bytes and urls. You ca
 !!! info
     LanceDB supports ingesting images directly from accessible links.
 
-
 ```python
 
 db = lancedb.connect(tmp_path)
@@ -240,4 +289,67 @@ print(actual.label)
 
 ```
 
+### Imagebind embeddings
+We have support for [imagebind](https://github.com/facebookresearch/ImageBind) model embeddings. You can download our version of the packaged model via - `pip install imagebind-packaged==0.1.2`.
+
+This function is registered as `imagebind` and supports Audio, Video and Text modalities(extending to Thermal,Depth,IMU data):
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"imagebind_huge"` | Name of the model. |
+| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
+| `normalize` | `bool` | `False` | set to `True` to normalize your inputs before model ingestion. |
+
+Below is an example demonstrating how the API works:
+
+```python
+db = lancedb.connect(tmp_path)
+registry = EmbeddingFunctionRegistry.get_instance()
+func = registry.get("imagebind").create()
+
+class ImageBindModel(LanceModel):
+    text: str
+    image_uri: str = func.SourceField()
+    audio_path: str
+    vector: Vector(func.ndims()) = func.VectorField()
+
+# add locally accessible image paths
+text_list=["A dog.", "A car", "A bird"]
+image_paths=[".assets/dog_image.jpg", ".assets/car_image.jpg", ".assets/bird_image.jpg"]
+audio_paths=[".assets/dog_audio.wav", ".assets/car_audio.wav", ".assets/bird_audio.wav"]
+
+# Load data
+inputs = [
+    {"text": a, "audio_path": b, "image_uri": c}
+    for a, b, c in zip(text_list, audio_paths, image_paths)
+]
+
+#create table and add data
+table = db.create_table("img_bind", schema=ImageBindModel)
+table.add(inputs)
+```
+
+Now, we can search using any modality:
+
+#### image search
+```python
+query_image = "./assets/dog_image2.jpg" #download an image and enter that path here
+actual = table.search(query_image).limit(1).to_pydantic(ImageBindModel)[0]
+print(actual.text == "dog")
+```
+#### audio search
+
+```python
+query_audio = "./assets/car_audio2.wav" #download an audio clip and enter path here
+actual = table.search(query_audio).limit(1).to_pydantic(ImageBindModel)[0]
+print(actual.text == "car")
+```
+#### Text search
+You can add any input query and fetch the result as follows:
+```python
+query = "an animal which flies and tweets" 
+actual = table.search(query).limit(1).to_pydantic(ImageBindModel)[0]
+print(actual.text == "bird")
+```
+
 If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue [on GitHub](https://github.com/lancedb/lancedb/issues).

docs/src/embeddings/embedding_explicit.md

@@ -1,141 +0,0 @@
-In this workflow, you define your own embedding function and pass it as a callable to LanceDB, invoking it in your code to generate the embeddings. Let's look at some examples.
-
-### Hugging Face
-
-!!! note
-    Currently, the Hugging Face method is only supported in the Python SDK.
-
-=== "Python"
-    The most popular open source option is to use the [sentence-transformers](https://www.sbert.net/) 
-    library, which can be installed via pip.
-
-    ```bash
-    pip install sentence-transformers
-    ```
-
-    The example below shows how to use the `paraphrase-albert-small-v2` model to generate embeddings 
-    for a given document.
-
-    ```python
-    from sentence_transformers import SentenceTransformer
-
-    name="paraphrase-albert-small-v2"
-    model = SentenceTransformer(name)
-
-    # used for both training and querying
-    def embed_func(batch):
-        return [model.encode(sentence) for sentence in batch]
-    ```
-
-### OpenAI
-
-Another popular alternative is to use an external API like OpenAI's [embeddings API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings).
-
-=== "Python"
-      ```python
-        import openai
-        import os
-
-        # Configuring the environment variable OPENAI_API_KEY
-        if "OPENAI_API_KEY" not in os.environ:
-        # OR set the key here as a variable
-        openai.api_key = "sk-..."
-
-        # verify that the API key is working
-        assert len(openai.Model.list()["data"]) > 0
-
-        def embed_func(c):
-            rs = openai.Embedding.create(input=c, engine="text-embedding-ada-002")
-            return [record["embedding"] for record in rs["data"]]
-      ```
-
-=== "JavaScript"
-      ```javascript
-        const lancedb = require("vectordb");
-
-        // You need to provide an OpenAI API key
-        const apiKey = "sk-..."
-        // The embedding function will create embeddings for the 'text' column
-        const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
-      ```
-
-## Applying an embedding function to data
-
-=== "Python"
-    Using an embedding function, you can apply it to raw data
-    to generate embeddings for each record.
-
-    Say you have a pandas DataFrame with a `text` column that you want embedded,
-    you can use the `with_embeddings` function to generate embeddings and add them to 
-    an existing table.
-
-    ```python
-     import pandas as pd
-     from lancedb.embeddings import with_embeddings
-
-     df = pd.DataFrame(
-        [
-            {"text": "pepperoni"},
-            {"text": "pineapple"}
-        ]
-    )
-     data = with_embeddings(embed_func, df)
-
-     # The output is used to create / append to a table
-     # db.create_table("my_table", data=data)
-    ```
-
-    If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
-
-    By default, LanceDB calls the function with batches of 1000 rows. This can be configured
-    using the `batch_size` parameter to `with_embeddings`.
-
-    LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
-    API call is reliable.
-
-=== "JavaScript"
-    Using an embedding function, you can apply it to raw data
-    to generate embeddings for each record.
-
-    Simply pass the embedding function created above and LanceDB will use it to generate
-    embeddings for your data.
-
-    ```javascript
-    const db = await lancedb.connect("data/sample-lancedb");
-    const data = [
-    { text: "pepperoni"},
-    { text: "pineapple"}
-    ]
-
-    const table = await db.createTable("vectors", data, embedding)
-    ```
-
-## Querying using an embedding function
-
-!!! warning
-    At query time, you **must** use the same embedding function you used to vectorize your data.
-    If you use a different embedding function, the embeddings will not reside in the same vector
-    space and the results will be nonsensical.
-
-=== "Python"
-     ```python
-     query = "What's the best pizza topping?"
-     query_vector = embed_func([query])[0]
-     results = (
-        tbl.search(query_vector)
-        .limit(10)
-        .to_pandas()
-     )
-     ```
-
-     The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
-
-=== "JavaScript"
-     ```javascript
-      const results = await table
-        .search("What's the best pizza topping?")
-        .limit(10)
-        .execute()
-     ```
-
-     The above snippet returns an array of records with the top 10 nearest neighbors to the query.

docs/src/embeddings/embedding_functions.md

@@ -3,61 +3,126 @@ Representing multi-modal data as vector embeddings is becoming a standard practi
 For this purpose, LanceDB introduces an **embedding functions API**, that allow you simply set up once, during the configuration stage of your project. After this, the table remembers it, effectively making the embedding functions *disappear in the background* so you don't have to worry about manually passing callables, and instead, simply focus on the rest of your data engineering pipeline.
 
 !!! warning
-    Using the implicit embeddings management approach means that you can forget about the manually passing around embedding
-    functions in your code, as long as you don't intend to change it at a later time. If your embedding function changes,
-    you'll have to re-configure your table with the new embedding function and regenerate the embeddings.
+    Using the embedding function registry means that you don't have to explicitly generate the embeddings yourself. 
+    However, if your embedding function changes, you'll have to re-configure your table with the new embedding function 
+    and regenerate the embeddings. In the future, we plan to support the ability to change the embedding function via
+    table metadata and have LanceDB automatically take care of regenerating the embeddings.
+
 
 ## 1. Define the embedding function
-We have some pre-defined embedding functions in the global registry, with more coming soon. Here's let's an implementation of CLIP as example.
-```
-registry = EmbeddingFunctionRegistry.get_instance()
-clip = registry.get("open-clip").create()
 
-```
-You can also define your own embedding function by implementing the `EmbeddingFunction` abstract base interface. It subclasses Pydantic Model which can be utilized to write complex schemas simply as we'll see next!
+=== "Python"
+    In the LanceDB python SDK, we define a global embedding function registry with
+    many different embedding models and even more coming soon. 
+    Here's let's an implementation of CLIP as example.
+
+    ```python
+    from lancedb.embeddings import get_registry
+
+    registry = get_registry()
+    clip = registry.get("open-clip").create()
+    ```
+
+    You can also define your own embedding function by implementing the `EmbeddingFunction` 
+    abstract base interface. It subclasses Pydantic Model which can be utilized to write complex schemas simply as we'll see next!
+
+=== "JavaScript""
+    In the TypeScript SDK, the choices are more limited. For now, only the OpenAI
+    embedding function is available.
+
+    ```javascript
+    const lancedb = require("vectordb");
+
+    // You need to provide an OpenAI API key
+    const apiKey = "sk-..."
+    // The embedding function will create embeddings for the 'text' column
+    const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
+    ```
 
 ## 2. Define the data model or schema
-The embedding function defined above abstracts away all the details about the models and dimensions required to define the schema. You can simply set a field as **source** or **vector** column. Here's how:
 
-```python
-class Pets(LanceModel):
-    vector: Vector(clip.ndims) = clip.VectorField()
-    image_uri: str = clip.SourceField()
-```
+=== "Python"
+    The embedding function defined above abstracts away all the details about the models and dimensions required to define the schema. You can simply set a field as **source** or **vector** column. Here's how:
 
-`VectorField` tells LanceDB to use the clip embedding function to generate query embeddings for the `vector` column and `SourceField` ensures that when adding data, we automatically use the specified embedding function to encode `image_uri`.
+    ```python
+    class Pets(LanceModel):
+        vector: Vector(clip.ndims) = clip.VectorField()
+        image_uri: str = clip.SourceField()
+    ```
 
-## 3. Create LanceDB table
-Now that we have chosen/defined our embedding function and the schema, we can create the table:
+    `VectorField` tells LanceDB to use the clip embedding function to generate query embeddings for the `vector` column and `SourceField` ensures that when adding data, we automatically use the specified embedding function to encode `image_uri`.
 
-```python
-db = lancedb.connect("~/lancedb")
-table = db.create_table("pets", schema=Pets)
+=== "JavaScript"
 
-```
+    For the TypeScript SDK, a schema can be inferred from input data, or an explicit
+    Arrow schema can be provided.
 
-That's it! We've provided all the information needed to embed the source and query inputs. We can now forget about the model and dimension details and start to build our VectorDB pipeline.
+## 3. Create table and add data
 
-## 4. Ingest lots of data and query your table
-Any new or incoming data can just be added and it'll be vectorized automatically.
+Now that we have chosen/defined our embedding function and the schema, 
+we can create the table and ingest data without needing to explicitly generate
+the embeddings at all:
 
-```python
-table.add([{"image_uri": u} for u in uris])
-```
+=== "Python"
+    ```python
+    db = lancedb.connect("~/lancedb")
+    table = db.create_table("pets", schema=Pets)
 
-Our OpenCLIP query embedding function supports querying via both text and images:
+    table.add([{"image_uri": u} for u in uris])
+    ```
 
-```python
-result = table.search("dog")
-```
+=== "JavaScript"
 
-Let's query an image:
+    ```javascript
+    const db = await lancedb.connect("data/sample-lancedb");
+    const data = [
+    { text: "pepperoni"},
+    { text: "pineapple"}
+    ]
 
-```python
-p = Path("path/to/images/samoyed_100.jpg")
-query_image = Image.open(p)
-table.search(query_image)
-```
+    const table = await db.createTable("vectors", data, embedding)
+    ```
+
+## 4. Querying your table
+Not only can you forget about the embeddings during ingestion, you also don't
+need to worry about it when you query the table:
+
+=== "Python"
+
+    Our OpenCLIP query embedding function supports querying via both text and images:
+
+    ```python
+    results = (
+        table.search("dog")
+        .limit(10)
+        .to_pandas()
+    )
+    ```
+
+    Or we can search using an image:
+
+    ```python
+    p = Path("path/to/images/samoyed_100.jpg")
+    query_image = Image.open(p)
+    results = (
+        table.search(query_image)
+        .limit(10)
+        .to_pandas()
+    )
+    ```
+
+    Both of the above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
+
+=== "JavaScript"
+
+    ```javascript
+    const results = await table
+    .search("What's the best pizza topping?")
+    .limit(10)
+    .execute()
+    ```    
+    
+    The above snippet returns an array of records with the top 10 nearest neighbors to the query.
 
 ---
 
@@ -100,4 +165,5 @@ rs[2].image
 
 ![](../assets/dog_clip_output.png)
 
-Now that you have the basic idea about implicit management via embedding functions, let's dive deeper into a [custom API](./api.md) that you can use to implement your own embedding functions.
+Now that you have the basic idea about LanceDB embedding functions and the embedding function registry,
+let's dive deeper into defining your own [custom functions](./custom_embedding_function.md).

docs/src/embeddings/index.md

@@ -1,8 +1,14 @@
-Due to the nature of vector embeddings, they can be used to represent any kind of data, from text to images to audio. This makes them a very powerful tool for machine learning practitioners. However, there's no one-size-fits-all solution for generating embeddings - there are many different libraries and APIs (both commercial and open source) that can be used to generate embeddings from structured/unstructured data.
+Due to the nature of vector embeddings, they can be used to represent any kind of data, from text to images to audio. 
+This makes them a very powerful tool for machine learning practitioners. 
+However, there's no one-size-fits-all solution for generating embeddings - there are many different libraries and APIs 
+(both commercial and open source) that can be used to generate embeddings from structured/unstructured data.
 
-LanceDB supports 2 methods of vectorizing your raw data into embeddings.
+LanceDB supports 3 methods of working with embeddings.
 
-1. **Explicit**: By manually calling LanceDB's `with_embedding` function to vectorize your data via an `embed_func` of your choice
-2. **Implicit**: Allow LanceDB to embed the data and queries in the background as they come in, by using the table's `EmbeddingRegistry` information
+1. You can manually generate embeddings for the data and queries. This is done outside of LanceDB.
+2. You can use the built-in [embedding functions](./embedding_functions.md) to embed the data and queries in the background.
+3. For python users, you can define your own [custom embedding function](./custom_embedding_function.md)
+   that extends the default embedding functions.
 
-See the [explicit](embedding_explicit.md) and [implicit](embedding_functions.md) embedding sections for more details.
+For python users, there is also a legacy [with_embeddings API](./legacy.md).
+It is retained for compatibility and will be removed in a future version.

docs/src/embeddings/legacy.md

@@ -0,0 +1,99 @@
+The legacy `with_embeddings` API is for Python only and is deprecated.
+
+### Hugging Face
+
+The most popular open source option is to use the [sentence-transformers](https://www.sbert.net/) 
+library, which can be installed via pip.
+
+```bash
+pip install sentence-transformers
+```
+
+The example below shows how to use the `paraphrase-albert-small-v2` model to generate embeddings 
+for a given document.
+
+```python
+from sentence_transformers import SentenceTransformer
+
+name="paraphrase-albert-small-v2"
+model = SentenceTransformer(name)
+
+# used for both training and querying
+def embed_func(batch):
+    return [model.encode(sentence) for sentence in batch]
+```
+
+
+### OpenAI
+
+Another popular alternative is to use an external API like OpenAI's [embeddings API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings).
+
+```python
+import openai
+import os
+
+# Configuring the environment variable OPENAI_API_KEY
+if "OPENAI_API_KEY" not in os.environ:
+# OR set the key here as a variable
+openai.api_key = "sk-..."
+
+client = openai.OpenAI()
+
+def embed_func(c):    
+    rs = client.embeddings.create(input=c, model="text-embedding-ada-002")
+    return [record.embedding for record in rs["data"]]
+```
+
+
+## Applying an embedding function to data
+
+Using an embedding function, you can apply it to raw data
+to generate embeddings for each record.
+
+Say you have a pandas DataFrame with a `text` column that you want embedded,
+you can use the `with_embeddings` function to generate embeddings and add them to 
+an existing table.
+
+```python
+    import pandas as pd
+    from lancedb.embeddings import with_embeddings
+
+    df = pd.DataFrame(
+        [
+            {"text": "pepperoni"},
+            {"text": "pineapple"}
+        ]
+    )
+    data = with_embeddings(embed_func, df)
+
+    # The output is used to create / append to a table
+    tbl = db.create_table("my_table", data=data)
+```
+
+If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
+
+By default, LanceDB calls the function with batches of 1000 rows. This can be configured
+using the `batch_size` parameter to `with_embeddings`.
+
+LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
+API call is reliable.
+
+## Querying using an embedding function
+
+!!! warning
+    At query time, you **must** use the same embedding function you used to vectorize your data.
+    If you use a different embedding function, the embeddings will not reside in the same vector
+    space and the results will be nonsensical.
+
+=== "Python"
+     ```python
+     query = "What's the best pizza topping?"
+     query_vector = embed_func([query])[0]
+     results = (
+        tbl.search(query_vector)
+        .limit(10)
+        .to_pandas()
+     )
+     ```
+
+     The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.

docs/src/examples/examples_rust.md

@@ -0,0 +1,3 @@
+# Examples: Rust
+
+Our Rust SDK is now stable. Examples are coming soon.

docs/src/examples/image_embeddings_roboflow.md

@@ -43,7 +43,7 @@ pip install lancedb
 We also need to install a specific commit of `tantivy`, a dependency of the LanceDB full text search engine we will use later in this guide:
 
 ```
-pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+pip install tantivy
 ```
 
 Create a new Python file and add the following code:

docs/src/examples/index.md

@@ -2,10 +2,11 @@
 
 ## Recipes and example code
 
-LanceDB provides language APIs, allowing you to embed a database in your language of choice. We currently provide Python and Javascript APIs, with the Rust API and examples actively being worked on and will be available soon.
+LanceDB provides language APIs, allowing you to embed a database in your language of choice.
 
 * 🐍 [Python](examples_python.md) examples
-* 👾 [JavaScript](exampled_js.md) examples
+* 👾 [JavaScript](examples_js.md) examples
+* 🦀 Rust examples (coming soon)
 
 ## Applications powered by LanceDB
 

docs/src/examples/modal_langchain.py

@@ -1,6 +1,5 @@
 import pickle
 import re
-import sys
 import zipfile
 from pathlib import Path
 
@@ -79,7 +78,10 @@ def qanda_langchain(query):
     download_docs()
     docs = store_docs()
 
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200,)
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=1000,
+        chunk_overlap=200,
+    )
     documents = text_splitter.split_documents(docs)
     embeddings = OpenAIEmbeddings()
 

docs/src/extra_js/init_ask_ai_widget.js

@@ -0,0 +1,79 @@
+// Creates an SVG robot icon (from Lucide)
+function robotSVG() {
+  var svg = document.createElementNS("http://www.w3.org/2000/svg", "svg");
+  svg.setAttribute("width", "24");
+  svg.setAttribute("height", "24");
+  svg.setAttribute("viewBox", "0 0 24 24");
+  svg.setAttribute("fill", "none");
+  svg.setAttribute("stroke", "currentColor");
+  svg.setAttribute("stroke-width", "2");
+  svg.setAttribute("stroke-linecap", "round");
+  svg.setAttribute("stroke-linejoin", "round");
+  svg.setAttribute("class", "lucide lucide-bot-message-square");
+
+  var path1 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path1.setAttribute("d", "M12 6V2H8");
+  svg.appendChild(path1);
+
+  var path2 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path2.setAttribute("d", "m8 18-4 4V8a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2Z");
+  svg.appendChild(path2);
+
+  var path3 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path3.setAttribute("d", "M2 12h2");
+  svg.appendChild(path3);
+
+  var path4 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path4.setAttribute("d", "M9 11v2");
+  svg.appendChild(path4);
+
+  var path5 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path5.setAttribute("d", "M15 11v2");
+  svg.appendChild(path5);
+
+  var path6 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path6.setAttribute("d", "M20 12h2");
+  svg.appendChild(path6);
+
+  return svg
+}
+
+// Creates the Fluidic Chatbot buttom
+function fluidicButton() {
+  var btn = document.createElement("a");
+  btn.href = "https://asklancedb.com";
+  btn.target = "_blank";
+  btn.style.position = "fixed";
+  btn.style.fontWeight = "bold";
+  btn.style.fontSize = ".8rem";
+  btn.style.right = "10px";
+  btn.style.bottom = "10px";
+  btn.style.width = "80px";
+  btn.style.height = "80px";
+  btn.style.background = "linear-gradient(135deg, #7C5EFF 0%, #625eff 100%)";
+  btn.style.color = "white";
+  btn.style.borderRadius = "5px";
+  btn.style.display = "flex";
+  btn.style.flexDirection = "column";
+  btn.style.justifyContent = "center";
+  btn.style.alignItems = "center";
+  btn.style.zIndex = "1000";
+  btn.style.opacity = "0";
+  btn.style.boxShadow = "0 0 0 rgba(0, 0, 0, 0)";
+  btn.style.transition = "opacity 0.2s ease-in, box-shadow 0.2s ease-in";
+
+  setTimeout(function() {
+      btn.style.opacity = "1";
+      btn.style.boxShadow = "0 0 .2rem #0000001a,0 .2rem .4rem #0003"
+  }, 0);
+
+  return btn
+}
+
+document.addEventListener("DOMContentLoaded", function() {
+  var btn = fluidicButton()
+  btn.appendChild(robotSVG());
+  var text = document.createTextNode("Ask AI");
+  btn.appendChild(text);
+  document.body.appendChild(btn);
+});

docs/src/faq.md

@@ -16,7 +16,7 @@ As we mention in our talk titled “[Lance, a modern columnar data format](https
 
 ### Why build in Rust? 🦀
 
-We believe that the Rust ecosystem has attained mainstream maturity and that Rust will form the underpinnings of large parts of the data and ML landscape in a few years. Performance, latency and reliability are paramount to a vector DB, and building in Rust allows us to iterate and release updates more rapidly due to Rust’s safety guarantees. Both Lance (the data format) and LanceDB (the database) are written entirely in Rust. We also provide Python and JavaScript client libraries to interact with the database. Our Rust API is a little rough around the edges right now, but is fast becoming on par with the Python and JS APIs.
+We believe that the Rust ecosystem has attained mainstream maturity and that Rust will form the underpinnings of large parts of the data and ML landscape in a few years. Performance, latency and reliability are paramount to a vector DB, and building in Rust allows us to iterate and release updates more rapidly due to Rust’s safety guarantees. Both Lance (the data format) and LanceDB (the database) are written entirely in Rust. We also provide Python, JavaScript, and Rust client libraries to interact with the database.
 
 ### What is the difference between LanceDB OSS and LanceDB Cloud?
 
@@ -40,11 +40,11 @@ LanceDB and its underlying data format, Lance, are built to scale to really larg
 
 No. LanceDB is blazing fast (due to its disk-based index) for even brute force kNN search, within reason. In our benchmarks, computing 100K pairs of 1000-dimension vectors takes less than 20ms. For small datasets of ~100K records or applications that can accept ~100ms latency, an ANN index is usually not necessary.
 
-For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index.
+For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index. See the [ANN indexes](ann_indexes.md) section for more details.
 
 ### Does LanceDB support full-text search?
 
-Yes, LanceDB supports full-text search (FTS) via [Tantivy](https://github.com/quickwit-oss/tantivy). Our current FTS integration is Python-only, and our goal is to push it down to the Rust level in future versions to enable much more powerful search capabilities available to our Python, JavaScript and Rust clients.
+Yes, LanceDB supports full-text search (FTS) via [Tantivy](https://github.com/quickwit-oss/tantivy). Our current FTS integration is Python-only, and our goal is to push it down to the Rust level in future versions to enable much more powerful search capabilities available to our Python, JavaScript and Rust clients.  Follow along in the [Github issue](https://github.com/lancedb/lance/issues/1195)
 
 ### How can I speed up data inserts?
 
@@ -69,3 +69,19 @@ MinIO supports an S3 compatible API. In order to connect to a MinIO instance, yo
 - Set the envvar `AWS_ENDPOINT` to the URL of your MinIO API
 - Set the envvars `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` with your MinIO credential
 - Call `lancedb.connect("s3://minio_bucket_name")`
+
+### Where can I find benchmarks for LanceDB?
+
+Refer to this [post](https://blog.lancedb.com/benchmarking-lancedb-92b01032874a) for recent benchmarks.
+
+### How much data can LanceDB practically manage without effecting performance?
+
+We target good performance on ~10-50 billion rows and ~10-30 TB of data.
+
+### Does LanceDB support concurrent operations?
+
+LanceDB can handle concurrent reads very well, and can scale horizontally. The main constraint is how well the [storage layer](https://lancedb.github.io/lancedb/concepts/storage/) you've chosen scales. For writes, we support concurrent writing, though too many concurrent writers can lead to failing writes as there is a limited number of times a writer retries a commit
+
+!!! info "Multiprocessing with LanceDB"
+
+    For multiprocessing you should probably not use ```fork``` as lance is multi-threaded internally and ```fork``` and multi-thread do not work well.[Refer to this discussion](https://discuss.python.org/t/concerns-regarding-deprecation-of-fork-with-alive-threads/33555)

docs/src/fts.md

@@ -1,6 +1,6 @@
 # Full-text search
 
-LanceDB provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy) (currently Python only), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. Our goal is to push the FTS integration down to the Rust level in the future, so that it's available for JavaScript users as well.
+LanceDB provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy) (currently Python only), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. Our goal is to push the FTS integration down to the Rust level in the future, so that it's available for Rust and JavaScript users as well.  Follow along at [this Github issue](https://github.com/lancedb/lance/issues/1195)
 
 A hybrid search solution combining vector and full-text search is also on the way.
 
@@ -75,25 +75,74 @@ applied on top of the full text search results. This can be invoked via the fami
 table.search("puppy").limit(10).where("meta='foo'").to_list()
 ```
 
-## Syntax
+## Sorting
 
-For full-text search you can perform either a phrase query like "the old man and the sea", 
-or a structured search query like "(Old AND Man) AND Sea".
-Double quotes are used to disambiguate.
+You can pre-sort the documents by specifying `ordering_field_names` when
+creating the full-text search index. Once pre-sorted, you can then specify
+`ordering_field_name` while searching to return results sorted by the given
+field. For example, 
 
-For example: 
+```
+table.create_fts_index(["text_field"], ordering_field_names=["sort_by_field"])
 
-If you intended "they could have been dogs OR cats" as a phrase query, this actually
-raises a syntax error since `OR` is a recognized operator. If you make `or` lower case,
-this avoids the syntax error. However, it is cumbersome to have to remember what will
-conflict with the query syntax. Instead, if you search using 
-`table.search('"they could have been dogs OR cats"')`, then the syntax checker avoids
-checking inside the quotes.
+(table.search("terms", ordering_field_name="sort_by_field")
+ .limit(20)
+ .to_list())
+```
 
+!!! note
+    If you wish to specify an ordering field at query time, you must also
+    have specified it during indexing time. Otherwise at query time, an
+    error will be raised that looks like `ValueError: The field does not exist: xxx`
+
+!!! note
+    The fields to sort on must be of typed unsigned integer, or else you will see 
+    an error during indexing that looks like 
+    `TypeError: argument 'value': 'float' object cannot be interpreted as an integer`.
+
+!!! note
+    You can specify multiple fields for ordering at indexing time.
+    But at query time only one ordering field is supported.
+
+
+## Phrase queries vs. terms queries
+
+For full-text search you can specify either a **phrase** query like `"the old man and the sea"`,
+or a **terms** search query like `"(Old AND Man) AND Sea"`. For more details on the terms
+query syntax, see Tantivy's [query parser rules](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html).
+
+!!! tip "Note"
+    The query parser will raise an exception on queries that are ambiguous. For example, in the query `they could have been dogs OR cats`, `OR` is capitalized so it's considered a keyword query operator. But it's ambiguous how the left part should be treated. So if you submit this search query as is, you'll get `Syntax Error: they could have been dogs OR cats`.
+
+    ```py
+    # This raises a syntax error
+    table.search("they could have been dogs OR cats")
+    ```
+
+    On the other hand, lowercasing `OR` to `or` will work, because there are no capitalized logical operators and
+    the query is treated as a phrase query.
+
+    ```py
+    # This works!
+    table.search("they could have been dogs or cats")
+    ```
+
+It can be cumbersome to have to remember what will cause a syntax error depending on the type of
+query you want to perform. To make this simpler, when you want to perform a phrase query, you can
+enforce it in one of two ways:
+
+1. Place the double-quoted query inside single quotes. For example, `table.search('"they could have been dogs OR cats"')` is treated as
+a phrase query.
+2. Explicitly declare the `phrase_query()` method. This is useful when you have a phrase query that
+itself contains double quotes. For example, `table.search('the cats OR dogs were not really "pets" at all').phrase_query()`
+is treated as a phrase query.
+
+In general, a query that's declared as a phrase query will be wrapped in double quotes during parsing, with nested
+double quotes replaced by single quotes.
 
 ## Configurations
 
-By default, LanceDB configures a 1GB heap size limit for creating the index. You can 
+By default, LanceDB configures a 1GB heap size limit for creating the index. You can
 reduce this if running on a smaller node, or increase this for faster performance while
 indexing a larger corpus.
 
@@ -109,7 +158,6 @@ table.create_fts_index(["text1", "text2"], writer_heap_size=heap, replace=True)
    If you add data after FTS index creation, it won't be reflected
    in search results until you do a full reindex.
 
-2. We currently only support local filesystem paths for the FTS index. 
+2. We currently only support local filesystem paths for the FTS index.
    This is a tantivy limitation. We've implemented an object store plugin
    but there's no way in tantivy-py to specify to use it.
-

docs/src/guides/storage.md

@@ -68,6 +68,82 @@ Alternatively, if you are using AWS SSO, you can use the `AWS_PROFILE` and `AWS_
 
 You can see a full list of environment variables [here](https://docs.rs/object_store/latest/object_store/aws/struct.AmazonS3Builder.html#method.from_env).
 
+!!! tip "Automatic cleanup for failed writes"
+
+    LanceDB uses [multi-part uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html) when writing data to S3 in order to maximize write speed. LanceDB will abort these uploads when it shuts down gracefully, such as when cancelled by keyboard interrupt. However, in the rare case that LanceDB crashes, it is possible that some data will be left lingering in your account. To cleanup this data, we recommend (as AWS themselves do) that you setup a lifecycle rule to delete in-progress uploads after 7 days. See the AWS guide:
+
+    **[Configuring a bucket lifecycle configuration to delete incomplete multipart uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpu-abort-incomplete-mpu-lifecycle-config.html)**
+
+#### AWS IAM Permissions
+
+If a bucket is private, then an IAM policy must be specified to allow access to it. For many development scenarios, using broad permissions such as a PowerUser account is more than sufficient for working with LanceDB. However, in many production scenarios, you may wish to have as narrow as possible permissions.
+
+For **read and write access**, LanceDB will need a policy such as:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+              "s3:PutObject",
+              "s3:GetObject",
+              "s3:DeleteObject",
+            ],
+            "Resource": "arn:aws:s3:::<bucket>/<prefix>/*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "s3:ListBucket",
+                "s3:GetBucketLocation"
+            ],
+            "Resource": "arn:aws:s3:::<bucket>",
+            "Condition": {
+                "StringLike": {
+                    "s3:prefix": [
+                        "<prefix>/*"
+                    ]
+                }
+            }
+        }
+    ]
+}
+```
+
+For **read-only access**, LanceDB will need a policy such as:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+              "s3:GetObject",
+            ],
+            "Resource": "arn:aws:s3:::<bucket>/<prefix>/*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "s3:ListBucket",
+                "s3:GetBucketLocation"
+            ],
+            "Resource": "arn:aws:s3:::<bucket>",
+            "Condition": {
+                "StringLike": {
+                    "s3:prefix": [
+                        "<prefix>/*"
+                    ]
+                }
+            }
+        }
+    ]
+}
+```
+
 #### S3-compatible stores
 
 LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you must specify two environment variables: `AWS_ENDPOINT` and `AWS_DEFAULT_REGION`. `AWS_ENDPOINT` should be the URL of the S3-compatible store, and `AWS_DEFAULT_REGION` should be the region to use.

docs/src/guides/tables.md

@@ -16,9 +16,22 @@ This guide will show how to create tables, insert data into them, and update the
     db = lancedb.connect("./.lancedb")
     ```
 
+=== "Javascript"
+
+    Initialize a VectorDB connection and create a table using one of the many methods listed below.
+
+    ```javascript
+    const lancedb = require("vectordb");
+
+    const uri = "data/sample-lancedb";
+    const db = await lancedb.connect(uri);
+    ```
+
     LanceDB allows ingesting data from various sources - `dict`, `list[dict]`, `pd.DataFrame`, `pa.Table` or a `Iterator[pa.RecordBatch]`. Let's take a look at some of the these.
 
-    ### From list of tuples or dictionaries
+### From list of tuples or dictionaries
+
+=== "Python"
 
     ```python
     import lancedb
@@ -32,7 +45,6 @@ This guide will show how to create tables, insert data into them, and update the
 
     db["my_table"].head()
     ```
-
     !!! info "Note"
         If the table already exists, LanceDB will raise an error by default. 
 
@@ -51,6 +63,27 @@ This guide will show how to create tables, insert data into them, and update the
         db.create_table("name", data, mode="overwrite")
         ```
 
+=== "Javascript"
+    You can create a LanceDB table in JavaScript using an array of JSON records as follows.
+
+    ```javascript
+    const tb = await db.createTable("my_table", [{
+        "vector": [3.1, 4.1],
+        "item": "foo",
+        "price": 10.0
+    }, {
+        "vector": [5.9, 26.5],
+        "item": "bar",
+        "price": 20.0
+    }]);
+    ```
+    !!! info "Note"
+    If the table already exists, LanceDB will raise an error by default. If you want to overwrite the table, you need to specify the `WriteMode` in the createTable function.
+
+    ```javascript
+    const table = await con.createTable(tableName, data, { writeMode: WriteMode.Overwrite })
+    ```
+
     ### From a Pandas DataFrame
 
     ```python
@@ -67,7 +100,9 @@ This guide will show how to create tables, insert data into them, and update the
     db["my_table"].head()
     ```
     !!! info "Note"
-        Data is converted to Arrow before being written to disk. For maximum control over how data is saved, either provide the PyArrow schema to convert to or else provide a PyArrow Table directly.
+    Data is converted to Arrow before being written to disk. For maximum control over how data is saved, either provide the PyArrow schema to convert to or else provide a PyArrow Table directly.
+
+    The **`vector`** column needs to be a [Vector](../python/pydantic.md#vector-field) (defined as [pyarrow.FixedSizeList](https://arrow.apache.org/docs/python/generated/pyarrow.list_.html)) type. 
 
     ```python
     custom_schema = pa.schema([
@@ -79,7 +114,7 @@ This guide will show how to create tables, insert data into them, and update the
     table = db.create_table("my_table", data, schema=custom_schema)
     ```
 
-    ### From a Polars DataFrame
+### From a Polars DataFrame
 
     LanceDB supports [Polars](https://pola.rs/), a modern, fast DataFrame library
     written in Rust. Just like in Pandas, the Polars integration is enabled by PyArrow
@@ -97,11 +132,168 @@ This guide will show how to create tables, insert data into them, and update the
     table = db.create_table("pl_table", data=data)
     ```
 
-    ### From PyArrow Tables
-    You can also create LanceDB tables directly from PyArrow tables
+### From an Arrow Table
+=== "Python"
+    You can also create LanceDB tables directly from Arrow tables. 
+    LanceDB supports float16 data type!
 
     ```python
-    table = pa.Table.from_arrays(
+    import pyarrows as pa
+    import numpy as np
+    
+    dim = 16
+    total = 2
+    schema = pa.schema(
+        [
+            pa.field("vector", pa.list_(pa.float16(), dim)),
+            pa.field("text", pa.string())
+        ]
+    )
+    data = pa.Table.from_arrays(
+        [
+            pa.array([np.random.randn(dim).astype(np.float16) for _ in range(total)],
+                    pa.list_(pa.float16(), dim)),
+            pa.array(["foo", "bar"])
+        ],
+        ["vector", "text"],
+    )
+    tbl = db.create_table("f16_tbl", data, schema=schema)
+    ```
+
+=== "Javascript"
+    You can also create LanceDB tables directly from Arrow tables. 
+    LanceDB supports Float16 data type!
+
+    ```javascript
+    --8<-- "docs/src/basic_legacy.ts:create_f16_table"
+    ```
+
+### From Pydantic Models
+
+When you create an empty table without data, you must specify the table schema.
+LanceDB supports creating tables by specifying a PyArrow schema or a specialized
+Pydantic model called `LanceModel`.
+
+For example, the following Content model specifies a table with 5 columns:
+`movie_id`, `vector`, `genres`, `title`, and `imdb_id`. When you create a table, you can
+pass the class as the value of the `schema` parameter to `create_table`.
+The `vector` column is a `Vector` type, which is a specialized Pydantic type that
+can be configured with the vector dimensions. It is also important to note that
+LanceDB only understands subclasses of `lancedb.pydantic.LanceModel`
+(which itself derives from `pydantic.BaseModel`).
+
+```python
+from lancedb.pydantic import Vector, LanceModel
+
+class Content(LanceModel):
+    movie_id: int
+    vector: Vector(128)
+    genres: str
+    title: str
+    imdb_id: int
+
+    @property
+    def imdb_url(self) -> str:
+        return f"https://www.imdb.com/title/tt{self.imdb_id}"
+
+import pyarrow as pa
+db = lancedb.connect("~/.lancedb")
+table_name = "movielens_small"
+table = db.create_table(table_name, schema=Content)
+```
+
+#### Nested schemas
+
+Sometimes your data model may contain nested objects.
+For example, you may want to store the document string
+and the document soure name as a nested Document object:
+
+```python
+class Document(BaseModel):
+    content: str
+    source: str
+```
+
+This can be used as the type of a LanceDB table column:
+
+```python
+class NestedSchema(LanceModel):
+    id: str
+    vector: Vector(1536)
+    document: Document
+
+tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite")
+```
+
+This creates a struct column called "document" that has two subfields 
+called "content" and "source":
+
+```
+In [28]: tbl.schema
+Out[28]:
+id: string not null
+vector: fixed_size_list<item: float>[1536] not null
+    child 0, item: float
+document: struct<content: string not null, source: string not null> not null
+    child 0, content: string not null
+    child 1, source: string not null    
+```
+
+#### Validators
+
+Note that neither Pydantic nor PyArrow automatically validates that input data
+is of the correct timezone, but this is easy to add as a custom field validator:
+
+```python
+from datetime import datetime
+from zoneinfo import ZoneInfo
+
+from lancedb.pydantic import LanceModel
+from pydantic import Field, field_validator, ValidationError, ValidationInfo
+
+tzname = "America/New_York"
+tz = ZoneInfo(tzname)
+
+class TestModel(LanceModel):
+    dt_with_tz: datetime = Field(json_schema_extra={"tz": tzname})
+
+    @field_validator('dt_with_tz')
+    @classmethod
+    def tz_must_match(cls, dt: datetime) -> datetime:
+        assert dt.tzinfo == tz
+        return dt        
+
+ok = TestModel(dt_with_tz=datetime.now(tz))
+
+try:
+    TestModel(dt_with_tz=datetime.now(ZoneInfo("Asia/Shanghai")))
+    assert 0 == 1, "this should raise ValidationError"
+except ValidationError:
+    print("A ValidationError was raised.")
+    pass
+```
+
+When you run this code it should print "A ValidationError was raised."
+
+#### Pydantic custom types
+
+LanceDB does NOT yet support converting pydantic custom types. If this is something you need,
+please file a feature request on the [LanceDB Github repo](https://github.com/lancedb/lancedb/issues/new).
+
+### Using Iterators / Writing Large Datasets
+
+It is recommended to use iterators to add large datasets in batches when creating your table in one go. This does not create multiple versions of your dataset unlike manually adding batches using `table.add()`
+
+LanceDB additionally supports PyArrow's `RecordBatch` Iterators or other generators producing supported data types.
+
+Here's an example using using `RecordBatch` iterator for creating tables.
+
+```python
+import pyarrow as pa
+
+def make_batches():
+    for i in range(5):
+        yield pa.RecordBatch.from_arrays(
             [
                 pa.array([[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],
                         pa.list_(pa.float32(), 4)),
@@ -111,186 +303,16 @@ This guide will show how to create tables, insert data into them, and update the
             ["vector", "item", "price"],
         )
 
-    db = lancedb.connect("db")
+schema = pa.schema([
+    pa.field("vector", pa.list_(pa.float32(), 4)),
+    pa.field("item", pa.utf8()),
+    pa.field("price", pa.float32()),
+])
 
-    tbl = db.create_table("my_table", table)
-    ```
+db.create_table("batched_tale", make_batches(), schema=schema)
+```
 
-    ### From Pydantic Models
-    When you create an empty table without data, you must specify the table schema.
-    LanceDB supports creating tables by specifying a PyArrow schema or a specialized
-    Pydantic model called `LanceModel`.
-
-    For example, the following Content model specifies a table with 5 columns:
-    `movie_id`, `vector`, `genres`, `title`, and `imdb_id`. When you create a table, you can
-    pass the class as the value of the `schema` parameter to `create_table`.
-    The `vector` column is a `Vector` type, which is a specialized Pydantic type that
-    can be configured with the vector dimensions. It is also important to note that
-    LanceDB only understands subclasses of `lancedb.pydantic.LanceModel`
-    (which itself derives from `pydantic.BaseModel`).
-
-    ```python
-    from lancedb.pydantic import Vector, LanceModel
-
-    class Content(LanceModel):
-        movie_id: int
-        vector: Vector(128)
-        genres: str
-        title: str
-        imdb_id: int
-
-        @property
-        def imdb_url(self) -> str:
-            return f"https://www.imdb.com/title/tt{self.imdb_id}"
-
-    import pyarrow as pa
-    db = lancedb.connect("~/.lancedb")
-    table_name = "movielens_small"
-    table = db.create_table(table_name, schema=Content)
-    ```
-
-    #### Nested schemas
-
-    Sometimes your data model may contain nested objects.
-    For example, you may want to store the document string
-    and the document soure name as a nested Document object:
-
-    ```python
-    class Document(BaseModel):
-        content: str
-        source: str
-    ```
-
-    This can be used as the type of a LanceDB table column:
-
-    ```python
-    class NestedSchema(LanceModel):
-        id: str
-        vector: Vector(1536)
-        document: Document
-
-    tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite")
-    ```
-
-    This creates a struct column called "document" that has two subfields 
-    called "content" and "source":
-
-    ```
-    In [28]: tbl.schema
-    Out[28]:
-    id: string not null
-    vector: fixed_size_list<item: float>[1536] not null
-        child 0, item: float
-    document: struct<content: string not null, source: string not null> not null
-        child 0, content: string not null
-        child 1, source: string not null    
-    ```
-
-    #### Validators
-
-    Note that neither Pydantic nor PyArrow automatically validates that input data
-    is of the correct timezone, but this is easy to add as a custom field validator:
-
-    ```python
-    from datetime import datetime
-    from zoneinfo import ZoneInfo
-
-    from lancedb.pydantic import LanceModel
-    from pydantic import Field, field_validator, ValidationError, ValidationInfo
-
-    tzname = "America/New_York"
-    tz = ZoneInfo(tzname)
-
-    class TestModel(LanceModel):
-        dt_with_tz: datetime = Field(json_schema_extra={"tz": tzname})
-
-        @field_validator('dt_with_tz')
-        @classmethod
-        def tz_must_match(cls, dt: datetime) -> datetime:
-            assert dt.tzinfo == tz
-            return dt        
-
-    ok = TestModel(dt_with_tz=datetime.now(tz))
-
-    try:
-        TestModel(dt_with_tz=datetime.now(ZoneInfo("Asia/Shanghai")))
-        assert 0 == 1, "this should raise ValidationError"
-    except ValidationError:
-        print("A ValidationError was raised.")
-        pass
-    ```
-
-    When you run this code it should print "A ValidationError was raised."
-
-    #### Pydantic custom types
-
-    LanceDB does NOT yet support converting pydantic custom types. If this is something you need,
-    please file a feature request on the [LanceDB Github repo](https://github.com/lancedb/lancedb/issues/new).
-
-    ### Using Iterators / Writing Large Datasets
-
-    It is recommended to use iterators to add large datasets in batches when creating your table in one go. This does not create multiple versions of your dataset unlike manually adding batches using `table.add()`
-
-    LanceDB additionally supports PyArrow's `RecordBatch` Iterators or other generators producing supported data types.
-
-    Here's an example using using `RecordBatch` iterator for creating tables.
-
-    ```python
-    import pyarrow as pa
-
-    def make_batches():
-        for i in range(5):
-            yield pa.RecordBatch.from_arrays(
-                [
-                    pa.array([[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],
-                            pa.list_(pa.float32(), 4)),
-                    pa.array(["foo", "bar"]),
-                    pa.array([10.0, 20.0]),
-                ],
-                ["vector", "item", "price"],
-            )
-
-    schema = pa.schema([
-        pa.field("vector", pa.list_(pa.float32(), 4)),
-        pa.field("item", pa.utf8()),
-        pa.field("price", pa.float32()),
-    ])
-
-    db.create_table("batched_tale", make_batches(), schema=schema)
-    ```
-
-    You can also use iterators of other types like Pandas DataFrame or Pylists directly in the above example.
-
-=== "JavaScript"
-    Initialize a VectorDB connection and create a table using one of the many methods listed below.
-
-    ```javascript
-    const lancedb = require("vectordb");
-
-    const uri = "data/sample-lancedb";
-    const db = await lancedb.connect(uri);
-    ```
-
-    You can create a LanceDB table in JavaScript using an array of JSON records as follows.
-
-    ```javascript
-    const tb = await db.createTable("my_table", [{
-        "vector": [3.1, 4.1],
-        "item": "foo",
-        "price": 10.0
-    }, {
-        "vector": [5.9, 26.5],
-        "item": "bar",
-        "price": 20.0
-    }]);
-    ```
-
-    !!! info "Note"
-    If the table already exists, LanceDB will raise an error by default. If you want to overwrite the table, you need to specify the `WriteMode` in the createTable function.
-
-    ```javascript
-    const table = await con.createTable(tableName, data, { writeMode: WriteMode.Overwrite })
-    ```
+You can also use iterators of other types like Pandas DataFrame or Pylists directly in the above example.
 
 ## Open existing tables
 
@@ -614,6 +636,70 @@ The `values` parameter is used to provide the new values for the columns as lite
 
     When rows are updated, they are moved out of the index. The row will still show up in ANN queries, but the query will not be as fast as it would be if the row was in the index. If you update a large proportion of rows, consider rebuilding the index afterwards.
 
+## Consistency
+
+In LanceDB OSS, users can set the `read_consistency_interval` parameter on connections to achieve different levels of read consistency. This parameter determines how frequently the database synchronizes with the underlying storage system to check for updates made by other processes. If another process updates a table, the database will not see the changes until the next synchronization.
+
+There are three possible settings for `read_consistency_interval`:
+
+1. **Unset (default)**: The database does not check for updates to tables made by other processes. This provides the best query performance, but means that clients may not see the most up-to-date data. This setting is suitable for applications where the data does not change during the lifetime of the table reference.
+2. **Zero seconds (Strong consistency)**: The database checks for updates on every read. This provides the strongest consistency guarantees, ensuring that all clients see the latest committed data. However, it has the most overhead. This setting is suitable when consistency matters more than having high QPS.
+3. **Custom interval (Eventual consistency)**: The database checks for updates at a custom interval, such as every 5 seconds. This provides eventual consistency, allowing for some lag between write and read operations. Performance wise, this is a middle ground between strong consistency and no consistency check. This setting is suitable for applications where immediate consistency is not critical, but clients should see updated data eventually.
+
+!!! tip "Consistency in LanceDB Cloud"
+
+    This is only tune-able in LanceDB OSS. In LanceDB Cloud, readers are always eventually consistent.
+
+=== "Python"
+    
+    To set strong consistency, use `timedelta(0)`:
+
+    ```python
+    from datetime import timedelta
+    db = lancedb.connect("./.lancedb",. read_consistency_interval=timedelta(0))
+    table = db.open_table("my_table")
+    ```
+
+    For eventual consistency, use a custom `timedelta`:
+
+    ```python
+    from datetime import timedelta
+    db = lancedb.connect("./.lancedb", read_consistency_interval=timedelta(seconds=5))
+    table = db.open_table("my_table")
+    ```
+
+    By default, a `Table` will never check for updates from other writers. To manually check for updates you can use `checkout_latest`:
+
+    ```python
+    db = lancedb.connect("./.lancedb")
+    table = db.open_table("my_table")
+    
+    # (Other writes happen to my_table from another process)
+
+    # Check for updates
+    table.checkout_latest()
+    ```
+
+=== "JavaScript/Typescript"
+
+    To set strong consistency, use `0`:
+
+    ```javascript
+    const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 0 });
+    const table = await db.openTable("my_table");
+    ```
+
+    For eventual consistency, specify the update interval as seconds:
+
+    ```javascript
+    const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 5 });
+    const table = await db.openTable("my_table");
+    ```
+
+<!-- Node doesn't yet support the version time travel: https://github.com/lancedb/lancedb/issues/1007 
+    Once it does, we can show manual consistency check for Node as well.
+-->
+
 ## What's next?
 
 Learn the best practices on creating an ANN index and getting the most out of it.

docs/src/hybrid_search/eval.md

@@ -0,0 +1,49 @@
+# Hybrid Search
+
+Hybrid Search is a broad (often misused) term. It can mean anything from combining multiple methods for searching, to applying ranking methods to better sort the results. In this blog, we use the definition of "hybrid search" to mean using a combination of keyword-based and vector search.
+
+## The challenge of (re)ranking search results
+Once you have a group of the most relevant search results from multiple search sources, you'd likely standardize the score and rank them accordingly. This process can also be seen as another independent step - reranking.
+There are two approaches for reranking search results from multiple sources.
+* <b>Score-based</b>: Calculate final relevance scores based on a weighted linear combination of individual search algorithm scores. Example - Weighted linear combination of semantic search & keyword-based search results.
+* <b>Relevance-based</b>: Discards the existing scores and calculates the relevance of each search result - query pair. Example - Cross Encoder models
+
+Even though there are many strategies for reranking search results, none works for all cases. Moreover, evaluating them itself is a challenge. Also, reranking can be dataset, application specific so it's hard to generalize.
+
+### Example evaluation of hybrid search with Reranking
+
+Here's some evaluation numbers from experiment comparing these re-rankers on about 800 queries. It is modified version of an evaluation script from [llama-index](https://github.com/run-llama/finetune-embedding/blob/main/evaluate.ipynb) that measures hit-rate at top-k.
+
+<b> With OpenAI ada2 embedding </b>
+
+Vector Search baseline - `0.64`
+
+| Reranker | Top-3 | Top-5 | Top-10 |
+| --- | --- | --- | --- |
+| Linear Combination | `0.73` | `0.74` | `0.85` |
+| Cross Encoder | `0.71` | `0.70` | `0.77` |
+| Cohere | `0.81` | `0.81` | `0.85` |
+| ColBERT | `0.68` | `0.68` | `0.73` |
+
+<p>
+<img src="https://github.com/AyushExel/assets/assets/15766192/d57b1780-ef27-414c-a5c3-73bee7808a45">
+</p>
+
+<b> With OpenAI embedding-v3-small </b>
+
+Vector Search baseline - `0.59`
+
+| Reranker | Top-3 | Top-5 | Top-10 |
+| --- | --- | --- | --- |
+| Linear Combination | `0.68` | `0.70` | `0.84` |
+| Cross Encoder | `0.72` | `0.72` | `0.79` |
+| Cohere | `0.79` | `0.79` | `0.84` |
+| ColBERT | `0.70` | `0.70` | `0.76` |
+
+<p>
+<img src="https://github.com/AyushExel/assets/assets/15766192/259adfd2-6ec6-4df6-a77d-1456598970dd">
+</p>
+
+### Conclusion
+
+The results show that the reranking methods are able to improve the search results. However, the improvement is not consistent across all rerankers. The choice of reranker depends on the dataset and the application. It is also important to note that the reranking methods are not a replacement for the search methods. They are complementary and should be used together to get the best results. The speed to recall tradeoff is also an important factor to consider when choosing the reranker.

docs/src/hybrid_search/hybrid_search.md

@@ -0,0 +1,242 @@
+# Hybrid Search
+
+LanceDB supports both semantic and keyword-based search (also termed full-text search, or FTS). In real world applications, it is often useful to combine these two approaches to get the best best results. For example, you may want to search for a document that is semantically similar to a query document, but also contains a specific keyword. This is an example of *hybrid search*, a search algorithm that combines multiple search techniques.
+
+## Hybrid search in LanceDB
+You can perform hybrid search in LanceDB by combining the results of semantic and full-text search via a reranking algorithm of your choice. LanceDB provides multiple rerankers out of the box. However, you can always write a custom reranker if your use case need more sophisticated logic .
+
+```python
+import os
+
+import lancedb
+import openai
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import LanceModel, Vector
+
+db = lancedb.connect("~/.lancedb")
+
+# Ingest embedding function in LanceDB table
+# Configuring the environment variable OPENAI_API_KEY
+if "OPENAI_API_KEY" not in os.environ:
+# OR set the key here as a variable
+    openai.api_key = "sk-..."
+embeddings = get_registry().get("openai").create()
+
+class Documents(LanceModel):
+    vector: Vector(embeddings.ndims()) = embeddings.VectorField()
+    text: str = embeddings.SourceField()
+
+table = db.create_table("documents", schema=Documents)
+
+data = [
+    { "text": "rebel spaceships striking from a hidden base"},
+    { "text": "have won their first victory against the evil Galactic Empire"},
+    { "text": "during the battle rebel spies managed to steal secret plans"},
+    { "text": "to the Empire's ultimate weapon the Death Star"}
+]
+
+# ingest docs with auto-vectorization
+table.add(data)
+
+# Create a fts index before the hybrid search
+table.create_fts_index("text")
+# hybrid search with default re-ranker
+results = table.search("flower moon", query_type="hybrid").to_pandas()
+```
+
+By default, LanceDB uses `LinearCombinationReranker(weight=0.7)` to combine and rerank the results of semantic and full-text search. You can customize the hyperparameters as needed or write your own custom reranker. Here's how you can use any of the available rerankers:
+
+
+### `rerank()` arguments
+* `normalize`: `str`, default `"score"`:
+    The method to normalize the scores. Can be "rank" or "score". If "rank", the scores are converted to ranks and then normalized. If "score", the scores are normalized directly.
+* `reranker`: `Reranker`, default `LinearCombinationReranker(weight=0.7)`.
+    The reranker to use. If not specified, the default reranker is used.
+
+
+## Available Rerankers
+LanceDB provides a number of re-rankers out of the box. You can use any of these re-rankers by passing them to the `rerank()` method. Here's a list of available re-rankers:
+
+### Linear Combination Reranker
+This is the default re-ranker used by LanceDB. It combines the results of semantic and full-text search using a linear combination of the scores. The weights for the linear combination can be specified. It defaults to 0.7, i.e, 70% weight for semantic search and 30% weight for full-text search.
+
+
+```python
+from lancedb.rerankers import LinearCombinationReranker
+
+reranker = LinearCombinationReranker(weight=0.3) # Use 0.3 as the weight for vector search
+
+results = table.search("rebel", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+### Arguments
+----------------
+* `weight`: `float`, default `0.7`:
+    The weight to use for the semantic search score. The weight for the full-text search score is `1 - weights`.
+* `fill`: `float`, default `1.0`:
+        The score to give to results that are only in one of the two result sets.This is treated as penalty, so a higher value means a lower score.
+        TODO: We should just hardcode this-- its pretty confusing as we invert scores to calculate final score
+* `return_score` : str, default `"relevance"`
+        options are "relevance" or "all"
+        The type of score to return. If "relevance", will return only the `_relevance_score. If "all", will return all scores from the vector and FTS search along with the relevance score.
+
+### Cohere Reranker
+This re-ranker uses the [Cohere](https://cohere.ai/) API to combine the results of semantic and full-text search. You can use this re-ranker by passing `CohereReranker()` to the `rerank()` method. Note that you'll need to set the `COHERE_API_KEY` environment variable to use this re-ranker.
+
+```python
+from lancedb.rerankers import CohereReranker
+
+reranker = CohereReranker()
+
+results = table.search("vampire weekend", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+### Arguments
+----------------
+* `model_name` : str, default `"rerank-english-v2.0"`
+        The name of the cross encoder model to use. Available cohere models are:
+        - rerank-english-v2.0
+        - rerank-multilingual-v2.0
+* `column` : str, default `"text"`
+        The name of the column to use as input to the cross encoder model.
+* `top_n` : str, default `None`
+        The number of results to return. If None, will return all results.
+
+!!! Note
+    Only returns `_relevance_score`. Does not support `return_score = "all"`.
+
+### Cross Encoder Reranker
+This reranker uses the [Sentence Transformers](https://www.sbert.net/) library to combine the results of semantic and full-text search. You can use it by passing `CrossEncoderReranker()` to the `rerank()` method.
+
+```python
+from lancedb.rerankers import CrossEncoderReranker
+
+reranker = CrossEncoderReranker()
+
+results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+
+### Arguments
+----------------
+* `model` : str, default `"cross-encoder/ms-marco-TinyBERT-L-6"`
+        The name of the cross encoder model to use. Available cross encoder models can be found [here](https://www.sbert.net/docs/pretrained_cross-encoders.html)
+* `column` : str, default `"text"`
+        The name of the column to use as input to the cross encoder model.
+* `device` : str, default `None`
+        The device to use for the cross encoder model. If None, will use "cuda" if available, otherwise "cpu".
+
+!!! Note
+    Only returns `_relevance_score`. Does not support `return_score = "all"`.
+
+
+### ColBERT Reranker
+This reranker uses the ColBERT model to combine the results of semantic and full-text search. You can use it by passing `ColbertrReranker()` to the `rerank()` method. 
+
+ColBERT reranker model calculates relevance of given docs against the query and don't take existing fts and vector search scores into account, so it currently only supports `return_score="relevance"`. By default, it looks for `text` column to rerank the results. But you can specify the column name to use as input to the cross encoder model as described below.
+
+```python
+from lancedb.rerankers import ColbertReranker
+
+reranker = ColbertReranker()
+
+results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+### Arguments
+----------------
+* `model_name` : `str`, default `"colbert-ir/colbertv2.0"`
+        The name of the cross encoder model to use.
+* `column` : `str`, default `"text"`
+        The name of the column to use as input to the cross encoder model.
+* `return_score` : `str`, default `"relevance"`
+        options are `"relevance"` or `"all"`. Only `"relevance"` is supported for now.
+
+!!! Note
+    Only returns `_relevance_score`. Does not support `return_score = "all"`.
+
+### OpenAI Reranker
+This reranker uses the OpenAI API to combine the results of semantic and full-text search. You can use it by passing `OpenaiReranker()` to the `rerank()` method.
+
+!!! Note
+    This prompts chat model to rerank results which is not a dedicated reranker model. This should be treated as experimental.
+
+!!! Tip
+    - You might run out of token limit so set the search `limits` based on your token limit.
+    - It is recommended to use gpt-4-turbo-preview, the default model, older models might lead to undesired behaviour
+
+```python
+from lancedb.rerankers import OpenaiReranker
+
+reranker = OpenaiReranker()
+
+results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+### Arguments
+----------------
+* `model_name` : `str`, default `"gpt-4-turbo-preview"`
+    The name of the cross encoder model to use.
+* `column` : `str`, default `"text"`
+    The name of the column to use as input to the cross encoder model.
+* `return_score` : `str`, default `"relevance"`
+    options are "relevance" or "all". Only "relevance" is supported for now.
+* `api_key` : `str`, default `None`
+    The API key to use. If None, will use the OPENAI_API_KEY environment variable.
+
+
+## Building Custom Rerankers
+You can build your own custom reranker by subclassing the `Reranker` class and implementing the `rerank_hybrid()` method. Here's an example of a custom reranker that combines the results of semantic and full-text search using a linear combination of the scores.
+
+The `Reranker` base interface comes with a `merge_results()` method that can be used to combine the results of semantic and full-text search. This is a vanilla merging algorithm that simply concatenates the results and removes the duplicates without taking the scores into consideration. It only keeps the first copy of the row encountered. This works well in cases that don't require the scores of semantic and full-text search to combine the results. If you want to use the scores or want to support `return_score="all"`, you'll need to implement your own merging algorithm.
+
+```python
+
+from lancedb.rerankers import Reranker
+import pyarrow as pa
+
+class MyReranker(Reranker):
+    def __init__(self, param1, param2, ..., return_score="relevance"):
+        super().__init__(return_score)
+        self.param1 = param1
+        self.param2 = param2
+    
+    def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table):
+        # Use the built-in merging function
+        combined_result = self.merge_results(vector_results, fts_results)
+        
+        # Do something with the combined results
+        # ...
+
+        # Return the combined results
+        return combined_result
+
+```
+
+### Example of a Custom Reranker
+For the sake of simplicity let's build custom reranker that just enchances the Cohere Reranker by accepting a filter query, and accept other CohereReranker params as kwags.
+
+```python
+
+from typing import List, Union
+import pandas as pd
+from lancedb.rerankers import CohereReranker
+
+class MofidifiedCohereReranker(CohereReranker):
+    def __init__(self, filters: Union[str, List[str]], **kwargs):
+        super().__init__(**kwargs)
+        filters = filters if isinstance(filters, list) else [filters]
+        self.filters = filters
+
+    def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table)-> pa.Table:
+        combined_result = super().rerank_hybrid(query, vector_results, fts_results)
+        df = combined_result.to_pandas()
+        for filter in self.filters:
+            df = df.query("not text.str.contains(@filter)")
+
+        return pa.Table.from_pandas(df)
+
+```
+
+!!! tip
+    The `vector_results` and `fts_results` are pyarrow tables. You can convert them to pandas dataframes using `to_pandas()` method and perform any operations you want. After you are done, you can convert the dataframe back to pyarrow table using `pa.Table.from_pandas()` method and return it.

docs/src/index.md

@@ -28,7 +28,7 @@ LanceDB **Cloud** is a SaaS (software-as-a-service) solution that runs serverles
 
 * Fast production-scale vector similarity, full-text & hybrid search and a SQL query interface (via [DataFusion](https://github.com/apache/arrow-datafusion))
 
-* Native Python and Javascript/Typescript support
+* Python, Javascript/Typescript, and Rust support
 
 * Store, query & manage multi-modal data (text, images, videos, point clouds, etc.), not just the embeddings and metadata
 
@@ -54,3 +54,4 @@ The following pages go deeper into the internal of LanceDB and how to use it.
 * [Ecosystem Integrations](integrations/index.md): Integrate LanceDB with other tools in the data ecosystem
 * [Python API Reference](python/python.md): Python OSS and Cloud API references
 * [JavaScript API Reference](javascript/modules.md): JavaScript OSS and Cloud API references
+* [Rust API Reference](https://docs.rs/lancedb/latest/lancedb/index.html): Rust API reference

docs/src/integrations/index.md

@@ -13,7 +13,7 @@ Get started using these examples and quick links.
 | Integrations | |
 |---|---:|
 | <h3> LlamaIndex </h3>LlamaIndex is a simple, flexible data framework for connecting custom data sources to large language models. Llama index integrates with LanceDB as the serverless VectorDB. <h3>[Lean More](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html) </h3> |<img src="../assets/llama-index.jpg" alt="image" width="150" height="auto">|
-| <h3>Langchain</h3>Langchain allows building applications with LLMs through composability <h3>[Lean More](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html) | <img src="../assets/langchain.png" alt="image" width="150" height="auto">|
+| <h3>Langchain</h3>Langchain allows building applications with LLMs through composability <h3>[Lean More](https://python.langchain.com/docs/integrations/vectorstores/lancedb) | <img src="../assets/langchain.png" alt="image" width="150" height="auto">|
 | <h3>Langchain TS</h3> Javascript bindings for Langchain. It integrates with LanceDB's serverless vectordb allowing you to build powerful AI applications through composibility using only serverless functions. <h3>[Learn More]( https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb) | <img src="../assets/langchain.png" alt="image" width="150" height="auto">|
 | <h3>Voxel51</h3>  It is an open source toolkit that enables you to build better computer vision workflows by improving the quality of your datasets and delivering insights about your models.<h3>[Learn More](./voxel51.md) | <img src="../assets/voxel.gif" alt="image" width="150" height="auto">|
 | <h3>PromptTools</h3>  Offers a set of free, open-source tools for testing and experimenting with models, prompts, and configurations. The core idea is to enable developers to evaluate prompts using familiar interfaces like code and notebooks. You can use it to experiment with different configurations of LanceDB, and test how LanceDB integrates with the LLM of your choice.<h3>[Learn More](./prompttools.md) | <img src="../assets/prompttools.jpeg" alt="image" width="150" height="auto">|

docs/src/notebooks/DisappearingEmbeddingFunction.ipynb

@@ -290,7 +290,7 @@
         "from lancedb.pydantic import LanceModel, Vector\n",
         "\n",
         "class Pets(LanceModel):\n",
-        "    vector: Vector(clip.ndims) = clip.VectorField()\n",
+        "    vector: Vector(clip.ndims()) = clip.VectorField()\n",
         "    image_uri: str = clip.SourceField()\n",
         "\n",
         "    @property\n",
@@ -360,7 +360,7 @@
         "    table = db.create_table(\"pets\", schema=Pets)\n",
         "    # use a sampling of 1000 images\n",
         "    p = Path(\"~/Downloads/images\").expanduser()\n",
-        "    uris = [str(f) for f in p.iterdir()]\n",
+        "    uris = [str(f) for f in p.glob(\"*.jpg\")]\n",
         "    uris = sample(uris, 1000)\n",
         "    table.add(pd.DataFrame({\"image_uri\": uris}))"
       ]
@@ -543,7 +543,7 @@
       ],
       "source": [
         "from PIL import Image\n",
-        "p = Path(\"/Users/changshe/Downloads/images/samoyed_100.jpg\")\n",
+        "p = Path(\"~/Downloads/images/samoyed_100.jpg\").expanduser()\n",
         "query_image = Image.open(p)\n",
         "query_image"
       ]

docs/src/notebooks/diffusiondb/datagen.py

@@ -23,10 +23,8 @@ from multiprocessing import Pool
 import lance
 import pyarrow as pa
 from datasets import load_dataset
-from PIL import Image
 from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast
 
-import lancedb
 
 MODEL_ID = "openai/clip-vit-base-patch32"
 

docs/src/notebooks/tables_guide.ipynb

@@ -13,7 +13,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 50,
+   "execution_count": 2,
    "id": "c1b4e34b-a49c-471d-a343-a5940bb5138a",
    "metadata": {},
    "outputs": [],
@@ -23,7 +23,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 3,
    "id": "4e5a8d07-d9a1-48c1-913a-8e0629289579",
    "metadata": {},
    "outputs": [],
@@ -44,7 +44,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 4,
    "id": "5df12f66-8d99-43ad-8d0b-22189ec0a6b9",
    "metadata": {},
    "outputs": [
@@ -62,7 +62,7 @@
        "long: [[-122.7,-74.1]]"
       ]
      },
-     "execution_count": 2,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -90,7 +90,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 5,
    "id": "f4d87ae9-0ccb-48eb-b31d-bb8f2370e47e",
    "metadata": {},
    "outputs": [
@@ -108,7 +108,7 @@
        "long: [[-122.7,-74.1]]"
       ]
      },
-     "execution_count": 3,
+     "execution_count": 5,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -135,10 +135,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 6,
    "id": "25f34bcf-fca0-4431-8601-eac95d1bd347",
    "metadata": {},
    "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[2024-01-31T18:59:33Z WARN  lance::dataset] No existing dataset at /Users/qian/Work/LanceDB/lancedb/docs/src/notebooks/.lancedb/table3.lance, it will be created\n"
+     ]
+    },
     {
      "data": {
       "text/plain": [
@@ -148,7 +155,7 @@
        "long: float"
       ]
      },
-     "execution_count": 8,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -171,45 +178,51 @@
    "id": "4df51925-7ca2-4005-9c72-38b3d26240c6",
    "metadata": {},
    "source": [
-    "### From PyArrow Tables\n",
+    "### From an Arrow Table\n",
     "\n",
-    "You can also create LanceDB tables directly from pyarrow tables"
+    "You can also create LanceDB tables directly from pyarrow tables. LanceDB supports float16 type."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 7,
    "id": "90a880f6-be43-4c9d-ba65-0b05197c0f6f",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "vector: fixed_size_list<item: float>[2]\n",
-       "  child 0, item: float\n",
-       "item: string\n",
-       "price: double"
+       "vector: fixed_size_list<item: halffloat>[16]\n",
+       "  child 0, item: halffloat\n",
+       "text: string"
       ]
      },
-     "execution_count": 12,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "table = pa.Table.from_arrays(\n",
-    "        [\n",
-    "            pa.array([[3.1, 4.1], [5.9, 26.5]],\n",
-    "                    pa.list_(pa.float32(), 2)),\n",
-    "            pa.array([\"foo\", \"bar\"]),\n",
-    "            pa.array([10.0, 20.0]),\n",
-    "        ],\n",
-    "        [\"vector\", \"item\", \"price\"],\n",
-    "    )\n",
+    "import numpy as np\n",
     "\n",
-    "db = lancedb.connect(\"db\")\n",
+    "dim = 16\n",
+    "total = 2\n",
+    "schema = pa.schema(\n",
+    "    [\n",
+    "        pa.field(\"vector\", pa.list_(pa.float16(), dim)),\n",
+    "        pa.field(\"text\", pa.string())\n",
+    "    ]\n",
+    ")\n",
+    "data = pa.Table.from_arrays(\n",
+    "    [\n",
+    "        pa.array([np.random.randn(dim).astype(np.float16) for _ in range(total)],\n",
+    "                pa.list_(pa.float16(), dim)),\n",
+    "        pa.array([\"foo\", \"bar\"])\n",
+    "    ],\n",
+    "    [\"vector\", \"text\"],\n",
+    ")\n",
     "\n",
-    "tbl = db.create_table(\"test1\", table, mode=\"overwrite\")\n",
+    "tbl = db.create_table(\"f16_tbl\", data, schema=schema)\n",
     "tbl.schema"
    ]
   },
@@ -225,7 +238,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 8,
    "id": "d81121d7-e4b7-447c-a48c-974b6ebb464a",
    "metadata": {},
    "outputs": [
@@ -240,7 +253,7 @@
        "imdb_id: int64 not null"
       ]
      },
-     "execution_count": 13,
+     "execution_count": 8,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -282,7 +295,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 9,
    "id": "bc247142-4e3c-41a2-b94c-8e00d2c2a508",
    "metadata": {},
    "outputs": [
@@ -292,7 +305,7 @@
        "LanceTable(table4)"
       ]
      },
-     "execution_count": 14,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -333,7 +346,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 10,
    "id": "25ad3523-e0c9-4c28-b3df-38189c4e0e5f",
    "metadata": {},
    "outputs": [
@@ -346,7 +359,7 @@
        "price: double not null"
       ]
      },
-     "execution_count": 16,
+     "execution_count": 10,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -385,7 +398,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 11,
    "id": "2814173a-eacc-4dd8-a64d-6312b44582cc",
    "metadata": {},
    "outputs": [],
@@ -411,7 +424,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 12,
    "id": "df9e13c0-41f6-437f-9dfa-2fd71d3d9c45",
    "metadata": {},
    "outputs": [
@@ -421,7 +434,7 @@
        "['table6', 'table4', 'table5', 'movielens_small']"
       ]
      },
-     "execution_count": 18,
+     "execution_count": 12,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -432,7 +445,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 13,
    "id": "9343f5ad-6024-42ee-ac2f-6c1471df8679",
    "metadata": {},
    "outputs": [
@@ -541,7 +554,7 @@
        "9  [5.9, 26.5]  bar   20.0"
       ]
      },
-     "execution_count": 20,
+     "execution_count": 13,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -564,7 +577,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 14,
    "id": "8a56250f-73a1-4c26-a6ad-5c7a0ce3a9ab",
    "metadata": {},
    "outputs": [],
@@ -590,7 +603,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 15,
    "id": "030c7057-b98e-4e2f-be14-b8c1f927f83c",
    "metadata": {},
    "outputs": [],
@@ -621,7 +634,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 16,
    "id": "e7a17de2-08d2-41b7-bd05-f63d1045ab1f",
    "metadata": {},
    "outputs": [
@@ -629,16 +642,16 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "32\n"
+      "22\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "17"
+       "12"
       ]
      },
-     "execution_count": 24,
+     "execution_count": 16,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -661,7 +674,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 17,
    "id": "fe3310bd-08f4-4a22-a63b-b3127d22f9f7",
    "metadata": {},
    "outputs": [
@@ -681,25 +694,20 @@
       "8    [3.1, 4.1]   foo   10.0\n",
       "9    [3.1, 4.1]   foo   10.0\n",
       "10   [3.1, 4.1]   foo   10.0\n",
-      "11   [3.1, 4.1]   foo   10.0\n",
-      "12   [3.1, 4.1]   foo   10.0\n",
-      "13   [3.1, 4.1]   foo   10.0\n",
-      "14   [3.1, 4.1]   foo   10.0\n",
-      "15   [3.1, 4.1]   foo   10.0\n",
-      "16   [3.1, 4.1]   foo   10.0\n"
+      "11   [3.1, 4.1]   foo   10.0\n"
      ]
     },
     {
      "ename": "OSError",
-     "evalue": "LanceError(IO): Error during planning: column foo does not exist",
+     "evalue": "LanceError(IO): Error during planning: column foo does not exist, /Users/runner/work/lance/lance/rust/lance-core/src/error.rs:212:23",
      "output_type": "error",
      "traceback": [
       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
       "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn[30], line 4\u001b[0m\n\u001b[1;32m      2\u001b[0m to_remove \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mstr\u001b[39m(v) \u001b[38;5;28;01mfor\u001b[39;00m v \u001b[38;5;129;01min\u001b[39;00m to_remove)\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28mprint\u001b[39m(tbl\u001b[38;5;241m.\u001b[39mto_pandas())\n\u001b[0;32m----> 4\u001b[0m \u001b[43mtbl\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mitem IN (\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mto_remove\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m)\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      5\u001b[0m tbl\u001b[38;5;241m.\u001b[39mto_pandas()\n",
-      "File \u001b[0;32m~/Documents/lancedb/lancedb/python/lancedb/table.py:610\u001b[0m, in \u001b[0;36mLanceTable.delete\u001b[0;34m(self, where)\u001b[0m\n\u001b[1;32m    609\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdelete\u001b[39m(\u001b[38;5;28mself\u001b[39m, where: \u001b[38;5;28mstr\u001b[39m):\n\u001b[0;32m--> 610\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dataset\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete\u001b[49m\u001b[43m(\u001b[49m\u001b[43mwhere\u001b[49m\u001b[43m)\u001b[49m\n",
-      "File \u001b[0;32m~/Documents/lancedb/lancedb/env/lib/python3.11/site-packages/lance/dataset.py:489\u001b[0m, in \u001b[0;36mLanceDataset.delete\u001b[0;34m(self, predicate)\u001b[0m\n\u001b[1;32m    487\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(predicate, pa\u001b[38;5;241m.\u001b[39mcompute\u001b[38;5;241m.\u001b[39mExpression):\n\u001b[1;32m    488\u001b[0m     predicate \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(predicate)\n\u001b[0;32m--> 489\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_ds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpredicate\u001b[49m\u001b[43m)\u001b[49m\n",
-      "\u001b[0;31mOSError\u001b[0m: LanceError(IO): Error during planning: column foo does not exist"
+      "Cell \u001b[0;32mIn[17], line 4\u001b[0m\n\u001b[1;32m      2\u001b[0m to_remove \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mstr\u001b[39m(v) \u001b[38;5;28;01mfor\u001b[39;00m v \u001b[38;5;129;01min\u001b[39;00m to_remove)\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28mprint\u001b[39m(tbl\u001b[38;5;241m.\u001b[39mto_pandas())\n\u001b[0;32m----> 4\u001b[0m \u001b[43mtbl\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mitem IN (\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mto_remove\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m)\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/Work/LanceDB/lancedb/docs/doc-venv/lib/python3.11/site-packages/lancedb/table.py:872\u001b[0m, in \u001b[0;36mLanceTable.delete\u001b[0;34m(self, where)\u001b[0m\n\u001b[1;32m    871\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdelete\u001b[39m(\u001b[38;5;28mself\u001b[39m, where: \u001b[38;5;28mstr\u001b[39m):\n\u001b[0;32m--> 872\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dataset\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete\u001b[49m\u001b[43m(\u001b[49m\u001b[43mwhere\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/Work/LanceDB/lancedb/docs/doc-venv/lib/python3.11/site-packages/lance/dataset.py:596\u001b[0m, in \u001b[0;36mLanceDataset.delete\u001b[0;34m(self, predicate)\u001b[0m\n\u001b[1;32m    594\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(predicate, pa\u001b[38;5;241m.\u001b[39mcompute\u001b[38;5;241m.\u001b[39mExpression):\n\u001b[1;32m    595\u001b[0m     predicate \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(predicate)\n\u001b[0;32m--> 596\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_ds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpredicate\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[0;31mOSError\u001b[0m: LanceError(IO): Error during planning: column foo does not exist, /Users/runner/work/lance/lance/rust/lance-core/src/error.rs:212:23"
      ]
     }
    ],
@@ -712,7 +720,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 43,
+   "execution_count": null,
    "id": "87d5bc21-847f-4c81-b56e-f6dbe5d05aac",
    "metadata": {},
    "outputs": [],
@@ -729,7 +737,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 44,
+   "execution_count": null,
    "id": "9cba4519-eb3a-4941-ab7e-873d762e750f",
    "metadata": {},
    "outputs": [],
@@ -742,7 +750,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 46,
+   "execution_count": null,
    "id": "5bdc9801-d5ed-4871-92d0-88b27108e788",
    "metadata": {},
    "outputs": [
@@ -817,7 +825,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.4"
+   "version": "3.11.7"
   }
  },
  "nbformat": 4,

docs/src/python/duckdb.md

@@ -1,6 +1,9 @@
 # DuckDB
 
-LanceDB is very well-integrated with [DuckDB](https://duckdb.org/), an in-process SQL OLAP database. This integration is done via [Arrow](https://duckdb.org/docs/guides/python/sql_on_arrow) .
+In Python, LanceDB tables can also be queried with [DuckDB](https://duckdb.org/), an in-process SQL OLAP database. This means you can write complex SQL queries to analyze your data in LanceDB.
+
+This integration is done via [Apache Arrow](https://duckdb.org/docs/guides/python/sql_on_arrow), which provides zero-copy data sharing between LanceDB and DuckDB. DuckDB is capable of passing down column selections and basic filters to LanceDB, reducing the amount of data that needs to be scanned to perform your query. Finally, the integration allows streaming data from LanceDB tables, allowing you to aggregate tables that won't fit into memory. All of this uses the same mechanism described in DuckDB's blog post *[DuckDB quacks Arrow](https://duckdb.org/2021/12/03/duck-arrow.html)*.
+
 
 We can demonstrate this by first installing `duckdb` and `lancedb`.
 
@@ -19,14 +22,15 @@ data = [
     {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}
 ]
 table = db.create_table("pd_table", data=data)
-arrow_table = table.to_arrow()
 ```
 
-DuckDB can directly query the `pyarrow.Table` object:
+To query the table, first call `to_lance` to convert the table to a "dataset", which is an object that can be queried by DuckDB. Then all you need to do is reference that dataset by the same name in your SQL query.
 
 ```python
 import duckdb
 
+arrow_table = table.to_lance()
+
 duckdb.query("SELECT * FROM arrow_table")
 ```
 

docs/src/python/python.md

@@ -24,6 +24,12 @@ pip install lancedb
 
 ::: lancedb.query.LanceQueryBuilder
 
+::: lancedb.query.LanceVectorQueryBuilder
+
+::: lancedb.query.LanceFtsQueryBuilder
+
+::: lancedb.query.LanceHybridQueryBuilder
+
 ## Embeddings
 
 ::: lancedb.embeddings.registry.EmbeddingFunctionRegistry
@@ -58,12 +64,26 @@ pip install lancedb
 
 ::: lancedb.schema.vector
 
+::: lancedb.merge.LanceMergeInsertBuilder
+
 ## Integrations
 
-### Pydantic
+## Pydantic
 
 ::: lancedb.pydantic.pydantic_to_schema
 
 ::: lancedb.pydantic.vector
 
 ::: lancedb.pydantic.LanceModel
+
+## Reranking
+
+::: lancedb.rerankers.linear_combination.LinearCombinationReranker
+
+::: lancedb.rerankers.cohere.CohereReranker
+
+::: lancedb.rerankers.colbert.ColbertReranker
+
+::: lancedb.rerankers.cross_encoder.CrossEncoderReranker
+
+::: lancedb.rerankers.openai.OpenaiReranker

docs/src/search.md

@@ -2,28 +2,27 @@
 
 A vector search finds the approximate or exact nearest neighbors to a given query vector.
 
-* In a recommendation system or search engine, you can find similar records to
-the one you searched.
-* In LLM and other AI applications,
-each data point can be represented by [embeddings generated from existing models](embeddings/index.md),
-following which the search returns the most relevant features.
+- In a recommendation system or search engine, you can find similar records to
+  the one you searched.
+- In LLM and other AI applications,
+  each data point can be represented by [embeddings generated from existing models](embeddings/index.md),
+  following which the search returns the most relevant features.
 
 ## Distance metrics
 
 Distance metrics are a measure of the similarity between a pair of vectors.
 Currently, LanceDB supports the following metrics:
 
-| Metric      | Description                          |
-| ----------- | ------------------------------------ |
-| `l2`        | [Euclidean / L2 distance](https://en.wikipedia.org/wiki/Euclidean_distance) |
-| `cosine`    | [Cosine Similarity](https://en.wikipedia.org/wiki/Cosine_similarity)|
-| `dot`       | [Dot Production](https://en.wikipedia.org/wiki/Dot_product) |
-
+| Metric   | Description                                                                 |
+| -------- | --------------------------------------------------------------------------- |
+| `l2`     | [Euclidean / L2 distance](https://en.wikipedia.org/wiki/Euclidean_distance) |
+| `cosine` | [Cosine Similarity](https://en.wikipedia.org/wiki/Cosine_similarity)        |
+| `dot`    | [Dot Production](https://en.wikipedia.org/wiki/Dot_product)                 |
 
 ## Exhaustive search (kNN)
 
-If you do not create a vector index, LanceDB exhaustively scans the *entire* vector space
-and compute the distance to every vector in order to find the exact nearest neighbors. This is effectively a kNN search.
+If you do not create a vector index, LanceDB exhaustively scans the _entire_ vector space
+and computes the distance to every vector in order to find the exact nearest neighbors. This is effectively a kNN search.
 
 <!-- Setup Code
 ```python
@@ -38,22 +37,9 @@ data = [{"vector": row, "item": f"item {i}"}
 db.create_table("my_vectors", data=data)
 ```
 -->
-<!-- Setup Code
-```javascript
-const vectordb_setup = require('vectordb')
-const db_setup = await vectordb_setup.connect('data/sample-lancedb')
-
-let data = []
-for (let i = 0; i < 10_000; i++) {
-     data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
-}
-await db_setup.createTable('my_vectors', data)
-```
--->
 
 === "Python"
 
-
     ```python
     import lancedb
     import numpy as np
@@ -70,17 +56,12 @@ await db_setup.createTable('my_vectors', data)
 === "JavaScript"
 
     ```javascript
-    const vectordb = require('vectordb')
-    const db = await vectordb.connect('data/sample-lancedb')
+    --8<-- "docs/src/search_legacy.ts:import"
 
-    const tbl = await db.openTable("my_vectors")
-
-    const results_1 = await tbl.search(Array(1536).fill(1.2))
-        .limit(10)
-        .execute()
+    --8<-- "docs/src/search_legacy.ts:search1"
     ```
 
-By default, `l2` will be used as metric type. You can specify the metric type as 
+By default, `l2` will be used as metric type. You can specify the metric type as
 `cosine` or `dot` if required.
 
 === "Python"
@@ -92,23 +73,19 @@ By default, `l2` will be used as metric type. You can specify the metric type as
         .to_list()
     ```
 
-
 === "JavaScript"
 
     ```javascript
-    const results_2 = await tbl.search(Array(1536).fill(1.2))
-        .metricType("cosine")
-        .limit(10)
-        .execute()
+    --8<-- "docs/src/search_legacy.ts:search2"
     ```
 
 ## Approximate nearest neighbor (ANN) search
 
 To perform scalable vector retrieval with acceptable latencies, it's common to build a vector index.
-While the exhaustive search is guaranteed to always return 100% recall, the approximate nature of 
+While the exhaustive search is guaranteed to always return 100% recall, the approximate nature of
 an ANN search means that using an index often involves a trade-off between recall and latency.
 
-See the [IVF_PQ index](./concepts/index_ivfpq.md.md) for a deeper description of how `IVF_PQ`
+See the [IVF_PQ index](./concepts/index_ivfpq.md) for a deeper description of how `IVF_PQ`
 indexes work in LanceDB.
 
 ## Output search results
@@ -117,7 +94,9 @@ LanceDB returns vector search results via different formats commonly used in pyt
 Let's create a LanceDB table with a nested schema:
 
 === "Python"
+
     ```python
+
     from datetime import datetime
     import lancedb
     from lancedb.pydantic import LanceModel, Vector
@@ -153,7 +132,7 @@ Let's create a LanceDB table with a nested schema:
     ### As a PyArrow table
 
     Using `to_arrow()` we can get the results back as a pyarrow Table.
-    This result table has the same columns as the LanceDB table, with 
+    This result table has the same columns as the LanceDB table, with
     the addition of an `_distance` column for vector search or a `score`
     column for full text search.
 
@@ -169,11 +148,11 @@ Let's create a LanceDB table with a nested schema:
     tbl.search(np.random.randn(1536)).to_pandas()
     ```
 
-    While other formats like Arrow/Pydantic/Python dicts have a natural 
-    way to handle nested schemas, pandas can only store nested data as a 
+    While other formats like Arrow/Pydantic/Python dicts have a natural
+    way to handle nested schemas, pandas can only store nested data as a
     python dict column, which makes it difficult to support nested references.
-    So for convenience, you can also tell LanceDB to flatten a nested schema 
-    when creating the pandas dataframe. 
+    So for convenience, you can also tell LanceDB to flatten a nested schema
+    when creating the pandas dataframe.
 
     ```python
     tbl.search(np.random.randn(1536)).to_pandas(flatten=True)
@@ -205,4 +184,3 @@ Let's create a LanceDB table with a nested schema:
 
     Note that in this case the extra `_distance` field is discarded since
     it's not part of the LanceSchema.
-

docs/src/search_legacy.ts

@@ -0,0 +1,41 @@
+// --8<-- [start:import]
+import * as lancedb from "vectordb";
+// --8<-- [end:import]
+import * as fs from "fs";
+
+async function setup() {
+  fs.rmSync("data/sample-lancedb", { recursive: true, force: true });
+  const db = await lancedb.connect("data/sample-lancedb");
+
+  let data = [];
+  for (let i = 0; i < 10_000; i++) {
+    data.push({
+      vector: Array(1536).fill(i),
+      id: `${i}`,
+      content: "",
+      longId: `${i}`,
+    });
+  }
+  await db.createTable("my_vectors", data);
+}
+
+async () => {
+  await setup();
+
+  // --8<-- [start:search1]
+  const db = await lancedb.connect("data/sample-lancedb");
+  const tbl = await db.openTable("my_vectors");
+
+  const results_1 = await tbl.search(Array(1536).fill(1.2)).limit(10).execute();
+  // --8<-- [end:search1]
+
+  // --8<-- [start:search2]
+  const results_2 = await tbl
+    .search(Array(1536).fill(1.2))
+    .metricType(lancedb.MetricType.Cosine)
+    .limit(10)
+    .execute();
+  // --8<-- [end:search2]
+
+  console.log("search: done");
+};

docs/src/sql.md

@@ -8,7 +8,7 @@ option that performs the filter prior to vector search. This can be useful to na
 the search space on a very large dataset to reduce query latency.
 
 <!-- Setup Code
-```python 
+```python
 import lancedb
 import numpy as np
 uri = "data/sample-lancedb"
@@ -21,7 +21,7 @@ tbl = db.create_table("my_vectors", data=data)
 ```
 -->
 <!-- Setup Code
-```javascript 
+```javascript
 const vectordb = require('vectordb')
 const db = await vectordb.connect('data/sample-lancedb')
 
@@ -34,6 +34,7 @@ const tbl = await db.createTable('myVectors', data)
 -->
 
 === "Python"
+
     ```py
     result = (
         tbl.search([0.5, 0.2])
@@ -44,12 +45,9 @@ const tbl = await db.createTable('myVectors', data)
     ```
 
 === "JavaScript"
+
     ```javascript
-    let result = await tbl.search(Array(1536).fill(0.5))
-        .limit(1)
-        .filter("id = 10")
-        .prefilter(true)
-        .execute()
+    --8<-- "docs/src/sql_legacy.ts:search"
     ```
 
 ## SQL filters
@@ -60,14 +58,14 @@ It can be used during vector search, update, and deletion operations.
 
 Currently, Lance supports a growing list of SQL expressions.
 
-* ``>``, ``>=``, ``<``, ``<=``, ``=``
-* ``AND``, ``OR``, ``NOT``
-* ``IS NULL``, ``IS NOT NULL``
-* ``IS TRUE``, ``IS NOT TRUE``, ``IS FALSE``, ``IS NOT FALSE``
-* ``IN``
-* ``LIKE``, ``NOT LIKE``
-* ``CAST``
-* ``regexp_match(column, pattern)``
+- `>`, `>=`, `<`, `<=`, `=`
+- `AND`, `OR`, `NOT`
+- `IS NULL`, `IS NOT NULL`
+- `IS TRUE`, `IS NOT TRUE`, `IS FALSE`, `IS NOT FALSE`
+- `IN`
+- `LIKE`, `NOT LIKE`
+- `CAST`
+- `regexp_match(column, pattern)`
 
 For example, the following filter string is acceptable:
 
@@ -82,29 +80,27 @@ For example, the following filter string is acceptable:
 === "Javascript"
 
     ```javascript
-    await tbl.search(Array(1536).fill(0))
-       .where("(item IN ('item 0', 'item 2')) AND (id > 10)")
-       .execute()
+    --8<-- "docs/src/sql_legacy.ts:vec_search"
     ```
 
-
 If your column name contains special characters or is a [SQL Keyword](https://docs.rs/sqlparser/latest/sqlparser/keywords/index.html),
 you can use backtick (`` ` ``) to escape it. For nested fields, each segment of the
 path must be wrapped in backticks.
 
 === "SQL"
+
     ```sql
     `CUBE` = 10 AND `column name with space` IS NOT NULL
       AND `nested with space`.`inner with space` < 2
     ```
 
-!!! warning
-    Field names containing periods (``.``) are not supported.
+!!!warning "Field names containing periods (`.`) are not supported."
 
 Literals for dates, timestamps, and decimals can be written by writing the string
 value after the type name. For example
 
 === "SQL"
+
     ```sql
     date_col = date '2021-01-01'
     and timestamp_col = timestamp '2021-01-01 00:00:00'
@@ -114,49 +110,47 @@ value after the type name. For example
 For timestamp columns, the precision can be specified as a number in the type
 parameter. Microsecond precision (6) is the default.
 
-| SQL              | Time unit    |
-|------------------|--------------|
-| ``timestamp(0)`` | Seconds      |
-| ``timestamp(3)`` | Milliseconds |
-| ``timestamp(6)`` | Microseconds |
-| ``timestamp(9)`` | Nanoseconds  |
+| SQL            | Time unit    |
+| -------------- | ------------ |
+| `timestamp(0)` | Seconds      |
+| `timestamp(3)` | Milliseconds |
+| `timestamp(6)` | Microseconds |
+| `timestamp(9)` | Nanoseconds  |
 
 LanceDB internally stores data in [Apache Arrow](https://arrow.apache.org/) format.
 The mapping from SQL types to Arrow types is:
 
-| SQL type | Arrow type |
-|----------|------------|
-| ``boolean`` | ``Boolean`` |
-| ``tinyint`` / ``tinyint unsigned`` | ``Int8`` / ``UInt8`` |
-| ``smallint`` / ``smallint unsigned`` | ``Int16`` / ``UInt16`` |
-| ``int`` or ``integer`` / ``int unsigned`` or ``integer unsigned`` | ``Int32`` / ``UInt32`` |
-| ``bigint`` / ``bigint unsigned`` | ``Int64`` / ``UInt64`` |
-| ``float`` | ``Float32`` |
-| ``double`` | ``Float64`` |
-| ``decimal(precision, scale)`` | ``Decimal128`` |
-| ``date`` | ``Date32`` |
-| ``timestamp`` | ``Timestamp`` [^1] |
-| ``string`` | ``Utf8`` |
-| ``binary`` | ``Binary`` |
+| SQL type                                                  | Arrow type         |
+| --------------------------------------------------------- | ------------------ |
+| `boolean`                                                 | `Boolean`          |
+| `tinyint` / `tinyint unsigned`                            | `Int8` / `UInt8`   |
+| `smallint` / `smallint unsigned`                          | `Int16` / `UInt16` |
+| `int` or `integer` / `int unsigned` or `integer unsigned` | `Int32` / `UInt32` |
+| `bigint` / `bigint unsigned`                              | `Int64` / `UInt64` |
+| `float`                                                   | `Float32`          |
+| `double`                                                  | `Float64`          |
+| `decimal(precision, scale)`                               | `Decimal128`       |
+| `date`                                                    | `Date32`           |
+| `timestamp`                                               | `Timestamp` [^1]   |
+| `string`                                                  | `Utf8`             |
+| `binary`                                                  | `Binary`           |
 
 [^1]: See precision mapping in previous table.
 
-
 ## Filtering without Vector Search
 
 You can also filter your data without search.
 
 === "Python"
-      ```python
-      tbl.search().where("id = 10").limit(10).to_arrow()
-      ```
+
+    ```python
+    tbl.search().where("id = 10").limit(10).to_arrow()
+    ```
 
 === "JavaScript"
-      ```javascript
-      await tbl.where('id = 10').limit(10).execute()
-      ```
 
-!!! warning
-    If your table is large, this could potentially return a very large
-    amount of data. Please be sure to use a `limit` clause unless
-    you're sure you want to return the whole result set.
+    ```javascript
+    --8<---- "docs/src/sql_legacy.ts:sql_search"
+    ```
+
+!!!warning "If your table is large, this could potentially return a very large amount of data. Please be sure to use a `limit` clause unless you're sure you want to return the whole result set."

docs/src/sql_legacy.ts

@@ -0,0 +1,38 @@
+import * as vectordb from "vectordb";
+
+(async () => {
+  const db = await vectordb.connect("data/sample-lancedb");
+
+  let data = [];
+  for (let i = 0; i < 10_000; i++) {
+    data.push({
+      vector: Array(1536).fill(i),
+      id: i,
+      item: `item ${i}`,
+      strId: `${i}`,
+    });
+  }
+  const tbl = await db.createTable("myVectors", data);
+
+  // --8<-- [start:search]
+  let result = await tbl
+    .search(Array(1536).fill(0.5))
+    .limit(1)
+    .filter("id = 10")
+    .prefilter(true)
+    .execute();
+  // --8<-- [end:search]
+
+  // --8<-- [start:vec_search]
+  await tbl
+    .search(Array(1536).fill(0))
+    .where("(item IN ('item 0', 'item 2')) AND (id > 10)")
+    .execute();
+  // --8<-- [end:vec_search]
+
+  // --8<-- [start:sql_search]
+  await tbl.filter("id = 10").limit(10).execute();
+  // --8<-- [end:sql_search]
+
+  console.log("SQL search: done");
+})();

docs/test/md_testing.js

@@ -1,54 +0,0 @@
-const glob = require("glob");
-const fs = require("fs");
-const path = require("path");
-
-const globString = "../src/**/*.md";
-
-const excludedGlobs = [
-  "../src/fts.md",
-  "../src/embedding.md",
-  "../src/examples/*.md",
-  "../src/guides/tables.md",
-  "../src/embeddings/*.md",
-];
-
-const nodePrefix = "javascript";
-const nodeFile = ".js";
-const nodeFolder = "node";
-const asyncPrefix = "(async () => {\n";
-const asyncSuffix = "})();";
-
-function* yieldLines(lines, prefix, suffix) {
-  let inCodeBlock = false;
-  for (const line of lines) {
-    if (line.trim().startsWith(prefix + nodePrefix)) {
-      inCodeBlock = true;
-    } else if (inCodeBlock && line.trim().startsWith(suffix)) {
-      inCodeBlock = false;
-      yield "\n";
-    } else if (inCodeBlock) {
-      yield line;
-    }
-  }
-}
-
-const files = glob.sync(globString, { recursive: true });
-const excludedFiles = glob.sync(excludedGlobs, { recursive: true });
-
-for (const file of files.filter((file) => !excludedFiles.includes(file))) {
-  const lines = [];
-  const data = fs.readFileSync(file, "utf-8");
-  const fileLines = data.split("\n");
-
-  for (const line of yieldLines(fileLines, "```", "```")) {
-    lines.push(line);
-  }
-
-  if (lines.length > 0) {
-    const fileName = path.basename(file, ".md");
-    const outPath = path.join(nodeFolder, fileName, `${fileName}${nodeFile}`);
-    console.log(outPath)
-    fs.mkdirSync(path.dirname(outPath), { recursive: true });
-    fs.writeFileSync(outPath, asyncPrefix + "\n" + lines.join("\n") + asyncSuffix);
-  }
-}

docs/test/md_testing.py

@@ -14,6 +14,7 @@ excluded_globs = [
     "../src/concepts/*.md",
     "../src/ann_indexes.md",
     "../src/basic.md",
+    "../src/hybrid_search/hybrid_search.md",
 ]
 
 python_prefix = "py"
@@ -48,6 +49,7 @@ def yield_lines(lines: Iterator[str], prefix: str, suffix: str):
             if not skip_test:
                 yield line[strip_length:]
 
+
 for file in filter(lambda file: file not in excluded_files, files):
     with open(file, "r") as f:
         lines = list(yield_lines(iter(f), "```", "```"))

docs/test/package.json

@@ -1,13 +0,0 @@
-{
-  "name": "lancedb-docs-test",
-  "version": "1.0.0",
-  "description": "",
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "fs": "^0.0.1-security",
-    "glob": "^10.2.7",
-    "path": "^0.12.7",
-    "vectordb": "https://gitpkg.now.sh/lancedb/lancedb/node?main"
-  }
-}

docs/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "include": [
+    "src/*.ts",
+  ],
+  "compilerOptions": {
+    "target": "es2022",
+    "module": "nodenext",
+    "declaration": true,
+    "outDir": "./dist",
+    "strict": true,
+    "allowJs": true,
+    "resolveJsonModule": true,
+  },
+  "exclude": [
+    "./dist/*",
+  ]
+}

node/.eslintrc.js

@@ -13,5 +13,10 @@ module.exports = {
   },
   rules: {
     "@typescript-eslint/method-signature-style": "off",
+    "@typescript-eslint/quotes": "off",
+    "@typescript-eslint/semi": "off",
+    "@typescript-eslint/explicit-function-return-type": "off",
+    "@typescript-eslint/space-before-function-paren": "off",
+    "@typescript-eslint/indent": "off",
   }
 }

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.4.4",
+  "version": "0.4.13",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.4.4",
+      "version": "0.4.13",
       "cpu": [
         "x64",
         "arm64"
@@ -18,9 +18,7 @@
         "win32"
       ],
       "dependencies": {
-        "@apache-arrow/ts": "^14.0.2",
         "@neon-rs/load": "^0.0.74",
-        "apache-arrow": "^14.0.2",
         "axios": "^1.4.0"
       },
       "devDependencies": {
@@ -33,6 +31,7 @@
         "@types/temp": "^0.9.1",
         "@types/uuid": "^9.0.3",
         "@typescript-eslint/eslint-plugin": "^5.59.1",
+        "apache-arrow-old": "npm:apache-arrow@13.0.0",
         "cargo-cp-artifact": "^0.1",
         "chai": "^4.3.7",
         "chai-as-promised": "^7.1.1",
@@ -53,11 +52,15 @@
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.4.4",
-        "@lancedb/vectordb-darwin-x64": "0.4.4",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.4.4",
-        "@lancedb/vectordb-linux-x64-gnu": "0.4.4",
-        "@lancedb/vectordb-win32-x64-msvc": "0.4.4"
+        "@lancedb/vectordb-darwin-arm64": "0.4.13",
+        "@lancedb/vectordb-darwin-x64": "0.4.13",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.4.13",
+        "@lancedb/vectordb-linux-x64-gnu": "0.4.13",
+        "@lancedb/vectordb-win32-x64-msvc": "0.4.13"
+      },
+      "peerDependencies": {
+        "@apache-arrow/ts": "^14.0.2",
+        "apache-arrow": "^14.0.2"
       }
     },
     "node_modules/@75lb/deep-merge": {
@@ -93,6 +96,7 @@
       "version": "14.0.2",
       "resolved": "https://registry.npmjs.org/@apache-arrow/ts/-/ts-14.0.2.tgz",
       "integrity": "sha512-CtwAvLkK0CZv7xsYeCo91ml6PvlfzAmAJZkRYuz2GNBwfYufj5SVi0iuSMwIMkcU/szVwvLdzORSLa5PlF/2ug==",
+      "peer": true,
       "dependencies": {
         "@types/command-line-args": "5.2.0",
         "@types/command-line-usage": "5.0.2",
@@ -109,7 +113,8 @@
     "node_modules/@apache-arrow/ts/node_modules/@types/node": {
       "version": "20.3.0",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz",
-      "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ=="
+      "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==",
+      "peer": true
     },
     "node_modules/@cargo-messages/android-arm-eabi": {
       "version": "0.0.160",
@@ -328,6 +333,66 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
+    "node_modules/@lancedb/vectordb-darwin-arm64": {
+      "version": "0.4.13",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.13.tgz",
+      "integrity": "sha512-JfroNCG8yKIU931Y+x8d0Fp8C9DHUSC5j+CjI+e5err7rTWtie4j3JbsXlWAnPFaFEOg0Xk3BWkSikCvhPGJGg==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-darwin-x64": {
+      "version": "0.4.13",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.4.13.tgz",
+      "integrity": "sha512-dG6IMvfpHpnHdbJ0UffzJ7cZfMiC02MjIi6YJzgx+hKz2UNXWNBIfTvvhqli85mZsGRXL1OYDdYv0K1YzNjXlA==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
+      "version": "0.4.13",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.4.13.tgz",
+      "integrity": "sha512-BRR1VzaMviXby7qmLm0axNZM8eUZF3ZqfvnDKdVRpC3LaRueD6pMXHuC2IUKaFkn7xktf+8BlDZb6foFNEj8bQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-linux-x64-gnu": {
+      "version": "0.4.13",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.4.13.tgz",
+      "integrity": "sha512-WnekZ7ZMlria+NODZ6aBCljCFQSe2bBNUS9ZpyFl/Y1vHduSQPuBxM6V7vp2QubC0daq/rifgjDob89DF+x3xw==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-win32-x64-msvc": {
+      "version": "0.4.13",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.4.13.tgz",
+      "integrity": "sha512-3NDpMWBL2ksDHXAraXhowiLqQcNWM5bdbeHwze4+InYMD54hyQ2ODNc+4usxp63Nya9biVnFS27yXULqkzIEqQ==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "win32"
+      ]
+    },
     "node_modules/@neon-rs/cli": {
       "version": "0.0.160",
       "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.160.tgz",
@@ -888,6 +953,7 @@
       "version": "14.0.2",
       "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-14.0.2.tgz",
       "integrity": "sha512-EBO2xJN36/XoY81nhLcwCJgFwkboDZeyNQ+OPsG7bCoQjc2BT0aTyH/MR6SrL+LirSNz+cYqjGRlupMMlP1aEg==",
+      "peer": true,
       "dependencies": {
         "@types/command-line-args": "5.2.0",
         "@types/command-line-usage": "5.0.2",
@@ -904,10 +970,39 @@
         "arrow2csv": "bin/arrow2csv.js"
       }
     },
+    "node_modules/apache-arrow-old": {
+      "name": "apache-arrow",
+      "version": "13.0.0",
+      "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-13.0.0.tgz",
+      "integrity": "sha512-3gvCX0GDawWz6KFNC28p65U+zGh/LZ6ZNKWNu74N6CQlKzxeoWHpi4CgEQsgRSEMuyrIIXi1Ea2syja7dwcHvw==",
+      "dev": true,
+      "dependencies": {
+        "@types/command-line-args": "5.2.0",
+        "@types/command-line-usage": "5.0.2",
+        "@types/node": "20.3.0",
+        "@types/pad-left": "2.1.1",
+        "command-line-args": "5.2.1",
+        "command-line-usage": "7.0.1",
+        "flatbuffers": "23.5.26",
+        "json-bignum": "^0.0.3",
+        "pad-left": "^2.1.0",
+        "tslib": "^2.5.3"
+      },
+      "bin": {
+        "arrow2csv": "bin/arrow2csv.js"
+      }
+    },
+    "node_modules/apache-arrow-old/node_modules/@types/node": {
+      "version": "20.3.0",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz",
+      "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==",
+      "dev": true
+    },
     "node_modules/apache-arrow/node_modules/@types/node": {
       "version": "20.3.0",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz",
-      "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ=="
+      "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==",
+      "peer": true
     },
     "node_modules/arg": {
       "version": "4.1.3",

node/package.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.4.4",
+  "version": "0.4.14",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
     "tsc": "tsc -b",
-    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json && tsc -b",
+    "build": "npm run tsc && cargo-cp-artifact --artifact cdylib lancedb-node index.node -- cargo build --message-format=json",
     "build-release": "npm run build -- --release",
     "test": "npm run tsc && mocha -recursive dist/test",
     "integration-test": "npm run tsc && mocha -recursive dist/integration_test",
@@ -17,7 +17,11 @@
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/lancedb/lancedb/node"
+    "url": "https://github.com/lancedb/lancedb.git"
+  },
+  "homepage": "https://lancedb.github.io/lancedb/",
+  "bugs": {
+    "url": "https://github.com/lancedb/lancedb/issues"
   },
   "keywords": [
     "data-format",
@@ -37,6 +41,7 @@
     "@types/temp": "^0.9.1",
     "@types/uuid": "^9.0.3",
     "@typescript-eslint/eslint-plugin": "^5.59.1",
+    "apache-arrow-old": "npm:apache-arrow@13.0.0",
     "cargo-cp-artifact": "^0.1",
     "chai": "^4.3.7",
     "chai-as-promised": "^7.1.1",
@@ -57,11 +62,13 @@
     "uuid": "^9.0.0"
   },
   "dependencies": {
-    "@apache-arrow/ts": "^14.0.2",
     "@neon-rs/load": "^0.0.74",
-    "apache-arrow": "^14.0.2",
     "axios": "^1.4.0"
   },
+  "peerDependencies": {
+    "@apache-arrow/ts": "^14.0.2",
+    "apache-arrow": "^14.0.2"
+  },
   "os": [
     "darwin",
     "linux",
@@ -81,10 +88,10 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.4.4",
-    "@lancedb/vectordb-darwin-x64": "0.4.4",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.4.4",
-    "@lancedb/vectordb-linux-x64-gnu": "0.4.4",
-    "@lancedb/vectordb-win32-x64-msvc": "0.4.4"
+    "@lancedb/vectordb-darwin-arm64": "0.4.14",
+    "@lancedb/vectordb-darwin-x64": "0.4.14",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.4.14",
+    "@lancedb/vectordb-linux-x64-gnu": "0.4.14",
+    "@lancedb/vectordb-win32-x64-msvc": "0.4.14"
   }
 }

node/src/arrow.ts

@@ -14,26 +14,30 @@
 
 import {
   Field,
-  type FixedSizeListBuilder,
-  Float32,
   makeBuilder,
   RecordBatchFileWriter,
   Utf8,
   type Vector,
   FixedSizeList,
   vectorFromArray,
-  type Schema,
+  Schema,
   Table as ArrowTable,
   RecordBatchStreamWriter,
   List,
-  Float64,
   RecordBatch,
   makeData,
   Struct,
-  type Float
+  Float,
+  DataType,
+  Binary,
+  Float32
 } from 'apache-arrow'
 import { type EmbeddingFunction } from './index'
+import { sanitizeSchema } from './sanitize'
 
+/*
+ * Options to control how a column should be converted to a vector array
+ */
 export class VectorColumnOptions {
   /** Vector column type. */
   type: Float = new Float32()
@@ -45,14 +49,50 @@ export class VectorColumnOptions {
 
 /** Options to control the makeArrowTable call. */
 export class MakeArrowTableOptions {
-  /** Provided schema. */
+  /*
+   * Schema of the data.
+   *
+   * If this is not provided then the data type will be inferred from the
+   * JS type.  Integer numbers will become int64, floating point numbers
+   * will become float64 and arrays will become variable sized lists with
+   * the data type inferred from the first element in the array.
+   *
+   * The schema must be specified if there are no records (e.g. to make
+   * an empty table)
+   */
   schema?: Schema
 
-  /** Vector columns */
+  /*
+   * Mapping from vector column name to expected type
+   *
+   * Lance expects vector columns to be fixed size list arrays (i.e. tensors)
+   * However, `makeArrowTable` will not infer this by default (it creates
+   * variable size list arrays).  This field can be used to indicate that a column
+   * should be treated as a vector column and converted to a fixed size list.
+   *
+   * The keys should be the names of the vector columns.  The value specifies the
+   * expected data type of the vector columns.
+   *
+   * If `schema` is provided then this field is ignored.
+   *
+   * By default, the column named "vector" will be assumed to be a float32
+   * vector column.
+   */
   vectorColumns: Record<string, VectorColumnOptions> = {
     vector: new VectorColumnOptions()
   }
 
+  /**
+   * If true then string columns will be encoded with dictionary encoding
+   *
+   * Set this to true if your string columns tend to repeat the same values
+   * often.  For more precise control use the `schema` property to specify the
+   * data type for individual columns.
+   *
+   * If `schema` is provided then this property is ignored.
+   */
+  dictionaryEncodeStrings: boolean = false
+
   constructor (values?: Partial<MakeArrowTableOptions>) {
     Object.assign(this, values)
   }
@@ -62,8 +102,29 @@ export class MakeArrowTableOptions {
  * An enhanced version of the {@link makeTable} function from Apache Arrow
  * that supports nested fields and embeddings columns.
  *
+ * This function converts an array of Record<String, any> (row-major JS objects)
+ * to an Arrow Table (a columnar structure)
+ *
  * Note that it currently does not support nulls.
  *
+ * If a schema is provided then it will be used to determine the resulting array
+ * types.  Fields will also be reordered to fit the order defined by the schema.
+ *
+ * If a schema is not provided then the types will be inferred and the field order
+ * will be controlled by the order of properties in the first record.
+ *
+ * If the input is empty then a schema must be provided to create an empty table.
+ *
+ * When a schema is not specified then data types will be inferred.  The inference
+ * rules are as follows:
+ *
+ *  - boolean => Bool
+ *  - number => Float64
+ *  - String => Utf8
+ *  - Buffer => Binary
+ *  - Record<String, any> => Struct
+ *  - Array<any> => List
+ *
  * @param data input data
  * @param options options to control the makeArrowTable call.
  *
@@ -86,8 +147,10 @@ export class MakeArrowTableOptions {
  *  ], { schema });
  * ```
  *
- * It guesses the vector columns if the schema is not provided. For example,
- * by default it assumes that the column named `vector` is a vector column.
+ * By default it assumes that the column named `vector` is a vector column
+ * and it will be converted into a fixed size list array of type float32.
+ * The `vectorColumns` option can be used to support other vector column
+ * names and data types.
  *
  * ```ts
  *
@@ -134,211 +197,322 @@ export function makeArrowTable (
   data: Array<Record<string, any>>,
   options?: Partial<MakeArrowTableOptions>
 ): ArrowTable {
-  if (data.length === 0) {
-    throw new Error('At least one record needs to be provided')
+  if (data.length === 0 && (options?.schema === undefined || options?.schema === null)) {
+    throw new Error('At least one record or a schema needs to be provided')
   }
+
   const opt = new MakeArrowTableOptions(options !== undefined ? options : {})
+  if (opt.schema !== undefined && opt.schema !== null) {
+    opt.schema = sanitizeSchema(opt.schema)
+  }
   const columns: Record<string, Vector> = {}
   // TODO: sample dataset to find missing columns
-  const columnNames = Object.keys(data[0])
+  // Prefer the field ordering of the schema, if present
+  const columnNames = ((opt.schema) != null) ? (opt.schema.names as string[]) : Object.keys(data[0])
   for (const colName of columnNames) {
-    const values = data.map((datum) => datum[colName])
-    let vector: Vector
+    if (data.length !== 0 && !Object.prototype.hasOwnProperty.call(data[0], colName)) {
+      // The field is present in the schema, but not in the data, skip it
+      continue
+    }
+    // Extract a single column from the records (transpose from row-major to col-major)
+    let values = data.map((datum) => datum[colName])
 
+    // By default (type === undefined) arrow will infer the type from the JS type
+    let type
     if (opt.schema !== undefined) {
-      // Explicit schema is provided, highest priority
-      vector = vectorFromArray(
-        values,
-        opt.schema?.fields.filter((f) => f.name === colName)[0]?.type
-      )
+      // If there is a schema provided, then use that for the type instead
+      type = opt.schema?.fields.filter((f) => f.name === colName)[0]?.type
+      if (DataType.isInt(type) && type.bitWidth === 64) {
+        // wrap in BigInt to avoid bug: https://github.com/apache/arrow/issues/40051
+        values = values.map((v) => {
+          if (v === null) {
+            return v
+          }
+          return BigInt(v)
+        })
+      }
     } else {
+      // Otherwise, check to see if this column is one of the vector columns
+      // defined by opt.vectorColumns and, if so, use the fixed size list type
       const vectorColumnOptions = opt.vectorColumns[colName]
       if (vectorColumnOptions !== undefined) {
-        const fslType = new FixedSizeList(
-          values[0].length,
-          new Field('item', vectorColumnOptions.type, false)
-        )
-        vector = vectorFromArray(values, fslType)
-      } else {
-        // Normal case
-        vector = vectorFromArray(values)
+        type = newVectorType(values[0].length, vectorColumnOptions.type)
       }
     }
-    columns[colName] = vector
+
+    try {
+      // Convert an Array of JS values to an arrow vector
+      columns[colName] = makeVector(values, type, opt.dictionaryEncodeStrings)
+    } catch (error: unknown) {
+      // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+      throw Error(`Could not convert column "${colName}" to Arrow: ${error}`)
+    }
   }
 
-  return new ArrowTable(columns)
+  if (opt.schema != null) {
+    // `new ArrowTable(columns)` infers a schema which may sometimes have
+    // incorrect nullability (it assumes nullable=true if there are 0 rows)
+    //
+    // `new ArrowTable(schema, columns)` will also fail because it will create a
+    // batch with an inferred schema and then complain that the batch schema
+    // does not match the provided schema.
+    //
+    // To work around this we first create a table with the wrong schema and
+    // then patch the schema of the batches so we can use
+    // `new ArrowTable(schema, batches)` which does not do any schema inference
+    const firstTable = new ArrowTable(columns)
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const batchesFixed = firstTable.batches.map(batch => new RecordBatch(opt.schema!, batch.data))
+    return new ArrowTable(opt.schema, batchesFixed)
+  } else {
+    return new ArrowTable(columns)
+  }
 }
 
-// Converts an Array of records into an Arrow Table, optionally applying an embeddings function to it.
+/**
+ * Create an empty Arrow table with the provided schema
+ */
+export function makeEmptyTable (schema: Schema): ArrowTable {
+  return makeArrowTable([], { schema })
+}
+
+// Helper function to convert Array<Array<any>> to a variable sized list array
+function makeListVector (lists: any[][]): Vector<any> {
+  if (lists.length === 0 || lists[0].length === 0) {
+    throw Error('Cannot infer list vector from empty array or empty list')
+  }
+  const sampleList = lists[0]
+  let inferredType
+  try {
+    const sampleVector = makeVector(sampleList)
+    inferredType = sampleVector.type
+  } catch (error: unknown) {
+    // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+    throw Error(`Cannot infer list vector.  Cannot infer inner type: ${error}`)
+  }
+
+  const listBuilder = makeBuilder({
+    type: new List(new Field('item', inferredType, true))
+  })
+  for (const list of lists) {
+    listBuilder.append(list)
+  }
+  return listBuilder.finish().toVector()
+}
+
+// Helper function to convert an Array of JS values to an Arrow Vector
+function makeVector (values: any[], type?: DataType, stringAsDictionary?: boolean): Vector<any> {
+  if (type !== undefined) {
+    // No need for inference, let Arrow create it
+    return vectorFromArray(values, type)
+  }
+  if (values.length === 0) {
+    throw Error('makeVector requires at least one value or the type must be specfied')
+  }
+  const sampleValue = values.find(val => val !== null && val !== undefined)
+  if (sampleValue === undefined) {
+    throw Error('makeVector cannot infer the type if all values are null or undefined')
+  }
+  if (Array.isArray(sampleValue)) {
+    // Default Arrow inference doesn't handle list types
+    return makeListVector(values)
+  } else if (Buffer.isBuffer(sampleValue)) {
+    // Default Arrow inference doesn't handle Buffer
+    return vectorFromArray(values, new Binary())
+  } else if (!(stringAsDictionary ?? false) && (typeof sampleValue === 'string' || sampleValue instanceof String)) {
+    // If the type is string then don't use Arrow's default inference unless dictionaries are requested
+    // because it will always use dictionary encoding for strings
+    return vectorFromArray(values, new Utf8())
+  } else {
+    // Convert a JS array of values to an arrow vector
+    return vectorFromArray(values)
+  }
+}
+
+async function applyEmbeddings<T> (table: ArrowTable, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<ArrowTable> {
+  if (embeddings == null) {
+    return table
+  }
+  if (schema !== undefined && schema !== null) {
+    schema = sanitizeSchema(schema)
+  }
+
+  // Convert from ArrowTable to Record<String, Vector>
+  const colEntries = [...Array(table.numCols).keys()].map((_, idx) => {
+    const name = table.schema.fields[idx].name
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const vec = table.getChildAt(idx)!
+    return [name, vec]
+  })
+  const newColumns = Object.fromEntries(colEntries)
+
+  const sourceColumn = newColumns[embeddings.sourceColumn]
+  const destColumn = embeddings.destColumn ?? 'vector'
+  const innerDestType = embeddings.embeddingDataType ?? new Float32()
+  if (sourceColumn === undefined) {
+    throw new Error(`Cannot apply embedding function because the source column '${embeddings.sourceColumn}' was not present in the data`)
+  }
+
+  if (table.numRows === 0) {
+    if (Object.prototype.hasOwnProperty.call(newColumns, destColumn)) {
+      // We have an empty table and it already has the embedding column so no work needs to be done
+      // Note: we don't return an error like we did below because this is a common occurrence.  For example,
+      // if we call convertToTable with 0 records and a schema that includes the embedding
+      return table
+    }
+    if (embeddings.embeddingDimension !== undefined) {
+      const destType = newVectorType(embeddings.embeddingDimension, innerDestType)
+      newColumns[destColumn] = makeVector([], destType)
+    } else if (schema != null) {
+      const destField = schema.fields.find(f => f.name === destColumn)
+      if (destField != null) {
+        newColumns[destColumn] = makeVector([], destField.type)
+      } else {
+        throw new Error(`Attempt to apply embeddings to an empty table failed because schema was missing embedding column '${destColumn}'`)
+      }
+    } else {
+      throw new Error('Attempt to apply embeddings to an empty table when the embeddings function does not specify `embeddingDimension`')
+    }
+  } else {
+    if (Object.prototype.hasOwnProperty.call(newColumns, destColumn)) {
+      throw new Error(`Attempt to apply embeddings to table failed because column ${destColumn} already existed`)
+    }
+    if (table.batches.length > 1) {
+      throw new Error('Internal error: `makeArrowTable` unexpectedly created a table with more than one batch')
+    }
+    const values = sourceColumn.toArray()
+    const vectors = await embeddings.embed(values as T[])
+    if (vectors.length !== values.length) {
+      throw new Error('Embedding function did not return an embedding for each input element')
+    }
+    const destType = newVectorType(vectors[0].length, innerDestType)
+    newColumns[destColumn] = makeVector(vectors, destType)
+  }
+
+  const newTable = new ArrowTable(newColumns)
+  if (schema != null) {
+    if (schema.fields.find(f => f.name === destColumn) === undefined) {
+      throw new Error(`When using embedding functions and specifying a schema the schema should include the embedding column but the column ${destColumn} was missing`)
+    }
+    return alignTable(newTable, schema)
+  }
+  return newTable
+}
+
+/*
+ * Convert an Array of records into an Arrow Table, optionally applying an
+ * embeddings function to it.
+ *
+ * This function calls `makeArrowTable` first to create the Arrow Table.
+ * Any provided `makeTableOptions` (e.g. a schema) will be passed on to
+ * that call.
+ *
+ * The embedding function will be passed a column of values (based on the
+ * `sourceColumn` of the embedding function) and expects to receive back
+ * number[][] which will be converted into a fixed size list column.  By
+ * default this will be a fixed size list of Float32 but that can be
+ * customized by the `embeddingDataType` property of the embedding function.
+ *
+ * If a schema is provided in `makeTableOptions` then it should include the
+ * embedding columns.  If no schema is provded then embedding columns will
+ * be placed at the end of the table, after all of the input columns.
+ */
 export async function convertToTable<T> (
   data: Array<Record<string, unknown>>,
-  embeddings?: EmbeddingFunction<T>
+  embeddings?: EmbeddingFunction<T>,
+  makeTableOptions?: Partial<MakeArrowTableOptions>
 ): Promise<ArrowTable> {
-  if (data.length === 0) {
-    throw new Error('At least one record needs to be provided')
-  }
-
-  const columns = Object.keys(data[0])
-  const records: Record<string, Vector> = {}
-
-  for (const columnsKey of columns) {
-    if (columnsKey === 'vector') {
-      const vectorSize = (data[0].vector as any[]).length
-      const listBuilder = newVectorBuilder(vectorSize)
-      for (const datum of data) {
-        if ((datum[columnsKey] as any[]).length !== vectorSize) {
-          throw new Error(`Invalid vector size, expected ${vectorSize}`)
-        }
-
-        listBuilder.append(datum[columnsKey])
-      }
-      records[columnsKey] = listBuilder.finish().toVector()
-    } else {
-      const values = []
-      for (const datum of data) {
-        values.push(datum[columnsKey])
-      }
-
-      if (columnsKey === embeddings?.sourceColumn) {
-        const vectors = await embeddings.embed(values as T[])
-        records.vector = vectorFromArray(
-          vectors,
-          newVectorType(vectors[0].length)
-        )
-      }
-
-      if (typeof values[0] === 'string') {
-        // `vectorFromArray` converts strings into dictionary vectors, forcing it back to a string column
-        records[columnsKey] = vectorFromArray(values, new Utf8())
-      } else if (Array.isArray(values[0])) {
-        const elementType = getElementType(values[0])
-        let innerType
-        if (elementType === 'string') {
-          innerType = new Utf8()
-        } else if (elementType === 'number') {
-          innerType = new Float64()
-        } else {
-          // TODO: pass in schema if it exists, else keep going to the next element
-          throw new Error(`Unsupported array element type ${elementType}`)
-        }
-        const listBuilder = makeBuilder({
-          type: new List(new Field('item', innerType, true))
-        })
-        for (const value of values) {
-          listBuilder.append(value)
-        }
-        records[columnsKey] = listBuilder.finish().toVector()
-      } else {
-        // TODO if this is a struct field then recursively align the subfields
-        records[columnsKey] = vectorFromArray(values)
-      }
-    }
-  }
-
-  return new ArrowTable(records)
-}
-
-function getElementType (arr: any[]): string {
-  if (arr.length === 0) {
-    return 'undefined'
-  }
-
-  return typeof arr[0]
-}
-
-// Creates a new Arrow ListBuilder that stores a Vector column
-function newVectorBuilder (dim: number): FixedSizeListBuilder<Float32> {
-  return makeBuilder({
-    type: newVectorType(dim)
-  })
+  const table = makeArrowTable(data, makeTableOptions)
+  return await applyEmbeddings(table, embeddings, makeTableOptions?.schema)
 }
 
 // Creates the Arrow Type for a Vector column with dimension `dim`
-function newVectorType (dim: number): FixedSizeList<Float32> {
+function newVectorType <T extends Float> (dim: number, innerType: T): FixedSizeList<T> {
   // Somewhere we always default to have the elements nullable, so we need to set it to true
   // otherwise we often get schema mismatches because the stored data always has schema with nullable elements
-  const children = new Field<Float32>('item', new Float32(), true)
+  const children = new Field<T>('item', innerType, true)
   return new FixedSizeList(dim, children)
 }
 
-// Converts an Array of records into Arrow IPC format
+/**
+ * Serialize an Array of records into a buffer using the Arrow IPC File serialization
+ *
+ * This function will call `convertToTable` and pass on `embeddings` and `schema`
+ *
+ * `schema` is required if data is empty
+ */
 export async function fromRecordsToBuffer<T> (
   data: Array<Record<string, unknown>>,
   embeddings?: EmbeddingFunction<T>,
   schema?: Schema
 ): Promise<Buffer> {
-  let table = await convertToTable(data, embeddings)
-  if (schema !== undefined) {
-    table = alignTable(table, schema)
+  if (schema !== undefined && schema !== null) {
+    schema = sanitizeSchema(schema)
   }
+  const table = await convertToTable(data, embeddings, { schema })
   const writer = RecordBatchFileWriter.writeAll(table)
   return Buffer.from(await writer.toUint8Array())
 }
 
-// Converts an Array of records into Arrow IPC stream format
+/**
+ * Serialize an Array of records into a buffer using the Arrow IPC Stream serialization
+ *
+ * This function will call `convertToTable` and pass on `embeddings` and `schema`
+ *
+ * `schema` is required if data is empty
+ */
 export async function fromRecordsToStreamBuffer<T> (
   data: Array<Record<string, unknown>>,
   embeddings?: EmbeddingFunction<T>,
   schema?: Schema
 ): Promise<Buffer> {
-  let table = await convertToTable(data, embeddings)
-  if (schema !== undefined) {
-    table = alignTable(table, schema)
+  if (schema !== null && schema !== undefined) {
+    schema = sanitizeSchema(schema)
   }
+  const table = await convertToTable(data, embeddings, { schema })
   const writer = RecordBatchStreamWriter.writeAll(table)
   return Buffer.from(await writer.toUint8Array())
 }
 
-// Converts an Arrow Table into Arrow IPC format
+/**
+ * Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
+ *
+ * This function will apply `embeddings` to the table in a manner similar to
+ * `convertToTable`.
+ *
+ * `schema` is required if the table is empty
+ */
 export async function fromTableToBuffer<T> (
   table: ArrowTable,
   embeddings?: EmbeddingFunction<T>,
   schema?: Schema
 ): Promise<Buffer> {
-  if (embeddings !== undefined) {
-    const source = table.getChild(embeddings.sourceColumn)
-
-    if (source === null) {
-      throw new Error(
-        `The embedding source column ${embeddings.sourceColumn} was not found in the Arrow Table`
-      )
-    }
-
-    const vectors = await embeddings.embed(source.toArray() as T[])
-    const column = vectorFromArray(vectors, newVectorType(vectors[0].length))
-    table = table.assign(new ArrowTable({ vector: column }))
+  if (schema !== null && schema !== undefined) {
+    schema = sanitizeSchema(schema)
   }
-  if (schema !== undefined) {
-    table = alignTable(table, schema)
-  }
-  const writer = RecordBatchFileWriter.writeAll(table)
+  const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema)
+  const writer = RecordBatchFileWriter.writeAll(tableWithEmbeddings)
   return Buffer.from(await writer.toUint8Array())
 }
 
-// Converts an Arrow Table into Arrow IPC stream format
+/**
+ * Serialize an Arrow Table into a buffer using the Arrow IPC Stream serialization
+ *
+ * This function will apply `embeddings` to the table in a manner similar to
+ * `convertToTable`.
+ *
+ * `schema` is required if the table is empty
+ */
 export async function fromTableToStreamBuffer<T> (
   table: ArrowTable,
   embeddings?: EmbeddingFunction<T>,
   schema?: Schema
 ): Promise<Buffer> {
-  if (embeddings !== undefined) {
-    const source = table.getChild(embeddings.sourceColumn)
-
-    if (source === null) {
-      throw new Error(
-        `The embedding source column ${embeddings.sourceColumn} was not found in the Arrow Table`
-      )
-    }
-
-    const vectors = await embeddings.embed(source.toArray() as T[])
-    const column = vectorFromArray(vectors, newVectorType(vectors[0].length))
-    table = table.assign(new ArrowTable({ vector: column }))
+  if (schema !== null && schema !== undefined) {
+    schema = sanitizeSchema(schema)
   }
-  if (schema !== undefined) {
-    table = alignTable(table, schema)
-  }
-  const writer = RecordBatchStreamWriter.writeAll(table)
+  const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema)
+  const writer = RecordBatchStreamWriter.writeAll(tableWithEmbeddings)
   return Buffer.from(await writer.toUint8Array())
 }
 
@@ -373,5 +547,5 @@ function alignTable (table: ArrowTable, schema: Schema): ArrowTable {
 
 // Creates an empty Arrow Table
 export function createEmptyTable (schema: Schema): ArrowTable {
-  return new ArrowTable(schema)
+  return new ArrowTable(sanitizeSchema(schema))
 }

node/src/embedding/embedding_function.ts

@@ -12,18 +12,53 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import { type Float } from 'apache-arrow'
+
 /**
  * An embedding function that automatically creates vector representation for a given column.
  */
 export interface EmbeddingFunction<T> {
   /**
-     * The name of the column that will be used as input for the Embedding Function.
-     */
+   * The name of the column that will be used as input for the Embedding Function.
+   */
   sourceColumn: string
 
   /**
-     * Creates a vector representation for the given values.
-     */
+   * The data type of the embedding
+   *
+   * The embedding function should return `number`.  This will be converted into
+   * an Arrow float array.  By default this will be Float32 but this property can
+   * be used to control the conversion.
+   */
+  embeddingDataType?: Float
+
+  /**
+   * The dimension of the embedding
+   *
+   * This is optional, normally this can be determined by looking at the results of
+   * `embed`.  If this is not specified, and there is an attempt to apply the embedding
+   * to an empty table, then that process will fail.
+   */
+  embeddingDimension?: number
+
+  /**
+   * The name of the column that will contain the embedding
+   *
+   * By default this is "vector"
+   */
+  destColumn?: string
+
+  /**
+   * Should the source column be excluded from the resulting table
+   *
+   * By default the source column is included.  Set this to true and
+   * only the embedding will be stored.
+   */
+  excludeSource?: boolean
+
+  /**
+   * Creates a vector representation for the given values.
+   */
   embed: (data: T[]) => Promise<number[][]>
 }
 

node/src/index.ts

@@ -24,6 +24,7 @@ import { RemoteConnection } from './remote'
 import { Query } from './query'
 import { isEmbeddingFunction } from './embedding/embedding_function'
 import { type Literal, toSQL } from './util'
+import { type HttpMiddleware } from './middleware'
 
 const {
   databaseNew,
@@ -37,18 +38,22 @@ const {
   tableCountRows,
   tableDelete,
   tableUpdate,
+  tableMergeInsert,
   tableCleanupOldVersions,
   tableCompactFiles,
   tableListIndices,
   tableIndexStats,
-  tableSchema
+  tableSchema,
+  tableAddColumns,
+  tableAlterColumns,
+  tableDropColumns
   // eslint-disable-next-line @typescript-eslint/no-var-requires
 } = require('../native.js')
 
 export { Query }
 export type { EmbeddingFunction }
 export { OpenAIEmbeddingFunction } from './embedding/openai'
-export { makeArrowTable, type MakeArrowTableOptions } from './arrow'
+export { convertToTable, makeArrowTable, type MakeArrowTableOptions } from './arrow'
 
 const defaultAwsRegion = 'us-west-2'
 
@@ -95,6 +100,19 @@ export interface ConnectionOptions {
    * This is useful for local testing.
    */
   hostOverride?: string
+
+  /**
+   * (For LanceDB OSS only): The interval, in seconds, at which to check for
+   * updates to the table from other processes. If None, then consistency is not
+   * checked. For performance reasons, this is the default. For strong
+   * consistency, set this to zero seconds. Then every read will check for
+   * updates from other processes. As a compromise, you can set this to a
+   * non-zero value for eventual consistency. If more than that interval
+   * has passed since the last check, then the table will be checked for updates.
+   * Note: this consistency only applies to read operations. Write operations are
+   * always consistent.
+   */
+  readConsistencyInterval?: number
 }
 
 function getAwsArgs (opts: ConnectionOptions): any[] {
@@ -159,22 +177,35 @@ export async function connect (
     opts = { uri: arg }
   } else {
     // opts = { uri: arg.uri, awsCredentials = arg.awsCredentials }
-    opts = Object.assign(
-      {
-        uri: '',
-        awsCredentials: undefined,
-        apiKey: undefined,
-        region: defaultAwsRegion
-      },
-      arg
-    )
+    const keys = Object.keys(arg)
+    if (keys.length === 1 && keys[0] === 'uri' && typeof arg.uri === 'string') {
+      opts = { uri: arg.uri }
+    } else {
+      opts = Object.assign(
+        {
+          uri: '',
+          awsCredentials: undefined,
+          awsRegion: defaultAwsRegion,
+          apiKey: undefined,
+          region: defaultAwsRegion
+        },
+        arg
+      )
+    }
   }
 
   if (opts.uri.startsWith('db://')) {
     // Remote connection
     return new RemoteConnection(opts)
   }
-  const db = await databaseNew(opts.uri)
+  const db = await databaseNew(
+    opts.uri,
+    opts.awsCredentials?.accessKeyId,
+    opts.awsCredentials?.secretKey,
+    opts.awsCredentials?.sessionToken,
+    opts.awsRegion,
+    opts.readConsistencyInterval
+  )
   return new LocalConnection(db, opts)
 }
 
@@ -272,6 +303,18 @@ export interface Connection {
    * @param name The name of the table to drop.
    */
   dropTable(name: string): Promise<void>
+
+  /**
+   * Instrument the behavior of this Connection with middleware.
+   *
+   * The middleware will be called in the order they are added.
+   *
+   * Currently this functionality is only supported for remote Connections.
+   *
+   * @param {HttpMiddleware} - Middleware which will instrument the Connection.
+   * @returns - this Connection instrumented by the passed middleware
+   */
+  withMiddleware(middleware: HttpMiddleware): Connection
 }
 
 /**
@@ -316,6 +359,7 @@ export interface Table<T = number[]> {
    *
    * @param column The column to index
    * @param replace If false, fail if an index already exists on the column
+   * it is always set to true for remote connections
    *
    * Scalar indices, like vector indices, can be used to speed up scans.  A scalar
    * index can speed up scans that contain filter expressions on the indexed column.
@@ -359,12 +403,12 @@ export interface Table<T = number[]> {
    * await table.createScalarIndex('my_col')
    * ```
    */
-  createScalarIndex: (column: string, replace: boolean) => Promise<void>
+  createScalarIndex: (column: string, replace?: boolean) => Promise<void>
 
   /**
    * Returns the number of rows in this table.
    */
-  countRows: () => Promise<number>
+  countRows: (filter?: string) => Promise<number>
 
   /**
    * Delete rows from this table.
@@ -433,6 +477,38 @@ export interface Table<T = number[]> {
    */
   update: (args: UpdateArgs | UpdateSqlArgs) => Promise<void>
 
+  /**
+   * Runs a "merge insert" operation on the table
+   *
+   * This operation can add rows, update rows, and remove rows all in a single
+   * transaction. It is a very generic tool that can be used to create
+   * behaviors like "insert if not exists", "update or insert (i.e. upsert)",
+   * or even replace a portion of existing data with new data (e.g. replace
+   * all data where month="january")
+   *
+   * The merge insert operation works by combining new data from a
+   * **source table** with existing data in a **target table** by using a
+   * join.  There are three categories of records.
+   *
+   * "Matched" records are records that exist in both the source table and
+   * the target table. "Not matched" records exist only in the source table
+   * (e.g. these are new data) "Not matched by source" records exist only
+   * in the target table (this is old data)
+   *
+   * The MergeInsertArgs can be used to customize what should happen for
+   * each category of data.
+   *
+   * Please note that the data may appear to be reordered as part of this
+   * operation.  This is because updated rows will be deleted from the
+   * dataset and then reinserted at the end with the new values.
+   *
+   * @param on a column to join on.  This is how records from the source
+   *           table and target table are matched.
+   * @param data the new data to insert
+   * @param args parameters controlling how the operation should behave
+   */
+  mergeInsert: (on: string, data: Array<Record<string, unknown>> | ArrowTable, args: MergeInsertArgs) => Promise<void>
+
   /**
    * List the indicies on this table.
    */
@@ -443,7 +519,74 @@ export interface Table<T = number[]> {
    */
   indexStats: (indexUuid: string) => Promise<IndexStats>
 
+  filter(value: string): Query<T>
+
   schema: Promise<Schema>
+
+  // TODO: Support BatchUDF
+  /**
+   * Add new columns with defined values.
+   *
+   * @param newColumnTransforms pairs of column names and the SQL expression to use
+   *                            to calculate the value of the new column. These
+   *                            expressions will be evaluated for each row in the
+   *                            table, and can reference existing columns in the table.
+   */
+  addColumns(newColumnTransforms: Array<{ name: string, valueSql: string }>): Promise<void>
+
+  /**
+   * Alter the name or nullability of columns.
+   *
+   * @param columnAlterations One or more alterations to apply to columns.
+   */
+  alterColumns(columnAlterations: ColumnAlteration[]): Promise<void>
+
+  /**
+   * Drop one or more columns from the dataset
+   *
+   * This is a metadata-only operation and does not remove the data from the
+   * underlying storage. In order to remove the data, you must subsequently
+   * call ``compact_files`` to rewrite the data without the removed columns and
+   * then call ``cleanup_files`` to remove the old files.
+   *
+   * @param columnNames The names of the columns to drop. These can be nested
+   *                    column references (e.g. "a.b.c") or top-level column
+   *                    names (e.g. "a").
+   */
+  dropColumns(columnNames: string[]): Promise<void>
+
+  /**
+   * Instrument the behavior of this Table with middleware.
+   *
+   * The middleware will be called in the order they are added.
+   *
+   * Currently this functionality is only supported for remote tables.
+   *
+   * @param {HttpMiddleware} - Middleware which will instrument the Table.
+   * @returns - this Table instrumented by the passed middleware
+   */
+  withMiddleware(middleware: HttpMiddleware): Table<T>
+}
+
+/**
+ * A definition of a column alteration. The alteration changes the column at
+ * `path` to have the new name `name`, to be nullable if `nullable` is true,
+ * and to have the data type `data_type`. At least one of `rename` or `nullable`
+ * must be provided.
+ */
+export interface ColumnAlteration {
+  /**
+   * The path to the column to alter. This is a dot-separated path to the column.
+   * If it is a top-level column then it is just the name of the column. If it is
+   * a nested column then it is the path to the column, e.g. "a.b.c" for a column
+   * `c` nested inside a column `b` nested inside a column `a`.
+   */
+  path: string
+  rename?: string
+  /**
+   * Set the new nullability. Note that a nullable column cannot be made non-nullable.
+   */
+  nullable?: boolean
 }
 
 export interface UpdateArgs {
@@ -474,6 +617,47 @@ export interface UpdateSqlArgs {
   valuesSql: Record<string, string>
 }
 
+export interface MergeInsertArgs {
+  /**
+   * If true then rows that exist in both the source table (new data) and
+   * the target table (old data) will be updated, replacing the old row
+   * with the corresponding matching row.
+   *
+   * If there are multiple matches then the behavior is undefined.
+   * Currently this causes multiple copies of the row to be created
+   * but that behavior is subject to change.
+   *
+   * Optionally, a filter can be specified.  This should be an SQL
+   * filter where fields with the prefix "target." refer to fields
+   * in the target table (old data) and fields with the prefix
+   * "source." refer to fields in the source table (new data).  For
+   * example, the filter "target.lastUpdated < source.lastUpdated" will
+   * only update matched rows when the incoming `lastUpdated` value is
+   * newer.
+   *
+   * Rows that do not match the filter will not be updated.  Rows that
+   * do not match the filter do become "not matched" rows.
+   */
+  whenMatchedUpdateAll?: string | boolean
+  /**
+   * If true then rows that exist only in the source table (new data)
+   * will be inserted into the target table.
+   */
+  whenNotMatchedInsertAll?: boolean
+  /**
+   * If true then rows that exist only in the target table (old data)
+   * will be deleted.
+   *
+   * If this is a string then it will be treated as an SQL filter and
+   * only rows that both do not match any row in the source table and
+   * match the given filter will be deleted.
+   *
+   * This can be used to replace a selection of existing data with
+   * new data.
+   */
+  whenNotMatchedBySourceDelete?: string | boolean
+}
+
 export interface VectorIndex {
   columns: string[]
   name: string
@@ -636,6 +820,10 @@ export class LocalConnection implements Connection {
   async dropTable (name: string): Promise<void> {
     await databaseDropTable.call(this._db, name)
   }
+
+  withMiddleware (middleware: HttpMiddleware): Connection {
+    return this
+  }
 }
 
 export class LocalTable<T = number[]> implements Table<T> {
@@ -761,15 +949,18 @@ export class LocalTable<T = number[]> implements Table<T> {
       })
   }
 
-  async createScalarIndex (column: string, replace: boolean): Promise<void> {
+  async createScalarIndex (column: string, replace?: boolean): Promise<void> {
+    if (replace === undefined) {
+      replace = true
+    }
     return tableCreateScalarIndex.call(this._tbl, column, replace)
   }
 
   /**
    * Returns the number of rows in this table.
    */
-  async countRows (): Promise<number> {
-    return tableCountRows.call(this._tbl)
+  async countRows (filter?: string): Promise<number> {
+    return tableCountRows.call(this._tbl, filter)
   }
 
   /**
@@ -812,6 +1003,46 @@ export class LocalTable<T = number[]> implements Table<T> {
       })
   }
 
+  async mergeInsert (on: string, data: Array<Record<string, unknown>> | ArrowTable, args: MergeInsertArgs): Promise<void> {
+    let whenMatchedUpdateAll = false
+    let whenMatchedUpdateAllFilt = null
+    if (args.whenMatchedUpdateAll !== undefined && args.whenMatchedUpdateAll !== null) {
+      whenMatchedUpdateAll = true
+      if (args.whenMatchedUpdateAll !== true) {
+        whenMatchedUpdateAllFilt = args.whenMatchedUpdateAll
+      }
+    }
+    const whenNotMatchedInsertAll = args.whenNotMatchedInsertAll ?? false
+    let whenNotMatchedBySourceDelete = false
+    let whenNotMatchedBySourceDeleteFilt = null
+    if (args.whenNotMatchedBySourceDelete !== undefined && args.whenNotMatchedBySourceDelete !== null) {
+      whenNotMatchedBySourceDelete = true
+      if (args.whenNotMatchedBySourceDelete !== true) {
+        whenNotMatchedBySourceDeleteFilt = args.whenNotMatchedBySourceDelete
+      }
+    }
+
+    const schema = await this.schema
+    let tbl: ArrowTable
+    if (data instanceof ArrowTable) {
+      tbl = data
+    } else {
+      tbl = makeArrowTable(data, { schema })
+    }
+    const buffer = await fromTableToBuffer(tbl, this._embeddings, schema)
+
+    this._tbl = await tableMergeInsert.call(
+      this._tbl,
+      on,
+      whenMatchedUpdateAll,
+      whenMatchedUpdateAllFilt,
+      whenNotMatchedInsertAll,
+      whenNotMatchedBySourceDelete,
+      whenNotMatchedBySourceDeleteFilt,
+      buffer
+    )
+  }
+
   /**
    * Clean up old versions of the table, freeing disk space.
    *
@@ -891,6 +1122,22 @@ export class LocalTable<T = number[]> implements Table<T> {
       return false
     }
   }
+
+  async addColumns (newColumnTransforms: Array<{ name: string, valueSql: string }>): Promise<void> {
+    return tableAddColumns.call(this._tbl, newColumnTransforms)
+  }
+
+  async alterColumns (columnAlterations: ColumnAlteration[]): Promise<void> {
+    return tableAlterColumns.call(this._tbl, columnAlterations)
+  }
+
+  async dropColumns (columnNames: string[]): Promise<void> {
+    return tableDropColumns.call(this._tbl, columnNames)
+  }
+
+  withMiddleware (middleware: HttpMiddleware): Table<T> {
+    return this
+  }
 }
 
 export interface CleanupStats {

node/src/middleware.ts

@@ -0,0 +1,58 @@
+// Copyright 2024 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * Middleware for Remote LanceDB Connection or Table
+ */
+export interface HttpMiddleware {
+  /**
+   * A callback that can be used to instrument the behavior of http requests to remote
+   * tables. It can be used to add headers, modify the request, or even short-circuit
+   * the request and return a response without making the request to the remote endpoint.
+   * It can also be used to modify the response from the remote endpoint.
+   *
+   * @param {RemoteResponse} res - Request to the remote endpoint
+   * @param {onRemoteRequestNext} next - Callback to advance the middleware chain
+   */
+  onRemoteRequest(
+    req: RemoteRequest,
+    next: (req: RemoteRequest) => Promise<RemoteResponse>,
+  ): Promise<RemoteResponse>
+};
+
+export enum Method {
+  GET,
+  POST
+}
+
+/**
+ * A LanceDB Remote HTTP Request
+ */
+export interface RemoteRequest {
+  uri: string
+  method: Method
+  headers: Map<string, string>
+  params?: Map<string, string>
+  body?: any
+}
+
+/**
+ * A LanceDB Remote HTTP Response
+ */
+export interface RemoteResponse {
+  status: number
+  statusText: string
+  headers: Map<string, string>
+  body: () => Promise<any>
+}

node/src/remote/client.ts

@@ -12,13 +12,101 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import axios, { type AxiosResponse } from 'axios'
+import axios, { type AxiosResponse, type ResponseType } from 'axios'
 
 import { tableFromIPC, type Table as ArrowTable } from 'apache-arrow'
 
+import { type RemoteResponse, type RemoteRequest, Method } from '../middleware'
+
+interface HttpLancedbClientMiddleware {
+  onRemoteRequest(
+    req: RemoteRequest,
+    next: (req: RemoteRequest) => Promise<RemoteResponse>,
+  ): Promise<RemoteResponse>
+}
+
+/**
+ * Invoke the middleware chain and at the end call the remote endpoint
+ */
+async function callWithMiddlewares (
+  req: RemoteRequest,
+  middlewares: HttpLancedbClientMiddleware[],
+  opts?: MiddlewareInvocationOptions
+): Promise<RemoteResponse> {
+  async function call (
+    i: number,
+    req: RemoteRequest
+  ): Promise<RemoteResponse> {
+    // if we have reached the end of the middleware chain, make the request
+    if (i > middlewares.length) {
+      const headers = Object.fromEntries(req.headers.entries())
+      const params = Object.fromEntries(req.params?.entries() ?? [])
+      const timeout = 10000
+      let res
+      if (req.method === Method.POST) {
+        res = await axios.post(
+          req.uri,
+          req.body,
+          {
+            headers,
+            params,
+            timeout,
+            responseType: opts?.responseType
+          }
+        )
+      } else {
+        res = await axios.get(
+          req.uri,
+          {
+            headers,
+            params,
+            timeout
+          }
+        )
+      }
+
+      return toLanceRes(res)
+    }
+
+    // call next middleware in chain
+    return await middlewares[i - 1].onRemoteRequest(
+      req,
+      async (req) => {
+        return await call(i + 1, req)
+      }
+    )
+  }
+
+  return await call(1, req)
+}
+
+interface MiddlewareInvocationOptions {
+  responseType?: ResponseType
+}
+
+/**
+ * Marshall the library response into a LanceDB response
+ */
+function toLanceRes (res: AxiosResponse): RemoteResponse {
+  const headers = new Map()
+  for (const h in res.headers) {
+    headers.set(h, res.headers[h])
+  }
+
+  return {
+    status: res.status,
+    statusText: res.statusText,
+    headers,
+    body: async () => {
+      return res.data
+    }
+  }
+}
+
 export class HttpLancedbClient {
   private readonly _url: string
   private readonly _apiKey: () => string
+  private readonly _middlewares: HttpLancedbClientMiddleware[]
 
   public constructor (
     url: string,
@@ -27,6 +115,7 @@ export class HttpLancedbClient {
   ) {
     this._url = url
     this._apiKey = () => apiKey
+    this._middlewares = []
   }
 
   get uri (): string {
@@ -43,74 +132,61 @@ export class HttpLancedbClient {
     columns?: string[],
     filter?: string
   ): Promise<ArrowTable<any>> {
-    const response = await axios.post(
-              `${this._url}/v1/table/${tableName}/query/`,
-              {
-                vector,
-                k,
-                nprobes,
-                refineFactor,
-                columns,
-                filter,
-                prefilter
-              },
-              {
-                headers: {
-                  'Content-Type': 'application/json',
-                  'x-api-key': this._apiKey(),
-                  ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
-                },
-                responseType: 'arraybuffer',
-                timeout: 10000
-              }
-    ).catch((err) => {
-      console.error('error: ', err)
-      if (err.response === undefined) {
-        throw new Error(`Network Error: ${err.message as string}`)
-      }
-      return err.response
-    })
-    if (response.status !== 200) {
-      const errorData = new TextDecoder().decode(response.data)
-      throw new Error(
-        `Server Error, status: ${response.status as number}, ` +
-        `message: ${response.statusText as string}: ${errorData}`
-      )
-    }
-
-    const table = tableFromIPC(response.data)
+    const result = await this.post(
+      `/v1/table/${tableName}/query/`,
+      {
+        vector,
+        k,
+        nprobes,
+        refineFactor,
+        columns,
+        filter,
+        prefilter
+      },
+      undefined,
+      undefined,
+      'arraybuffer'
+    )
+    const table = tableFromIPC(await result.body())
     return table
   }
 
   /**
    * Sent GET request.
    */
-  public async get (path: string, params?: Record<string, string | number>): Promise<AxiosResponse> {
-    const response = await axios.get(
-      `${this._url}${path}`,
-      {
-        headers: {
-          'Content-Type': 'application/json',
-          'x-api-key': this._apiKey(),
-          ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
-        },
-        params,
-        timeout: 10000
-      }
-    ).catch((err) => {
+  public async get (path: string, params?: Record<string, string>): Promise<RemoteResponse> {
+    const req = {
+      uri: `${this._url}${path}`,
+      method: Method.GET,
+      headers: new Map(Object.entries({
+        'Content-Type': 'application/json',
+        'x-api-key': this._apiKey(),
+        ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
+      })),
+      params: new Map(Object.entries(params ?? {}))
+    }
+
+    let response
+    try {
+      response = await callWithMiddlewares(req, this._middlewares)
+      return response
+    } catch (err: any) {
       console.error('error: ', err)
       if (err.response === undefined) {
         throw new Error(`Network Error: ${err.message as string}`)
       }
-      return err.response
-    })
+
+      response = toLanceRes(err.response)
+    }
+
     if (response.status !== 200) {
-      const errorData = new TextDecoder().decode(response.data)
+      const errorData = new TextDecoder().decode(await response.body())
       throw new Error(
-        `Server Error, status: ${response.status as number}, ` +
-        `message: ${response.statusText as string}: ${errorData}`
+        `Server Error, status: ${response.status}, ` +
+        `message: ${response.statusText}: ${errorData}`
       )
     }
+
     return response
   }
 
@@ -120,35 +196,65 @@ export class HttpLancedbClient {
   public async post (
     path: string,
     data?: any,
-    params?: Record<string, string | number>,
-    content?: string | undefined
-  ): Promise<AxiosResponse> {
-    const response = await axios.post(
-        `${this._url}${path}`,
-        data,
-        {
-          headers: {
-            'Content-Type': content ?? 'application/json',
-            'x-api-key': this._apiKey(),
-            ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
-          },
-          params,
-          timeout: 30000
-        }
-    ).catch((err) => {
+    params?: Record<string, string>,
+    content?: string | undefined,
+    responseType?: ResponseType | undefined
+  ): Promise<RemoteResponse> {
+    const req = {
+      uri: `${this._url}${path}`,
+      method: Method.POST,
+      headers: new Map(Object.entries({
+        'Content-Type': content ?? 'application/json',
+        'x-api-key': this._apiKey(),
+        ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
+      })),
+      params: new Map(Object.entries(params ?? {})),
+      body: data
+    }
+
+    let response
+    try {
+      response = await callWithMiddlewares(req, this._middlewares, { responseType })
+
+      // return response
+    } catch (err: any) {
       console.error('error: ', err)
       if (err.response === undefined) {
         throw new Error(`Network Error: ${err.message as string}`)
       }
-      return err.response
-    })
+      response = toLanceRes(err.response)
+    }
+
     if (response.status !== 200) {
-      const errorData = new TextDecoder().decode(response.data)
+      const errorData = new TextDecoder().decode(await response.body())
       throw new Error(
-          `Server Error, status: ${response.status as number}, ` +
-          `message: ${response.statusText as string}: ${errorData}`
+        `Server Error, status: ${response.status}, ` +
+        `message: ${response.statusText}: ${errorData}`
       )
     }
+
     return response
   }
+
+  /**
+   * Instrument this client with middleware
+   * @param mw - The middleware that instruments the client
+   * @returns - an instance of this client instrumented with the middleware
+   */
+  public withMiddleware (mw: HttpLancedbClientMiddleware): HttpLancedbClient {
+    const wrapped = this.clone()
+    wrapped._middlewares.push(mw)
+    return wrapped
+  }
+
+  /**
+   * Make a clone of this client
+   */
+  private clone (): HttpLancedbClient {
+    const clone = new HttpLancedbClient(this._url, this._apiKey(), this._dbName)
+    for (const mw of this._middlewares) {
+      clone._middlewares.push(mw)
+    }
+    return clone
+  }
 }

node/src/remote/index.ts

@@ -24,7 +24,9 @@ import {
   type IndexStats,
   type UpdateArgs,
   type UpdateSqlArgs,
-  makeArrowTable
+  makeArrowTable,
+  type MergeInsertArgs,
+  type ColumnAlteration
 } from '../index'
 import { Query } from '../query'
 
@@ -37,12 +39,13 @@ import {
   fromTableToStreamBuffer
 } from '../arrow'
 import { toSQL } from '../util'
+import { type HttpMiddleware } from '../middleware'
 
 /**
  * Remote connection.
  */
 export class RemoteConnection implements Connection {
-  private readonly _client: HttpLancedbClient
+  private _client: HttpLancedbClient
   private readonly _dbName: string
 
   constructor (opts: ConnectionOptions) {
@@ -82,10 +85,11 @@ export class RemoteConnection implements Connection {
     limit: number = 10
   ): Promise<string[]> {
     const response = await this._client.get('/v1/table/', {
-      limit,
+      limit: `${limit}`,
       page_token: pageToken
     })
-    return response.data.tables
+    const body = await response.body()
+    return body.tables
   }
 
   async openTable (name: string): Promise<Table>
@@ -161,7 +165,7 @@ export class RemoteConnection implements Connection {
       throw new Error(
         `Server Error, status: ${res.status}, ` +
           // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-          `message: ${res.statusText}: ${res.data}`
+          `message: ${res.statusText}: ${await res.body()}`
       )
     }
 
@@ -175,6 +179,17 @@ export class RemoteConnection implements Connection {
   async dropTable (name: string): Promise<void> {
     await this._client.post(`/v1/table/${name}/drop/`)
   }
+
+  withMiddleware (middleware: HttpMiddleware): Connection {
+    const wrapped = this.clone()
+    wrapped._client = wrapped._client.withMiddleware(middleware)
+    return wrapped
+  }
+
+  private clone (): RemoteConnection {
+    const clone: RemoteConnection = Object.create(RemoteConnection.prototype)
+    return Object.assign(clone, this)
+  }
 }
 
 export class RemoteQuery<T = number[]> extends Query<T> {
@@ -227,7 +242,7 @@ export class RemoteQuery<T = number[]> extends Query<T> {
 // we are using extend until we have next next version release
 // Table and Connection has both been refactored to interfaces
 export class RemoteTable<T = number[]> implements Table<T> {
-  private readonly _client: HttpLancedbClient
+  private _client: HttpLancedbClient
   private readonly _embeddings?: EmbeddingFunction<T>
   private readonly _name: string
 
@@ -254,15 +269,15 @@ export class RemoteTable<T = number[]> implements Table<T> {
   get schema (): Promise<any> {
     return this._client
       .post(`/v1/table/${this._name}/describe/`)
-      .then((res) => {
+      .then(async (res) => {
         if (res.status !== 200) {
           throw new Error(
             `Server Error, status: ${res.status}, ` +
               // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-              `message: ${res.statusText}: ${res.data}`
+              `message: ${res.statusText}: ${await res.body()}`
           )
         }
-        return res.data?.schema
+        return (await res.body())?.schema
       })
   }
 
@@ -270,6 +285,59 @@ export class RemoteTable<T = number[]> implements Table<T> {
     return new RemoteQuery(query, this._client, this._name) //, this._embeddings_new)
   }
 
+  filter (where: string): Query<T> {
+    throw new Error('Not implemented')
+  }
+
+  async mergeInsert (on: string, data: Array<Record<string, unknown>> | ArrowTable, args: MergeInsertArgs): Promise<void> {
+    let tbl: ArrowTable
+    if (data instanceof ArrowTable) {
+      tbl = data
+    } else {
+      tbl = makeArrowTable(data, await this.schema)
+    }
+
+    const queryParams: any = {
+      on
+    }
+    if (args.whenMatchedUpdateAll !== false && args.whenMatchedUpdateAll !== null && args.whenMatchedUpdateAll !== undefined) {
+      queryParams.when_matched_update_all = 'true'
+      if (typeof args.whenMatchedUpdateAll === 'string') {
+        queryParams.when_matched_update_all_filt = args.whenMatchedUpdateAll
+      }
+    } else {
+      queryParams.when_matched_update_all = 'false'
+    }
+    if (args.whenNotMatchedInsertAll ?? false) {
+      queryParams.when_not_matched_insert_all = 'true'
+    } else {
+      queryParams.when_not_matched_insert_all = 'false'
+    }
+    if (args.whenNotMatchedBySourceDelete !== false && args.whenNotMatchedBySourceDelete !== null && args.whenNotMatchedBySourceDelete !== undefined) {
+      queryParams.when_not_matched_by_source_delete = 'true'
+      if (typeof args.whenNotMatchedBySourceDelete === 'string') {
+        queryParams.when_not_matched_by_source_delete_filt = args.whenNotMatchedBySourceDelete
+      }
+    } else {
+      queryParams.when_not_matched_by_source_delete = 'false'
+    }
+
+    const buffer = await fromTableToStreamBuffer(tbl, this._embeddings)
+    const res = await this._client.post(
+      `/v1/table/${this._name}/merge_insert/`,
+      buffer,
+      queryParams,
+      'application/vnd.apache.arrow.stream'
+    )
+    if (res.status !== 200) {
+      throw new Error(
+        `Server Error, status: ${res.status}, ` +
+          // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+          `message: ${res.statusText}: ${await res.body()}`
+      )
+    }
+  }
+
   async add (data: Array<Record<string, unknown>> | ArrowTable): Promise<number> {
     let tbl: ArrowTable
     if (data instanceof ArrowTable) {
@@ -291,7 +359,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
       throw new Error(
         `Server Error, status: ${res.status}, ` +
           // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-          `message: ${res.statusText}: ${res.data}`
+          `message: ${res.statusText}: ${await res.body()}`
       )
     }
     return tbl.numRows
@@ -317,7 +385,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
       throw new Error(
         `Server Error, status: ${res.status}, ` +
           // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-          `message: ${res.statusText}: ${res.data}`
+          `message: ${res.statusText}: ${await res.body()}`
       )
     }
     return tbl.numRows
@@ -342,7 +410,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
     }
 
     const column = indexParams.column ?? 'vector'
-    const indexType = 'vector' // only vector index is supported for remote connections
+    const indexType = 'vector'
     const metricType = indexParams.metric_type ?? 'L2'
     const indexCacheSize = indexParams.index_cache_size ?? null
 
@@ -360,18 +428,35 @@ export class RemoteTable<T = number[]> implements Table<T> {
       throw new Error(
         `Server Error, status: ${res.status}, ` +
           // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-          `message: ${res.statusText}: ${res.data}`
+          `message: ${res.statusText}: ${await res.body()}`
       )
     }
   }
 
-  async createScalarIndex (column: string, replace: boolean): Promise<void> {
-    throw new Error('Not implemented')
+  async createScalarIndex (column: string): Promise<void> {
+    const indexType = 'scalar'
+
+    const data = {
+      column,
+      index_type: indexType,
+      replace: true
+    }
+    const res = await this._client.post(
+      `/v1/table/${this._name}/create_scalar_index/`,
+      data
+    )
+    if (res.status !== 200) {
+      throw new Error(
+        `Server Error, status: ${res.status}, ` +
+          // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+          `message: ${res.statusText}: ${await res.body()}`
+      )
+    }
   }
 
   async countRows (): Promise<number> {
     const result = await this._client.post(`/v1/table/${this._name}/describe/`)
-    return result.data?.stats?.num_rows
+    return (await result.body())?.stats?.num_rows
   }
 
   async delete (filter: string): Promise<void> {
@@ -404,7 +489,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
     const results = await this._client.post(
       `/v1/table/${this._name}/index/list/`
     )
-    return results.data.indexes?.map((index: any) => ({
+    return (await results.body()).indexes?.map((index: any) => ({
       columns: index.columns,
       name: index.index_name,
       uuid: index.index_uuid
@@ -415,9 +500,33 @@ export class RemoteTable<T = number[]> implements Table<T> {
     const results = await this._client.post(
       `/v1/table/${this._name}/index/${indexUuid}/stats/`
     )
+    const body = await results.body()
     return {
-      numIndexedRows: results.data.num_indexed_rows,
-      numUnindexedRows: results.data.num_unindexed_rows
+      numIndexedRows: body?.num_indexed_rows,
+      numUnindexedRows: body?.num_unindexed_rows
     }
   }
+
+  async addColumns (newColumnTransforms: Array<{ name: string, valueSql: string }>): Promise<void> {
+    throw new Error('Add columns is not yet supported in LanceDB Cloud.')
+  }
+
+  async alterColumns (columnAlterations: ColumnAlteration[]): Promise<void> {
+    throw new Error('Alter columns is not yet supported in LanceDB Cloud.')
+  }
+
+  async dropColumns (columnNames: string[]): Promise<void> {
+    throw new Error('Drop columns is not yet supported in LanceDB Cloud.')
+  }
+
+  withMiddleware(middleware: HttpMiddleware): Table<T> {
+    const wrapped = this.clone()
+    wrapped._client = wrapped._client.withMiddleware(middleware)
+    return wrapped
+  }
+
+  private clone (): RemoteTable<T> {
+    const clone: RemoteTable<T> = Object.create(RemoteTable.prototype)
+    return Object.assign(clone, this)
+  }
 }

node/src/sanitize.ts

@@ -0,0 +1,508 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The utilities in this file help sanitize data from the user's arrow
+// library into the types expected by vectordb's arrow library.  Node
+// generally allows for mulitple versions of the same library (and sometimes
+// even multiple copies of the same version) to be installed at the same
+// time.  However, arrow-js uses instanceof which expected that the input
+// comes from the exact same library instance.  This is not always the case
+// and so we must sanitize the input to ensure that it is compatible.
+
+import {
+  Field,
+  Utf8,
+  FixedSizeBinary,
+  FixedSizeList,
+  Schema,
+  List,
+  Struct,
+  Float,
+  Bool,
+  Date_,
+  Decimal,
+  DataType,
+  Dictionary,
+  Binary,
+  Float32,
+  Interval,
+  Map_,
+  Duration,
+  Union,
+  Time,
+  Timestamp,
+  Type,
+  Null,
+  Int,
+  type Precision,
+  type DateUnit,
+  Int8,
+  Int16,
+  Int32,
+  Int64,
+  Uint8,
+  Uint16,
+  Uint32,
+  Uint64,
+  Float16,
+  Float64,
+  DateDay,
+  DateMillisecond,
+  DenseUnion,
+  SparseUnion,
+  TimeNanosecond,
+  TimeMicrosecond,
+  TimeMillisecond,
+  TimeSecond,
+  TimestampNanosecond,
+  TimestampMicrosecond,
+  TimestampMillisecond,
+  TimestampSecond,
+  IntervalDayTime,
+  IntervalYearMonth,
+  DurationNanosecond,
+  DurationMicrosecond,
+  DurationMillisecond,
+  DurationSecond,
+} from "apache-arrow";
+import type { IntBitWidth, TimeBitWidth } from "apache-arrow/type";
+
+function sanitizeMetadata(
+  metadataLike?: unknown,
+): Map<string, string> | undefined {
+  if (metadataLike === undefined || metadataLike === null) {
+    return undefined;
+  }
+  if (!(metadataLike instanceof Map)) {
+    throw Error("Expected metadata, if present, to be a Map<string, string>");
+  }
+  for (const item of metadataLike) {
+    if (!(typeof item[0] === "string" || !(typeof item[1] === "string"))) {
+      throw Error(
+        "Expected metadata, if present, to be a Map<string, string> but it had non-string keys or values",
+      );
+    }
+  }
+  return metadataLike as Map<string, string>;
+}
+
+function sanitizeInt(typeLike: object) {
+  if (
+    !("bitWidth" in typeLike) ||
+    typeof typeLike.bitWidth !== "number" ||
+    !("isSigned" in typeLike) ||
+    typeof typeLike.isSigned !== "boolean"
+  ) {
+    throw Error(
+      "Expected an Int Type to have a `bitWidth` and `isSigned` property",
+    );
+  }
+  return new Int(typeLike.isSigned, typeLike.bitWidth as IntBitWidth);
+}
+
+function sanitizeFloat(typeLike: object) {
+  if (!("precision" in typeLike) || typeof typeLike.precision !== "number") {
+    throw Error("Expected a Float Type to have a `precision` property");
+  }
+  return new Float(typeLike.precision as Precision);
+}
+
+function sanitizeDecimal(typeLike: object) {
+  if (
+    !("scale" in typeLike) ||
+    typeof typeLike.scale !== "number" ||
+    !("precision" in typeLike) ||
+    typeof typeLike.precision !== "number" ||
+    !("bitWidth" in typeLike) ||
+    typeof typeLike.bitWidth !== "number"
+  ) {
+    throw Error(
+      "Expected a Decimal Type to have `scale`, `precision`, and `bitWidth` properties",
+    );
+  }
+  return new Decimal(typeLike.scale, typeLike.precision, typeLike.bitWidth);
+}
+
+function sanitizeDate(typeLike: object) {
+  if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
+    throw Error("Expected a Date type to have a `unit` property");
+  }
+  return new Date_(typeLike.unit as DateUnit);
+}
+
+function sanitizeTime(typeLike: object) {
+  if (
+    !("unit" in typeLike) ||
+    typeof typeLike.unit !== "number" ||
+    !("bitWidth" in typeLike) ||
+    typeof typeLike.bitWidth !== "number"
+  ) {
+    throw Error(
+      "Expected a Time type to have `unit` and `bitWidth` properties",
+    );
+  }
+  return new Time(typeLike.unit, typeLike.bitWidth as TimeBitWidth);
+}
+
+function sanitizeTimestamp(typeLike: object) {
+  if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
+    throw Error("Expected a Timestamp type to have a `unit` property");
+  }
+  let timezone = null;
+  if ("timezone" in typeLike && typeof typeLike.timezone === "string") {
+    timezone = typeLike.timezone;
+  }
+  return new Timestamp(typeLike.unit, timezone);
+}
+
+function sanitizeTypedTimestamp(
+  typeLike: object,
+  Datatype:
+    | typeof TimestampNanosecond
+    | typeof TimestampMicrosecond
+    | typeof TimestampMillisecond
+    | typeof TimestampSecond,
+) {
+  let timezone = null;
+  if ("timezone" in typeLike && typeof typeLike.timezone === "string") {
+    timezone = typeLike.timezone;
+  }
+  return new Datatype(timezone);
+}
+
+function sanitizeInterval(typeLike: object) {
+  if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
+    throw Error("Expected an Interval type to have a `unit` property");
+  }
+  return new Interval(typeLike.unit);
+}
+
+function sanitizeList(typeLike: object) {
+  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
+    throw Error(
+      "Expected a List type to have an array-like `children` property",
+    );
+  }
+  if (typeLike.children.length !== 1) {
+    throw Error("Expected a List type to have exactly one child");
+  }
+  return new List(sanitizeField(typeLike.children[0]));
+}
+
+function sanitizeStruct(typeLike: object) {
+  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
+    throw Error(
+      "Expected a Struct type to have an array-like `children` property",
+    );
+  }
+  return new Struct(typeLike.children.map((child) => sanitizeField(child)));
+}
+
+function sanitizeUnion(typeLike: object) {
+  if (
+    !("typeIds" in typeLike) ||
+    !("mode" in typeLike) ||
+    typeof typeLike.mode !== "number"
+  ) {
+    throw Error(
+      "Expected a Union type to have `typeIds` and `mode` properties",
+    );
+  }
+  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
+    throw Error(
+      "Expected a Union type to have an array-like `children` property",
+    );
+  }
+
+  return new Union(
+    typeLike.mode,
+    typeLike.typeIds as any,
+    typeLike.children.map((child) => sanitizeField(child)),
+  );
+}
+
+function sanitizeTypedUnion(
+  typeLike: object,
+  UnionType: typeof DenseUnion | typeof SparseUnion,
+) {
+  if (!("typeIds" in typeLike)) {
+    throw Error(
+      "Expected a DenseUnion/SparseUnion type to have a `typeIds` property",
+    );
+  }
+  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
+    throw Error(
+      "Expected a DenseUnion/SparseUnion type to have an array-like `children` property",
+    );
+  }
+
+  return new UnionType(
+    typeLike.typeIds as any,
+    typeLike.children.map((child) => sanitizeField(child)),
+  );
+}
+
+function sanitizeFixedSizeBinary(typeLike: object) {
+  if (!("byteWidth" in typeLike) || typeof typeLike.byteWidth !== "number") {
+    throw Error(
+      "Expected a FixedSizeBinary type to have a `byteWidth` property",
+    );
+  }
+  return new FixedSizeBinary(typeLike.byteWidth);
+}
+
+function sanitizeFixedSizeList(typeLike: object) {
+  if (!("listSize" in typeLike) || typeof typeLike.listSize !== "number") {
+    throw Error("Expected a FixedSizeList type to have a `listSize` property");
+  }
+  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
+    throw Error(
+      "Expected a FixedSizeList type to have an array-like `children` property",
+    );
+  }
+  if (typeLike.children.length !== 1) {
+    throw Error("Expected a FixedSizeList type to have exactly one child");
+  }
+  return new FixedSizeList(
+    typeLike.listSize,
+    sanitizeField(typeLike.children[0]),
+  );
+}
+
+function sanitizeMap(typeLike: object) {
+  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
+    throw Error(
+      "Expected a Map type to have an array-like `children` property",
+    );
+  }
+  if (!("keysSorted" in typeLike) || typeof typeLike.keysSorted !== "boolean") {
+    throw Error("Expected a Map type to have a `keysSorted` property");
+  }
+  return new Map_(
+    typeLike.children.map((field) => sanitizeField(field)) as any,
+    typeLike.keysSorted,
+  );
+}
+
+function sanitizeDuration(typeLike: object) {
+  if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
+    throw Error("Expected a Duration type to have a `unit` property");
+  }
+  return new Duration(typeLike.unit);
+}
+
+function sanitizeDictionary(typeLike: object) {
+  if (!("id" in typeLike) || typeof typeLike.id !== "number") {
+    throw Error("Expected a Dictionary type to have an `id` property");
+  }
+  if (!("indices" in typeLike) || typeof typeLike.indices !== "object") {
+    throw Error("Expected a Dictionary type to have an `indices` property");
+  }
+  if (!("dictionary" in typeLike) || typeof typeLike.dictionary !== "object") {
+    throw Error("Expected a Dictionary type to have an `dictionary` property");
+  }
+  if (!("isOrdered" in typeLike) || typeof typeLike.isOrdered !== "boolean") {
+    throw Error("Expected a Dictionary type to have an `isOrdered` property");
+  }
+  return new Dictionary(
+    sanitizeType(typeLike.dictionary),
+    sanitizeType(typeLike.indices) as any,
+    typeLike.id,
+    typeLike.isOrdered,
+  );
+}
+
+function sanitizeType(typeLike: unknown): DataType<any> {
+  if (typeof typeLike !== "object" || typeLike === null) {
+    throw Error("Expected a Type but object was null/undefined");
+  }
+  if (!("typeId" in typeLike) || !(typeof typeLike.typeId !== "function")) {
+    throw Error("Expected a Type to have a typeId function");
+  }
+  let typeId: Type;
+  if (typeof typeLike.typeId === "function") {
+    typeId = (typeLike.typeId as () => unknown)() as Type;
+  } else if (typeof typeLike.typeId === "number") {
+    typeId = typeLike.typeId as Type;
+  } else {
+    throw Error("Type's typeId property was not a function or number");
+  }
+
+  switch (typeId) {
+    case Type.NONE:
+      throw Error("Received a Type with a typeId of NONE");
+    case Type.Null:
+      return new Null();
+    case Type.Int:
+      return sanitizeInt(typeLike);
+    case Type.Float:
+      return sanitizeFloat(typeLike);
+    case Type.Binary:
+      return new Binary();
+    case Type.Utf8:
+      return new Utf8();
+    case Type.Bool:
+      return new Bool();
+    case Type.Decimal:
+      return sanitizeDecimal(typeLike);
+    case Type.Date:
+      return sanitizeDate(typeLike);
+    case Type.Time:
+      return sanitizeTime(typeLike);
+    case Type.Timestamp:
+      return sanitizeTimestamp(typeLike);
+    case Type.Interval:
+      return sanitizeInterval(typeLike);
+    case Type.List:
+      return sanitizeList(typeLike);
+    case Type.Struct:
+      return sanitizeStruct(typeLike);
+    case Type.Union:
+      return sanitizeUnion(typeLike);
+    case Type.FixedSizeBinary:
+      return sanitizeFixedSizeBinary(typeLike);
+    case Type.FixedSizeList:
+      return sanitizeFixedSizeList(typeLike);
+    case Type.Map:
+      return sanitizeMap(typeLike);
+    case Type.Duration:
+      return sanitizeDuration(typeLike);
+    case Type.Dictionary:
+      return sanitizeDictionary(typeLike);
+    case Type.Int8:
+      return new Int8();
+    case Type.Int16:
+      return new Int16();
+    case Type.Int32:
+      return new Int32();
+    case Type.Int64:
+      return new Int64();
+    case Type.Uint8:
+      return new Uint8();
+    case Type.Uint16:
+      return new Uint16();
+    case Type.Uint32:
+      return new Uint32();
+    case Type.Uint64:
+      return new Uint64();
+    case Type.Float16:
+      return new Float16();
+    case Type.Float32:
+      return new Float32();
+    case Type.Float64:
+      return new Float64();
+    case Type.DateMillisecond:
+      return new DateMillisecond();
+    case Type.DateDay:
+      return new DateDay();
+    case Type.TimeNanosecond:
+      return new TimeNanosecond();
+    case Type.TimeMicrosecond:
+      return new TimeMicrosecond();
+    case Type.TimeMillisecond:
+      return new TimeMillisecond();
+    case Type.TimeSecond:
+      return new TimeSecond();
+    case Type.TimestampNanosecond:
+      return sanitizeTypedTimestamp(typeLike, TimestampNanosecond);
+    case Type.TimestampMicrosecond:
+      return sanitizeTypedTimestamp(typeLike, TimestampMicrosecond);
+    case Type.TimestampMillisecond:
+      return sanitizeTypedTimestamp(typeLike, TimestampMillisecond);
+    case Type.TimestampSecond:
+      return sanitizeTypedTimestamp(typeLike, TimestampSecond);
+    case Type.DenseUnion:
+      return sanitizeTypedUnion(typeLike, DenseUnion);
+    case Type.SparseUnion:
+      return sanitizeTypedUnion(typeLike, SparseUnion);
+    case Type.IntervalDayTime:
+      return new IntervalDayTime();
+    case Type.IntervalYearMonth:
+      return new IntervalYearMonth();
+    case Type.DurationNanosecond:
+      return new DurationNanosecond();
+    case Type.DurationMicrosecond:
+      return new DurationMicrosecond();
+    case Type.DurationMillisecond:
+      return new DurationMillisecond();
+    case Type.DurationSecond:
+      return new DurationSecond();
+  }
+}
+
+function sanitizeField(fieldLike: unknown): Field {
+  if (fieldLike instanceof Field) {
+    return fieldLike;
+  }
+  if (typeof fieldLike !== "object" || fieldLike === null) {
+    throw Error("Expected a Field but object was null/undefined");
+  }
+  if (
+    !("type" in fieldLike) ||
+    !("name" in fieldLike) ||
+    !("nullable" in fieldLike)
+  ) {
+    throw Error(
+      "The field passed in is missing a `type`/`name`/`nullable` property",
+    );
+  }
+  const type = sanitizeType(fieldLike.type);
+  const name = fieldLike.name;
+  if (!(typeof name === "string")) {
+    throw Error("The field passed in had a non-string `name` property");
+  }
+  const nullable = fieldLike.nullable;
+  if (!(typeof nullable === "boolean")) {
+    throw Error("The field passed in had a non-boolean `nullable` property");
+  }
+  let metadata;
+  if ("metadata" in fieldLike) {
+    metadata = sanitizeMetadata(fieldLike.metadata);
+  }
+  return new Field(name, type, nullable, metadata);
+}
+
+/**
+ * Convert something schemaLike into a Schema instance
+ *
+ * This method is often needed even when the caller is using a Schema
+ * instance because they might be using a different instance of apache-arrow
+ * than lancedb is using.
+ */
+export function sanitizeSchema(schemaLike: unknown): Schema {
+  if (schemaLike instanceof Schema) {
+    return schemaLike;
+  }
+  if (typeof schemaLike !== "object" || schemaLike === null) {
+    throw Error("Expected a Schema but object was null/undefined");
+  }
+  if (!("fields" in schemaLike)) {
+    throw Error(
+      "The schema passed in does not appear to be a schema (no 'fields' property)",
+    );
+  }
+  let metadata;
+  if ("metadata" in schemaLike) {
+    metadata = sanitizeMetadata(schemaLike.metadata);
+  }
+  if (!Array.isArray(schemaLike.fields)) {
+    throw Error(
+      "The schema passed in had a 'fields' property but it was not an array",
+    );
+  }
+  const sanitizedFields = schemaLike.fields.map((field) =>
+    sanitizeField(field),
+  );
+  return new Schema(sanitizedFields, metadata);
+}

node/src/test/arrow.test.ts

@@ -13,9 +13,10 @@
 // limitations under the License.
 
 import { describe } from 'mocha'
-import { assert } from 'chai'
+import { assert, expect, use as chaiUse } from 'chai'
+import * as chaiAsPromised from 'chai-as-promised'
 
-import { fromTableToBuffer, makeArrowTable } from '../arrow'
+import { convertToTable, fromTableToBuffer, makeArrowTable, makeEmptyTable } from '../arrow'
 import {
   Field,
   FixedSizeList,
@@ -24,21 +25,91 @@ import {
   Int32,
   tableFromIPC,
   Schema,
-  Float64
+  Float64,
+  type Table,
+  Binary,
+  Bool,
+  Utf8,
+  Struct,
+  List,
+  DataType,
+  Dictionary,
+  Int64,
+  MetadataVersion
 } from 'apache-arrow'
+import {
+  Dictionary as OldDictionary,
+  Field as OldField,
+  FixedSizeList as OldFixedSizeList,
+  Float32 as OldFloat32,
+  Int32 as OldInt32,
+  Struct as OldStruct,
+  Schema as OldSchema,
+  TimestampNanosecond as OldTimestampNanosecond,
+  Utf8 as OldUtf8
+} from 'apache-arrow-old'
+import { type EmbeddingFunction } from '../embedding/embedding_function'
 
-describe('Apache Arrow tables', function () {
-  it('customized schema', async function () {
+chaiUse(chaiAsPromised)
+
+function sampleRecords (): Array<Record<string, any>> {
+  return [
+    {
+      binary: Buffer.alloc(5),
+      boolean: false,
+      number: 7,
+      string: 'hello',
+      struct: { x: 0, y: 0 },
+      list: ['anime', 'action', 'comedy']
+    }
+  ]
+}
+
+// Helper method to verify various ways to create a table
+async function checkTableCreation (tableCreationMethod: (records: any, recordsReversed: any, schema: Schema) => Promise<Table>): Promise<void> {
+  const records = sampleRecords()
+  const recordsReversed = [{
+    list: ['anime', 'action', 'comedy'],
+    struct: { x: 0, y: 0 },
+    string: 'hello',
+    number: 7,
+    boolean: false,
+    binary: Buffer.alloc(5)
+  }]
+  const schema = new Schema([
+    new Field('binary', new Binary(), false),
+    new Field('boolean', new Bool(), false),
+    new Field('number', new Float64(), false),
+    new Field('string', new Utf8(), false),
+    new Field('struct', new Struct([
+      new Field('x', new Float64(), false),
+      new Field('y', new Float64(), false)
+    ])),
+    new Field('list', new List(new Field('item', new Utf8(), false)), false)
+  ])
+
+  const table = await tableCreationMethod(records, recordsReversed, schema)
+  schema.fields.forEach((field, idx) => {
+    const actualField = table.schema.fields[idx]
+    assert.isFalse(actualField.nullable)
+    assert.equal(table.getChild(field.name)?.type.toString(), field.type.toString())
+    assert.equal(table.getChildAt(idx)?.type.toString(), field.type.toString())
+  })
+}
+
+describe('The function makeArrowTable', function () {
+  it('will use data types from a provided schema instead of inference', async function () {
     const schema = new Schema([
       new Field('a', new Int32()),
       new Field('b', new Float32()),
-      new Field('c', new FixedSizeList(3, new Field('item', new Float16())))
+      new Field('c', new FixedSizeList(3, new Field('item', new Float16()))),
+      new Field('d', new Int64())
     ])
     const table = makeArrowTable(
       [
-        { a: 1, b: 2, c: [1, 2, 3] },
-        { a: 4, b: 5, c: [4, 5, 6] },
-        { a: 7, b: 8, c: [7, 8, 9] }
+        { a: 1, b: 2, c: [1, 2, 3], d: 9 },
+        { a: 4, b: 5, c: [4, 5, 6], d: 10 },
+        { a: 7, b: 8, c: [7, 8, 9], d: null }
       ],
       { schema }
     )
@@ -52,13 +123,13 @@ describe('Apache Arrow tables', function () {
     assert.deepEqual(actualSchema, schema)
   })
 
-  it('default vector column', async function () {
+  it('will assume the column `vector` is FixedSizeList<Float32> by default', async function () {
     const schema = new Schema([
       new Field('a', new Float64()),
       new Field('b', new Float64()),
       new Field(
         'vector',
-        new FixedSizeList(3, new Field('item', new Float32()))
+        new FixedSizeList(3, new Field('item', new Float32(), true))
       )
     ])
     const table = makeArrowTable([
@@ -76,12 +147,12 @@ describe('Apache Arrow tables', function () {
     assert.deepEqual(actualSchema, schema)
   })
 
-  it('2 vector columns', async function () {
+  it('can support multiple vector columns', async function () {
     const schema = new Schema([
       new Field('a', new Float64()),
       new Field('b', new Float64()),
-      new Field('vec1', new FixedSizeList(3, new Field('item', new Float16()))),
-      new Field('vec2', new FixedSizeList(3, new Field('item', new Float16())))
+      new Field('vec1', new FixedSizeList(3, new Field('item', new Float16(), true))),
+      new Field('vec2', new FixedSizeList(3, new Field('item', new Float16(), true)))
     ])
     const table = makeArrowTable(
       [
@@ -105,4 +176,185 @@ describe('Apache Arrow tables', function () {
     const actualSchema = actual.schema
     assert.deepEqual(actualSchema, schema)
   })
+
+  it('will allow different vector column types', async function () {
+    const table = makeArrowTable(
+      [
+        { fp16: [1], fp32: [1], fp64: [1] }
+      ],
+      {
+        vectorColumns: {
+          fp16: { type: new Float16() },
+          fp32: { type: new Float32() },
+          fp64: { type: new Float64() }
+        }
+      }
+    )
+
+    assert.equal(table.getChild('fp16')?.type.children[0].type.toString(), new Float16().toString())
+    assert.equal(table.getChild('fp32')?.type.children[0].type.toString(), new Float32().toString())
+    assert.equal(table.getChild('fp64')?.type.children[0].type.toString(), new Float64().toString())
+  })
+
+  it('will use dictionary encoded strings if asked', async function () {
+    const table = makeArrowTable([{ str: 'hello' }])
+    assert.isTrue(DataType.isUtf8(table.getChild('str')?.type))
+
+    const tableWithDict = makeArrowTable([{ str: 'hello' }], { dictionaryEncodeStrings: true })
+    assert.isTrue(DataType.isDictionary(tableWithDict.getChild('str')?.type))
+
+    const schema = new Schema([
+      new Field('str', new Dictionary(new Utf8(), new Int32()))
+    ])
+
+    const tableWithDict2 = makeArrowTable([{ str: 'hello' }], { schema })
+    assert.isTrue(DataType.isDictionary(tableWithDict2.getChild('str')?.type))
+  })
+
+  it('will infer data types correctly', async function () {
+    await checkTableCreation(async (records) => makeArrowTable(records))
+  })
+
+  it('will allow a schema to be provided', async function () {
+    await checkTableCreation(async (records, _, schema) => makeArrowTable(records, { schema }))
+  })
+
+  it('will use the field order of any provided schema', async function () {
+    await checkTableCreation(async (_, recordsReversed, schema) => makeArrowTable(recordsReversed, { schema }))
+  })
+
+  it('will make an empty table', async function () {
+    await checkTableCreation(async (_, __, schema) => makeArrowTable([], { schema }))
+  })
+})
+
+class DummyEmbedding implements EmbeddingFunction<string> {
+  public readonly sourceColumn = 'string'
+  public readonly embeddingDimension = 2
+  public readonly embeddingDataType = new Float16()
+
+  async embed (data: string[]): Promise<number[][]> {
+    return data.map(
+      () => [0.0, 0.0]
+    )
+  }
+}
+
+class DummyEmbeddingWithNoDimension implements EmbeddingFunction<string> {
+  public readonly sourceColumn = 'string'
+
+  async embed (data: string[]): Promise<number[][]> {
+    return data.map(
+      () => [0.0, 0.0]
+    )
+  }
+}
+
+describe('convertToTable', function () {
+  it('will infer data types correctly', async function () {
+    await checkTableCreation(async (records) => await convertToTable(records))
+  })
+
+  it('will allow a schema to be provided', async function () {
+    await checkTableCreation(async (records, _, schema) => await convertToTable(records, undefined, { schema }))
+  })
+
+  it('will use the field order of any provided schema', async function () {
+    await checkTableCreation(async (_, recordsReversed, schema) => await convertToTable(recordsReversed, undefined, { schema }))
+  })
+
+  it('will make an empty table', async function () {
+    await checkTableCreation(async (_, __, schema) => await convertToTable([], undefined, { schema }))
+  })
+
+  it('will apply embeddings', async function () {
+    const records = sampleRecords()
+    const table = await convertToTable(records, new DummyEmbedding())
+    assert.isTrue(DataType.isFixedSizeList(table.getChild('vector')?.type))
+    assert.equal(table.getChild('vector')?.type.children[0].type.toString(), new Float16().toString())
+  })
+
+  it('will fail if missing the embedding source column', async function () {
+    return await expect(convertToTable([{ id: 1 }], new DummyEmbedding())).to.be.rejectedWith("'string' was not present")
+  })
+
+  it('use embeddingDimension if embedding missing from table', async function () {
+    const schema = new Schema([
+      new Field('string', new Utf8(), false)
+    ])
+    // Simulate getting an empty Arrow table (minus embedding) from some other source
+    // In other words, we aren't starting with records
+    const table = makeEmptyTable(schema)
+
+    // If the embedding specifies the dimension we are fine
+    await fromTableToBuffer(table, new DummyEmbedding())
+
+    // We can also supply a schema and should be ok
+    const schemaWithEmbedding = new Schema([
+      new Field('string', new Utf8(), false),
+      new Field('vector', new FixedSizeList(2, new Field('item', new Float16(), false)), false)
+    ])
+    await fromTableToBuffer(table, new DummyEmbeddingWithNoDimension(), schemaWithEmbedding)
+
+    // Otherwise we will get an error
+    return await expect(fromTableToBuffer(table, new DummyEmbeddingWithNoDimension())).to.be.rejectedWith('does not specify `embeddingDimension`')
+  })
+
+  it('will apply embeddings to an empty table', async function () {
+    const schema = new Schema([
+      new Field('string', new Utf8(), false),
+      new Field('vector', new FixedSizeList(2, new Field('item', new Float16(), false)), false)
+    ])
+    const table = await convertToTable([], new DummyEmbedding(), { schema })
+    assert.isTrue(DataType.isFixedSizeList(table.getChild('vector')?.type))
+    assert.equal(table.getChild('vector')?.type.children[0].type.toString(), new Float16().toString())
+  })
+
+  it('will complain if embeddings present but schema missing embedding column', async function () {
+    const schema = new Schema([
+      new Field('string', new Utf8(), false)
+    ])
+    return await expect(convertToTable([], new DummyEmbedding(), { schema })).to.be.rejectedWith('column vector was missing')
+  })
+
+  it('will provide a nice error if run twice', async function () {
+    const records = sampleRecords()
+    const table = await convertToTable(records, new DummyEmbedding())
+    // fromTableToBuffer will try and apply the embeddings again
+    return await expect(fromTableToBuffer(table, new DummyEmbedding())).to.be.rejectedWith('already existed')
+  })
+})
+
+describe('makeEmptyTable', function () {
+  it('will make an empty table', async function () {
+    await checkTableCreation(async (_, __, schema) => makeEmptyTable(schema))
+  })
+})
+
+describe('when using two versions of arrow', function () {
+  it('can still import data', async function() {
+    const schema = new OldSchema([
+      new OldField('id', new OldInt32()),
+      new OldField('vector', new OldFixedSizeList(1024, new OldField("item", new OldFloat32(), true))),
+      new OldField('struct', new OldStruct([
+        new OldField('nested', new OldDictionary(new OldUtf8(), new OldInt32(), 1, true)),
+        new OldField('ts_with_tz', new OldTimestampNanosecond("some_tz")),
+        new OldField('ts_no_tz', new OldTimestampNanosecond(null))
+      ]))
+    ]) as any
+    // We use arrow version 13 to emulate a "foreign arrow" and this version doesn't have metadataVersion
+    // In theory, this wouldn't matter.  We don't rely on that property.  However, it causes deepEqual to
+    // fail so we patch it back in
+    schema.metadataVersion = MetadataVersion.V5
+    const table = makeArrowTable(
+      [],
+      { schema }
+    )
+
+    const buf = await fromTableToBuffer(table)
+    assert.isAbove(buf.byteLength, 0)
+    const actual = tableFromIPC(buf)
+    const actualSchema = actual.schema
+    assert.deepEqual(actualSchema, schema)
+  })
 })

node/src/test/test.ts

@@ -37,8 +37,10 @@ import {
   Utf8,
   Table as ArrowTable,
   vectorFromArray,
+  Float64,
   Float32,
-  Float16
+  Float16,
+  Int64
 } from 'apache-arrow'
 
 const expect = chai.expect
@@ -126,6 +128,11 @@ describe('LanceDB client', function () {
       assertResults(results)
       results = await table.where('id % 2 = 0').execute()
       assertResults(results)
+
+      // Should reject a bad filter
+      await expect(table.filter('id % 2 = 0 AND').execute()).to.be.rejectedWith(
+        /.*sql parser error: Expected an expression:, found: EOF.*/
+      )
     })
 
     it('uses a filter / where clause', async function () {
@@ -196,7 +203,7 @@ describe('LanceDB client', function () {
       const table = await con.openTable('vectors')
       const results = await table
         .search([0.1, 0.1])
-        .select(['is_active'])
+        .select(['is_active', 'vector'])
         .execute()
       assert.equal(results.length, 2)
       // vector and _distance are always returned
@@ -281,7 +288,8 @@ describe('LanceDB client', function () {
 
     it('create a table from an Arrow Table', async function () {
       const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
+      // Also test the connect function with an object
+      const con = await lancedb.connect({ uri: dir })
 
       const i32s = new Int32Array(new Array<number>(10))
       const i32 = makeVector(i32s)
@@ -294,6 +302,7 @@ describe('LanceDB client', function () {
       })
       assert.equal(table.name, 'vectors')
       assert.equal(await table.countRows(), 10)
+      assert.equal(await table.countRows('vector IS NULL'), 0)
       assert.deepEqual(await con.tableNames(), ['vectors'])
     })
 
@@ -369,6 +378,7 @@ describe('LanceDB client', function () {
       const table = await con.createTable('f16', data)
       assert.equal(table.name, 'f16')
       assert.equal(await table.countRows(), total)
+      assert.equal(await table.countRows('id < 5'), 5)
       assert.deepEqual(await con.tableNames(), ['f16'])
       assert.deepEqual(await table.schema, schema)
 
@@ -391,24 +401,6 @@ describe('LanceDB client', function () {
       })
     }).timeout(120000)
 
-    it('fails to create a new table when the vector column is missing', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-
-      const data = [
-        {
-          id: 1,
-          price: 10
-        }
-      ]
-
-      const create = con.createTable('missing_vector', data)
-      await expect(create).to.be.rejectedWith(
-        Error,
-        "column 'vector' is missing"
-      )
-    })
-
     it('use overwrite flag to overwrite existing table', async function () {
       const dir = await track().mkdir('lancejs')
       const con = await lancedb.connect(dir)
@@ -549,6 +541,54 @@ describe('LanceDB client', function () {
       assert.equal(await table.countRows(), 2)
     })
 
+    it('can merge insert records into the table', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const data = [{ id: 1, age: 1 }, { id: 2, age: 1 }]
+      const table = await con.createTable('my_table', data)
+
+      // insert if not exists
+      let newData = [{ id: 2, age: 2 }, { id: 3, age: 2 }]
+      await table.mergeInsert('id', newData, {
+        whenNotMatchedInsertAll: true
+      })
+      assert.equal(await table.countRows(), 3)
+      assert.equal(await table.countRows('age = 2'), 1)
+
+      // conditional update
+      newData = [{ id: 2, age: 3 }, { id: 3, age: 3 }]
+      await table.mergeInsert('id', newData, {
+        whenMatchedUpdateAll: 'target.age = 1'
+      })
+      assert.equal(await table.countRows(), 3)
+      assert.equal(await table.countRows('age = 1'), 1)
+      assert.equal(await table.countRows('age = 3'), 1)
+
+      newData = [{ id: 3, age: 4 }, { id: 4, age: 4 }]
+      await table.mergeInsert('id', newData, {
+        whenNotMatchedInsertAll: true,
+        whenMatchedUpdateAll: true
+      })
+      assert.equal(await table.countRows(), 4)
+      assert.equal((await table.filter('age = 4').execute()).length, 2)
+
+      newData = [{ id: 5, age: 5 }]
+      await table.mergeInsert('id', newData, {
+        whenNotMatchedInsertAll: true,
+        whenMatchedUpdateAll: true,
+        whenNotMatchedBySourceDelete: 'age < 4'
+      })
+      assert.equal(await table.countRows(), 3)
+
+      await table.mergeInsert('id', newData, {
+        whenNotMatchedInsertAll: true,
+        whenMatchedUpdateAll: true,
+        whenNotMatchedBySourceDelete: true
+      })
+      assert.equal(await table.countRows(), 1)
+    })
+
     it('can update records in the table', async function () {
       const uri = await createTestDB()
       const con = await lancedb.connect(uri)
@@ -711,11 +751,11 @@ describe('LanceDB client', function () {
         num_sub_vectors: 2
       })
       await expect(createIndex).to.be.rejectedWith(
-        /VectorIndex requires the column data type to be fixed size list of float32s/
+        "index cannot be created on the column `name` which has data type Utf8"
       )
     })
 
-    it('it should fail when the column is not a vector', async function () {
+    it('it should fail when num_partitions is invalid', async function () {
       const uri = await createTestDB(32, 300)
       const con = await lancedb.connect(uri)
       const table = await con.openTable('vectors')
@@ -1025,3 +1065,63 @@ describe('Compact and cleanup', function () {
     assert.equal(await table.countRows(), 3)
   })
 })
+
+describe('schema evolution', function () {
+  // Create a new sample table
+  it('can add a new column to the schema', async function () {
+    const dir = await track().mkdir('lancejs')
+    const con = await lancedb.connect(dir)
+    const table = await con.createTable('vectors', [
+      { id: 1n, vector: [0.1, 0.2] }
+    ])
+
+    await table.addColumns([{ name: 'price', valueSql: 'cast(10.0 as float)' }])
+
+    const expectedSchema = new Schema([
+      new Field('id', new Int64()),
+      new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true))),
+      new Field('price', new Float32())
+    ])
+    expect(await table.schema).to.deep.equal(expectedSchema)
+  })
+
+  it('can alter the columns in the schema', async function () {
+    const dir = await track().mkdir('lancejs')
+    const con = await lancedb.connect(dir)
+    const schema = new Schema([
+      new Field('id', new Int64(), false),
+      new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true))),
+      new Field('price', new Float64(), false)
+    ])
+    const table = await con.createTable('vectors', [
+      { id: 1n, vector: [0.1, 0.2], price: 10.0 }
+    ])
+    expect(await table.schema).to.deep.equal(schema)
+
+    await table.alterColumns([
+      { path: 'id', rename: 'new_id' },
+      { path: 'price', nullable: true }
+    ])
+
+    const expectedSchema = new Schema([
+      new Field('new_id', new Int64(), false),
+      new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true))),
+      new Field('price', new Float64(), true)
+    ])
+    expect(await table.schema).to.deep.equal(expectedSchema)
+  })
+
+  it('can drop a column from the schema', async function () {
+    const dir = await track().mkdir('lancejs')
+    const con = await lancedb.connect(dir)
+    const table = await con.createTable('vectors', [
+      { id: 1n, vector: [0.1, 0.2] }
+    ])
+    await table.dropColumns(['vector'])
+
+    const expectedSchema = new Schema([
+      new Field('id', new Int64(), false)
+    ])
+    expect(await table.schema).to.deep.equal(expectedSchema)
+  })
+})

node/tsconfig.json

@@ -9,6 +9,6 @@
     "declaration": true,
     "outDir": "./dist",
     "strict": true,
-    // "esModuleInterop": true,
+    "sourceMap": true,
   }
 }

nodejs/.eslintignore

@@ -0,0 +1,3 @@
+**/dist/**/*
+**/native.js
+**/native.d.ts

nodejs/.eslintrc.js

@@ -1,22 +0,0 @@
-module.exports = {
-  env: {
-    browser: true,
-    es2021: true,
-  },
-  extends: [
-    "eslint:recommended",
-    "plugin:@typescript-eslint/recommended-type-checked",
-    "plugin:@typescript-eslint/stylistic-type-checked",
-  ],
-  overrides: [],
-  parserOptions: {
-    project: "./tsconfig.json",
-    ecmaVersion: "latest",
-    sourceType: "module",
-  },
-  rules: {
-    "@typescript-eslint/method-signature-style": "off",
-    "@typescript-eslint/no-explicit-any": "off",
-  },
-  ignorePatterns: ["node_modules/", "dist/", "build/", "vectordb/native.*"],
-};

nodejs/.prettierignore

@@ -0,0 +1 @@
+.eslintignore

nodejs/Cargo.toml

@@ -1,27 +1,28 @@
 [package]
-name = "vectordb-nodejs"
-edition = "2021"
+name = "lancedb-nodejs"
+edition.workspace = true
 version = "0.0.0"
 license.workspace = true
+description.workspace = true
 repository.workspace = true
+keywords.workspace = true
+categories.workspace = true
 
 [lib]
 crate-type = ["cdylib"]
 
 [dependencies]
 arrow-ipc.workspace = true
-napi = { version = "2.14", default-features = false, features = [
+futures.workspace = true
+lancedb = { path = "../rust/lancedb" }
+napi = { version = "2.15", default-features = false, features = [
     "napi7",
-    "async"
+    "async",
 ] }
-napi-derive = "2.14"
-vectordb = { path = "../rust/vectordb" }
-lance.workspace = true
-lance-linalg.workspace = true
+napi-derive = "2"
+
+# Prevent dynamic linking of lzma, which comes from datafusion
+lzma-sys = { version = "*", features = ["static"] }
 
 [build-dependencies]
 napi-build = "2.1"
-
-[profile.release]
-lto = true
-strip = "symbols"

nodejs/README.md

@@ -2,7 +2,6 @@
 
 It will replace the NodeJS SDK when it is ready.
 
-
 ## Development
 
 ```sh
@@ -10,9 +9,35 @@ npm run build
 npm t
 ```
 
-Generating docs
+### Running lint / format
 
+LanceDb uses eslint for linting. VSCode does not need any plugins to use eslint. However, it
+may need some additional configuration. Make sure that eslint.experimental.useFlatConfig is
+set to true. Also, if your vscode root folder is the repo root then you will need to set
+the eslint.workingDirectories to ["nodejs"]. To manually lint your code you can run:
+
+```sh
+npm run lint
 ```
+
+LanceDb uses prettier for formatting. If you are using VSCode you will need to install the
+"Prettier - Code formatter" extension. You should then configure it to be the default formatter
+for typescript and you should enable format on save. To manually check your code's format you
+can run:
+
+```sh
+npm run chkformat
+```
+
+If you need to manually format your code you can run:
+
+```sh
+npx prettier --write .
+```
+
+### Generating docs
+
+```sh
 npm run docs
 
 cd ../docs

nodejs/__test__/arrow.test.ts

@@ -12,7 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { makeArrowTable, toBuffer } from "../vectordb/arrow";
+import {
+  convertToTable,
+  fromTableToBuffer,
+  makeArrowTable,
+  makeEmptyTable,
+} from "../dist/arrow";
 import {
   Field,
   FixedSizeList,
@@ -22,85 +27,444 @@ import {
   tableFromIPC,
   Schema,
   Float64,
+  type Table,
+  Binary,
+  Bool,
+  Utf8,
+  Struct,
+  List,
+  DataType,
+  Dictionary,
+  Int64,
+  Float,
+  Precision,
+  MetadataVersion,
 } from "apache-arrow";
+import {
+  Dictionary as OldDictionary,
+  Field as OldField,
+  FixedSizeList as OldFixedSizeList,
+  Float32 as OldFloat32,
+  Int32 as OldInt32,
+  Struct as OldStruct,
+  Schema as OldSchema,
+  TimestampNanosecond as OldTimestampNanosecond,
+  Utf8 as OldUtf8,
+} from "apache-arrow-old";
+import { type EmbeddingFunction } from "../dist/embedding/embedding_function";
 
-test("customized schema", function () {
-  const schema = new Schema([
-    new Field("a", new Int32(), true),
-    new Field("b", new Float32(), true),
-    new Field(
-      "c",
-      new FixedSizeList(3, new Field("item", new Float16())),
-      true
-    ),
-  ]);
-  const table = makeArrowTable(
-    [
-      { a: 1, b: 2, c: [1, 2, 3] },
-      { a: 4, b: 5, c: [4, 5, 6] },
-      { a: 7, b: 8, c: [7, 8, 9] },
-    ],
-    { schema }
-  );
-
-  expect(table.schema.toString()).toEqual(schema.toString());
-
-  const buf = toBuffer(table);
-  expect(buf.byteLength).toBeGreaterThan(0);
-
-  const actual = tableFromIPC(buf);
-  expect(actual.numRows).toBe(3);
-  const actualSchema = actual.schema;
-  expect(actualSchema.toString()).toStrictEqual(schema.toString());
-});
-
-test("default vector column", function () {
-  const schema = new Schema([
-    new Field("a", new Float64(), true),
-    new Field("b", new Float64(), true),
-    new Field("vector", new FixedSizeList(3, new Field("item", new Float32()))),
-  ]);
-  const table = makeArrowTable([
-    { a: 1, b: 2, vector: [1, 2, 3] },
-    { a: 4, b: 5, vector: [4, 5, 6] },
-    { a: 7, b: 8, vector: [7, 8, 9] },
-  ]);
-
-  const buf = toBuffer(table);
-  expect(buf.byteLength).toBeGreaterThan(0);
-
-  const actual = tableFromIPC(buf);
-  expect(actual.numRows).toBe(3);
-  const actualSchema = actual.schema;
-  expect(actualSchema.toString()).toEqual(actualSchema.toString());
-});
-
-test("2 vector columns", function () {
-  const schema = new Schema([
-    new Field("a", new Float64()),
-    new Field("b", new Float64()),
-    new Field("vec1", new FixedSizeList(3, new Field("item", new Float16()))),
-    new Field("vec2", new FixedSizeList(3, new Field("item", new Float16()))),
-  ]);
-  const table = makeArrowTable(
-    [
-      { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
-      { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
-      { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] },
-    ],
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function sampleRecords(): Array<Record<string, any>> {
+  return [
     {
-      vectorColumns: {
-        vec1: { type: new Float16() },
-        vec2: { type: new Float16() },
-      },
+      binary: Buffer.alloc(5),
+      boolean: false,
+      number: 7,
+      string: "hello",
+      struct: { x: 0, y: 0 },
+      list: ["anime", "action", "comedy"],
+    },
+  ];
+}
+
+// Helper method to verify various ways to create a table
+async function checkTableCreation(
+  tableCreationMethod: (
+    records: Record<string, unknown>[],
+    recordsReversed: Record<string, unknown>[],
+    schema: Schema,
+  ) => Promise<Table>,
+  infersTypes: boolean,
+): Promise<void> {
+  const records = sampleRecords();
+  const recordsReversed = [
+    {
+      list: ["anime", "action", "comedy"],
+      struct: { x: 0, y: 0 },
+      string: "hello",
+      number: 7,
+      boolean: false,
+      binary: Buffer.alloc(5),
+    },
+  ];
+  const schema = new Schema([
+    new Field("binary", new Binary(), false),
+    new Field("boolean", new Bool(), false),
+    new Field("number", new Float64(), false),
+    new Field("string", new Utf8(), false),
+    new Field(
+      "struct",
+      new Struct([
+        new Field("x", new Float64(), false),
+        new Field("y", new Float64(), false),
+      ]),
+    ),
+    new Field("list", new List(new Field("item", new Utf8(), false)), false),
+  ]);
+
+  const table = await tableCreationMethod(records, recordsReversed, schema);
+  schema.fields.forEach((field, idx) => {
+    const actualField = table.schema.fields[idx];
+    // Type inference always assumes nullable=true
+    if (infersTypes) {
+      expect(actualField.nullable).toBe(true);
+    } else {
+      expect(actualField.nullable).toBe(false);
     }
-  );
+    expect(table.getChild(field.name)?.type.toString()).toEqual(
+      field.type.toString(),
+    );
+    expect(table.getChildAt(idx)?.type.toString()).toEqual(
+      field.type.toString(),
+    );
+  });
+}
 
-  const buf = toBuffer(table);
-  expect(buf.byteLength).toBeGreaterThan(0);
+describe("The function makeArrowTable", function () {
+  it("will use data types from a provided schema instead of inference", async function () {
+    const schema = new Schema([
+      new Field("a", new Int32()),
+      new Field("b", new Float32()),
+      new Field("c", new FixedSizeList(3, new Field("item", new Float16()))),
+      new Field("d", new Int64()),
+    ]);
+    const table = makeArrowTable(
+      [
+        { a: 1, b: 2, c: [1, 2, 3], d: 9 },
+        { a: 4, b: 5, c: [4, 5, 6], d: 10 },
+        { a: 7, b: 8, c: [7, 8, 9], d: null },
+      ],
+      { schema },
+    );
 
-  const actual = tableFromIPC(buf);
-  expect(actual.numRows).toBe(3);
-  const actualSchema = actual.schema;
-  expect(actualSchema.toString()).toEqual(schema.toString());
+    const buf = await fromTableToBuffer(table);
+    expect(buf.byteLength).toBeGreaterThan(0);
+
+    const actual = tableFromIPC(buf);
+    expect(actual.numRows).toBe(3);
+    const actualSchema = actual.schema;
+    expect(actualSchema).toEqual(schema);
+  });
+
+  it("will assume the column `vector` is FixedSizeList<Float32> by default", async function () {
+    const schema = new Schema([
+      new Field("a", new Float(Precision.DOUBLE), true),
+      new Field("b", new Float(Precision.DOUBLE), true),
+      new Field(
+        "vector",
+        new FixedSizeList(
+          3,
+          new Field("item", new Float(Precision.SINGLE), true),
+        ),
+        true,
+      ),
+    ]);
+    const table = makeArrowTable([
+      { a: 1, b: 2, vector: [1, 2, 3] },
+      { a: 4, b: 5, vector: [4, 5, 6] },
+      { a: 7, b: 8, vector: [7, 8, 9] },
+    ]);
+
+    const buf = await fromTableToBuffer(table);
+    expect(buf.byteLength).toBeGreaterThan(0);
+
+    const actual = tableFromIPC(buf);
+    expect(actual.numRows).toBe(3);
+    const actualSchema = actual.schema;
+    expect(actualSchema).toEqual(schema);
+  });
+
+  it("can support multiple vector columns", async function () {
+    const schema = new Schema([
+      new Field("a", new Float(Precision.DOUBLE), true),
+      new Field("b", new Float(Precision.DOUBLE), true),
+      new Field(
+        "vec1",
+        new FixedSizeList(3, new Field("item", new Float16(), true)),
+        true,
+      ),
+      new Field(
+        "vec2",
+        new FixedSizeList(3, new Field("item", new Float16(), true)),
+        true,
+      ),
+    ]);
+    const table = makeArrowTable(
+      [
+        { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
+        { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
+        { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] },
+      ],
+      {
+        vectorColumns: {
+          vec1: { type: new Float16() },
+          vec2: { type: new Float16() },
+        },
+      },
+    );
+
+    const buf = await fromTableToBuffer(table);
+    expect(buf.byteLength).toBeGreaterThan(0);
+
+    const actual = tableFromIPC(buf);
+    expect(actual.numRows).toBe(3);
+    const actualSchema = actual.schema;
+    expect(actualSchema).toEqual(schema);
+  });
+
+  it("will allow different vector column types", async function () {
+    const table = makeArrowTable([{ fp16: [1], fp32: [1], fp64: [1] }], {
+      vectorColumns: {
+        fp16: { type: new Float16() },
+        fp32: { type: new Float32() },
+        fp64: { type: new Float64() },
+      },
+    });
+
+    expect(table.getChild("fp16")?.type.children[0].type.toString()).toEqual(
+      new Float16().toString(),
+    );
+    expect(table.getChild("fp32")?.type.children[0].type.toString()).toEqual(
+      new Float32().toString(),
+    );
+    expect(table.getChild("fp64")?.type.children[0].type.toString()).toEqual(
+      new Float64().toString(),
+    );
+  });
+
+  it("will use dictionary encoded strings if asked", async function () {
+    const table = makeArrowTable([{ str: "hello" }]);
+    expect(DataType.isUtf8(table.getChild("str")?.type)).toBe(true);
+
+    const tableWithDict = makeArrowTable([{ str: "hello" }], {
+      dictionaryEncodeStrings: true,
+    });
+    expect(DataType.isDictionary(tableWithDict.getChild("str")?.type)).toBe(
+      true,
+    );
+
+    const schema = new Schema([
+      new Field("str", new Dictionary(new Utf8(), new Int32())),
+    ]);
+
+    const tableWithDict2 = makeArrowTable([{ str: "hello" }], { schema });
+    expect(DataType.isDictionary(tableWithDict2.getChild("str")?.type)).toBe(
+      true,
+    );
+  });
+
+  it("will infer data types correctly", async function () {
+    await checkTableCreation(async (records) => makeArrowTable(records), true);
+  });
+
+  it("will allow a schema to be provided", async function () {
+    await checkTableCreation(
+      async (records, _, schema) => makeArrowTable(records, { schema }),
+      false,
+    );
+  });
+
+  it("will use the field order of any provided schema", async function () {
+    await checkTableCreation(
+      async (_, recordsReversed, schema) =>
+        makeArrowTable(recordsReversed, { schema }),
+      false,
+    );
+  });
+
+  it("will make an empty table", async function () {
+    await checkTableCreation(
+      async (_, __, schema) => makeArrowTable([], { schema }),
+      false,
+    );
+  });
+});
+
+class DummyEmbedding implements EmbeddingFunction<string> {
+  public readonly sourceColumn = "string";
+  public readonly embeddingDimension = 2;
+  public readonly embeddingDataType = new Float16();
+
+  async embed(data: string[]): Promise<number[][]> {
+    return data.map(() => [0.0, 0.0]);
+  }
+}
+
+class DummyEmbeddingWithNoDimension implements EmbeddingFunction<string> {
+  public readonly sourceColumn = "string";
+
+  async embed(data: string[]): Promise<number[][]> {
+    return data.map(() => [0.0, 0.0]);
+  }
+}
+
+describe("convertToTable", function () {
+  it("will infer data types correctly", async function () {
+    await checkTableCreation(
+      async (records) => await convertToTable(records),
+      true,
+    );
+  });
+
+  it("will allow a schema to be provided", async function () {
+    await checkTableCreation(
+      async (records, _, schema) =>
+        await convertToTable(records, undefined, { schema }),
+      false,
+    );
+  });
+
+  it("will use the field order of any provided schema", async function () {
+    await checkTableCreation(
+      async (_, recordsReversed, schema) =>
+        await convertToTable(recordsReversed, undefined, { schema }),
+      false,
+    );
+  });
+
+  it("will make an empty table", async function () {
+    await checkTableCreation(
+      async (_, __, schema) => await convertToTable([], undefined, { schema }),
+      false,
+    );
+  });
+
+  it("will apply embeddings", async function () {
+    const records = sampleRecords();
+    const table = await convertToTable(records, new DummyEmbedding());
+    expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe(true);
+    expect(table.getChild("vector")?.type.children[0].type.toString()).toEqual(
+      new Float16().toString(),
+    );
+  });
+
+  it("will fail if missing the embedding source column", async function () {
+    await expect(
+      convertToTable([{ id: 1 }], new DummyEmbedding()),
+    ).rejects.toThrow("'string' was not present");
+  });
+
+  it("use embeddingDimension if embedding missing from table", async function () {
+    const schema = new Schema([new Field("string", new Utf8(), false)]);
+    // Simulate getting an empty Arrow table (minus embedding) from some other source
+    // In other words, we aren't starting with records
+    const table = makeEmptyTable(schema);
+
+    // If the embedding specifies the dimension we are fine
+    await fromTableToBuffer(table, new DummyEmbedding());
+
+    // We can also supply a schema and should be ok
+    const schemaWithEmbedding = new Schema([
+      new Field("string", new Utf8(), false),
+      new Field(
+        "vector",
+        new FixedSizeList(2, new Field("item", new Float16(), false)),
+        false,
+      ),
+    ]);
+    await fromTableToBuffer(
+      table,
+      new DummyEmbeddingWithNoDimension(),
+      schemaWithEmbedding,
+    );
+
+    // Otherwise we will get an error
+    await expect(
+      fromTableToBuffer(table, new DummyEmbeddingWithNoDimension()),
+    ).rejects.toThrow("does not specify `embeddingDimension`");
+  });
+
+  it("will apply embeddings to an empty table", async function () {
+    const schema = new Schema([
+      new Field("string", new Utf8(), false),
+      new Field(
+        "vector",
+        new FixedSizeList(2, new Field("item", new Float16(), false)),
+        false,
+      ),
+    ]);
+    const table = await convertToTable([], new DummyEmbedding(), { schema });
+    expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe(true);
+    expect(table.getChild("vector")?.type.children[0].type.toString()).toEqual(
+      new Float16().toString(),
+    );
+  });
+
+  it("will complain if embeddings present but schema missing embedding column", async function () {
+    const schema = new Schema([new Field("string", new Utf8(), false)]);
+    await expect(
+      convertToTable([], new DummyEmbedding(), { schema }),
+    ).rejects.toThrow("column vector was missing");
+  });
+
+  it("will provide a nice error if run twice", async function () {
+    const records = sampleRecords();
+    const table = await convertToTable(records, new DummyEmbedding());
+    // fromTableToBuffer will try and apply the embeddings again
+    await expect(
+      fromTableToBuffer(table, new DummyEmbedding()),
+    ).rejects.toThrow("already existed");
+  });
+});
+
+describe("makeEmptyTable", function () {
+  it("will make an empty table", async function () {
+    await checkTableCreation(
+      async (_, __, schema) => makeEmptyTable(schema),
+      false,
+    );
+  });
+});
+
+describe("when using two versions of arrow", function () {
+  it("can still import data", async function () {
+    const schema = new OldSchema([
+      new OldField("id", new OldInt32()),
+      new OldField(
+        "vector",
+        new OldFixedSizeList(
+          1024,
+          new OldField("item", new OldFloat32(), true),
+        ),
+      ),
+      new OldField(
+        "struct",
+        new OldStruct([
+          new OldField(
+            "nested",
+            new OldDictionary(new OldUtf8(), new OldInt32(), 1, true),
+          ),
+          new OldField("ts_with_tz", new OldTimestampNanosecond("some_tz")),
+          new OldField("ts_no_tz", new OldTimestampNanosecond(null)),
+        ]),
+      ),
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    ]) as any;
+    schema.metadataVersion = MetadataVersion.V5;
+    const table = makeArrowTable([], { schema });
+
+    const buf = await fromTableToBuffer(table);
+    expect(buf.byteLength).toBeGreaterThan(0);
+    const actual = tableFromIPC(buf);
+    const actualSchema = actual.schema;
+    expect(actualSchema.fields.length).toBe(3);
+
+    // Deep equality gets hung up on some very minor unimportant differences
+    // between arrow version 13 and 15 which isn't really what we're testing for
+    // and so we do our own comparison that just checks name/type/nullability
+    function compareFields(lhs: Field, rhs: Field) {
+      expect(lhs.name).toEqual(rhs.name);
+      expect(lhs.nullable).toEqual(rhs.nullable);
+      expect(lhs.typeId).toEqual(rhs.typeId);
+      if ("children" in lhs.type && lhs.type.children !== null) {
+        const lhsChildren = lhs.type.children as Field[];
+        lhsChildren.forEach((child: Field, idx) => {
+          compareFields(child, rhs.type.children[idx]);
+        });
+      }
+    }
+    actualSchema.fields.forEach((field, idx) => {
+      compareFields(field, actualSchema.fields[idx]);
+    });
+  });
 });