[python] Bump version: 0.6.4 → 0.6.5

changed the error msg for query with wrong vector dim thus need this
change to pass the nodejs tests.

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.10
+current_version = 0.4.13
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True
@@ -9,4 +9,4 @@ tag_name = v{new_version}
 
 [bumpversion:file:rust/ffi/node/Cargo.toml]
 
-[bumpversion:file:rust/vectordb/Cargo.toml]
+[bumpversion:file:rust/lancedb/Cargo.toml]

.github/workflows/build_linux_wheel/action.yml

@@ -0,0 +1,58 @@
+# We create a composite action to be re-used both for testing and for releasing
+name: build-linux-wheel
+description: "Build a manylinux wheel for lance"
+inputs:
+  python-minor-version:
+    description: "8, 9, 10, 11, 12"
+    required: true
+  args:
+    description: "--release"
+    required: false
+    default: ""
+  arm-build:
+    description: "Build for arm64 instead of x86_64"
+    # Note: this does *not* mean the host is arm64, since we might be cross-compiling.
+    required: false
+    default: "false"
+runs:
+  using: "composite"
+  steps:
+    - name: CONFIRM ARM BUILD
+      shell: bash
+      run: |
+        echo "ARM BUILD: ${{ inputs.arm-build }}"
+    - name: Build x86_64 Manylinux wheel
+      if: ${{ inputs.arm-build == 'false' }}
+      uses: PyO3/maturin-action@v1
+      with:
+        command: build
+        working-directory: python
+        target: x86_64-unknown-linux-gnu
+        manylinux: "2_17"
+        args: ${{ inputs.args }}
+        before-script-linux: |
+          set -e
+          yum install -y openssl-devel \
+            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
+            && unzip /tmp/protoc.zip -d /usr/local \
+            && rm /tmp/protoc.zip
+    - name: Build Arm Manylinux Wheel
+      if: ${{ inputs.arm-build == 'true' }}
+      uses: PyO3/maturin-action@v1
+      with:
+        command: build
+        working-directory: python
+        target: aarch64-unknown-linux-gnu
+        manylinux: "2_24"
+        args: ${{ inputs.args }}
+        before-script-linux: |
+          set -e
+          apt install -y unzip
+          if [ $(uname -m) = "x86_64" ]; then
+            PROTOC_ARCH="x86_64"
+          else
+            PROTOC_ARCH="aarch_64"
+          fi
+          curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$PROTOC_ARCH.zip > /tmp/protoc.zip \
+            && unzip /tmp/protoc.zip -d /usr/local \
+            && rm /tmp/protoc.zip

.github/workflows/build_mac_wheel/action.yml

@@ -0,0 +1,25 @@
+# We create a composite action to be re-used both for testing and for releasing
+name: build_wheel
+description: "Build a lance wheel"
+inputs:
+  python-minor-version:
+    description: "8, 9, 10, 11"
+    required: true
+  args:
+    description: "--release"
+    required: false
+    default: ""
+runs:
+  using: "composite"
+  steps:
+    - name: Install macos dependency
+      shell: bash
+      run: |
+        brew install protobuf
+    - name: Build wheel
+      uses: PyO3/maturin-action@v1
+      with:
+        command: build
+        args: ${{ inputs.args }}
+        working-directory: python
+        interpreter: 3.${{ inputs.python-minor-version }}

.github/workflows/build_windows_wheel/action.yml

@@ -0,0 +1,33 @@
+# We create a composite action to be re-used both for testing and for releasing
+name: build_wheel
+description: "Build a lance wheel"
+inputs:
+  python-minor-version:
+    description: "8, 9, 10, 11"
+    required: true
+  args:
+    description: "--release"
+    required: false
+    default: ""
+runs:
+  using: "composite"
+  steps:
+    - name: Install Protoc v21.12
+      working-directory: C:\
+      run: |
+        New-Item -Path 'C:\protoc' -ItemType Directory
+        Set-Location C:\protoc
+        Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+        7z x protoc.zip
+        Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+      shell: powershell
+    - name: Build wheel
+      uses: PyO3/maturin-action@v1
+      with:
+        command: build
+        args: ${{ inputs.args }}
+        working-directory: python
+    - uses: actions/upload-artifact@v3
+      with:
+        name: windows-wheels
+        path: python\target\wheels

.github/workflows/cargo-publish.yml

@@ -26,4 +26,4 @@ jobs:
           sudo apt install -y protobuf-compiler libssl-dev
       - name: Publish the package
         run: |
-          cargo publish -p vectordb --all-features --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          cargo publish -p lancedb --all-features --token ${{ secrets.CARGO_REGISTRY_TOKEN }}

.github/workflows/docs.yml

@@ -24,10 +24,14 @@ jobs:
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-22.04
+    runs-on: buildjet-8vcpu-ubuntu-2204
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Install dependecies needed for ubuntu
+        run: |
+          sudo apt install -y protobuf-compiler libssl-dev
+          rustup update && rustup default        
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

.github/workflows/docs_test.yml

@@ -24,16 +24,22 @@ env:
 jobs:
   test-python:
     name: Test doc python code
-    runs-on: "ubuntu-latest"
+    runs-on: "buildjet-8vcpu-ubuntu-2204"
     steps:
     - name: Checkout
       uses: actions/checkout@v4
+    - name: Install dependecies needed for ubuntu
+      run: |
+        sudo apt install -y protobuf-compiler libssl-dev
+        rustup update && rustup default
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
         python-version: 3.11
         cache: "pip"
         cache-dependency-path: "docs/test/requirements.txt"
+    - name: Rust cache
+      uses: swatinem/rust-cache@v2        
     - name: Build Python
       working-directory: docs/test
       run:
@@ -48,8 +54,8 @@ jobs:
         for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
   test-node:
     name: Test doc nodejs code
-    runs-on: "ubuntu-latest"
-    timeout-minutes: 45
+    runs-on: "buildjet-8vcpu-ubuntu-2204"
+    timeout-minutes: 60
     strategy:
       fail-fast: false
     steps:
@@ -65,6 +71,7 @@ jobs:
     - name: Install dependecies needed for ubuntu
       run: |
         sudo apt install -y protobuf-compiler libssl-dev
+        rustup update && rustup default
     - name: Rust cache
       uses: swatinem/rust-cache@v2
     - name: Install node dependencies

.github/workflows/node.yml

@@ -24,27 +24,6 @@ env:
   RUST_BACKTRACE: "1"
 
 jobs:
-  lint:
-    name: Lint
-    runs-on: ubuntu-22.04
-    defaults:
-      run:
-        shell: bash
-        working-directory: node
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 20
-        cache: 'npm'
-        cache-dependency-path: node/package-lock.json
-    - name: Lint
-      run: |
-        npm ci
-        npm run lint
   linux:
     name: Linux (Node ${{ matrix.node-version }})
     timeout-minutes: 30

.github/workflows/nodejs.yml

@@ -49,6 +49,7 @@ jobs:
         cargo clippy --all --all-features -- -D warnings
         npm ci
         npm run lint
+        npm run chkformat
   linux:
     name: Linux (NodeJS ${{ matrix.node-version }})
     timeout-minutes: 30
@@ -111,4 +112,3 @@ jobs:
     - name: Test
       run: |
         npm run test
-

.github/workflows/pypi-publish.yml

@@ -2,30 +2,91 @@ name: PyPI Publish
 
 on:
   release:
-    types: [ published ]
+    types: [published]
 
 jobs:
-  publish:
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the python-make-release action
-    if: startsWith(github.ref, 'refs/tags/python-v')
-    defaults:
-      run:
-        shell: bash
-        working-directory: python
+  linux:
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        python-minor-version: ["8"]
+        platform:
+          - x86_64
+          - aarch64
+    runs-on: "ubuntu-22.04"
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
-      - name: Build distribution
-        run: |
-          ls -la
-          pip install wheel setuptools --upgrade
-          python setup.py sdist bdist_wheel
-      - name: Publish
-        uses: pypa/gh-action-pypi-publish@v1.8.5
+          python-version: 3.${{ matrix.python-minor-version }}
+      - uses: ./.github/workflows/build_linux_wheel
         with:
-          password: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
-          packages-dir: python/dist
+          python-minor-version: ${{ matrix.python-minor-version }}
+          args: "--release --strip"
+          arm-build: ${{ matrix.platform == 'aarch64' }}
+      - uses: ./.github/workflows/upload_wheel
+        with:
+          token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
+          repo: "pypi"
+  mac:
+    timeout-minutes: 60
+    runs-on: ${{ matrix.config.runner }}
+    strategy:
+      matrix:
+        python-minor-version: ["8"]
+        config:
+          - target: x86_64-apple-darwin
+            runner: macos-13
+          - target: aarch64-apple-darwin
+            runner: macos-14
+    env:
+      MACOSX_DEPLOYMENT_TARGET: 10.15
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.12
+      - uses: ./.github/workflows/build_mac_wheel
+        with:
+          python-minor-version: ${{ matrix.python-minor-version }}
+          args: "--release --strip --target ${{ matrix.config.target }}"
+      - uses: ./.github/workflows/upload_wheel
+        with:
+          python-minor-version: ${{ matrix.python-minor-version }}
+          token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
+          repo: "pypi"
+  windows:
+    timeout-minutes: 60
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        python-minor-version: ["8"]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.${{ matrix.python-minor-version }}
+      - uses: ./.github/workflows/build_windows_wheel
+        with:
+          python-minor-version: ${{ matrix.python-minor-version }}
+          args: "--release --strip"
+          vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }}
+      - uses: ./.github/workflows/upload_wheel
+        with:
+          python-minor-version: ${{ matrix.python-minor-version }}
+          token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
+          repo: "pypi"

.github/workflows/python.yml

@@ -14,11 +14,9 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  linux:
+  lint:
+    name: "Lint"
     timeout-minutes: 30
-    strategy:
-      matrix:
-        python-minor-version: [ "8", "11" ]
     runs-on: "ubuntu-22.04"
     defaults:
       run:
@@ -32,31 +30,117 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-        python-version: 3.${{ matrix.python-minor-version }}
-    - name: Install lancedb
+          python-version: "3.11"
+      - name: Install ruff
         run: |
-        pip install -e .[tests]
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock ruff
+          pip install ruff==0.2.2
       - name: Format check
         run: ruff format --check .
       - name: Lint
         run: ruff .
-    - name: Run tests
-      run: pytest -m "not slow" -x -v --durations=30 tests
-    - name: doctest
-      run: pytest --doctest-modules lancedb
+  doctest:
+    name: "Doctest"
+    timeout-minutes: 30
+    runs-on: "ubuntu-22.04"
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: "pip"
+      - name: Install protobuf
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+      - name: Install
+        run: |
+          pip install -e .[tests,dev,embeddings]
+          pip install tantivy
+          pip install mlx
+      - name: Doctest
+        run: pytest --doctest-modules python/lancedb
+  linux:
+    name: "Linux: python-3.${{ matrix.python-minor-version }}"
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        python-minor-version: ["8", "11"]
+    runs-on: "ubuntu-22.04"
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Install protobuf
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.${{ matrix.python-minor-version }}
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+      - uses: ./.github/workflows/build_linux_wheel
+      - uses: ./.github/workflows/run_tests
+      # Make sure wheels are not included in the Rust cache
+      - name: Delete wheels
+        run: rm -rf target/wheels
   platform:
-    name: "Platform: ${{ matrix.config.name }}"
+    name: "Mac: ${{ matrix.config.name }}"
     timeout-minutes: 30
     strategy:
       matrix:
         config:
-          - name: x86 Mac
+          - name: x86
             runner: macos-13
-          - name: Arm Mac
+          - name: Arm
             runner: macos-14
-          - name: x86 Windows
+    runs-on: "${{ matrix.config.runner }}"
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+      - uses: ./.github/workflows/build_mac_wheel
+      - uses: ./.github/workflows/run_tests
+      # Make sure wheels are not included in the Rust cache
+      - name: Delete wheels
+        run: rm -rf target/wheels
+  windows:
+    name: "Windows: ${{ matrix.config.name }}"
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        config:
+          - name: x86
             runner: windows-latest
     runs-on: "${{ matrix.config.runner }}"
     defaults:
@@ -72,13 +156,14 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.11"
-    - name: Install lancedb
-      run: |
-        pip install -e .[tests]
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock
-    - name: Run tests
-      run: pytest -m "not slow" -x -v --durations=30 tests
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+      - uses: ./.github/workflows/build_windows_wheel
+      - uses: ./.github/workflows/run_tests
+      # Make sure wheels are not included in the Rust cache
+      - name: Delete wheels
+        run: rm -rf target/wheels
   pydantic1x:
     timeout-minutes: 30
     runs-on: "ubuntu-22.04"
@@ -91,6 +176,10 @@ jobs:
         with:
           fetch-depth: 0
           lfs: true
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -99,9 +188,6 @@ jobs:
         run: |
           pip install "pydantic<2"
           pip install -e .[tests]
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock
+          pip install tantivy
       - name: Run tests
-      run: pytest -m "not slow" -x -v --durations=30 tests
-    - name: doctest
-      run: pytest --doctest-modules lancedb
+        run: pytest -m "not slow" -x -v --durations=30 python/tests

.github/workflows/run_tests/action.yml

@@ -0,0 +1,17 @@
+name: run-tests
+
+description: "Install lance wheel and run unit tests"
+inputs:
+  python-minor-version:
+    required: true
+    description: "8 9 10 11 12"
+runs:
+  using: "composite"
+  steps:
+    - name: Install lancedb
+      shell: bash
+      run: |
+        pip3 install $(ls target/wheels/lancedb-*.whl)[tests,dev]
+    - name: pytest
+      shell: bash
+      run: pytest -m "not slow" -x -v --durations=30 python/python/tests

.github/workflows/rust.yml

@@ -119,3 +119,4 @@ jobs:
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
           cargo build
           cargo test
+  

.github/workflows/upload_wheel/action.yml

@@ -0,0 +1,29 @@
+name: upload-wheel
+
+description: "Upload wheels to Pypi"
+inputs:
+  os:
+    required: true
+    description: "ubuntu-22.04 or macos-13"
+  repo:
+    required: false
+    description: "pypi or testpypi"
+    default: "pypi"
+  token:
+    required: true
+    description: "release token for the repo"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install dependencies
+      shell: bash
+      run: |
+        python -m pip install --upgrade pip
+        pip install twine
+    - name: Publish wheel
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ inputs.token }}
+      shell: bash
+      run: twine upload --repository ${{ inputs.repo }} target/wheels/lancedb-*.whl

.gitignore

@@ -22,6 +22,11 @@ python/dist
 
 **/.hypothesis
 
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
 ## Javascript
 *.node
 **/node_modules
@@ -34,4 +39,6 @@ dist
 ## Rust
 target
 
+**/sccache.log
+
 Cargo.lock

.pre-commit-config.yaml

@@ -5,17 +5,14 @@ repos:
     -   id: check-yaml
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
--   repo: https://github.com/psf/black
-    rev: 22.12.0
-    hooks:
-    -   id: black
 -   repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.0.277
+    rev: v0.2.2
     hooks:
     - id: ruff
--   repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+- repo: https://github.com/pre-commit/mirrors-prettier
+  rev: v3.1.0
   hooks:
-    - id: isort
-      name: isort (python)
+    - id: prettier
+      files: "nodejs/.*"
+      exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["rust/ffi/node", "rust/vectordb", "nodejs"]
+members = ["rust/ffi/node", "rust/lancedb", "nodejs", "python"]
 # Python package needs to be built by maturin.
 exclude = ["python"]
 resolver = "2"
@@ -14,10 +14,10 @@ keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
 
 [workspace.dependencies]
-lance = { "version" = "=0.9.18", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.9.18" }
-lance-linalg = { "version" = "=0.9.18" }
-lance-testing = { "version" = "=0.9.18" }
+lance = { "version" = "=0.10.5", "features" = ["dynamodb"] }
+lance-index = { "version" = "=0.10.5" }
+lance-linalg = { "version" = "=0.10.5" }
+lance-testing = { "version" = "=0.10.5" }
 # Note that this one does not include pyarrow
 arrow = { version = "50.0", optional = false }
 arrow-array = "50.0"
@@ -28,13 +28,14 @@ arrow-schema = "50.0"
 arrow-arith = "50.0"
 arrow-cast = "50.0"
 async-trait = "0"
-chrono = "0.4.23"
+chrono = "0.4.35"
 half = { "version" = "=2.3.1", default-features = false, features = [
     "num-traits",
 ] }
 futures = "0"
 log = "0.4"
 object_store = "0.9.0"
+pin-project = "1.0.7"
 snafu = "0.7.4"
 url = "2"
 num-traits = "0.2"

dockerfiles/Dockerfile

@@ -0,0 +1,27 @@
+#Simple base dockerfile that supports basic dependencies required to run lance with FTS and Hybrid Search
+#Usage docker build -t lancedb:latest -f Dockerfile .
+FROM python:3.10-slim-buster
+
+# Install Rust
+RUN apt-get update && apt-get install -y curl build-essential && \
+  curl https://sh.rustup.rs -sSf | sh -s -- -y
+
+# Set the environment variable for Rust
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install protobuf compiler
+RUN apt-get install -y protobuf-compiler && \
+  apt-get clean && \
+  rm -rf /var/lib/apt/lists/*
+
+RUN apt-get -y update &&\
+  apt-get -y upgrade && \
+  apt-get -y install git
+
+
+# Verify installations
+RUN python --version && \
+  rustc --version && \
+  protoc --version
+
+RUN pip install tantivy lancedb

docs/mkdocs.yml

@@ -27,7 +27,6 @@ theme:
     - content.tabs.link
     - content.action.edit
     - toc.follow
-    # - toc.integrate
     - navigation.top
     - navigation.tabs
     - navigation.tabs.sticky
@@ -140,12 +139,14 @@ nav:
       - Serverless Website Chatbot: examples/serverless_website_chatbot.md
       - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
       - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
+    - 🦀 Rust:
+      - Overview: examples/examples_rust.md
   - 🔧 CLI & Config: cli_config.md
   - 💭 FAQs: faq.md
   - ⚙️ API reference:
     - 🐍 Python: python/python.md
     - 👾 JavaScript: javascript/modules.md
-    - 🦀 Rust: https://docs.rs/vectordb/latest/vectordb/
+    - 🦀 Rust: https://docs.rs/lancedb/latest/lancedb/
   - ☁️ LanceDB Cloud:
     - Overview: cloud/index.md
     - API reference:
@@ -189,21 +190,21 @@ nav:
   - Pydantic: python/pydantic.md
   - Voxel51: integrations/voxel51.md
   - PromptTools: integrations/prompttools.md
-- Python examples:
+- Examples:
   - examples/index.md
   - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
   - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
   - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
   - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
   - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
-- Javascript examples:
-  - Overview: examples/examples_js.md
-  - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
+  - YouTube Transcript Search (JS): examples/youtube_transcript_bot_with_nodejs.md
   - Serverless Chatbot from any website: examples/serverless_website_chatbot.md
   - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
 - API reference:
+  - Overview: api_reference.md
   - Python: python/python.md
   - Javascript: javascript/modules.md
+  - Rust: https://docs.rs/lancedb/latest/lancedb/index.html
 - LanceDB Cloud:
     - Overview: cloud/index.md
     - API reference:

docs/src/ann_indexes.md

@@ -7,20 +7,11 @@ for brute-force scanning of the entire vector space.
 A vector index is faster but less accurate than exhaustive search (kNN or flat search).
 LanceDB provides many parameters to fine-tune the index's size, the speed of queries, and the accuracy of results.
 
-Currently, LanceDB does _not_ automatically create the ANN index.
-LanceDB has optimized code for kNN as well. For many use-cases, datasets under 100K vectors won't require index creation at all.
-If you can live with <100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
+## Disk-based Index
 
-In the future we will look to automatically create and configure the ANN index as data comes in.
-
-## Types of Index
-
-Lance can support multiple index types, the most widely used one is `IVF_PQ`.
-
-- `IVF_PQ`: use **Inverted File Index (IVF)** to first divide the dataset into `N` partitions,
-  and then use **Product Quantization** to compress vectors in each partition.
-- `DiskANN` (**Experimental**): organize the vector as a on-disk graph, where the vertices approximately
-  represent the nearest neighbors of each vector.
+Lance provides an `IVF_PQ` disk-based index. It uses **Inverted File Index (IVF)** to first divide
+the dataset into `N` partitions, and then applies **Product Quantization** to compress vectors in each partition.
+See the [indexing](concepts/index_ivfpq.md) concepts guide for more information on how this works.
 
 ## Creating an IVF_PQ Index
 
@@ -55,12 +46,34 @@ Lance supports `IVF_PQ` index type by default.
     --8<-- "docs/src/ann_indexes.ts:ingest"
     ```
 
-- **metric** (default: "L2"): The distance metric to use. By default it uses euclidean distance "`L2`".
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/ivf_pq.rs:create_index"
+    ```
+
+    IVF_PQ index parameters are more fully defined in the [crate docs](https://docs.rs/lancedb/latest/lancedb/index/vector/struct.IvfPqIndexBuilder.html).
+
+The following IVF_PQ paramters can be specified:
+
+- **distance_type**: The distance metric to use. By default it uses euclidean distance "`L2`".
   We also support "cosine" and "dot" distance as well.
-- **num_partitions** (default: 256): The number of partitions of the index.
-- **num_sub_vectors** (default: 96): The number of sub-vectors (M) that will be created during Product Quantization (PQ).
-  For D dimensional vector, it will be divided into `M` of `D/M` sub-vectors, each of which is presented by
-  a single PQ code.
+- **num_partitions**: The number of partitions in the index. The default is the square root
+  of the number of rows.
+
+!!! note
+
+    In the synchronous python SDK and node's `vectordb` the default is 256. This default has
+    changed in the asynchronous python SDK and node's `lancedb`.
+
+- **num_sub_vectors**: The number of sub-vectors (M) that will be created during Product Quantization (PQ).
+  For D dimensional vector, it will be divided into `M` subvectors with dimension `D/M`, each of which is replaced by
+  a single PQ code. The default is the dimension of the vector divided by 16.
+
+!!! note
+
+    In the synchronous python SDK and node's `vectordb` the default is currently 96. This default has
+    changed in the asynchronous python SDK and node's `lancedb`.
 
 <figure markdown>
   ![IVF PQ](./assets/ivf_pq.png)
@@ -88,7 +101,7 @@ You can specify the GPU device to train IVF partitions via
      )
      ```
 
-=== "Macos"
+=== "MacOS"
 
      <!-- skip-test -->
      ```python
@@ -100,7 +113,7 @@ You can specify the GPU device to train IVF partitions via
      )
      ```
 
-Trouble shootings:
+Troubleshooting:
 
 If you see `AssertionError: Torch not compiled with CUDA enabled`, you need to [install
 PyTorch with CUDA support](https://pytorch.org/get-started/locally/).
@@ -143,6 +156,14 @@ There are a couple of parameters that can be used to fine-tune the search:
     --8<-- "docs/src/ann_indexes.ts:search1"
     ```
 
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/ivf_pq.rs:search1"
+    ```
+
+    Vector search options are more fully defined in the [crate docs](https://docs.rs/lancedb/latest/lancedb/query/struct.Query.html#method.nearest_to).
+
 The search will return the data requested in addition to the distance of each item.
 
 ### Filtering (where clause)
@@ -187,13 +208,21 @@ You can select the columns returned by the query using a select clause.
 
 ## FAQ
 
+### Why do I need to manually create an index?
+
+Currently, LanceDB does _not_ automatically create the ANN index.
+LanceDB is well-optimized for kNN (exhaustive search) via a disk-based index. For many use-cases,
+datasets of the order of ~100K vectors don't require index creation. If you can live with up to
+100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
+
 ### When is it necessary to create an ANN vector index?
 
-`LanceDB` has manually-tuned SIMD code for computing vector distances.
-In our benchmarks, computing 100K pairs of 1K dimension vectors takes **less than 20ms**.
-For small datasets (< 100K rows) or applications that can accept 100ms latency, vector indices are usually not necessary.
+`LanceDB` comes out-of-the-box with highly optimized SIMD code for computing vector similarity.
+In our benchmarks, computing distances for 100K pairs of 1K dimension vectors takes **less than 20ms**.
+We observe that for small datasets (~100K rows) or for applications that can accept 100ms latency,
+vector indices are usually not necessary.
 
-For large-scale or higher dimension vectors, it is beneficial to create vector index.
+For large-scale or higher dimension vectors, it can beneficial to create vector index for performance.
 
 ### How big is my index, and how many memory will it take?
 

docs/src/api_reference.md

@@ -0,0 +1,7 @@
+# API Reference
+
+The API reference for the LanceDB client SDKs are available at the following locations:
+
+- [Python](python/python.md)
+- [JavaScript](javascript/modules.md)
+- [Rust](https://docs.rs/lancedb/latest/lancedb/index.html)

docs/src/basic.md

@@ -3,7 +3,7 @@
 !!! info "LanceDB can be run in a number of ways:"
 
     * Embedded within an existing backend (like your Django, Flask, Node.js or FastAPI application)
-    * Connected to directly from a client application like a Jupyter notebook for analytical workloads
+    * Directly from a client application like a Jupyter notebook for analytical workloads
     * Deployed as a remote serverless database
 
 ![](assets/lancedb_embedded_explanation.png)
@@ -24,13 +24,11 @@
 
 === "Rust"
 
-    !!! warning "Rust SDK is experimental, might introduce breaking changes in the near future"
-
     ```shell
-    cargo add vectordb
+    cargo add lancedb
     ```
 
-    !!! info "To use the vectordb create, you first need to install protobuf."
+    !!! info "To use the lancedb create, you first need to install protobuf."
 
     === "macOS"
 
@@ -44,9 +42,9 @@
         sudo apt install -y protobuf-compiler libssl-dev
         ```
 
-    !!! info "Please also make sure you're using the same version of Arrow as in the [vectordb crate](https://github.com/lancedb/lancedb/blob/main/Cargo.toml)"
+    !!! info "Please also make sure you're using the same version of Arrow as in the [lancedb crate](https://github.com/lancedb/lancedb/blob/main/Cargo.toml)"
 
-## How to connect to a database
+## Connect to a database
 
 === "Python"
 
@@ -69,17 +67,23 @@
     ```rust
     #[tokio::main]
     async fn main() -> Result<()> {
-        --8<-- "rust/vectordb/examples/simple.rs:connect"
+        --8<-- "rust/lancedb/examples/simple.rs:connect"
     }
     ```
 
-    !!! info "See [examples/simple.rs](https://github.com/lancedb/lancedb/tree/main/rust/vectordb/examples/simple.rs) for a full working example."
+    !!! info "See [examples/simple.rs](https://github.com/lancedb/lancedb/tree/main/rust/lancedb/examples/simple.rs) for a full working example."
 
 LanceDB will create the directory if it doesn't exist (including parent directories).
 
 If you need a reminder of the uri, you can call `db.uri()`.
 
-## How to create a table
+## Create a table
+
+### Create a table from initial data
+
+If you have data to insert into the table at creation time, you can simultaneously create a
+table and insert the data into it.  The schema of the data will be used as the schema of the
+table.
 
 === "Python"
 
@@ -115,20 +119,27 @@ If you need a reminder of the uri, you can call `db.uri()`.
 === "Rust"
 
     ```rust
-    use arrow_schema::{DataType, Schema, Field};
-    use arrow_array::{RecordBatch, RecordBatchIterator};
-
-    --8<-- "rust/vectordb/examples/simple.rs:create_table"
+    --8<-- "rust/lancedb/examples/simple.rs:create_table"
     ```
 
-    If the table already exists, LanceDB will raise an error by default.
+    If the table already exists, LanceDB will raise an error by default.  See
+    [the mode option](https://docs.rs/lancedb/latest/lancedb/connection/struct.CreateTableBuilder.html#method.mode)
+    for details on how to overwrite (or open) existing tables instead.
 
-!!! info "Under the hood, LanceDB is converting the input data into an Apache Arrow table and persisting it to disk in [Lance format](https://www.github.com/lancedb/lance)."
+    !!! Providing table records in Rust
 
-### Creating an empty table
+        The Rust SDK currently expects data to be provided as an Arrow
+        [RecordBatchReader](https://docs.rs/arrow-array/latest/arrow_array/trait.RecordBatchReader.html)
+        Support for additional formats (such as serde or polars) is on the roadmap.
+
+!!! info "Under the hood, LanceDB reads in the Apache Arrow data and persists it to disk using the [Lance format](https://www.github.com/lancedb/lance)."
+
+### Create an empty table
 
 Sometimes you may not have the data to insert into the table at creation time.
-In this case, you can create an empty table and specify the schema.
+In this case, you can create an empty table and specify the schema, so that you can add
+data to the table at a later time (as long as it conforms to the schema).  This is
+similar to a `CREATE TABLE` statement in SQL.
 
 === "Python"
 
@@ -147,12 +158,12 @@ In this case, you can create an empty table and specify the schema.
 === "Rust"
 
     ```rust
-    --8<-- "rust/vectordb/examples/simple.rs:create_empty_table"
+    --8<-- "rust/lancedb/examples/simple.rs:create_empty_table"
     ```
 
-## How to open an existing table
+## Open an existing table
 
-Once created, you can open a table using the following code:
+Once created, you can open a table as follows:
 
 === "Python"
 
@@ -169,7 +180,7 @@ Once created, you can open a table using the following code:
 === "Rust"
 
     ```rust
-    --8<-- "rust/vectordb/examples/simple.rs:open_with_existing_file"
+    --8<-- "rust/lancedb/examples/simple.rs:open_existing_tbl"
     ```
 
 If you forget the name of your table, you can always get a listing of all table names:
@@ -189,12 +200,12 @@ If you forget the name of your table, you can always get a listing of all table
 === "Rust"
 
     ```rust
-    --8<-- "rust/vectordb/examples/simple.rs:list_names"
+    --8<-- "rust/lancedb/examples/simple.rs:list_names"
     ```
 
-## How to add data to a table
+## Add data to a table
 
-After a table has been created, you can always add more data to it using
+After a table has been created, you can always add more data to it as follows:
 
 === "Python"
 
@@ -219,12 +230,12 @@ After a table has been created, you can always add more data to it using
 === "Rust"
 
     ```rust
-    --8<-- "rust/vectordb/examples/simple.rs:add"
+    --8<-- "rust/lancedb/examples/simple.rs:add"
     ```
 
-## How to search for (approximate) nearest neighbors
+## Search for nearest neighbors
 
-Once you've embedded the query, you can find its nearest neighbors using the following code:
+Once you've embedded the query, you can find its nearest neighbors as follows:
 
 === "Python"
 
@@ -245,11 +256,20 @@ Once you've embedded the query, you can find its nearest neighbors using the fol
     ```rust
     use futures::TryStreamExt;
 
-    --8<-- "rust/vectordb/examples/simple.rs:search"
+    --8<-- "rust/lancedb/examples/simple.rs:search"
     ```
 
+    !!! Query vectors in Rust
+        Rust does not yet support automatic execution of embedding functions.  You will need to
+        calculate embeddings yourself.  Support for this is on the roadmap and can be tracked at
+        https://github.com/lancedb/lancedb/issues/994
+
+        Query vectors can be provided as Arrow arrays or a Vec/slice of Rust floats.
+        Support for additional formats (e.g. `polars::series::Series`) is on the roadmap.
+
 By default, LanceDB runs a brute-force scan over dataset to find the K nearest neighbours (KNN).
 For tables with more than 50K vectors, creating an ANN index is recommended to speed up search performance.
+LanceDB allows you to create an ANN index on a table as follows:
 
 === "Python"
 
@@ -266,12 +286,17 @@ For tables with more than 50K vectors, creating an ANN index is recommended to s
 === "Rust"
 
     ```rust
-     --8<-- "rust/vectordb/examples/simple.rs:create_index"
+     --8<-- "rust/lancedb/examples/simple.rs:create_index"
     ```
 
-Check [Approximate Nearest Neighbor (ANN) Indexes](/ann_indices.md) section for more details.
+!!! note "Why do I need to create an index manually?"
+    LanceDB does not automatically create the ANN index for two reasons. The first is that it's optimized
+    for really fast retrievals via a disk-based index, and the second is that data and query workloads can
+    be very diverse, so there's no one-size-fits-all index configuration. LanceDB provides many parameters
+    to fine-tune index size, query latency and accuracy. See the section on
+    [ANN indexes](ann_indexes.md) for more details.
 
-## How to delete rows from a table
+## Delete rows from a table
 
 Use the `delete()` method on tables to delete rows from a table. To choose
 which rows to delete, provide a filter that matches on the metadata columns.
@@ -292,12 +317,13 @@ This can delete any number of rows that match the filter.
 === "Rust"
 
     ```rust
-    --8<-- "rust/vectordb/examples/simple.rs:delete"
+    --8<-- "rust/lancedb/examples/simple.rs:delete"
     ```
 
 The deletion predicate is a SQL expression that supports the same expressions
-as the `where()` clause on a search. They can be as simple or complex as needed.
-To see what expressions are supported, see the [SQL filters](sql.md) section.
+as the `where()` clause (`only_if()` in Rust) on a search. They can be as
+simple or complex as needed.  To see what expressions are supported, see the
+[SQL filters](sql.md) section.
 
 === "Python"
 
@@ -307,7 +333,11 @@ To see what expressions are supported, see the [SQL filters](sql.md) section.
 
       Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
 
-## How to remove a table
+=== "Rust"
+
+      Read more: [lancedb::Table::delete](https://docs.rs/lancedb/latest/lancedb/table/struct.Table.html#method.delete)
+
+## Drop a table
 
 Use the `drop_table()` method on the database to remove a table.
 
@@ -333,7 +363,7 @@ Use the `drop_table()` method on the database to remove a table.
 === "Rust"
 
     ```rust
-    --8<-- "rust/vectordb/examples/simple.rs:drop_table"
+    --8<-- "rust/lancedb/examples/simple.rs:drop_table"
     ```
 
 !!! note "Bundling `vectordb` apps with Webpack"

docs/src/concepts/index_ivfpq.md

@@ -31,7 +31,7 @@ As an example, consider starting with 128-dimensional vector consisting of 32-bi
 
 While PQ helps with reducing the size of the index, IVF primarily addresses search performance. The primary purpose of an inverted file index is to facilitate rapid and effective nearest neighbor search by narrowing down the search space.
 
-In IVF, the PQ vector space is divided into *Voronoi cells*, which are essentially partitions that consist of all the points in the space that are within a threshold distance of the given region's seed point. These seed points are used to create an inverted index that correlates each centroid with a list of vectors in the space, allowing a search to be restricted to just a subset of vectors in the index.
+In IVF, the PQ vector space is divided into *Voronoi cells*, which are essentially partitions that consist of all the points in the space that are within a threshold distance of the given region's seed point. These seed points are initialized by running K-means over the stored vectors. The centroids of K-means turn into the seed points which then each define a region. These regions are then are used to create an inverted index that correlates each centroid with a list of vectors in the space, allowing a search to be restricted to just a subset of vectors in the index.
 
 ![](../assets/ivfpq_ivf_desc.webp)
 
@@ -81,24 +81,4 @@ The above query will perform a search on the table `tbl` using the given query v
 * `to_pandas()`: Convert the results to a pandas DataFrame
 
 And there you have it! You now understand what an IVF-PQ index is, and how to create and query it in LanceDB.
-
-
-## FAQ
-
-### When is it necessary to create a vector index?
-
-LanceDB has manually-tuned SIMD code for computing vector distances. In our benchmarks, computing 100K pairs of 1K dimension vectors takes **<20ms**. For small datasets (<100K rows) or applications that can accept up to 100ms latency, vector indices are usually not necessary.
-
-For large-scale or higher dimension vectors, it is beneficial to create vector index.
-
-### How big is my index, and how much memory will it take?
-
-In LanceDB, all vector indices are disk-based, meaning that when responding to a vector query, only the relevant pages from the index file are loaded from disk and cached in memory. Additionally, each sub-vector is usually encoded into 1 byte PQ code.
-
-For example, with 1024-dimension vectors, if we choose `num_sub_vectors = 64`, each sub-vector has `1024 / 64 = 16` float32 numbers. Product quantization can lead to approximately `16 * sizeof(float32) / 1 = 64` times of space reduction.
-
-### How to choose `num_partitions` and `num_sub_vectors` for IVF_PQ index?
-
-`num_partitions` is used to decide how many partitions the first level IVF index uses. Higher number of partitions could lead to more efficient I/O during queries and better accuracy, but it takes much more time to train. On SIFT-1M dataset, our benchmark shows that keeping each partition 1K-4K rows lead to a good latency/recall.
-
-`num_sub_vectors` specifies how many PQ short codes to generate on each vector. Because PQ is a lossy compression of the original vector, a higher `num_sub_vectors` usually results in less space distortion, and thus yields better accuracy. However, a higher `num_sub_vectors` also causes heavier I/O and more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.
+To see how to create an IVF-PQ index in LanceDB, take a look at the [ANN indexes](../ann_indexes.md) section.

docs/src/embeddings/default_embedding_functions.md

@@ -47,6 +47,7 @@ LanceDB registers the OpenAI embeddings function in the registry by default, as
 | Parameter | Type | Default Value | Description |
 |---|---|---|---|
 | `name` | `str` | `"text-embedding-ada-002"` | The name of the model. |
+| `dim` | `int` |  Model default   | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it |
 
 
 ```python
@@ -175,7 +176,8 @@ Supported Embedding modelIDs are:
 * `cohere.embed-english-v3`
 * `cohere.embed-multilingual-v3`
 
-Supported paramters (to be passed in `create` method) are:
+Supported parameters (to be passed in `create` method) are:
+
 | Parameter | Type | Default Value | Description |
 |---|---|---|---|
 | **name** | str | "amazon.titan-embed-text-v1" | The model ID of the bedrock model to use. Supported base models for Text Embeddings: amazon.titan-embed-text-v1, cohere.embed-english-v3, cohere.embed-multilingual-v3 |
@@ -222,7 +224,6 @@ This embedding function supports ingesting images as both bytes and urls. You ca
 !!! info
     LanceDB supports ingesting images directly from accessible links.
 
-
 ```python
 
 db = lancedb.connect(tmp_path)
@@ -288,4 +289,67 @@ print(actual.label)
 
 ```
 
+### Imagebind embeddings
+We have support for [imagebind](https://github.com/facebookresearch/ImageBind) model embeddings. You can download our version of the packaged model via - `pip install imagebind-packaged==0.1.2`.
+
+This function is registered as `imagebind` and supports Audio, Video and Text modalities(extending to Thermal,Depth,IMU data):
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"imagebind_huge"` | Name of the model. |
+| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
+| `normalize` | `bool` | `False` | set to `True` to normalize your inputs before model ingestion. |
+
+Below is an example demonstrating how the API works:
+
+```python
+db = lancedb.connect(tmp_path)
+registry = EmbeddingFunctionRegistry.get_instance()
+func = registry.get("imagebind").create()
+
+class ImageBindModel(LanceModel):
+    text: str
+    image_uri: str = func.SourceField()
+    audio_path: str
+    vector: Vector(func.ndims()) = func.VectorField()
+
+# add locally accessible image paths
+text_list=["A dog.", "A car", "A bird"]
+image_paths=[".assets/dog_image.jpg", ".assets/car_image.jpg", ".assets/bird_image.jpg"]
+audio_paths=[".assets/dog_audio.wav", ".assets/car_audio.wav", ".assets/bird_audio.wav"]
+
+# Load data
+inputs = [
+    {"text": a, "audio_path": b, "image_uri": c}
+    for a, b, c in zip(text_list, audio_paths, image_paths)
+]
+
+#create table and add data
+table = db.create_table("img_bind", schema=ImageBindModel)
+table.add(inputs)
+```
+
+Now, we can search using any modality:
+
+#### image search
+```python
+query_image = "./assets/dog_image2.jpg" #download an image and enter that path here
+actual = table.search(query_image).limit(1).to_pydantic(ImageBindModel)[0]
+print(actual.text == "dog")
+```
+#### audio search
+
+```python
+query_audio = "./assets/car_audio2.wav" #download an audio clip and enter path here
+actual = table.search(query_audio).limit(1).to_pydantic(ImageBindModel)[0]
+print(actual.text == "car")
+```
+#### Text search
+You can add any input query and fetch the result as follows:
+```python
+query = "an animal which flies and tweets" 
+actual = table.search(query).limit(1).to_pydantic(ImageBindModel)[0]
+print(actual.text == "bird")
+```
+
 If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue [on GitHub](https://github.com/lancedb/lancedb/issues).

docs/src/examples/examples_rust.md

@@ -0,0 +1,3 @@
+# Examples: Rust
+
+Our Rust SDK is now stable. Examples are coming soon.

docs/src/examples/image_embeddings_roboflow.md

@@ -43,7 +43,7 @@ pip install lancedb
 We also need to install a specific commit of `tantivy`, a dependency of the LanceDB full text search engine we will use later in this guide:
 
 ```
-pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+pip install tantivy
 ```
 
 Create a new Python file and add the following code:

docs/src/examples/index.md

@@ -2,10 +2,11 @@
 
 ## Recipes and example code
 
-LanceDB provides language APIs, allowing you to embed a database in your language of choice. We currently provide Python and Javascript APIs, with the Rust API and examples actively being worked on and will be available soon.
+LanceDB provides language APIs, allowing you to embed a database in your language of choice.
 
 * 🐍 [Python](examples_python.md) examples
-* 👾 [JavaScript](exampled_js.md) examples
+* 👾 [JavaScript](examples_js.md) examples
+* 🦀 Rust examples (coming soon)
 
 ## Applications powered by LanceDB
 

docs/src/faq.md

@@ -16,7 +16,7 @@ As we mention in our talk titled “[Lance, a modern columnar data format](https
 
 ### Why build in Rust? 🦀
 
-We believe that the Rust ecosystem has attained mainstream maturity and that Rust will form the underpinnings of large parts of the data and ML landscape in a few years. Performance, latency and reliability are paramount to a vector DB, and building in Rust allows us to iterate and release updates more rapidly due to Rust’s safety guarantees. Both Lance (the data format) and LanceDB (the database) are written entirely in Rust. We also provide Python and JavaScript client libraries to interact with the database. Our Rust API is a little rough around the edges right now, but is fast becoming on par with the Python and JS APIs.
+We believe that the Rust ecosystem has attained mainstream maturity and that Rust will form the underpinnings of large parts of the data and ML landscape in a few years. Performance, latency and reliability are paramount to a vector DB, and building in Rust allows us to iterate and release updates more rapidly due to Rust’s safety guarantees. Both Lance (the data format) and LanceDB (the database) are written entirely in Rust. We also provide Python, JavaScript, and Rust client libraries to interact with the database.
 
 ### What is the difference between LanceDB OSS and LanceDB Cloud?
 
@@ -40,11 +40,11 @@ LanceDB and its underlying data format, Lance, are built to scale to really larg
 
 No. LanceDB is blazing fast (due to its disk-based index) for even brute force kNN search, within reason. In our benchmarks, computing 100K pairs of 1000-dimension vectors takes less than 20ms. For small datasets of ~100K records or applications that can accept ~100ms latency, an ANN index is usually not necessary.
 
-For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index.
+For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index. See the [ANN indexes](ann_indexes.md) section for more details.
 
 ### Does LanceDB support full-text search?
 
-Yes, LanceDB supports full-text search (FTS) via [Tantivy](https://github.com/quickwit-oss/tantivy). Our current FTS integration is Python-only, and our goal is to push it down to the Rust level in future versions to enable much more powerful search capabilities available to our Python, JavaScript and Rust clients.
+Yes, LanceDB supports full-text search (FTS) via [Tantivy](https://github.com/quickwit-oss/tantivy). Our current FTS integration is Python-only, and our goal is to push it down to the Rust level in future versions to enable much more powerful search capabilities available to our Python, JavaScript and Rust clients.  Follow along in the [Github issue](https://github.com/lancedb/lance/issues/1195)
 
 ### How can I speed up data inserts?
 

docs/src/fts.md

@@ -1,6 +1,6 @@
 # Full-text search
 
-LanceDB provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy) (currently Python only), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. Our goal is to push the FTS integration down to the Rust level in the future, so that it's available for JavaScript users as well.
+LanceDB provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy) (currently Python only), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. Our goal is to push the FTS integration down to the Rust level in the future, so that it's available for Rust and JavaScript users as well.  Follow along at [this Github issue](https://github.com/lancedb/lance/issues/1195)
 
 A hybrid search solution combining vector and full-text search is also on the way.
 
@@ -75,21 +75,70 @@ applied on top of the full text search results. This can be invoked via the fami
 table.search("puppy").limit(10).where("meta='foo'").to_list()
 ```
 
-## Syntax
+## Sorting
 
-For full-text search you can perform either a phrase query like "the old man and the sea", 
-or a structured search query like "(Old AND Man) AND Sea".
-Double quotes are used to disambiguate.
+You can pre-sort the documents by specifying `ordering_field_names` when
+creating the full-text search index. Once pre-sorted, you can then specify
+`ordering_field_name` while searching to return results sorted by the given
+field. For example, 
 
-For example: 
+```
+table.create_fts_index(["text_field"], ordering_field_names=["sort_by_field"])
 
-If you intended "they could have been dogs OR cats" as a phrase query, this actually
-raises a syntax error since `OR` is a recognized operator. If you make `or` lower case,
-this avoids the syntax error. However, it is cumbersome to have to remember what will
-conflict with the query syntax. Instead, if you search using 
-`table.search('"they could have been dogs OR cats"')`, then the syntax checker avoids
-checking inside the quotes.
+(table.search("terms", ordering_field_name="sort_by_field")
+ .limit(20)
+ .to_list())
+```
 
+!!! note
+    If you wish to specify an ordering field at query time, you must also
+    have specified it during indexing time. Otherwise at query time, an
+    error will be raised that looks like `ValueError: The field does not exist: xxx`
+
+!!! note
+    The fields to sort on must be of typed unsigned integer, or else you will see 
+    an error during indexing that looks like 
+    `TypeError: argument 'value': 'float' object cannot be interpreted as an integer`.
+
+!!! note
+    You can specify multiple fields for ordering at indexing time.
+    But at query time only one ordering field is supported.
+
+
+## Phrase queries vs. terms queries
+
+For full-text search you can specify either a **phrase** query like `"the old man and the sea"`,
+or a **terms** search query like `"(Old AND Man) AND Sea"`. For more details on the terms
+query syntax, see Tantivy's [query parser rules](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html).
+
+!!! tip "Note"
+    The query parser will raise an exception on queries that are ambiguous. For example, in the query `they could have been dogs OR cats`, `OR` is capitalized so it's considered a keyword query operator. But it's ambiguous how the left part should be treated. So if you submit this search query as is, you'll get `Syntax Error: they could have been dogs OR cats`.
+
+    ```py
+    # This raises a syntax error
+    table.search("they could have been dogs OR cats")
+    ```
+
+    On the other hand, lowercasing `OR` to `or` will work, because there are no capitalized logical operators and
+    the query is treated as a phrase query.
+
+    ```py
+    # This works!
+    table.search("they could have been dogs or cats")
+    ```
+
+It can be cumbersome to have to remember what will cause a syntax error depending on the type of
+query you want to perform. To make this simpler, when you want to perform a phrase query, you can
+enforce it in one of two ways:
+
+1. Place the double-quoted query inside single quotes. For example, `table.search('"they could have been dogs OR cats"')` is treated as
+a phrase query.
+2. Explicitly declare the `phrase_query()` method. This is useful when you have a phrase query that
+itself contains double quotes. For example, `table.search('the cats OR dogs were not really "pets" at all').phrase_query()`
+is treated as a phrase query.
+
+In general, a query that's declared as a phrase query will be wrapped in double quotes during parsing, with nested
+double quotes replaced by single quotes.
 
 ## Configurations
 
@@ -112,4 +161,3 @@ table.create_fts_index(["text1", "text2"], writer_heap_size=heap, replace=True)
 2. We currently only support local filesystem paths for the FTS index.
    This is a tantivy limitation. We've implemented an object store plugin
    but there's no way in tantivy-py to specify to use it.
-

docs/src/guides/tables.md

@@ -168,24 +168,24 @@ This guide will show how to create tables, insert data into them, and update the
     --8<-- "docs/src/basic_legacy.ts:create_f16_table"
     ```
 
-    ### From Pydantic Models
+### From Pydantic Models
 
-    When you create an empty table without data, you must specify the table schema.
-    LanceDB supports creating tables by specifying a PyArrow schema or a specialized
-    Pydantic model called `LanceModel`.
+When you create an empty table without data, you must specify the table schema.
+LanceDB supports creating tables by specifying a PyArrow schema or a specialized
+Pydantic model called `LanceModel`.
 
-    For example, the following Content model specifies a table with 5 columns:
-    `movie_id`, `vector`, `genres`, `title`, and `imdb_id`. When you create a table, you can
-    pass the class as the value of the `schema` parameter to `create_table`.
-    The `vector` column is a `Vector` type, which is a specialized Pydantic type that
-    can be configured with the vector dimensions. It is also important to note that
-    LanceDB only understands subclasses of `lancedb.pydantic.LanceModel`
-    (which itself derives from `pydantic.BaseModel`).
+For example, the following Content model specifies a table with 5 columns:
+`movie_id`, `vector`, `genres`, `title`, and `imdb_id`. When you create a table, you can
+pass the class as the value of the `schema` parameter to `create_table`.
+The `vector` column is a `Vector` type, which is a specialized Pydantic type that
+can be configured with the vector dimensions. It is also important to note that
+LanceDB only understands subclasses of `lancedb.pydantic.LanceModel`
+(which itself derives from `pydantic.BaseModel`).
 
-    ```python
-    from lancedb.pydantic import Vector, LanceModel
+```python
+from lancedb.pydantic import Vector, LanceModel
 
-    class Content(LanceModel):
+class Content(LanceModel):
     movie_id: int
     vector: Vector(128)
     genres: str
@@ -196,65 +196,65 @@ This guide will show how to create tables, insert data into them, and update the
     def imdb_url(self) -> str:
         return f"https://www.imdb.com/title/tt{self.imdb_id}"
 
-    import pyarrow as pa
-    db = lancedb.connect("~/.lancedb")
-    table_name = "movielens_small"
-    table = db.create_table(table_name, schema=Content)
-    ```
+import pyarrow as pa
+db = lancedb.connect("~/.lancedb")
+table_name = "movielens_small"
+table = db.create_table(table_name, schema=Content)
+```
 
-    #### Nested schemas
+#### Nested schemas
 
-    Sometimes your data model may contain nested objects.
-    For example, you may want to store the document string
-    and the document soure name as a nested Document object:
+Sometimes your data model may contain nested objects.
+For example, you may want to store the document string
+and the document soure name as a nested Document object:
 
-    ```python
-    class Document(BaseModel):
+```python
+class Document(BaseModel):
     content: str
     source: str
-    ```
+```
 
-    This can be used as the type of a LanceDB table column:
+This can be used as the type of a LanceDB table column:
 
-    ```python
-    class NestedSchema(LanceModel):
+```python
+class NestedSchema(LanceModel):
     id: str
     vector: Vector(1536)
     document: Document
 
-    tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite")
-    ```
+tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite")
+```
 
-    This creates a struct column called "document" that has two subfields 
-    called "content" and "source":
+This creates a struct column called "document" that has two subfields 
+called "content" and "source":
 
-    ```
-    In [28]: tbl.schema
-    Out[28]:
-    id: string not null
-    vector: fixed_size_list<item: float>[1536] not null
+```
+In [28]: tbl.schema
+Out[28]:
+id: string not null
+vector: fixed_size_list<item: float>[1536] not null
     child 0, item: float
-    document: struct<content: string not null, source: string not null> not null
+document: struct<content: string not null, source: string not null> not null
     child 0, content: string not null
     child 1, source: string not null    
-    ```
+```
 
-    #### Validators
+#### Validators
 
-    Note that neither Pydantic nor PyArrow automatically validates that input data
-    is of the correct timezone, but this is easy to add as a custom field validator:
+Note that neither Pydantic nor PyArrow automatically validates that input data
+is of the correct timezone, but this is easy to add as a custom field validator:
 
-    ```python
-    from datetime import datetime
-    from zoneinfo import ZoneInfo
+```python
+from datetime import datetime
+from zoneinfo import ZoneInfo
 
-    from lancedb.pydantic import LanceModel
-    from pydantic import Field, field_validator, ValidationError, ValidationInfo
+from lancedb.pydantic import LanceModel
+from pydantic import Field, field_validator, ValidationError, ValidationInfo
 
-    tzname = "America/New_York"
-    tz = ZoneInfo(tzname)
+tzname = "America/New_York"
+tz = ZoneInfo(tzname)
 
-    class TestModel(LanceModel):
+class TestModel(LanceModel):
     dt_with_tz: datetime = Field(json_schema_extra={"tz": tzname})
 
     @field_validator('dt_with_tz')
@@ -263,35 +263,35 @@ This guide will show how to create tables, insert data into them, and update the
         assert dt.tzinfo == tz
         return dt        
 
-    ok = TestModel(dt_with_tz=datetime.now(tz))
+ok = TestModel(dt_with_tz=datetime.now(tz))
 
-    try:
+try:
     TestModel(dt_with_tz=datetime.now(ZoneInfo("Asia/Shanghai")))
     assert 0 == 1, "this should raise ValidationError"
-    except ValidationError:
+except ValidationError:
     print("A ValidationError was raised.")
     pass
-    ```
+```
 
-    When you run this code it should print "A ValidationError was raised."
+When you run this code it should print "A ValidationError was raised."
 
-    #### Pydantic custom types
+#### Pydantic custom types
 
-    LanceDB does NOT yet support converting pydantic custom types. If this is something you need,
-    please file a feature request on the [LanceDB Github repo](https://github.com/lancedb/lancedb/issues/new).
+LanceDB does NOT yet support converting pydantic custom types. If this is something you need,
+please file a feature request on the [LanceDB Github repo](https://github.com/lancedb/lancedb/issues/new).
 
-    ### Using Iterators / Writing Large Datasets
+### Using Iterators / Writing Large Datasets
 
-    It is recommended to use iterators to add large datasets in batches when creating your table in one go. This does not create multiple versions of your dataset unlike manually adding batches using `table.add()`
+It is recommended to use iterators to add large datasets in batches when creating your table in one go. This does not create multiple versions of your dataset unlike manually adding batches using `table.add()`
 
-    LanceDB additionally supports PyArrow's `RecordBatch` Iterators or other generators producing supported data types.
+LanceDB additionally supports PyArrow's `RecordBatch` Iterators or other generators producing supported data types.
 
-    Here's an example using using `RecordBatch` iterator for creating tables.
+Here's an example using using `RecordBatch` iterator for creating tables.
 
-    ```python
-    import pyarrow as pa
+```python
+import pyarrow as pa
 
-    def make_batches():
+def make_batches():
     for i in range(5):
         yield pa.RecordBatch.from_arrays(
             [
@@ -303,16 +303,16 @@ This guide will show how to create tables, insert data into them, and update the
             ["vector", "item", "price"],
         )
 
-    schema = pa.schema([
+schema = pa.schema([
     pa.field("vector", pa.list_(pa.float32(), 4)),
     pa.field("item", pa.utf8()),
     pa.field("price", pa.float32()),
-    ])
+])
 
-    db.create_table("batched_tale", make_batches(), schema=schema)
-    ```
+db.create_table("batched_tale", make_batches(), schema=schema)
+```
 
-    You can also use iterators of other types like Pandas DataFrame or Pylists directly in the above example.
+You can also use iterators of other types like Pandas DataFrame or Pylists directly in the above example.
 
 ## Open existing tables
 
@@ -636,6 +636,70 @@ The `values` parameter is used to provide the new values for the columns as lite
 
     When rows are updated, they are moved out of the index. The row will still show up in ANN queries, but the query will not be as fast as it would be if the row was in the index. If you update a large proportion of rows, consider rebuilding the index afterwards.
 
+## Consistency
+
+In LanceDB OSS, users can set the `read_consistency_interval` parameter on connections to achieve different levels of read consistency. This parameter determines how frequently the database synchronizes with the underlying storage system to check for updates made by other processes. If another process updates a table, the database will not see the changes until the next synchronization.
+
+There are three possible settings for `read_consistency_interval`:
+
+1. **Unset (default)**: The database does not check for updates to tables made by other processes. This provides the best query performance, but means that clients may not see the most up-to-date data. This setting is suitable for applications where the data does not change during the lifetime of the table reference.
+2. **Zero seconds (Strong consistency)**: The database checks for updates on every read. This provides the strongest consistency guarantees, ensuring that all clients see the latest committed data. However, it has the most overhead. This setting is suitable when consistency matters more than having high QPS.
+3. **Custom interval (Eventual consistency)**: The database checks for updates at a custom interval, such as every 5 seconds. This provides eventual consistency, allowing for some lag between write and read operations. Performance wise, this is a middle ground between strong consistency and no consistency check. This setting is suitable for applications where immediate consistency is not critical, but clients should see updated data eventually.
+
+!!! tip "Consistency in LanceDB Cloud"
+
+    This is only tune-able in LanceDB OSS. In LanceDB Cloud, readers are always eventually consistent.
+
+=== "Python"
+    
+    To set strong consistency, use `timedelta(0)`:
+
+    ```python
+    from datetime import timedelta
+    db = lancedb.connect("./.lancedb",. read_consistency_interval=timedelta(0))
+    table = db.open_table("my_table")
+    ```
+
+    For eventual consistency, use a custom `timedelta`:
+
+    ```python
+    from datetime import timedelta
+    db = lancedb.connect("./.lancedb", read_consistency_interval=timedelta(seconds=5))
+    table = db.open_table("my_table")
+    ```
+
+    By default, a `Table` will never check for updates from other writers. To manually check for updates you can use `checkout_latest`:
+
+    ```python
+    db = lancedb.connect("./.lancedb")
+    table = db.open_table("my_table")
+    
+    # (Other writes happen to my_table from another process)
+
+    # Check for updates
+    table.checkout_latest()
+    ```
+
+=== "JavaScript/Typescript"
+
+    To set strong consistency, use `0`:
+
+    ```javascript
+    const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 0 });
+    const table = await db.openTable("my_table");
+    ```
+
+    For eventual consistency, specify the update interval as seconds:
+
+    ```javascript
+    const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 5 });
+    const table = await db.openTable("my_table");
+    ```
+
+<!-- Node doesn't yet support the version time travel: https://github.com/lancedb/lancedb/issues/1007 
+    Once it does, we can show manual consistency check for Node as well.
+-->
+
 ## What's next?
 
 Learn the best practices on creating an ANN index and getting the most out of it.

docs/src/index.md

@@ -28,7 +28,7 @@ LanceDB **Cloud** is a SaaS (software-as-a-service) solution that runs serverles
 
 * Fast production-scale vector similarity, full-text & hybrid search and a SQL query interface (via [DataFusion](https://github.com/apache/arrow-datafusion))
 
-* Native Python and Javascript/Typescript support
+* Python, Javascript/Typescript, and Rust support
 
 * Store, query & manage multi-modal data (text, images, videos, point clouds, etc.), not just the embeddings and metadata
 
@@ -54,3 +54,4 @@ The following pages go deeper into the internal of LanceDB and how to use it.
 * [Ecosystem Integrations](integrations/index.md): Integrate LanceDB with other tools in the data ecosystem
 * [Python API Reference](python/python.md): Python OSS and Cloud API references
 * [JavaScript API Reference](javascript/modules.md): JavaScript OSS and Cloud API references
+* [Rust API Reference](https://docs.rs/lancedb/latest/lancedb/index.html): Rust API reference

docs/src/integrations/index.md

@@ -13,7 +13,7 @@ Get started using these examples and quick links.
 | Integrations | |
 |---|---:|
 | <h3> LlamaIndex </h3>LlamaIndex is a simple, flexible data framework for connecting custom data sources to large language models. Llama index integrates with LanceDB as the serverless VectorDB. <h3>[Lean More](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html) </h3> |<img src="../assets/llama-index.jpg" alt="image" width="150" height="auto">|
-| <h3>Langchain</h3>Langchain allows building applications with LLMs through composability <h3>[Lean More](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html) | <img src="../assets/langchain.png" alt="image" width="150" height="auto">|
+| <h3>Langchain</h3>Langchain allows building applications with LLMs through composability <h3>[Lean More](https://python.langchain.com/docs/integrations/vectorstores/lancedb) | <img src="../assets/langchain.png" alt="image" width="150" height="auto">|
 | <h3>Langchain TS</h3> Javascript bindings for Langchain. It integrates with LanceDB's serverless vectordb allowing you to build powerful AI applications through composibility using only serverless functions. <h3>[Learn More]( https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb) | <img src="../assets/langchain.png" alt="image" width="150" height="auto">|
 | <h3>Voxel51</h3>  It is an open source toolkit that enables you to build better computer vision workflows by improving the quality of your datasets and delivering insights about your models.<h3>[Learn More](./voxel51.md) | <img src="../assets/voxel.gif" alt="image" width="150" height="auto">|
 | <h3>PromptTools</h3>  Offers a set of free, open-source tools for testing and experimenting with models, prompts, and configurations. The core idea is to enable developers to evaluate prompts using familiar interfaces like code and notebooks. You can use it to experiment with different configurations of LanceDB, and test how LanceDB integrates with the LLM of your choice.<h3>[Learn More](./prompttools.md) | <img src="../assets/prompttools.jpeg" alt="image" width="150" height="auto">|

docs/src/python/python.md

@@ -24,6 +24,12 @@ pip install lancedb
 
 ::: lancedb.query.LanceQueryBuilder
 
+::: lancedb.query.LanceVectorQueryBuilder
+
+::: lancedb.query.LanceFtsQueryBuilder
+
+::: lancedb.query.LanceHybridQueryBuilder
+
 ## Embeddings
 
 ::: lancedb.embeddings.registry.EmbeddingFunctionRegistry
@@ -62,10 +68,22 @@ pip install lancedb
 
 ## Integrations
 
-### Pydantic
+## Pydantic
 
 ::: lancedb.pydantic.pydantic_to_schema
 
 ::: lancedb.pydantic.vector
 
 ::: lancedb.pydantic.LanceModel
+
+## Reranking
+
+::: lancedb.rerankers.linear_combination.LinearCombinationReranker
+
+::: lancedb.rerankers.cohere.CohereReranker
+
+::: lancedb.rerankers.colbert.ColbertReranker
+
+::: lancedb.rerankers.cross_encoder.CrossEncoderReranker
+
+::: lancedb.rerankers.openai.OpenaiReranker

docs/src/search.md

@@ -22,7 +22,7 @@ Currently, LanceDB supports the following metrics:
 ## Exhaustive search (kNN)
 
 If you do not create a vector index, LanceDB exhaustively scans the _entire_ vector space
-and compute the distance to every vector in order to find the exact nearest neighbors. This is effectively a kNN search.
+and computes the distance to every vector in order to find the exact nearest neighbors. This is effectively a kNN search.
 
 <!-- Setup Code
 ```python
@@ -85,7 +85,7 @@ To perform scalable vector retrieval with acceptable latencies, it's common to b
 While the exhaustive search is guaranteed to always return 100% recall, the approximate nature of
 an ANN search means that using an index often involves a trade-off between recall and latency.
 
-See the [IVF_PQ index](./concepts/index_ivfpq.md.md) for a deeper description of how `IVF_PQ`
+See the [IVF_PQ index](./concepts/index_ivfpq.md) for a deeper description of how `IVF_PQ`
 indexes work in LanceDB.
 
 ## Output search results
@@ -184,4 +184,3 @@ Let's create a LanceDB table with a nested schema:
 
     Note that in this case the extra `_distance` field is discarded since
     it's not part of the LanceSchema.
-

node/.eslintrc.js

@@ -13,5 +13,10 @@ module.exports = {
   },
   rules: {
     "@typescript-eslint/method-signature-style": "off",
+    "@typescript-eslint/quotes": "off",
+    "@typescript-eslint/semi": "off",
+    "@typescript-eslint/explicit-function-return-type": "off",
+    "@typescript-eslint/space-before-function-paren": "off",
+    "@typescript-eslint/indent": "off",
   }
 }

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.4.10",
+  "version": "0.4.13",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.4.10",
+      "version": "0.4.13",
       "cpu": [
         "x64",
         "arm64"
@@ -18,9 +18,7 @@
         "win32"
       ],
       "dependencies": {
-        "@apache-arrow/ts": "^14.0.2",
         "@neon-rs/load": "^0.0.74",
-        "apache-arrow": "^14.0.2",
         "axios": "^1.4.0"
       },
       "devDependencies": {
@@ -33,6 +31,7 @@
         "@types/temp": "^0.9.1",
         "@types/uuid": "^9.0.3",
         "@typescript-eslint/eslint-plugin": "^5.59.1",
+        "apache-arrow-old": "npm:apache-arrow@13.0.0",
         "cargo-cp-artifact": "^0.1",
         "chai": "^4.3.7",
         "chai-as-promised": "^7.1.1",
@@ -53,11 +52,15 @@
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.4.10",
-        "@lancedb/vectordb-darwin-x64": "0.4.10",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.4.10",
-        "@lancedb/vectordb-linux-x64-gnu": "0.4.10",
-        "@lancedb/vectordb-win32-x64-msvc": "0.4.10"
+        "@lancedb/vectordb-darwin-arm64": "0.4.13",
+        "@lancedb/vectordb-darwin-x64": "0.4.13",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.4.13",
+        "@lancedb/vectordb-linux-x64-gnu": "0.4.13",
+        "@lancedb/vectordb-win32-x64-msvc": "0.4.13"
+      },
+      "peerDependencies": {
+        "@apache-arrow/ts": "^14.0.2",
+        "apache-arrow": "^14.0.2"
       }
     },
     "node_modules/@75lb/deep-merge": {
@@ -93,6 +96,7 @@
       "version": "14.0.2",
       "resolved": "https://registry.npmjs.org/@apache-arrow/ts/-/ts-14.0.2.tgz",
       "integrity": "sha512-CtwAvLkK0CZv7xsYeCo91ml6PvlfzAmAJZkRYuz2GNBwfYufj5SVi0iuSMwIMkcU/szVwvLdzORSLa5PlF/2ug==",
+      "peer": true,
       "dependencies": {
         "@types/command-line-args": "5.2.0",
         "@types/command-line-usage": "5.0.2",
@@ -109,7 +113,8 @@
     "node_modules/@apache-arrow/ts/node_modules/@types/node": {
       "version": "20.3.0",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz",
-      "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ=="
+      "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==",
+      "peer": true
     },
     "node_modules/@cargo-messages/android-arm-eabi": {
       "version": "0.0.160",
@@ -329,9 +334,9 @@
       }
     },
     "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.4.10",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.10.tgz",
-      "integrity": "sha512-y/uHOGb0g15pvqv5tdTyZ6oN+0QVpBmZDzKFWW6pPbuSZjB2uPqcs+ti0RB+AUdmS21kavVQqaNsw/HLKEGrHA==",
+      "version": "0.4.13",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.13.tgz",
+      "integrity": "sha512-JfroNCG8yKIU931Y+x8d0Fp8C9DHUSC5j+CjI+e5err7rTWtie4j3JbsXlWAnPFaFEOg0Xk3BWkSikCvhPGJGg==",
       "cpu": [
         "arm64"
       ],
@@ -341,9 +346,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.4.10",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.4.10.tgz",
-      "integrity": "sha512-XbfR58OkQpAe0xMSTrwJh9ZjGSzG9EZ7zwO6HfYem8PxcLYAcC6eWRWoSG/T0uObyrPTcYYyvHsp0eNQWYBFAQ==",
+      "version": "0.4.13",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.4.13.tgz",
+      "integrity": "sha512-dG6IMvfpHpnHdbJ0UffzJ7cZfMiC02MjIi6YJzgx+hKz2UNXWNBIfTvvhqli85mZsGRXL1OYDdYv0K1YzNjXlA==",
       "cpu": [
         "x64"
       ],
@@ -353,9 +358,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.4.10",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.4.10.tgz",
-      "integrity": "sha512-x40WKH9b+KxorRmKr9G7fv8p5mMj8QJQvRMA0v6v+nbZHr2FLlAZV+9mvhHOnm4AGIkPP5335cUgv6Qz6hgwkQ==",
+      "version": "0.4.13",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.4.13.tgz",
+      "integrity": "sha512-BRR1VzaMviXby7qmLm0axNZM8eUZF3ZqfvnDKdVRpC3LaRueD6pMXHuC2IUKaFkn7xktf+8BlDZb6foFNEj8bQ==",
       "cpu": [
         "arm64"
       ],
@@ -365,9 +370,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.4.10",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.4.10.tgz",
-      "integrity": "sha512-CTGPpuzlqq2nVjUxI9gAJOT1oBANIovtIaFsOmBSnEAHgX7oeAxKy2b6L/kJzsgqSzvR5vfLwYcWFrr6ZmBxSA==",
+      "version": "0.4.13",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.4.13.tgz",
+      "integrity": "sha512-WnekZ7ZMlria+NODZ6aBCljCFQSe2bBNUS9ZpyFl/Y1vHduSQPuBxM6V7vp2QubC0daq/rifgjDob89DF+x3xw==",
       "cpu": [
         "x64"
       ],
@@ -377,9 +382,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.4.10",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.4.10.tgz",
-      "integrity": "sha512-Fd7r74coZyrKzkfXg4WthqOL+uKyJyPTia6imcrMNqKOlTGdKmHf02Qi2QxWZrFaabkRYo4Tpn5FeRJ3yYX8CA==",
+      "version": "0.4.13",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.4.13.tgz",
+      "integrity": "sha512-3NDpMWBL2ksDHXAraXhowiLqQcNWM5bdbeHwze4+InYMD54hyQ2ODNc+4usxp63Nya9biVnFS27yXULqkzIEqQ==",
       "cpu": [
         "x64"
       ],
@@ -948,6 +953,7 @@
       "version": "14.0.2",
       "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-14.0.2.tgz",
       "integrity": "sha512-EBO2xJN36/XoY81nhLcwCJgFwkboDZeyNQ+OPsG7bCoQjc2BT0aTyH/MR6SrL+LirSNz+cYqjGRlupMMlP1aEg==",
+      "peer": true,
       "dependencies": {
         "@types/command-line-args": "5.2.0",
         "@types/command-line-usage": "5.0.2",
@@ -964,10 +970,39 @@
         "arrow2csv": "bin/arrow2csv.js"
       }
     },
+    "node_modules/apache-arrow-old": {
+      "name": "apache-arrow",
+      "version": "13.0.0",
+      "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-13.0.0.tgz",
+      "integrity": "sha512-3gvCX0GDawWz6KFNC28p65U+zGh/LZ6ZNKWNu74N6CQlKzxeoWHpi4CgEQsgRSEMuyrIIXi1Ea2syja7dwcHvw==",
+      "dev": true,
+      "dependencies": {
+        "@types/command-line-args": "5.2.0",
+        "@types/command-line-usage": "5.0.2",
+        "@types/node": "20.3.0",
+        "@types/pad-left": "2.1.1",
+        "command-line-args": "5.2.1",
+        "command-line-usage": "7.0.1",
+        "flatbuffers": "23.5.26",
+        "json-bignum": "^0.0.3",
+        "pad-left": "^2.1.0",
+        "tslib": "^2.5.3"
+      },
+      "bin": {
+        "arrow2csv": "bin/arrow2csv.js"
+      }
+    },
+    "node_modules/apache-arrow-old/node_modules/@types/node": {
+      "version": "20.3.0",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz",
+      "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==",
+      "dev": true
+    },
     "node_modules/apache-arrow/node_modules/@types/node": {
       "version": "20.3.0",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz",
-      "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ=="
+      "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==",
+      "peer": true
     },
     "node_modules/arg": {
       "version": "4.1.3",

node/package.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.4.10",
+  "version": "0.4.13",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
     "tsc": "tsc -b",
-    "build": "npm run tsc && cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json",
+    "build": "npm run tsc && cargo-cp-artifact --artifact cdylib lancedb-node index.node -- cargo build --message-format=json",
     "build-release": "npm run build -- --release",
     "test": "npm run tsc && mocha -recursive dist/test",
     "integration-test": "npm run tsc && mocha -recursive dist/integration_test",
@@ -41,6 +41,7 @@
     "@types/temp": "^0.9.1",
     "@types/uuid": "^9.0.3",
     "@typescript-eslint/eslint-plugin": "^5.59.1",
+    "apache-arrow-old": "npm:apache-arrow@13.0.0",
     "cargo-cp-artifact": "^0.1",
     "chai": "^4.3.7",
     "chai-as-promised": "^7.1.1",
@@ -61,11 +62,13 @@
     "uuid": "^9.0.0"
   },
   "dependencies": {
-    "@apache-arrow/ts": "^14.0.2",
     "@neon-rs/load": "^0.0.74",
-    "apache-arrow": "^14.0.2",
     "axios": "^1.4.0"
   },
+  "peerDependencies": {
+    "@apache-arrow/ts": "^14.0.2",
+    "apache-arrow": "^14.0.2"
+  },
   "os": [
     "darwin",
     "linux",
@@ -85,10 +88,10 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.4.10",
-    "@lancedb/vectordb-darwin-x64": "0.4.10",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.4.10",
-    "@lancedb/vectordb-linux-x64-gnu": "0.4.10",
-    "@lancedb/vectordb-win32-x64-msvc": "0.4.10"
+    "@lancedb/vectordb-darwin-arm64": "0.4.13",
+    "@lancedb/vectordb-darwin-x64": "0.4.13",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.4.13",
+    "@lancedb/vectordb-linux-x64-gnu": "0.4.13",
+    "@lancedb/vectordb-win32-x64-msvc": "0.4.13"
   }
 }

node/src/arrow.ts

@@ -20,19 +20,20 @@ import {
   type Vector,
   FixedSizeList,
   vectorFromArray,
-  type Schema,
+  Schema,
   Table as ArrowTable,
   RecordBatchStreamWriter,
   List,
   RecordBatch,
   makeData,
   Struct,
-  type Float,
+  Float,
   DataType,
   Binary,
   Float32
 } from 'apache-arrow'
 import { type EmbeddingFunction } from './index'
+import { sanitizeSchema } from './sanitize'
 
 /*
  * Options to control how a column should be converted to a vector array
@@ -201,10 +202,13 @@ export function makeArrowTable (
   }
 
   const opt = new MakeArrowTableOptions(options !== undefined ? options : {})
+  if (opt.schema !== undefined && opt.schema !== null) {
+    opt.schema = sanitizeSchema(opt.schema)
+  }
   const columns: Record<string, Vector> = {}
   // TODO: sample dataset to find missing columns
   // Prefer the field ordering of the schema, if present
-  const columnNames = ((options?.schema) != null) ? (options?.schema?.names as string[]) : Object.keys(data[0])
+  const columnNames = ((opt.schema) != null) ? (opt.schema.names as string[]) : Object.keys(data[0])
   for (const colName of columnNames) {
     if (data.length !== 0 && !Object.prototype.hasOwnProperty.call(data[0], colName)) {
       // The field is present in the schema, but not in the data, skip it
@@ -329,6 +333,9 @@ async function applyEmbeddings<T> (table: ArrowTable, embeddings?: EmbeddingFunc
   if (embeddings == null) {
     return table
   }
+  if (schema !== undefined && schema !== null) {
+    schema = sanitizeSchema(schema)
+  }
 
   // Convert from ArrowTable to Record<String, Vector>
   const colEntries = [...Array(table.numCols).keys()].map((_, idx) => {
@@ -439,6 +446,9 @@ export async function fromRecordsToBuffer<T> (
   embeddings?: EmbeddingFunction<T>,
   schema?: Schema
 ): Promise<Buffer> {
+  if (schema !== undefined && schema !== null) {
+    schema = sanitizeSchema(schema)
+  }
   const table = await convertToTable(data, embeddings, { schema })
   const writer = RecordBatchFileWriter.writeAll(table)
   return Buffer.from(await writer.toUint8Array())
@@ -456,6 +466,9 @@ export async function fromRecordsToStreamBuffer<T> (
   embeddings?: EmbeddingFunction<T>,
   schema?: Schema
 ): Promise<Buffer> {
+  if (schema !== null && schema !== undefined) {
+    schema = sanitizeSchema(schema)
+  }
   const table = await convertToTable(data, embeddings, { schema })
   const writer = RecordBatchStreamWriter.writeAll(table)
   return Buffer.from(await writer.toUint8Array())
@@ -474,6 +487,9 @@ export async function fromTableToBuffer<T> (
   embeddings?: EmbeddingFunction<T>,
   schema?: Schema
 ): Promise<Buffer> {
+  if (schema !== null && schema !== undefined) {
+    schema = sanitizeSchema(schema)
+  }
   const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema)
   const writer = RecordBatchFileWriter.writeAll(tableWithEmbeddings)
   return Buffer.from(await writer.toUint8Array())
@@ -492,6 +508,9 @@ export async function fromTableToStreamBuffer<T> (
   embeddings?: EmbeddingFunction<T>,
   schema?: Schema
 ): Promise<Buffer> {
+  if (schema !== null && schema !== undefined) {
+    schema = sanitizeSchema(schema)
+  }
   const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema)
   const writer = RecordBatchStreamWriter.writeAll(tableWithEmbeddings)
   return Buffer.from(await writer.toUint8Array())
@@ -528,5 +547,5 @@ function alignTable (table: ArrowTable, schema: Schema): ArrowTable {
 
 // Creates an empty Arrow Table
 export function createEmptyTable (schema: Schema): ArrowTable {
-  return new ArrowTable(schema)
+  return new ArrowTable(sanitizeSchema(schema))
 }

node/src/index.ts

@@ -42,7 +42,10 @@ const {
   tableCompactFiles,
   tableListIndices,
   tableIndexStats,
-  tableSchema
+  tableSchema,
+  tableAddColumns,
+  tableAlterColumns,
+  tableDropColumns
   // eslint-disable-next-line @typescript-eslint/no-var-requires
 } = require('../native.js')
 
@@ -96,6 +99,19 @@ export interface ConnectionOptions {
    * This is useful for local testing.
    */
   hostOverride?: string
+
+  /**
+   * (For LanceDB OSS only): The interval, in seconds, at which to check for
+   * updates to the table from other processes. If None, then consistency is not
+   * checked. For performance reasons, this is the default. For strong
+   * consistency, set this to zero seconds. Then every read will check for
+   * updates from other processes. As a compromise, you can set this to a
+   * non-zero value for eventual consistency. If more than that interval
+   * has passed since the last check, then the table will be checked for updates.
+   * Note: this consistency only applies to read operations. Write operations are
+   * always consistent.
+   */
+  readConsistencyInterval?: number
 }
 
 function getAwsArgs (opts: ConnectionOptions): any[] {
@@ -160,6 +176,10 @@ export async function connect (
     opts = { uri: arg }
   } else {
     // opts = { uri: arg.uri, awsCredentials = arg.awsCredentials }
+    const keys = Object.keys(arg)
+    if (keys.length === 1 && keys[0] === 'uri' && typeof arg.uri === 'string') {
+      opts = { uri: arg.uri }
+    } else {
       opts = Object.assign(
         {
           uri: '',
@@ -171,6 +191,7 @@ export async function connect (
         arg
       )
     }
+  }
 
   if (opts.uri.startsWith('db://')) {
     // Remote connection
@@ -181,7 +202,8 @@ export async function connect (
     opts.awsCredentials?.accessKeyId,
     opts.awsCredentials?.secretKey,
     opts.awsCredentials?.sessionToken,
-    opts.awsRegion
+    opts.awsRegion,
+    opts.readConsistencyInterval
   )
   return new LocalConnection(db, opts)
 }
@@ -324,6 +346,7 @@ export interface Table<T = number[]> {
    *
    * @param column The column to index
    * @param replace If false, fail if an index already exists on the column
+   * it is always set to true for remote connections
    *
    * Scalar indices, like vector indices, can be used to speed up scans.  A scalar
    * index can speed up scans that contain filter expressions on the indexed column.
@@ -367,7 +390,7 @@ export interface Table<T = number[]> {
    * await table.createScalarIndex('my_col')
    * ```
    */
-  createScalarIndex: (column: string, replace: boolean) => Promise<void>
+  createScalarIndex: (column: string, replace?: boolean) => Promise<void>
 
   /**
    * Returns the number of rows in this table.
@@ -486,6 +509,59 @@ export interface Table<T = number[]> {
   filter(value: string): Query<T>
 
   schema: Promise<Schema>
+
+  // TODO: Support BatchUDF
+  /**
+   * Add new columns with defined values.
+   *
+   * @param newColumnTransforms pairs of column names and the SQL expression to use
+   *                            to calculate the value of the new column. These
+   *                            expressions will be evaluated for each row in the
+   *                            table, and can reference existing columns in the table.
+   */
+  addColumns(newColumnTransforms: Array<{ name: string, valueSql: string }>): Promise<void>
+
+  /**
+   * Alter the name or nullability of columns.
+   *
+   * @param columnAlterations One or more alterations to apply to columns.
+   */
+  alterColumns(columnAlterations: ColumnAlteration[]): Promise<void>
+
+  /**
+   * Drop one or more columns from the dataset
+   *
+   * This is a metadata-only operation and does not remove the data from the
+   * underlying storage. In order to remove the data, you must subsequently
+   * call ``compact_files`` to rewrite the data without the removed columns and
+   * then call ``cleanup_files`` to remove the old files.
+   *
+   * @param columnNames The names of the columns to drop. These can be nested
+   *                    column references (e.g. "a.b.c") or top-level column
+   *                    names (e.g. "a").
+   */
+  dropColumns(columnNames: string[]): Promise<void>
+}
+
+/**
+ * A definition of a column alteration. The alteration changes the column at
+ * `path` to have the new name `name`, to be nullable if `nullable` is true,
+ * and to have the data type `data_type`. At least one of `rename` or `nullable`
+ * must be provided.
+ */
+export interface ColumnAlteration {
+  /**
+   * The path to the column to alter. This is a dot-separated path to the column.
+   * If it is a top-level column then it is just the name of the column. If it is
+   * a nested column then it is the path to the column, e.g. "a.b.c" for a column
+   * `c` nested inside a column `b` nested inside a column `a`.
+   */
+  path: string
+  rename?: string
+  /**
+   * Set the new nullability. Note that a nullable column cannot be made non-nullable.
+   */
+  nullable?: boolean
 }
 
 export interface UpdateArgs {
@@ -844,7 +920,10 @@ export class LocalTable<T = number[]> implements Table<T> {
       })
   }
 
-  async createScalarIndex (column: string, replace: boolean): Promise<void> {
+  async createScalarIndex (column: string, replace?: boolean): Promise<void> {
+    if (replace === undefined) {
+      replace = true
+    }
     return tableCreateScalarIndex.call(this._tbl, column, replace)
   }
 
@@ -1014,6 +1093,18 @@ export class LocalTable<T = number[]> implements Table<T> {
       return false
     }
   }
+
+  async addColumns (newColumnTransforms: Array<{ name: string, valueSql: string }>): Promise<void> {
+    return tableAddColumns.call(this._tbl, newColumnTransforms)
+  }
+
+  async alterColumns (columnAlterations: ColumnAlteration[]): Promise<void> {
+    return tableAlterColumns.call(this._tbl, columnAlterations)
+  }
+
+  async dropColumns (columnNames: string[]): Promise<void> {
+    return tableDropColumns.call(this._tbl, columnNames)
+  }
 }
 
 export interface CleanupStats {

node/src/remote/index.ts

@@ -25,7 +25,8 @@ import {
   type UpdateArgs,
   type UpdateSqlArgs,
   makeArrowTable,
-  type MergeInsertArgs
+  type MergeInsertArgs,
+  type ColumnAlteration
 } from '../index'
 import { Query } from '../query'
 
@@ -396,7 +397,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
     }
 
     const column = indexParams.column ?? 'vector'
-    const indexType = 'vector' // only vector index is supported for remote connections
+    const indexType = 'vector'
     const metricType = indexParams.metric_type ?? 'L2'
     const indexCacheSize = indexParams.index_cache_size ?? null
 
@@ -419,8 +420,25 @@ export class RemoteTable<T = number[]> implements Table<T> {
     }
   }
 
-  async createScalarIndex (column: string, replace: boolean): Promise<void> {
-    throw new Error('Not implemented')
+  async createScalarIndex (column: string): Promise<void> {
+    const indexType = 'scalar'
+
+    const data = {
+      column,
+      index_type: indexType,
+      replace: true
+    }
+    const res = await this._client.post(
+      `/v1/table/${this._name}/create_scalar_index/`,
+      data
+    )
+    if (res.status !== 200) {
+      throw new Error(
+        `Server Error, status: ${res.status}, ` +
+          // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+          `message: ${res.statusText}: ${res.data}`
+      )
+    }
   }
 
   async countRows (): Promise<number> {
@@ -474,4 +492,16 @@ export class RemoteTable<T = number[]> implements Table<T> {
       numUnindexedRows: results.data.num_unindexed_rows
     }
   }
+
+  async addColumns (newColumnTransforms: Array<{ name: string, valueSql: string }>): Promise<void> {
+    throw new Error('Add columns is not yet supported in LanceDB Cloud.')
+  }
+
+  async alterColumns (columnAlterations: ColumnAlteration[]): Promise<void> {
+    throw new Error('Alter columns is not yet supported in LanceDB Cloud.')
+  }
+
+  async dropColumns (columnNames: string[]): Promise<void> {
+    throw new Error('Drop columns is not yet supported in LanceDB Cloud.')
+  }
 }

node/src/sanitize.ts

@@ -0,0 +1,508 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The utilities in this file help sanitize data from the user's arrow
+// library into the types expected by vectordb's arrow library.  Node
+// generally allows for mulitple versions of the same library (and sometimes
+// even multiple copies of the same version) to be installed at the same
+// time.  However, arrow-js uses instanceof which expected that the input
+// comes from the exact same library instance.  This is not always the case
+// and so we must sanitize the input to ensure that it is compatible.
+
+import {
+  Field,
+  Utf8,
+  FixedSizeBinary,
+  FixedSizeList,
+  Schema,
+  List,
+  Struct,
+  Float,
+  Bool,
+  Date_,
+  Decimal,
+  DataType,
+  Dictionary,
+  Binary,
+  Float32,
+  Interval,
+  Map_,
+  Duration,
+  Union,
+  Time,
+  Timestamp,
+  Type,
+  Null,
+  Int,
+  type Precision,
+  type DateUnit,
+  Int8,
+  Int16,
+  Int32,
+  Int64,
+  Uint8,
+  Uint16,
+  Uint32,
+  Uint64,
+  Float16,
+  Float64,
+  DateDay,
+  DateMillisecond,
+  DenseUnion,
+  SparseUnion,
+  TimeNanosecond,
+  TimeMicrosecond,
+  TimeMillisecond,
+  TimeSecond,
+  TimestampNanosecond,
+  TimestampMicrosecond,
+  TimestampMillisecond,
+  TimestampSecond,
+  IntervalDayTime,
+  IntervalYearMonth,
+  DurationNanosecond,
+  DurationMicrosecond,
+  DurationMillisecond,
+  DurationSecond,
+} from "apache-arrow";
+import type { IntBitWidth, TimeBitWidth } from "apache-arrow/type";
+
+function sanitizeMetadata(
+  metadataLike?: unknown,
+): Map<string, string> | undefined {
+  if (metadataLike === undefined || metadataLike === null) {
+    return undefined;
+  }
+  if (!(metadataLike instanceof Map)) {
+    throw Error("Expected metadata, if present, to be a Map<string, string>");
+  }
+  for (const item of metadataLike) {
+    if (!(typeof item[0] === "string" || !(typeof item[1] === "string"))) {
+      throw Error(
+        "Expected metadata, if present, to be a Map<string, string> but it had non-string keys or values",
+      );
+    }
+  }
+  return metadataLike as Map<string, string>;
+}
+
+function sanitizeInt(typeLike: object) {
+  if (
+    !("bitWidth" in typeLike) ||
+    typeof typeLike.bitWidth !== "number" ||
+    !("isSigned" in typeLike) ||
+    typeof typeLike.isSigned !== "boolean"
+  ) {
+    throw Error(
+      "Expected an Int Type to have a `bitWidth` and `isSigned` property",
+    );
+  }
+  return new Int(typeLike.isSigned, typeLike.bitWidth as IntBitWidth);
+}
+
+function sanitizeFloat(typeLike: object) {
+  if (!("precision" in typeLike) || typeof typeLike.precision !== "number") {
+    throw Error("Expected a Float Type to have a `precision` property");
+  }
+  return new Float(typeLike.precision as Precision);
+}
+
+function sanitizeDecimal(typeLike: object) {
+  if (
+    !("scale" in typeLike) ||
+    typeof typeLike.scale !== "number" ||
+    !("precision" in typeLike) ||
+    typeof typeLike.precision !== "number" ||
+    !("bitWidth" in typeLike) ||
+    typeof typeLike.bitWidth !== "number"
+  ) {
+    throw Error(
+      "Expected a Decimal Type to have `scale`, `precision`, and `bitWidth` properties",
+    );
+  }
+  return new Decimal(typeLike.scale, typeLike.precision, typeLike.bitWidth);
+}
+
+function sanitizeDate(typeLike: object) {
+  if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
+    throw Error("Expected a Date type to have a `unit` property");
+  }
+  return new Date_(typeLike.unit as DateUnit);
+}
+
+function sanitizeTime(typeLike: object) {
+  if (
+    !("unit" in typeLike) ||
+    typeof typeLike.unit !== "number" ||
+    !("bitWidth" in typeLike) ||
+    typeof typeLike.bitWidth !== "number"
+  ) {
+    throw Error(
+      "Expected a Time type to have `unit` and `bitWidth` properties",
+    );
+  }
+  return new Time(typeLike.unit, typeLike.bitWidth as TimeBitWidth);
+}
+
+function sanitizeTimestamp(typeLike: object) {
+  if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
+    throw Error("Expected a Timestamp type to have a `unit` property");
+  }
+  let timezone = null;
+  if ("timezone" in typeLike && typeof typeLike.timezone === "string") {
+    timezone = typeLike.timezone;
+  }
+  return new Timestamp(typeLike.unit, timezone);
+}
+
+function sanitizeTypedTimestamp(
+  typeLike: object,
+  Datatype:
+    | typeof TimestampNanosecond
+    | typeof TimestampMicrosecond
+    | typeof TimestampMillisecond
+    | typeof TimestampSecond,
+) {
+  let timezone = null;
+  if ("timezone" in typeLike && typeof typeLike.timezone === "string") {
+    timezone = typeLike.timezone;
+  }
+  return new Datatype(timezone);
+}
+
+function sanitizeInterval(typeLike: object) {
+  if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
+    throw Error("Expected an Interval type to have a `unit` property");
+  }
+  return new Interval(typeLike.unit);
+}
+
+function sanitizeList(typeLike: object) {
+  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
+    throw Error(
+      "Expected a List type to have an array-like `children` property",
+    );
+  }
+  if (typeLike.children.length !== 1) {
+    throw Error("Expected a List type to have exactly one child");
+  }
+  return new List(sanitizeField(typeLike.children[0]));
+}
+
+function sanitizeStruct(typeLike: object) {
+  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
+    throw Error(
+      "Expected a Struct type to have an array-like `children` property",
+    );
+  }
+  return new Struct(typeLike.children.map((child) => sanitizeField(child)));
+}
+
+function sanitizeUnion(typeLike: object) {
+  if (
+    !("typeIds" in typeLike) ||
+    !("mode" in typeLike) ||
+    typeof typeLike.mode !== "number"
+  ) {
+    throw Error(
+      "Expected a Union type to have `typeIds` and `mode` properties",
+    );
+  }
+  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
+    throw Error(
+      "Expected a Union type to have an array-like `children` property",
+    );
+  }
+
+  return new Union(
+    typeLike.mode,
+    typeLike.typeIds as any,
+    typeLike.children.map((child) => sanitizeField(child)),
+  );
+}
+
+function sanitizeTypedUnion(
+  typeLike: object,
+  UnionType: typeof DenseUnion | typeof SparseUnion,
+) {
+  if (!("typeIds" in typeLike)) {
+    throw Error(
+      "Expected a DenseUnion/SparseUnion type to have a `typeIds` property",
+    );
+  }
+  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
+    throw Error(
+      "Expected a DenseUnion/SparseUnion type to have an array-like `children` property",
+    );
+  }
+
+  return new UnionType(
+    typeLike.typeIds as any,
+    typeLike.children.map((child) => sanitizeField(child)),
+  );
+}
+
+function sanitizeFixedSizeBinary(typeLike: object) {
+  if (!("byteWidth" in typeLike) || typeof typeLike.byteWidth !== "number") {
+    throw Error(
+      "Expected a FixedSizeBinary type to have a `byteWidth` property",
+    );
+  }
+  return new FixedSizeBinary(typeLike.byteWidth);
+}
+
+function sanitizeFixedSizeList(typeLike: object) {
+  if (!("listSize" in typeLike) || typeof typeLike.listSize !== "number") {
+    throw Error("Expected a FixedSizeList type to have a `listSize` property");
+  }
+  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
+    throw Error(
+      "Expected a FixedSizeList type to have an array-like `children` property",
+    );
+  }
+  if (typeLike.children.length !== 1) {
+    throw Error("Expected a FixedSizeList type to have exactly one child");
+  }
+  return new FixedSizeList(
+    typeLike.listSize,
+    sanitizeField(typeLike.children[0]),
+  );
+}
+
+function sanitizeMap(typeLike: object) {
+  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
+    throw Error(
+      "Expected a Map type to have an array-like `children` property",
+    );
+  }
+  if (!("keysSorted" in typeLike) || typeof typeLike.keysSorted !== "boolean") {
+    throw Error("Expected a Map type to have a `keysSorted` property");
+  }
+  return new Map_(
+    typeLike.children.map((field) => sanitizeField(field)) as any,
+    typeLike.keysSorted,
+  );
+}
+
+function sanitizeDuration(typeLike: object) {
+  if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
+    throw Error("Expected a Duration type to have a `unit` property");
+  }
+  return new Duration(typeLike.unit);
+}
+
+function sanitizeDictionary(typeLike: object) {
+  if (!("id" in typeLike) || typeof typeLike.id !== "number") {
+    throw Error("Expected a Dictionary type to have an `id` property");
+  }
+  if (!("indices" in typeLike) || typeof typeLike.indices !== "object") {
+    throw Error("Expected a Dictionary type to have an `indices` property");
+  }
+  if (!("dictionary" in typeLike) || typeof typeLike.dictionary !== "object") {
+    throw Error("Expected a Dictionary type to have an `dictionary` property");
+  }
+  if (!("isOrdered" in typeLike) || typeof typeLike.isOrdered !== "boolean") {
+    throw Error("Expected a Dictionary type to have an `isOrdered` property");
+  }
+  return new Dictionary(
+    sanitizeType(typeLike.dictionary),
+    sanitizeType(typeLike.indices) as any,
+    typeLike.id,
+    typeLike.isOrdered,
+  );
+}
+
+function sanitizeType(typeLike: unknown): DataType<any> {
+  if (typeof typeLike !== "object" || typeLike === null) {
+    throw Error("Expected a Type but object was null/undefined");
+  }
+  if (!("typeId" in typeLike) || !(typeof typeLike.typeId !== "function")) {
+    throw Error("Expected a Type to have a typeId function");
+  }
+  let typeId: Type;
+  if (typeof typeLike.typeId === "function") {
+    typeId = (typeLike.typeId as () => unknown)() as Type;
+  } else if (typeof typeLike.typeId === "number") {
+    typeId = typeLike.typeId as Type;
+  } else {
+    throw Error("Type's typeId property was not a function or number");
+  }
+
+  switch (typeId) {
+    case Type.NONE:
+      throw Error("Received a Type with a typeId of NONE");
+    case Type.Null:
+      return new Null();
+    case Type.Int:
+      return sanitizeInt(typeLike);
+    case Type.Float:
+      return sanitizeFloat(typeLike);
+    case Type.Binary:
+      return new Binary();
+    case Type.Utf8:
+      return new Utf8();
+    case Type.Bool:
+      return new Bool();
+    case Type.Decimal:
+      return sanitizeDecimal(typeLike);
+    case Type.Date:
+      return sanitizeDate(typeLike);
+    case Type.Time:
+      return sanitizeTime(typeLike);
+    case Type.Timestamp:
+      return sanitizeTimestamp(typeLike);
+    case Type.Interval:
+      return sanitizeInterval(typeLike);
+    case Type.List:
+      return sanitizeList(typeLike);
+    case Type.Struct:
+      return sanitizeStruct(typeLike);
+    case Type.Union:
+      return sanitizeUnion(typeLike);
+    case Type.FixedSizeBinary:
+      return sanitizeFixedSizeBinary(typeLike);
+    case Type.FixedSizeList:
+      return sanitizeFixedSizeList(typeLike);
+    case Type.Map:
+      return sanitizeMap(typeLike);
+    case Type.Duration:
+      return sanitizeDuration(typeLike);
+    case Type.Dictionary:
+      return sanitizeDictionary(typeLike);
+    case Type.Int8:
+      return new Int8();
+    case Type.Int16:
+      return new Int16();
+    case Type.Int32:
+      return new Int32();
+    case Type.Int64:
+      return new Int64();
+    case Type.Uint8:
+      return new Uint8();
+    case Type.Uint16:
+      return new Uint16();
+    case Type.Uint32:
+      return new Uint32();
+    case Type.Uint64:
+      return new Uint64();
+    case Type.Float16:
+      return new Float16();
+    case Type.Float32:
+      return new Float32();
+    case Type.Float64:
+      return new Float64();
+    case Type.DateMillisecond:
+      return new DateMillisecond();
+    case Type.DateDay:
+      return new DateDay();
+    case Type.TimeNanosecond:
+      return new TimeNanosecond();
+    case Type.TimeMicrosecond:
+      return new TimeMicrosecond();
+    case Type.TimeMillisecond:
+      return new TimeMillisecond();
+    case Type.TimeSecond:
+      return new TimeSecond();
+    case Type.TimestampNanosecond:
+      return sanitizeTypedTimestamp(typeLike, TimestampNanosecond);
+    case Type.TimestampMicrosecond:
+      return sanitizeTypedTimestamp(typeLike, TimestampMicrosecond);
+    case Type.TimestampMillisecond:
+      return sanitizeTypedTimestamp(typeLike, TimestampMillisecond);
+    case Type.TimestampSecond:
+      return sanitizeTypedTimestamp(typeLike, TimestampSecond);
+    case Type.DenseUnion:
+      return sanitizeTypedUnion(typeLike, DenseUnion);
+    case Type.SparseUnion:
+      return sanitizeTypedUnion(typeLike, SparseUnion);
+    case Type.IntervalDayTime:
+      return new IntervalDayTime();
+    case Type.IntervalYearMonth:
+      return new IntervalYearMonth();
+    case Type.DurationNanosecond:
+      return new DurationNanosecond();
+    case Type.DurationMicrosecond:
+      return new DurationMicrosecond();
+    case Type.DurationMillisecond:
+      return new DurationMillisecond();
+    case Type.DurationSecond:
+      return new DurationSecond();
+  }
+}
+
+function sanitizeField(fieldLike: unknown): Field {
+  if (fieldLike instanceof Field) {
+    return fieldLike;
+  }
+  if (typeof fieldLike !== "object" || fieldLike === null) {
+    throw Error("Expected a Field but object was null/undefined");
+  }
+  if (
+    !("type" in fieldLike) ||
+    !("name" in fieldLike) ||
+    !("nullable" in fieldLike)
+  ) {
+    throw Error(
+      "The field passed in is missing a `type`/`name`/`nullable` property",
+    );
+  }
+  const type = sanitizeType(fieldLike.type);
+  const name = fieldLike.name;
+  if (!(typeof name === "string")) {
+    throw Error("The field passed in had a non-string `name` property");
+  }
+  const nullable = fieldLike.nullable;
+  if (!(typeof nullable === "boolean")) {
+    throw Error("The field passed in had a non-boolean `nullable` property");
+  }
+  let metadata;
+  if ("metadata" in fieldLike) {
+    metadata = sanitizeMetadata(fieldLike.metadata);
+  }
+  return new Field(name, type, nullable, metadata);
+}
+
+/**
+ * Convert something schemaLike into a Schema instance
+ *
+ * This method is often needed even when the caller is using a Schema
+ * instance because they might be using a different instance of apache-arrow
+ * than lancedb is using.
+ */
+export function sanitizeSchema(schemaLike: unknown): Schema {
+  if (schemaLike instanceof Schema) {
+    return schemaLike;
+  }
+  if (typeof schemaLike !== "object" || schemaLike === null) {
+    throw Error("Expected a Schema but object was null/undefined");
+  }
+  if (!("fields" in schemaLike)) {
+    throw Error(
+      "The schema passed in does not appear to be a schema (no 'fields' property)",
+    );
+  }
+  let metadata;
+  if ("metadata" in schemaLike) {
+    metadata = sanitizeMetadata(schemaLike.metadata);
+  }
+  if (!Array.isArray(schemaLike.fields)) {
+    throw Error(
+      "The schema passed in had a 'fields' property but it was not an array",
+    );
+  }
+  const sanitizedFields = schemaLike.fields.map((field) =>
+    sanitizeField(field),
+  );
+  return new Schema(sanitizedFields, metadata);
+}

node/src/test/arrow.test.ts

@@ -34,8 +34,20 @@ import {
   List,
   DataType,
   Dictionary,
-  Int64
+  Int64,
+  MetadataVersion
 } from 'apache-arrow'
+import {
+  Dictionary as OldDictionary,
+  Field as OldField,
+  FixedSizeList as OldFixedSizeList,
+  Float32 as OldFloat32,
+  Int32 as OldInt32,
+  Struct as OldStruct,
+  Schema as OldSchema,
+  TimestampNanosecond as OldTimestampNanosecond,
+  Utf8 as OldUtf8
+} from 'apache-arrow-old'
 import { type EmbeddingFunction } from '../embedding/embedding_function'
 
 chaiUse(chaiAsPromised)
@@ -318,3 +330,31 @@ describe('makeEmptyTable', function () {
     await checkTableCreation(async (_, __, schema) => makeEmptyTable(schema))
   })
 })
+
+describe('when using two versions of arrow', function () {
+  it('can still import data', async function() {
+    const schema = new OldSchema([
+      new OldField('id', new OldInt32()),
+      new OldField('vector', new OldFixedSizeList(1024, new OldField("item", new OldFloat32(), true))),
+      new OldField('struct', new OldStruct([
+        new OldField('nested', new OldDictionary(new OldUtf8(), new OldInt32(), 1, true)),
+        new OldField('ts_with_tz', new OldTimestampNanosecond("some_tz")),
+        new OldField('ts_no_tz', new OldTimestampNanosecond(null))
+      ]))
+    ]) as any
+    // We use arrow version 13 to emulate a "foreign arrow" and this version doesn't have metadataVersion
+    // In theory, this wouldn't matter.  We don't rely on that property.  However, it causes deepEqual to
+    // fail so we patch it back in
+    schema.metadataVersion = MetadataVersion.V5
+    const table = makeArrowTable(
+      [],
+      { schema }
+    )
+
+    const buf = await fromTableToBuffer(table)
+    assert.isAbove(buf.byteLength, 0)
+    const actual = tableFromIPC(buf)
+    const actualSchema = actual.schema
+    assert.deepEqual(actualSchema, schema)
+  })
+})

node/src/test/test.ts

@@ -37,8 +37,10 @@ import {
   Utf8,
   Table as ArrowTable,
   vectorFromArray,
+  Float64,
   Float32,
-  Float16
+  Float16,
+  Int64
 } from 'apache-arrow'
 
 const expect = chai.expect
@@ -126,6 +128,11 @@ describe('LanceDB client', function () {
       assertResults(results)
       results = await table.where('id % 2 = 0').execute()
       assertResults(results)
+
+      // Should reject a bad filter
+      await expect(table.filter('id % 2 = 0 AND').execute()).to.be.rejectedWith(
+        /.*sql parser error: Expected an expression:, found: EOF.*/
+      )
     })
 
     it('uses a filter / where clause', async function () {
@@ -196,7 +203,7 @@ describe('LanceDB client', function () {
       const table = await con.openTable('vectors')
       const results = await table
         .search([0.1, 0.1])
-        .select(['is_active'])
+        .select(['is_active', 'vector'])
         .execute()
       assert.equal(results.length, 2)
       // vector and _distance are always returned
@@ -281,7 +288,8 @@ describe('LanceDB client', function () {
 
     it('create a table from an Arrow Table', async function () {
       const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
+      // Also test the connect function with an object
+      const con = await lancedb.connect({ uri: dir })
 
       const i32s = new Int32Array(new Array<number>(10))
       const i32 = makeVector(i32s)
@@ -743,11 +751,11 @@ describe('LanceDB client', function () {
         num_sub_vectors: 2
       })
       await expect(createIndex).to.be.rejectedWith(
-        /VectorIndex requires the column data type to be fixed size list of float32s/
+        "index cannot be created on the column `name` which has data type Utf8"
       )
     })
 
-    it('it should fail when the column is not a vector', async function () {
+    it('it should fail when num_partitions is invalid', async function () {
       const uri = await createTestDB(32, 300)
       const con = await lancedb.connect(uri)
       const table = await con.openTable('vectors')
@@ -1057,3 +1065,63 @@ describe('Compact and cleanup', function () {
     assert.equal(await table.countRows(), 3)
   })
 })
+
+describe('schema evolution', function () {
+  // Create a new sample table
+  it('can add a new column to the schema', async function () {
+    const dir = await track().mkdir('lancejs')
+    const con = await lancedb.connect(dir)
+    const table = await con.createTable('vectors', [
+      { id: 1n, vector: [0.1, 0.2] }
+    ])
+
+    await table.addColumns([{ name: 'price', valueSql: 'cast(10.0 as float)' }])
+
+    const expectedSchema = new Schema([
+      new Field('id', new Int64()),
+      new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true))),
+      new Field('price', new Float32())
+    ])
+    expect(await table.schema).to.deep.equal(expectedSchema)
+  })
+
+  it('can alter the columns in the schema', async function () {
+    const dir = await track().mkdir('lancejs')
+    const con = await lancedb.connect(dir)
+    const schema = new Schema([
+      new Field('id', new Int64(), false),
+      new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true))),
+      new Field('price', new Float64(), false)
+    ])
+    const table = await con.createTable('vectors', [
+      { id: 1n, vector: [0.1, 0.2], price: 10.0 }
+    ])
+    expect(await table.schema).to.deep.equal(schema)
+
+    await table.alterColumns([
+      { path: 'id', rename: 'new_id' },
+      { path: 'price', nullable: true }
+    ])
+
+    const expectedSchema = new Schema([
+      new Field('new_id', new Int64(), false),
+      new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true))),
+      new Field('price', new Float64(), true)
+    ])
+    expect(await table.schema).to.deep.equal(expectedSchema)
+  })
+
+  it('can drop a column from the schema', async function () {
+    const dir = await track().mkdir('lancejs')
+    const con = await lancedb.connect(dir)
+    const table = await con.createTable('vectors', [
+      { id: 1n, vector: [0.1, 0.2] }
+    ])
+    await table.dropColumns(['vector'])
+
+    const expectedSchema = new Schema([
+      new Field('id', new Int64(), false)
+    ])
+    expect(await table.schema).to.deep.equal(expectedSchema)
+  })
+})

nodejs/.eslintignore

@@ -0,0 +1,3 @@
+**/dist/**/*
+**/native.js
+**/native.d.ts

nodejs/.eslintrc.js

@@ -1,22 +0,0 @@
-module.exports = {
-  env: {
-    browser: true,
-    es2021: true,
-  },
-  extends: [
-    "eslint:recommended",
-    "plugin:@typescript-eslint/recommended-type-checked",
-    "plugin:@typescript-eslint/stylistic-type-checked",
-  ],
-  overrides: [],
-  parserOptions: {
-    project: "./tsconfig.json",
-    ecmaVersion: "latest",
-    sourceType: "module",
-  },
-  rules: {
-    "@typescript-eslint/method-signature-style": "off",
-    "@typescript-eslint/no-explicit-any": "off",
-  },
-  ignorePatterns: ["node_modules/", "dist/", "build/", "vectordb/native.*"],
-};

nodejs/.prettierignore

@@ -0,0 +1 @@
+.eslintignore

nodejs/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "vectordb-nodejs"
+name = "lancedb-nodejs"
 edition.workspace = true
 version = "0.0.0"
 license.workspace = true
@@ -14,12 +14,10 @@ crate-type = ["cdylib"]
 [dependencies]
 arrow-ipc.workspace = true
 futures.workspace = true
-lance-linalg.workspace = true
-lance.workspace = true
-vectordb = { path = "../rust/vectordb" }
+lancedb = { path = "../rust/lancedb" }
 napi = { version = "2.15", default-features = false, features = [
     "napi7",
-    "async"
+    "async",
 ] }
 napi-derive = "2"
 

nodejs/README.md

@@ -2,7 +2,6 @@
 
 It will replace the NodeJS SDK when it is ready.
 
-
 ## Development
 
 ```sh
@@ -10,9 +9,35 @@ npm run build
 npm t
 ```
 
-Generating docs
+### Running lint / format
 
+LanceDb uses eslint for linting. VSCode does not need any plugins to use eslint. However, it
+may need some additional configuration. Make sure that eslint.experimental.useFlatConfig is
+set to true. Also, if your vscode root folder is the repo root then you will need to set
+the eslint.workingDirectories to ["nodejs"]. To manually lint your code you can run:
+
+```sh
+npm run lint
 ```
+
+LanceDb uses prettier for formatting. If you are using VSCode you will need to install the
+"Prettier - Code formatter" extension. You should then configure it to be the default formatter
+for typescript and you should enable format on save. To manually check your code's format you
+can run:
+
+```sh
+npm run chkformat
+```
+
+If you need to manually format your code you can run:
+
+```sh
+npx prettier --write .
+```
+
+### Generating docs
+
+```sh
 npm run docs
 
 cd ../docs

nodejs/__test__/arrow.test.ts

@@ -12,9 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { makeArrowTable, toBuffer } from "../vectordb/arrow";
 import {
-  Int64,
+  convertToTable,
+  fromTableToBuffer,
+  makeArrowTable,
+  makeEmptyTable,
+} from "../dist/arrow";
+import {
   Field,
   FixedSizeList,
   Float16,
@@ -23,43 +27,137 @@ import {
   tableFromIPC,
   Schema,
   Float64,
+  type Table,
+  Binary,
+  Bool,
+  Utf8,
+  Struct,
+  List,
+  DataType,
+  Dictionary,
+  Int64,
+  Float,
+  Precision,
+  MetadataVersion,
 } from "apache-arrow";
+import {
+  Dictionary as OldDictionary,
+  Field as OldField,
+  FixedSizeList as OldFixedSizeList,
+  Float32 as OldFloat32,
+  Int32 as OldInt32,
+  Struct as OldStruct,
+  Schema as OldSchema,
+  TimestampNanosecond as OldTimestampNanosecond,
+  Utf8 as OldUtf8,
+} from "apache-arrow-old";
+import { type EmbeddingFunction } from "../dist/embedding/embedding_function";
 
-test("customized schema", function () {
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function sampleRecords(): Array<Record<string, any>> {
+  return [
+    {
+      binary: Buffer.alloc(5),
+      boolean: false,
+      number: 7,
+      string: "hello",
+      struct: { x: 0, y: 0 },
+      list: ["anime", "action", "comedy"],
+    },
+  ];
+}
+
+// Helper method to verify various ways to create a table
+async function checkTableCreation(
+  tableCreationMethod: (
+    records: Record<string, unknown>[],
+    recordsReversed: Record<string, unknown>[],
+    schema: Schema,
+  ) => Promise<Table>,
+  infersTypes: boolean,
+): Promise<void> {
+  const records = sampleRecords();
+  const recordsReversed = [
+    {
+      list: ["anime", "action", "comedy"],
+      struct: { x: 0, y: 0 },
+      string: "hello",
+      number: 7,
+      boolean: false,
+      binary: Buffer.alloc(5),
+    },
+  ];
   const schema = new Schema([
-    new Field("a", new Int32(), true),
-    new Field("b", new Float32(), true),
+    new Field("binary", new Binary(), false),
+    new Field("boolean", new Bool(), false),
+    new Field("number", new Float64(), false),
+    new Field("string", new Utf8(), false),
     new Field(
-      "c",
-      new FixedSizeList(3, new Field("item", new Float16())),
-      true
+      "struct",
+      new Struct([
+        new Field("x", new Float64(), false),
+        new Field("y", new Float64(), false),
+      ]),
     ),
+    new Field("list", new List(new Field("item", new Utf8(), false)), false),
+  ]);
+
+  const table = await tableCreationMethod(records, recordsReversed, schema);
+  schema.fields.forEach((field, idx) => {
+    const actualField = table.schema.fields[idx];
+    // Type inference always assumes nullable=true
+    if (infersTypes) {
+      expect(actualField.nullable).toBe(true);
+    } else {
+      expect(actualField.nullable).toBe(false);
+    }
+    expect(table.getChild(field.name)?.type.toString()).toEqual(
+      field.type.toString(),
+    );
+    expect(table.getChildAt(idx)?.type.toString()).toEqual(
+      field.type.toString(),
+    );
+  });
+}
+
+describe("The function makeArrowTable", function () {
+  it("will use data types from a provided schema instead of inference", async function () {
+    const schema = new Schema([
+      new Field("a", new Int32()),
+      new Field("b", new Float32()),
+      new Field("c", new FixedSizeList(3, new Field("item", new Float16()))),
+      new Field("d", new Int64()),
     ]);
     const table = makeArrowTable(
       [
-      { a: 1, b: 2, c: [1, 2, 3] },
-      { a: 4, b: 5, c: [4, 5, 6] },
-      { a: 7, b: 8, c: [7, 8, 9] },
+        { a: 1, b: 2, c: [1, 2, 3], d: 9 },
+        { a: 4, b: 5, c: [4, 5, 6], d: 10 },
+        { a: 7, b: 8, c: [7, 8, 9], d: null },
       ],
-    { schema }
+      { schema },
     );
 
-  expect(table.schema.toString()).toEqual(schema.toString());
-
-  const buf = toBuffer(table);
+    const buf = await fromTableToBuffer(table);
     expect(buf.byteLength).toBeGreaterThan(0);
 
     const actual = tableFromIPC(buf);
     expect(actual.numRows).toBe(3);
     const actualSchema = actual.schema;
-  expect(actualSchema.toString()).toStrictEqual(schema.toString());
-});
+    expect(actualSchema).toEqual(schema);
+  });
 
-test("default vector column", function () {
+  it("will assume the column `vector` is FixedSizeList<Float32> by default", async function () {
     const schema = new Schema([
-    new Field("a", new Float64(), true),
-    new Field("b", new Float64(), true),
-    new Field("vector", new FixedSizeList(3, new Field("item", new Float32()))),
+      new Field("a", new Float(Precision.DOUBLE), true),
+      new Field("b", new Float(Precision.DOUBLE), true),
+      new Field(
+        "vector",
+        new FixedSizeList(
+          3,
+          new Field("item", new Float(Precision.SINGLE), true),
+        ),
+        true,
+      ),
     ]);
     const table = makeArrowTable([
       { a: 1, b: 2, vector: [1, 2, 3] },
@@ -67,21 +165,29 @@ test("default vector column", function () {
       { a: 7, b: 8, vector: [7, 8, 9] },
     ]);
 
-  const buf = toBuffer(table);
+    const buf = await fromTableToBuffer(table);
     expect(buf.byteLength).toBeGreaterThan(0);
 
     const actual = tableFromIPC(buf);
     expect(actual.numRows).toBe(3);
     const actualSchema = actual.schema;
-  expect(actualSchema.toString()).toEqual(actualSchema.toString());
-});
+    expect(actualSchema).toEqual(schema);
+  });
 
-test("2 vector columns", function () {
+  it("can support multiple vector columns", async function () {
     const schema = new Schema([
-    new Field("a", new Float64()),
-    new Field("b", new Float64()),
-    new Field("vec1", new FixedSizeList(3, new Field("item", new Float16()))),
-    new Field("vec2", new FixedSizeList(3, new Field("item", new Float16()))),
+      new Field("a", new Float(Precision.DOUBLE), true),
+      new Field("b", new Float(Precision.DOUBLE), true),
+      new Field(
+        "vec1",
+        new FixedSizeList(3, new Field("item", new Float16(), true)),
+        true,
+      ),
+      new Field(
+        "vec2",
+        new FixedSizeList(3, new Field("item", new Float16(), true)),
+        true,
+      ),
     ]);
     const table = makeArrowTable(
       [
@@ -94,27 +200,271 @@ test("2 vector columns", function () {
           vec1: { type: new Float16() },
           vec2: { type: new Float16() },
         },
-    }
+      },
     );
 
-  const buf = toBuffer(table);
+    const buf = await fromTableToBuffer(table);
     expect(buf.byteLength).toBeGreaterThan(0);
 
     const actual = tableFromIPC(buf);
     expect(actual.numRows).toBe(3);
     const actualSchema = actual.schema;
-  expect(actualSchema.toString()).toEqual(schema.toString());
+    expect(actualSchema).toEqual(schema);
+  });
+
+  it("will allow different vector column types", async function () {
+    const table = makeArrowTable([{ fp16: [1], fp32: [1], fp64: [1] }], {
+      vectorColumns: {
+        fp16: { type: new Float16() },
+        fp32: { type: new Float32() },
+        fp64: { type: new Float64() },
+      },
+    });
+
+    expect(table.getChild("fp16")?.type.children[0].type.toString()).toEqual(
+      new Float16().toString(),
+    );
+    expect(table.getChild("fp32")?.type.children[0].type.toString()).toEqual(
+      new Float32().toString(),
+    );
+    expect(table.getChild("fp64")?.type.children[0].type.toString()).toEqual(
+      new Float64().toString(),
+    );
+  });
+
+  it("will use dictionary encoded strings if asked", async function () {
+    const table = makeArrowTable([{ str: "hello" }]);
+    expect(DataType.isUtf8(table.getChild("str")?.type)).toBe(true);
+
+    const tableWithDict = makeArrowTable([{ str: "hello" }], {
+      dictionaryEncodeStrings: true,
+    });
+    expect(DataType.isDictionary(tableWithDict.getChild("str")?.type)).toBe(
+      true,
+    );
+
+    const schema = new Schema([
+      new Field("str", new Dictionary(new Utf8(), new Int32())),
+    ]);
+
+    const tableWithDict2 = makeArrowTable([{ str: "hello" }], { schema });
+    expect(DataType.isDictionary(tableWithDict2.getChild("str")?.type)).toBe(
+      true,
+    );
+  });
+
+  it("will infer data types correctly", async function () {
+    await checkTableCreation(async (records) => makeArrowTable(records), true);
+  });
+
+  it("will allow a schema to be provided", async function () {
+    await checkTableCreation(
+      async (records, _, schema) => makeArrowTable(records, { schema }),
+      false,
+    );
+  });
+
+  it("will use the field order of any provided schema", async function () {
+    await checkTableCreation(
+      async (_, recordsReversed, schema) =>
+        makeArrowTable(recordsReversed, { schema }),
+      false,
+    );
+  });
+
+  it("will make an empty table", async function () {
+    await checkTableCreation(
+      async (_, __, schema) => makeArrowTable([], { schema }),
+      false,
+    );
+  });
 });
 
-test("handles int64", function() {
-  // https://github.com/lancedb/lancedb/issues/960
-  const schema = new Schema([
-    new Field("x", new Int64(), true)
+class DummyEmbedding implements EmbeddingFunction<string> {
+  public readonly sourceColumn = "string";
+  public readonly embeddingDimension = 2;
+  public readonly embeddingDataType = new Float16();
+
+  async embed(data: string[]): Promise<number[][]> {
+    return data.map(() => [0.0, 0.0]);
+  }
+}
+
+class DummyEmbeddingWithNoDimension implements EmbeddingFunction<string> {
+  public readonly sourceColumn = "string";
+
+  async embed(data: string[]): Promise<number[][]> {
+    return data.map(() => [0.0, 0.0]);
+  }
+}
+
+describe("convertToTable", function () {
+  it("will infer data types correctly", async function () {
+    await checkTableCreation(
+      async (records) => await convertToTable(records),
+      true,
+    );
+  });
+
+  it("will allow a schema to be provided", async function () {
+    await checkTableCreation(
+      async (records, _, schema) =>
+        await convertToTable(records, undefined, { schema }),
+      false,
+    );
+  });
+
+  it("will use the field order of any provided schema", async function () {
+    await checkTableCreation(
+      async (_, recordsReversed, schema) =>
+        await convertToTable(recordsReversed, undefined, { schema }),
+      false,
+    );
+  });
+
+  it("will make an empty table", async function () {
+    await checkTableCreation(
+      async (_, __, schema) => await convertToTable([], undefined, { schema }),
+      false,
+    );
+  });
+
+  it("will apply embeddings", async function () {
+    const records = sampleRecords();
+    const table = await convertToTable(records, new DummyEmbedding());
+    expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe(true);
+    expect(table.getChild("vector")?.type.children[0].type.toString()).toEqual(
+      new Float16().toString(),
+    );
+  });
+
+  it("will fail if missing the embedding source column", async function () {
+    await expect(
+      convertToTable([{ id: 1 }], new DummyEmbedding()),
+    ).rejects.toThrow("'string' was not present");
+  });
+
+  it("use embeddingDimension if embedding missing from table", async function () {
+    const schema = new Schema([new Field("string", new Utf8(), false)]);
+    // Simulate getting an empty Arrow table (minus embedding) from some other source
+    // In other words, we aren't starting with records
+    const table = makeEmptyTable(schema);
+
+    // If the embedding specifies the dimension we are fine
+    await fromTableToBuffer(table, new DummyEmbedding());
+
+    // We can also supply a schema and should be ok
+    const schemaWithEmbedding = new Schema([
+      new Field("string", new Utf8(), false),
+      new Field(
+        "vector",
+        new FixedSizeList(2, new Field("item", new Float16(), false)),
+        false,
+      ),
     ]);
-  const table = makeArrowTable([
-    { x: 1 },
-    { x: 2 },
-    { x: 3 }
-  ], { schema });
-  expect(table.schema).toEqual(schema);
-})
+    await fromTableToBuffer(
+      table,
+      new DummyEmbeddingWithNoDimension(),
+      schemaWithEmbedding,
+    );
+
+    // Otherwise we will get an error
+    await expect(
+      fromTableToBuffer(table, new DummyEmbeddingWithNoDimension()),
+    ).rejects.toThrow("does not specify `embeddingDimension`");
+  });
+
+  it("will apply embeddings to an empty table", async function () {
+    const schema = new Schema([
+      new Field("string", new Utf8(), false),
+      new Field(
+        "vector",
+        new FixedSizeList(2, new Field("item", new Float16(), false)),
+        false,
+      ),
+    ]);
+    const table = await convertToTable([], new DummyEmbedding(), { schema });
+    expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe(true);
+    expect(table.getChild("vector")?.type.children[0].type.toString()).toEqual(
+      new Float16().toString(),
+    );
+  });
+
+  it("will complain if embeddings present but schema missing embedding column", async function () {
+    const schema = new Schema([new Field("string", new Utf8(), false)]);
+    await expect(
+      convertToTable([], new DummyEmbedding(), { schema }),
+    ).rejects.toThrow("column vector was missing");
+  });
+
+  it("will provide a nice error if run twice", async function () {
+    const records = sampleRecords();
+    const table = await convertToTable(records, new DummyEmbedding());
+    // fromTableToBuffer will try and apply the embeddings again
+    await expect(
+      fromTableToBuffer(table, new DummyEmbedding()),
+    ).rejects.toThrow("already existed");
+  });
+});
+
+describe("makeEmptyTable", function () {
+  it("will make an empty table", async function () {
+    await checkTableCreation(
+      async (_, __, schema) => makeEmptyTable(schema),
+      false,
+    );
+  });
+});
+
+describe("when using two versions of arrow", function () {
+  it("can still import data", async function () {
+    const schema = new OldSchema([
+      new OldField("id", new OldInt32()),
+      new OldField(
+        "vector",
+        new OldFixedSizeList(
+          1024,
+          new OldField("item", new OldFloat32(), true),
+        ),
+      ),
+      new OldField(
+        "struct",
+        new OldStruct([
+          new OldField(
+            "nested",
+            new OldDictionary(new OldUtf8(), new OldInt32(), 1, true),
+          ),
+          new OldField("ts_with_tz", new OldTimestampNanosecond("some_tz")),
+          new OldField("ts_no_tz", new OldTimestampNanosecond(null)),
+        ]),
+      ),
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    ]) as any;
+    schema.metadataVersion = MetadataVersion.V5;
+    const table = makeArrowTable([], { schema });
+
+    const buf = await fromTableToBuffer(table);
+    expect(buf.byteLength).toBeGreaterThan(0);
+    const actual = tableFromIPC(buf);
+    const actualSchema = actual.schema;
+    expect(actualSchema.fields.length).toBe(3);
+
+    // Deep equality gets hung up on some very minor unimportant differences
+    // between arrow version 13 and 15 which isn't really what we're testing for
+    // and so we do our own comparison that just checks name/type/nullability
+    function compareFields(lhs: Field, rhs: Field) {
+      expect(lhs.name).toEqual(rhs.name);
+      expect(lhs.nullable).toEqual(rhs.nullable);
+      expect(lhs.typeId).toEqual(rhs.typeId);
+      if ("children" in lhs.type && lhs.type.children !== null) {
+        const lhsChildren = lhs.type.children as Field[];
+        lhsChildren.forEach((child: Field, idx) => {
+          compareFields(child, rhs.type.children[idx]);
+        });
+      }
+    }
+    actualSchema.fields.forEach((field, idx) => {
+      compareFields(field, actualSchema.fields[idx]);
+    });
+  });
+});

nodejs/__test__/connection.test.ts

@@ -0,0 +1,88 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import * as tmp from "tmp";
+
+import { Connection, connect } from "../dist/index.js";
+
+describe("when connecting", () => {
+  let tmpDir: tmp.DirResult;
+  beforeEach(() => (tmpDir = tmp.dirSync({ unsafeCleanup: true })));
+  afterEach(() => tmpDir.removeCallback());
+
+  it("should connect", async () => {
+    const db = await connect(tmpDir.name);
+    expect(db.display()).toBe(
+      `NativeDatabase(uri=${tmpDir.name}, read_consistency_interval=None)`,
+    );
+  });
+
+  it("should allow read consistency interval to be specified", async () => {
+    const db = await connect(tmpDir.name, { readConsistencyInterval: 5 });
+    expect(db.display()).toBe(
+      `NativeDatabase(uri=${tmpDir.name}, read_consistency_interval=5s)`,
+    );
+  });
+});
+
+describe("given a connection", () => {
+  let tmpDir: tmp.DirResult;
+  let db: Connection;
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    db = await connect(tmpDir.name);
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  it("should raise an error if opening a non-existent table", async () => {
+    await expect(db.openTable("non-existent")).rejects.toThrow("was not found");
+  });
+
+  it("should raise an error if any operation is tried after it is closed", async () => {
+    expect(db.isOpen()).toBe(true);
+    await db.close();
+    expect(db.isOpen()).toBe(false);
+    await expect(db.tableNames()).rejects.toThrow("Connection is closed");
+  });
+
+  it("should fail if creating table twice, unless overwrite is true", async () => {
+    let tbl = await db.createTable("test", [{ id: 1 }, { id: 2 }]);
+    await expect(tbl.countRows()).resolves.toBe(2);
+    await expect(
+      db.createTable("test", [{ id: 1 }, { id: 2 }]),
+    ).rejects.toThrow();
+    tbl = await db.createTable("test", [{ id: 3 }], { mode: "overwrite" });
+    await expect(tbl.countRows()).resolves.toBe(1);
+  });
+
+  it("should respect limit and page token when listing tables", async () => {
+    const db = await connect(tmpDir.name);
+
+    await db.createTable("b", [{ id: 1 }]);
+    await db.createTable("a", [{ id: 1 }]);
+    await db.createTable("c", [{ id: 1 }]);
+
+    let tables = await db.tableNames();
+    expect(tables).toEqual(["a", "b", "c"]);
+
+    tables = await db.tableNames({ limit: 1 });
+    expect(tables).toEqual(["a"]);
+
+    tables = await db.tableNames({ limit: 1, startAfter: "a" });
+    expect(tables).toEqual(["b"]);
+
+    tables = await db.tableNames({ startAfter: "a" });
+    expect(tables).toEqual(["b", "c"]);
+  });
+});

nodejs/__test__/index.test.ts

@@ -1,34 +0,0 @@
-// Copyright 2024 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import * as os from "os";
-import * as path from "path";
-import * as fs from "fs";
-
-import { Schema, Field, Float64 } from "apache-arrow";
-import { connect } from "../dist/index.js";
-
-test("open database", async () => {
-  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "test-open"));
-
-  const db = await connect(tmpDir);
-  let tableNames = await db.tableNames();
-  expect(tableNames).toStrictEqual([]);
-
-  const tbl = await db.createTable("test", [{ id: 1 }, { id: 2 }]);
-  expect(await db.tableNames()).toStrictEqual(["test"]);
-
-  const schema = tbl.schema;
-  expect(schema).toEqual(new Schema([new Field("id", new Float64(), true)]));
-});

nodejs/__test__/table.test.ts

@@ -12,27 +12,91 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import * as os from "os";
-import * as path from "path";
 import * as fs from "fs";
+import * as path from "path";
+import * as tmp from "tmp";
 
-import { connect } from "../dist";
-import { Schema, Field, Float32, Int32, FixedSizeList } from "apache-arrow";
+import { Table, connect } from "../dist";
+import {
+  Schema,
+  Field,
+  Float32,
+  Int32,
+  FixedSizeList,
+  Int64,
+  Float64,
+} from "apache-arrow";
 import { makeArrowTable } from "../dist/arrow";
+import { Index } from "../dist/indices";
 
-describe("Test creating index", () => {
-  let tmpDir: string;
+describe("Given a table", () => {
+  let tmpDir: tmp.DirResult;
+  let table: Table;
+  const schema = new Schema([new Field("id", new Float64(), true)]);
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    const conn = await connect(tmpDir.name);
+    table = await conn.createEmptyTable("some_table", schema);
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  it("be displayable", async () => {
+    expect(table.display()).toMatch(
+      /NativeTable\(some_table, uri=.*, read_consistency_interval=None\)/,
+    );
+    table.close();
+    expect(table.display()).toBe("ClosedTable(some_table)");
+  });
+
+  it("should let me add data", async () => {
+    await table.add([{ id: 1 }, { id: 2 }]);
+    await table.add([{ id: 1 }]);
+    await expect(table.countRows()).resolves.toBe(3);
+  });
+
+  it("should overwrite data if asked", async () => {
+    await table.add([{ id: 1 }, { id: 2 }]);
+    await table.add([{ id: 1 }], { mode: "overwrite" });
+    await expect(table.countRows()).resolves.toBe(1);
+  });
+
+  it("should let me close the table", async () => {
+    expect(table.isOpen()).toBe(true);
+    table.close();
+    expect(table.isOpen()).toBe(false);
+    expect(table.countRows()).rejects.toThrow("Table some_table is closed");
+  });
+
+  it("should let me update values", async () => {
+    await table.add([{ id: 1 }]);
+    expect(await table.countRows("id == 1")).toBe(1);
+    expect(await table.countRows("id == 7")).toBe(0);
+    await table.update({ id: "7" });
+    expect(await table.countRows("id == 1")).toBe(0);
+    expect(await table.countRows("id == 7")).toBe(1);
+    await table.add([{ id: 2 }]);
+    // Test Map as input
+    await table.update(new Map(Object.entries({ id: "10" })), {
+      where: "id % 2 == 0",
+    });
+    expect(await table.countRows("id == 2")).toBe(0);
+    expect(await table.countRows("id == 7")).toBe(1);
+    expect(await table.countRows("id == 10")).toBe(1);
+  });
+});
+
+describe("When creating an index", () => {
+  let tmpDir: tmp.DirResult;
   const schema = new Schema([
     new Field("id", new Int32(), true),
     new Field("vec", new FixedSizeList(32, new Field("item", new Float32()))),
   ]);
+  let tbl: Table;
+  let queryVec: number[];
 
-  beforeEach(() => {
-    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "index-"));
-  });
-
-  test("create vector index with no column", async () => {
-    const db = await connect(tmpDir);
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    const db = await connect(tmpDir.name);
     const data = makeArrowTable(
       Array(300)
         .fill(1)
@@ -44,57 +108,90 @@ describe("Test creating index", () => {
         })),
       {
         schema,
-      }
+      },
     );
-    const tbl = await db.createTable("test", data);
-    await tbl.createIndex().build();
+    queryVec = data.toArray()[5].vec.toJSON();
+    tbl = await db.createTable("test", data);
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  it("should create a vector index on vector columns", async () => {
+    await tbl.createIndex("vec");
 
     // check index directory
-    const indexDir = path.join(tmpDir, "test.lance", "_indices");
+    const indexDir = path.join(tmpDir.name, "test.lance", "_indices");
     expect(fs.readdirSync(indexDir)).toHaveLength(1);
-    // TODO: check index type.
+    const indices = await tbl.listIndices();
+    expect(indices.length).toBe(1);
+    expect(indices[0]).toEqual({
+      indexType: "IvfPq",
+      columns: ["vec"],
+    });
 
     // Search without specifying the column
-    let query_vector = data.toArray()[5].vec.toJSON();
-    let rst = await tbl.query().nearestTo(query_vector).limit(2).toArrow();
+    let rst = await tbl
+      .query()
+      .limit(2)
+      .nearestTo(queryVec)
+      .distanceType("DoT")
+      .toArrow();
+    expect(rst.numRows).toBe(2);
+
+    // Search using `vectorSearch`
+    rst = await tbl.vectorSearch(queryVec).limit(2).toArrow();
     expect(rst.numRows).toBe(2);
 
     // Search with specifying the column
-    let rst2 = await tbl.search(query_vector, "vec").limit(2).toArrow();
+    const rst2 = await tbl
+      .query()
+      .limit(2)
+      .nearestTo(queryVec)
+      .column("vec")
+      .toArrow();
     expect(rst2.numRows).toBe(2);
     expect(rst.toString()).toEqual(rst2.toString());
   });
 
-  test("no vector column available", async () => {
-    const db = await connect(tmpDir);
-    const tbl = await db.createTable(
-      "no_vec",
-      makeArrowTable([
-        { id: 1, val: 2 },
-        { id: 2, val: 3 },
-      ])
-    );
-    await expect(tbl.createIndex().build()).rejects.toThrow(
-      "No vector column found"
-    );
+  it("should allow parameters to be specified", async () => {
+    await tbl.createIndex("vec", {
+      config: Index.ivfPq({
+        numPartitions: 10,
+      }),
+    });
 
-    await tbl.createIndex("val").build();
-    const indexDir = path.join(tmpDir, "no_vec.lance", "_indices");
+    // TODO: Verify parameters when we can load index config as part of list indices
+  });
+
+  it("should allow me to replace (or not) an existing index", async () => {
+    await tbl.createIndex("id");
+    // Default is replace=true
+    await tbl.createIndex("id");
+    await expect(tbl.createIndex("id", { replace: false })).rejects.toThrow(
+      "already exists",
+    );
+    await tbl.createIndex("id", { replace: true });
+  });
+
+  test("should create a scalar index on scalar columns", async () => {
+    await tbl.createIndex("id");
+    const indexDir = path.join(tmpDir.name, "test.lance", "_indices");
     expect(fs.readdirSync(indexDir)).toHaveLength(1);
 
-    for await (const r of tbl.query().filter("id > 1").select(["id"])) {
-      expect(r.numRows).toBe(1);
+    for await (const r of tbl.query().where("id > 1").select(["id"])) {
+      expect(r.numRows).toBe(298);
     }
   });
 
+  // TODO: Move this test to the query API test (making sure we can reject queries
+  // when the dimension is incorrect)
   test("two columns with different dimensions", async () => {
-    const db = await connect(tmpDir);
+    const db = await connect(tmpDir.name);
     const schema = new Schema([
       new Field("id", new Int32(), true),
       new Field("vec", new FixedSizeList(32, new Field("item", new Float32()))),
       new Field(
         "vec2",
-        new FixedSizeList(64, new Field("item", new Float32()))
+        new FixedSizeList(64, new Field("item", new Float32())),
       ),
     ]);
     const tbl = await db.createTable(
@@ -111,73 +208,214 @@ describe("Test creating index", () => {
               .fill(1)
               .map(() => Math.random()),
           })),
-        { schema }
-      )
+        { schema },
+      ),
     );
 
     // Only build index over v1
-    await expect(tbl.createIndex().build()).rejects.toThrow(
-      /.*More than one vector columns found.*/
-    );
-    tbl
-      .createIndex("vec")
-      .ivf_pq({ num_partitions: 2, num_sub_vectors: 2 })
-      .build();
+    await tbl.createIndex("vec", {
+      config: Index.ivfPq({ numPartitions: 2, numSubVectors: 2 }),
+    });
 
     const rst = await tbl
       .query()
+      .limit(2)
       .nearestTo(
         Array(32)
           .fill(1)
-          .map(() => Math.random())
+          .map(() => Math.random()),
       )
-      .limit(2)
       .toArrow();
     expect(rst.numRows).toBe(2);
 
     // Search with specifying the column
     await expect(
       tbl
-        .search(
+        .query()
+        .limit(2)
+        .nearestTo(
           Array(64)
             .fill(1)
             .map(() => Math.random()),
-          "vec"
         )
-        .limit(2)
-        .toArrow()
-    ).rejects.toThrow(/.*does not match the dimension.*/);
+        .column("vec")
+        .toArrow(),
+    ).rejects.toThrow(/.* query dim=64, expected vector dim=32.*/);
 
     const query64 = Array(64)
       .fill(1)
       .map(() => Math.random());
-    const rst64_1 = await tbl.query().nearestTo(query64).limit(2).toArrow();
-    const rst64_2 = await tbl.search(query64, "vec2").limit(2).toArrow();
-    expect(rst64_1.toString()).toEqual(rst64_2.toString());
-    expect(rst64_1.numRows).toBe(2);
-  });
-
-  test("create scalar index", async () => {
-    const db = await connect(tmpDir);
-    const data = makeArrowTable(
-      Array(300)
-        .fill(1)
-        .map((_, i) => ({
-          id: i,
-          vec: Array(32)
-            .fill(1)
-            .map(() => Math.random()),
-        })),
-      {
-        schema,
-      }
-    );
-    const tbl = await db.createTable("test", data);
-    await tbl.createIndex("id").build();
-
-    // check index directory
-    const indexDir = path.join(tmpDir, "test.lance", "_indices");
-    expect(fs.readdirSync(indexDir)).toHaveLength(1);
-    // TODO: check index type.
+    const rst64Query = await tbl.query().limit(2).nearestTo(query64).toArrow();
+    const rst64Search = await tbl
+      .query()
+      .limit(2)
+      .nearestTo(query64)
+      .column("vec2")
+      .toArrow();
+    expect(rst64Query.toString()).toEqual(rst64Search.toString());
+    expect(rst64Query.numRows).toBe(2);
+  });
+});
+
+describe("Read consistency interval", () => {
+  let tmpDir: tmp.DirResult;
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  // const intervals = [undefined, 0, 0.1];
+  const intervals = [0];
+  test.each(intervals)("read consistency interval %p", async (interval) => {
+    const db = await connect(tmpDir.name);
+    const table = await db.createTable("my_table", [{ id: 1 }]);
+
+    const db2 = await connect(tmpDir.name, {
+      readConsistencyInterval: interval,
+    });
+    const table2 = await db2.openTable("my_table");
+    expect(await table2.countRows()).toEqual(await table.countRows());
+
+    await table.add([{ id: 2 }]);
+
+    if (interval === undefined) {
+      expect(await table2.countRows()).toEqual(1);
+      // TODO: once we implement time travel we can uncomment this part of the test.
+      // await table2.checkout_latest();
+      // expect(await table2.countRows()).toEqual(2);
+    } else if (interval === 0) {
+      expect(await table2.countRows()).toEqual(2);
+    } else {
+      // interval == 0.1
+      expect(await table2.countRows()).toEqual(1);
+      await new Promise((r) => setTimeout(r, 100));
+      expect(await table2.countRows()).toEqual(2);
+    }
+  });
+});
+
+describe("schema evolution", function () {
+  let tmpDir: tmp.DirResult;
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => {
+    tmpDir.removeCallback();
+  });
+
+  // Create a new sample table
+  it("can add a new column to the schema", async function () {
+    const con = await connect(tmpDir.name);
+    const table = await con.createTable("vectors", [
+      { id: 1n, vector: [0.1, 0.2] },
+    ]);
+
+    await table.addColumns([
+      { name: "price", valueSql: "cast(10.0 as float)" },
+    ]);
+
+    const expectedSchema = new Schema([
+      new Field("id", new Int64(), true),
+      new Field(
+        "vector",
+        new FixedSizeList(2, new Field("item", new Float32(), true)),
+        true,
+      ),
+      new Field("price", new Float32(), false),
+    ]);
+    expect(await table.schema()).toEqual(expectedSchema);
+  });
+
+  it("can alter the columns in the schema", async function () {
+    const con = await connect(tmpDir.name);
+    const schema = new Schema([
+      new Field("id", new Int64(), true),
+      new Field(
+        "vector",
+        new FixedSizeList(2, new Field("item", new Float32(), true)),
+        true,
+      ),
+      new Field("price", new Float64(), false),
+    ]);
+    const table = await con.createTable("vectors", [
+      { id: 1n, vector: [0.1, 0.2] },
+    ]);
+    // Can create a non-nullable column only through addColumns at the moment.
+    await table.addColumns([
+      { name: "price", valueSql: "cast(10.0 as double)" },
+    ]);
+    expect(await table.schema()).toEqual(schema);
+
+    await table.alterColumns([
+      { path: "id", rename: "new_id" },
+      { path: "price", nullable: true },
+    ]);
+
+    const expectedSchema = new Schema([
+      new Field("new_id", new Int64(), true),
+      new Field(
+        "vector",
+        new FixedSizeList(2, new Field("item", new Float32(), true)),
+        true,
+      ),
+      new Field("price", new Float64(), true),
+    ]);
+    expect(await table.schema()).toEqual(expectedSchema);
+  });
+
+  it("can drop a column from the schema", async function () {
+    const con = await connect(tmpDir.name);
+    const table = await con.createTable("vectors", [
+      { id: 1n, vector: [0.1, 0.2] },
+    ]);
+    await table.dropColumns(["vector"]);
+
+    const expectedSchema = new Schema([new Field("id", new Int64(), true)]);
+    expect(await table.schema()).toEqual(expectedSchema);
+  });
+});
+
+describe("when dealing with versioning", () => {
+  let tmpDir: tmp.DirResult;
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => {
+    tmpDir.removeCallback();
+  });
+
+  it("can travel in time", async () => {
+    // Setup
+    const con = await connect(tmpDir.name);
+    const table = await con.createTable("vectors", [
+      { id: 1n, vector: [0.1, 0.2] },
+    ]);
+    const version = await table.version();
+    await table.add([{ id: 2n, vector: [0.1, 0.2] }]);
+    expect(await table.countRows()).toBe(2);
+    // Make sure we can rewind
+    await table.checkout(version);
+    expect(await table.countRows()).toBe(1);
+    // Can't add data in time travel mode
+    await expect(table.add([{ id: 3n, vector: [0.1, 0.2] }])).rejects.toThrow(
+      "table cannot be modified when a specific version is checked out",
+    );
+    // Can go back to normal mode
+    await table.checkoutLatest();
+    expect(await table.countRows()).toBe(2);
+    // Should be able to add data again
+    await table.add([{ id: 2n, vector: [0.1, 0.2] }]);
+    expect(await table.countRows()).toBe(3);
+    // Now checkout and restore
+    await table.checkout(version);
+    await table.restore();
+    expect(await table.countRows()).toBe(1);
+    // Should be able to add data
+    await table.add([{ id: 2n, vector: [0.1, 0.2] }]);
+    expect(await table.countRows()).toBe(2);
+    // Can't use restore if not checked out
+    await expect(table.restore()).rejects.toThrow(
+      "checkout before running restore",
+    );
   });
 });

nodejs/__test__/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./dist/spec",
+    "module": "commonjs",
+    "target": "es2022",
+    "types": ["jest", "node"]
+  },
+  "include": ["**/*"]
+}

nodejs/eslint.config.js

@@ -0,0 +1,28 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+// @ts-check
+
+const eslint = require("@eslint/js");
+const tseslint = require("typescript-eslint");
+const eslintConfigPrettier = require("eslint-config-prettier");
+const jsdoc = require("eslint-plugin-jsdoc");
+
+module.exports = tseslint.config(
+  eslint.configs.recommended,
+  jsdoc.configs["flat/recommended"],
+  eslintConfigPrettier,
+  ...tseslint.configs.recommended,
+  {
+    rules: {
+      "@typescript-eslint/naming-convention": "error",
+      "jsdoc/require-returns": "off",
+      "jsdoc/require-param": "off",
+      "jsdoc/require-jsdoc": [
+        "error",
+        {
+          publicOnly: true,
+        },
+      ],
+    },
+    plugins: jsdoc,
+  },
+);

nodejs/jest.config.js

@@ -1,7 +1,7 @@
 /** @type {import('ts-jest').JestConfigWithTsJest} */
 module.exports = {
-  preset: 'ts-jest',
-  testEnvironment: 'node',
+  preset: "ts-jest",
+  testEnvironment: "node",
   moduleDirectories: ["node_modules", "./dist"],
   moduleFileExtensions: ["js", "ts"],
 };

nodejs/lancedb/arrow.ts

@@ -0,0 +1,647 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {
+  Field,
+  makeBuilder,
+  RecordBatchFileWriter,
+  Utf8,
+  type Vector,
+  FixedSizeList,
+  vectorFromArray,
+  type Schema,
+  Table as ArrowTable,
+  RecordBatchStreamWriter,
+  List,
+  RecordBatch,
+  makeData,
+  Struct,
+  type Float,
+  DataType,
+  Binary,
+  Float32,
+  type makeTable,
+} from "apache-arrow";
+import { type EmbeddingFunction } from "./embedding/embedding_function";
+import { sanitizeSchema } from "./sanitize";
+
+/** Data type accepted by NodeJS SDK */
+export type Data = Record<string, unknown>[] | ArrowTable;
+
+/*
+ * Options to control how a column should be converted to a vector array
+ */
+export class VectorColumnOptions {
+  /** Vector column type. */
+  type: Float = new Float32();
+
+  constructor(values?: Partial<VectorColumnOptions>) {
+    Object.assign(this, values);
+  }
+}
+
+/** Options to control the makeArrowTable call. */
+export class MakeArrowTableOptions {
+  /*
+   * Schema of the data.
+   *
+   * If this is not provided then the data type will be inferred from the
+   * JS type.  Integer numbers will become int64, floating point numbers
+   * will become float64 and arrays will become variable sized lists with
+   * the data type inferred from the first element in the array.
+   *
+   * The schema must be specified if there are no records (e.g. to make
+   * an empty table)
+   */
+  schema?: Schema;
+
+  /*
+   * Mapping from vector column name to expected type
+   *
+   * Lance expects vector columns to be fixed size list arrays (i.e. tensors)
+   * However, `makeArrowTable` will not infer this by default (it creates
+   * variable size list arrays).  This field can be used to indicate that a column
+   * should be treated as a vector column and converted to a fixed size list.
+   *
+   * The keys should be the names of the vector columns.  The value specifies the
+   * expected data type of the vector columns.
+   *
+   * If `schema` is provided then this field is ignored.
+   *
+   * By default, the column named "vector" will be assumed to be a float32
+   * vector column.
+   */
+  vectorColumns: Record<string, VectorColumnOptions> = {
+    vector: new VectorColumnOptions(),
+  };
+
+  /**
+   * If true then string columns will be encoded with dictionary encoding
+   *
+   * Set this to true if your string columns tend to repeat the same values
+   * often.  For more precise control use the `schema` property to specify the
+   * data type for individual columns.
+   *
+   * If `schema` is provided then this property is ignored.
+   */
+  dictionaryEncodeStrings: boolean = false;
+
+  constructor(values?: Partial<MakeArrowTableOptions>) {
+    Object.assign(this, values);
+  }
+}
+
+/**
+ * An enhanced version of the {@link makeTable} function from Apache Arrow
+ * that supports nested fields and embeddings columns.
+ *
+ * This function converts an array of Record<String, any> (row-major JS objects)
+ * to an Arrow Table (a columnar structure)
+ *
+ * Note that it currently does not support nulls.
+ *
+ * If a schema is provided then it will be used to determine the resulting array
+ * types.  Fields will also be reordered to fit the order defined by the schema.
+ *
+ * If a schema is not provided then the types will be inferred and the field order
+ * will be controlled by the order of properties in the first record.  If a type
+ * is inferred it will always be nullable.
+ *
+ * If the input is empty then a schema must be provided to create an empty table.
+ *
+ * When a schema is not specified then data types will be inferred.  The inference
+ * rules are as follows:
+ *
+ *  - boolean => Bool
+ *  - number => Float64
+ *  - String => Utf8
+ *  - Buffer => Binary
+ *  - Record<String, any> => Struct
+ *  - Array<any> => List
+ * @example
+ * import { fromTableToBuffer, makeArrowTable } from "../arrow";
+ * import { Field, FixedSizeList, Float16, Float32, Int32, Schema } from "apache-arrow";
+ *
+ * const schema = new Schema([
+ *   new Field("a", new Int32()),
+ *   new Field("b", new Float32()),
+ *   new Field("c", new FixedSizeList(3, new Field("item", new Float16()))),
+ *  ]);
+ *  const table = makeArrowTable([
+ *    { a: 1, b: 2, c: [1, 2, 3] },
+ *    { a: 4, b: 5, c: [4, 5, 6] },
+ *    { a: 7, b: 8, c: [7, 8, 9] },
+ *  ], { schema });
+ * ```
+ *
+ * By default it assumes that the column named `vector` is a vector column
+ * and it will be converted into a fixed size list array of type float32.
+ * The `vectorColumns` option can be used to support other vector column
+ * names and data types.
+ *
+ * ```ts
+ *
+ * const schema = new Schema([
+    new Field("a", new Float64()),
+    new Field("b", new Float64()),
+    new Field(
+      "vector",
+      new FixedSizeList(3, new Field("item", new Float32()))
+    ),
+  ]);
+  const table = makeArrowTable([
+    { a: 1, b: 2, vector: [1, 2, 3] },
+    { a: 4, b: 5, vector: [4, 5, 6] },
+    { a: 7, b: 8, vector: [7, 8, 9] },
+  ]);
+  assert.deepEqual(table.schema, schema);
+ * ```
+ *
+ * You can specify the vector column types and names using the options as well
+ *
+ * ```typescript
+ *
+ * const schema = new Schema([
+    new Field('a', new Float64()),
+    new Field('b', new Float64()),
+    new Field('vec1', new FixedSizeList(3, new Field('item', new Float16()))),
+    new Field('vec2', new FixedSizeList(3, new Field('item', new Float16())))
+  ]);
+ * const table = makeArrowTable([
+    { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
+    { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
+    { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] }
+  ], {
+    vectorColumns: {
+      vec1: { type: new Float16() },
+      vec2: { type: new Float16() }
+    }
+  }
+ * assert.deepEqual(table.schema, schema)
+ * ```
+ */
+export function makeArrowTable(
+  data: Array<Record<string, unknown>>,
+  options?: Partial<MakeArrowTableOptions>,
+): ArrowTable {
+  if (
+    data.length === 0 &&
+    (options?.schema === undefined || options?.schema === null)
+  ) {
+    throw new Error("At least one record or a schema needs to be provided");
+  }
+
+  const opt = new MakeArrowTableOptions(options !== undefined ? options : {});
+  if (opt.schema !== undefined && opt.schema !== null) {
+    opt.schema = sanitizeSchema(opt.schema);
+  }
+  const columns: Record<string, Vector> = {};
+  // TODO: sample dataset to find missing columns
+  // Prefer the field ordering of the schema, if present
+  const columnNames =
+    opt.schema != null ? (opt.schema.names as string[]) : Object.keys(data[0]);
+  for (const colName of columnNames) {
+    if (
+      data.length !== 0 &&
+      !Object.prototype.hasOwnProperty.call(data[0], colName)
+    ) {
+      // The field is present in the schema, but not in the data, skip it
+      continue;
+    }
+    // Extract a single column from the records (transpose from row-major to col-major)
+    let values = data.map((datum) => datum[colName]);
+
+    // By default (type === undefined) arrow will infer the type from the JS type
+    let type;
+    if (opt.schema !== undefined) {
+      // If there is a schema provided, then use that for the type instead
+      type = opt.schema?.fields.filter((f) => f.name === colName)[0]?.type;
+      if (DataType.isInt(type) && type.bitWidth === 64) {
+        // wrap in BigInt to avoid bug: https://github.com/apache/arrow/issues/40051
+        values = values.map((v) => {
+          if (v === null) {
+            return v;
+          }
+          if (typeof v === "bigint") {
+            return v;
+          }
+          if (typeof v === "number") {
+            return BigInt(v);
+          }
+          throw new Error(
+            `Expected BigInt or number for column ${colName}, got ${typeof v}`,
+          );
+        });
+      }
+    } else {
+      // Otherwise, check to see if this column is one of the vector columns
+      // defined by opt.vectorColumns and, if so, use the fixed size list type
+      const vectorColumnOptions = opt.vectorColumns[colName];
+      if (vectorColumnOptions !== undefined) {
+        const firstNonNullValue = values.find((v) => v !== null);
+        if (Array.isArray(firstNonNullValue)) {
+          type = newVectorType(
+            firstNonNullValue.length,
+            vectorColumnOptions.type,
+          );
+        } else {
+          throw new Error(
+            `Column ${colName} is expected to be a vector column but first non-null value is not an array.  Could not determine size of vector column`,
+          );
+        }
+      }
+    }
+
+    try {
+      // Convert an Array of JS values to an arrow vector
+      columns[colName] = makeVector(values, type, opt.dictionaryEncodeStrings);
+    } catch (error: unknown) {
+      // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+      throw Error(`Could not convert column "${colName}" to Arrow: ${error}`);
+    }
+  }
+
+  if (opt.schema != null) {
+    // `new ArrowTable(columns)` infers a schema which may sometimes have
+    // incorrect nullability (it assumes nullable=true always)
+    //
+    // `new ArrowTable(schema, columns)` will also fail because it will create a
+    // batch with an inferred schema and then complain that the batch schema
+    // does not match the provided schema.
+    //
+    // To work around this we first create a table with the wrong schema and
+    // then patch the schema of the batches so we can use
+    // `new ArrowTable(schema, batches)` which does not do any schema inference
+    const firstTable = new ArrowTable(columns);
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const batchesFixed = firstTable.batches.map(
+      (batch) => new RecordBatch(opt.schema!, batch.data),
+    );
+    return new ArrowTable(opt.schema, batchesFixed);
+  } else {
+    return new ArrowTable(columns);
+  }
+}
+
+/**
+ * Create an empty Arrow table with the provided schema
+ */
+export function makeEmptyTable(schema: Schema): ArrowTable {
+  return makeArrowTable([], { schema });
+}
+
+/**
+ * Helper function to convert Array<Array<any>> to a variable sized list array
+ */
+// @ts-expect-error (Vector<unknown> is not assignable to Vector<any>)
+function makeListVector(lists: unknown[][]): Vector<unknown> {
+  if (lists.length === 0 || lists[0].length === 0) {
+    throw Error("Cannot infer list vector from empty array or empty list");
+  }
+  const sampleList = lists[0];
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  let inferredType: any;
+  try {
+    const sampleVector = makeVector(sampleList);
+    inferredType = sampleVector.type;
+  } catch (error: unknown) {
+    // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+    throw Error(`Cannot infer list vector.  Cannot infer inner type: ${error}`);
+  }
+
+  const listBuilder = makeBuilder({
+    type: new List(new Field("item", inferredType, true)),
+  });
+  for (const list of lists) {
+    listBuilder.append(list);
+  }
+  return listBuilder.finish().toVector();
+}
+
+/** Helper function to convert an Array of JS values to an Arrow Vector */
+function makeVector(
+  values: unknown[],
+  type?: DataType,
+  stringAsDictionary?: boolean,
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+): Vector<any> {
+  if (type !== undefined) {
+    // No need for inference, let Arrow create it
+    return vectorFromArray(values, type);
+  }
+  if (values.length === 0) {
+    throw Error(
+      "makeVector requires at least one value or the type must be specfied",
+    );
+  }
+  const sampleValue = values.find((val) => val !== null && val !== undefined);
+  if (sampleValue === undefined) {
+    throw Error(
+      "makeVector cannot infer the type if all values are null or undefined",
+    );
+  }
+  if (Array.isArray(sampleValue)) {
+    // Default Arrow inference doesn't handle list types
+    return makeListVector(values as unknown[][]);
+  } else if (Buffer.isBuffer(sampleValue)) {
+    // Default Arrow inference doesn't handle Buffer
+    return vectorFromArray(values, new Binary());
+  } else if (
+    !(stringAsDictionary ?? false) &&
+    (typeof sampleValue === "string" || sampleValue instanceof String)
+  ) {
+    // If the type is string then don't use Arrow's default inference unless dictionaries are requested
+    // because it will always use dictionary encoding for strings
+    return vectorFromArray(values, new Utf8());
+  } else {
+    // Convert a JS array of values to an arrow vector
+    return vectorFromArray(values);
+  }
+}
+
+/** Helper function to apply embeddings to an input table */
+async function applyEmbeddings<T>(
+  table: ArrowTable,
+  embeddings?: EmbeddingFunction<T>,
+  schema?: Schema,
+): Promise<ArrowTable> {
+  if (embeddings == null) {
+    return table;
+  }
+
+  if (schema !== undefined && schema !== null) {
+    schema = sanitizeSchema(schema);
+  }
+
+  // Convert from ArrowTable to Record<String, Vector>
+  const colEntries = [...Array(table.numCols).keys()].map((_, idx) => {
+    const name = table.schema.fields[idx].name;
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const vec = table.getChildAt(idx)!;
+    return [name, vec];
+  });
+  const newColumns = Object.fromEntries(colEntries);
+
+  const sourceColumn = newColumns[embeddings.sourceColumn];
+  const destColumn = embeddings.destColumn ?? "vector";
+  const innerDestType = embeddings.embeddingDataType ?? new Float32();
+  if (sourceColumn === undefined) {
+    throw new Error(
+      `Cannot apply embedding function because the source column '${embeddings.sourceColumn}' was not present in the data`,
+    );
+  }
+
+  if (table.numRows === 0) {
+    if (Object.prototype.hasOwnProperty.call(newColumns, destColumn)) {
+      // We have an empty table and it already has the embedding column so no work needs to be done
+      // Note: we don't return an error like we did below because this is a common occurrence.  For example,
+      // if we call convertToTable with 0 records and a schema that includes the embedding
+      return table;
+    }
+    if (embeddings.embeddingDimension !== undefined) {
+      const destType = newVectorType(
+        embeddings.embeddingDimension,
+        innerDestType,
+      );
+      newColumns[destColumn] = makeVector([], destType);
+    } else if (schema != null) {
+      const destField = schema.fields.find((f) => f.name === destColumn);
+      if (destField != null) {
+        newColumns[destColumn] = makeVector([], destField.type);
+      } else {
+        throw new Error(
+          `Attempt to apply embeddings to an empty table failed because schema was missing embedding column '${destColumn}'`,
+        );
+      }
+    } else {
+      throw new Error(
+        "Attempt to apply embeddings to an empty table when the embeddings function does not specify `embeddingDimension`",
+      );
+    }
+  } else {
+    if (Object.prototype.hasOwnProperty.call(newColumns, destColumn)) {
+      throw new Error(
+        `Attempt to apply embeddings to table failed because column ${destColumn} already existed`,
+      );
+    }
+    if (table.batches.length > 1) {
+      throw new Error(
+        "Internal error: `makeArrowTable` unexpectedly created a table with more than one batch",
+      );
+    }
+    const values = sourceColumn.toArray();
+    const vectors = await embeddings.embed(values as T[]);
+    if (vectors.length !== values.length) {
+      throw new Error(
+        "Embedding function did not return an embedding for each input element",
+      );
+    }
+    const destType = newVectorType(vectors[0].length, innerDestType);
+    newColumns[destColumn] = makeVector(vectors, destType);
+  }
+
+  const newTable = new ArrowTable(newColumns);
+  if (schema != null) {
+    if (schema.fields.find((f) => f.name === destColumn) === undefined) {
+      throw new Error(
+        `When using embedding functions and specifying a schema the schema should include the embedding column but the column ${destColumn} was missing`,
+      );
+    }
+    return alignTable(newTable, schema);
+  }
+  return newTable;
+}
+
+/**
+ * Convert an Array of records into an Arrow Table, optionally applying an
+ * embeddings function to it.
+ *
+ * This function calls `makeArrowTable` first to create the Arrow Table.
+ * Any provided `makeTableOptions` (e.g. a schema) will be passed on to
+ * that call.
+ *
+ * The embedding function will be passed a column of values (based on the
+ * `sourceColumn` of the embedding function) and expects to receive back
+ * number[][] which will be converted into a fixed size list column.  By
+ * default this will be a fixed size list of Float32 but that can be
+ * customized by the `embeddingDataType` property of the embedding function.
+ *
+ * If a schema is provided in `makeTableOptions` then it should include the
+ * embedding columns.  If no schema is provded then embedding columns will
+ * be placed at the end of the table, after all of the input columns.
+ */
+export async function convertToTable<T>(
+  data: Array<Record<string, unknown>>,
+  embeddings?: EmbeddingFunction<T>,
+  makeTableOptions?: Partial<MakeArrowTableOptions>,
+): Promise<ArrowTable> {
+  const table = makeArrowTable(data, makeTableOptions);
+  return await applyEmbeddings(table, embeddings, makeTableOptions?.schema);
+}
+
+/** Creates the Arrow Type for a Vector column with dimension `dim` */
+function newVectorType<T extends Float>(
+  dim: number,
+  innerType: T,
+): FixedSizeList<T> {
+  // in Lance we always default to have the elements nullable, so we need to set it to true
+  // otherwise we often get schema mismatches because the stored data always has schema with nullable elements
+  const children = new Field<T>("item", innerType, true);
+  return new FixedSizeList(dim, children);
+}
+
+/**
+ * Serialize an Array of records into a buffer using the Arrow IPC File serialization
+ *
+ * This function will call `convertToTable` and pass on `embeddings` and `schema`
+ *
+ * `schema` is required if data is empty
+ */
+export async function fromRecordsToBuffer<T>(
+  data: Array<Record<string, unknown>>,
+  embeddings?: EmbeddingFunction<T>,
+  schema?: Schema,
+): Promise<Buffer> {
+  if (schema !== undefined && schema !== null) {
+    schema = sanitizeSchema(schema);
+  }
+  const table = await convertToTable(data, embeddings, { schema });
+  const writer = RecordBatchFileWriter.writeAll(table);
+  return Buffer.from(await writer.toUint8Array());
+}
+
+/**
+ * Serialize an Array of records into a buffer using the Arrow IPC Stream serialization
+ *
+ * This function will call `convertToTable` and pass on `embeddings` and `schema`
+ *
+ * `schema` is required if data is empty
+ */
+export async function fromRecordsToStreamBuffer<T>(
+  data: Array<Record<string, unknown>>,
+  embeddings?: EmbeddingFunction<T>,
+  schema?: Schema,
+): Promise<Buffer> {
+  if (schema !== undefined && schema !== null) {
+    schema = sanitizeSchema(schema);
+  }
+  const table = await convertToTable(data, embeddings, { schema });
+  const writer = RecordBatchStreamWriter.writeAll(table);
+  return Buffer.from(await writer.toUint8Array());
+}
+
+/**
+ * Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
+ *
+ * This function will apply `embeddings` to the table in a manner similar to
+ * `convertToTable`.
+ *
+ * `schema` is required if the table is empty
+ */
+export async function fromTableToBuffer<T>(
+  table: ArrowTable,
+  embeddings?: EmbeddingFunction<T>,
+  schema?: Schema,
+): Promise<Buffer> {
+  if (schema !== undefined && schema !== null) {
+    schema = sanitizeSchema(schema);
+  }
+  const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
+  const writer = RecordBatchFileWriter.writeAll(tableWithEmbeddings);
+  return Buffer.from(await writer.toUint8Array());
+}
+
+/**
+ * Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
+ *
+ * This function will apply `embeddings` to the table in a manner similar to
+ * `convertToTable`.
+ *
+ * `schema` is required if the table is empty
+ */
+export async function fromDataToBuffer<T>(
+  data: Data,
+  embeddings?: EmbeddingFunction<T>,
+  schema?: Schema,
+): Promise<Buffer> {
+  if (schema !== undefined && schema !== null) {
+    schema = sanitizeSchema(schema);
+  }
+  if (data instanceof ArrowTable) {
+    return fromTableToBuffer(data, embeddings, schema);
+  } else {
+    const table = await convertToTable(data);
+    return fromTableToBuffer(table, embeddings, schema);
+  }
+}
+
+/**
+ * Serialize an Arrow Table into a buffer using the Arrow IPC Stream serialization
+ *
+ * This function will apply `embeddings` to the table in a manner similar to
+ * `convertToTable`.
+ *
+ * `schema` is required if the table is empty
+ */
+export async function fromTableToStreamBuffer<T>(
+  table: ArrowTable,
+  embeddings?: EmbeddingFunction<T>,
+  schema?: Schema,
+): Promise<Buffer> {
+  const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
+  const writer = RecordBatchStreamWriter.writeAll(tableWithEmbeddings);
+  return Buffer.from(await writer.toUint8Array());
+}
+
+/**
+ * Reorder the columns in `batch` so that they agree with the field order in `schema`
+ */
+function alignBatch(batch: RecordBatch, schema: Schema): RecordBatch {
+  const alignedChildren = [];
+  for (const field of schema.fields) {
+    const indexInBatch = batch.schema.fields?.findIndex(
+      (f) => f.name === field.name,
+    );
+    if (indexInBatch < 0) {
+      throw new Error(
+        `The column ${field.name} was not found in the Arrow Table`,
+      );
+    }
+    alignedChildren.push(batch.data.children[indexInBatch]);
+  }
+  const newData = makeData({
+    type: new Struct(schema.fields),
+    length: batch.numRows,
+    nullCount: batch.nullCount,
+    children: alignedChildren,
+  });
+  return new RecordBatch(schema, newData);
+}
+
+/**
+ * Reorder the columns in `table` so that they agree with the field order in `schema`
+ */
+function alignTable(table: ArrowTable, schema: Schema): ArrowTable {
+  const alignedBatches = table.batches.map((batch) =>
+    alignBatch(batch, schema),
+  );
+  return new ArrowTable(schema, alignedBatches);
+}
+
+/**
+ * Create an empty table with the given schema
+ */
+export function createEmptyTable(schema: Schema): ArrowTable {
+  return new ArrowTable(sanitizeSchema(schema));
+}

nodejs/lancedb/connection.ts

@@ -0,0 +1,176 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { fromTableToBuffer, makeArrowTable, makeEmptyTable } from "./arrow";
+import { Connection as LanceDbConnection } from "./native";
+import { Table } from "./table";
+import { Table as ArrowTable, Schema } from "apache-arrow";
+
+export interface CreateTableOptions {
+  /**
+   * The mode to use when creating the table.
+   *
+   * If this is set to "create" and the table already exists then either
+   * an error will be thrown or, if existOk is true, then nothing will
+   * happen.  Any provided data will be ignored.
+   *
+   * If this is set to "overwrite" then any existing table will be replaced.
+   */
+  mode: "create" | "overwrite";
+  /**
+   * If this is true and the table already exists and the mode is "create"
+   * then no error will be raised.
+   */
+  existOk: boolean;
+}
+
+export interface TableNamesOptions {
+  /**
+   * If present, only return names that come lexicographically after the
+   * supplied value.
+   *
+   * This can be combined with limit to implement pagination by setting this to
+   * the last table name from the previous page.
+   */
+  startAfter?: string;
+  /** An optional limit to the number of results to return. */
+  limit?: number;
+}
+
+/**
+ * A LanceDB Connection that allows you to open tables and create new ones.
+ *
+ * Connection could be local against filesystem or remote against a server.
+ *
+ * A Connection is intended to be a long lived object and may hold open
+ * resources such as HTTP connection pools.  This is generally fine and
+ * a single connection should be shared if it is going to be used many
+ * times. However, if you are finished with a connection, you may call
+ * close to eagerly free these resources.  Any call to a Connection
+ * method after it has been closed will result in an error.
+ *
+ * Closing a connection is optional.  Connections will automatically
+ * be closed when they are garbage collected.
+ *
+ * Any created tables are independent and will continue to work even if
+ * the underlying connection has been closed.
+ */
+export class Connection {
+  readonly inner: LanceDbConnection;
+
+  constructor(inner: LanceDbConnection) {
+    this.inner = inner;
+  }
+
+  /** Return true if the connection has not been closed */
+  isOpen(): boolean {
+    return this.inner.isOpen();
+  }
+
+  /**
+   * Close the connection, releasing any underlying resources.
+   *
+   * It is safe to call this method multiple times.
+   *
+   * Any attempt to use the connection after it is closed will result in an error.
+   */
+  close(): void {
+    this.inner.close();
+  }
+
+  /** Return a brief description of the connection */
+  display(): string {
+    return this.inner.display();
+  }
+
+  /**
+   * List all the table names in this database.
+   *
+   * Tables will be returned in lexicographical order.
+   * @param {Partial<TableNamesOptions>} options - options to control the
+   * paging / start point
+   */
+  async tableNames(options?: Partial<TableNamesOptions>): Promise<string[]> {
+    return this.inner.tableNames(options?.startAfter, options?.limit);
+  }
+
+  /**
+   * Open a table in the database.
+   * @param {string} name - The name of the table
+   */
+  async openTable(name: string): Promise<Table> {
+    const innerTable = await this.inner.openTable(name);
+    return new Table(innerTable);
+  }
+
+  /**
+   * Creates a new Table and initialize it with new data.
+   * @param {string} name - The name of the table.
+   * @param {Record<string, unknown>[] | ArrowTable} data - Non-empty Array of Records
+   * to be inserted into the table
+   */
+  async createTable(
+    name: string,
+    data: Record<string, unknown>[] | ArrowTable,
+    options?: Partial<CreateTableOptions>,
+  ): Promise<Table> {
+    let mode: string = options?.mode ?? "create";
+    const existOk = options?.existOk ?? false;
+
+    if (mode === "create" && existOk) {
+      mode = "exist_ok";
+    }
+
+    let table: ArrowTable;
+    if (data instanceof ArrowTable) {
+      table = data;
+    } else {
+      table = makeArrowTable(data);
+    }
+    const buf = await fromTableToBuffer(table);
+    const innerTable = await this.inner.createTable(name, buf, mode);
+    return new Table(innerTable);
+  }
+
+  /**
+   * Creates a new empty Table
+   * @param {string} name - The name of the table.
+   * @param {Schema} schema - The schema of the table
+   */
+  async createEmptyTable(
+    name: string,
+    schema: Schema,
+    options?: Partial<CreateTableOptions>,
+  ): Promise<Table> {
+    let mode: string = options?.mode ?? "create";
+    const existOk = options?.existOk ?? false;
+
+    if (mode === "create" && existOk) {
+      mode = "exist_ok";
+    }
+
+    const table = makeEmptyTable(schema);
+    const buf = await fromTableToBuffer(table);
+    const innerTable = await this.inner.createEmptyTable(name, buf, mode);
+    return new Table(innerTable);
+  }
+
+  /**
+   * Drop an existing table.
+   * @param {string} name The name of the table to drop.
+   */
+  async dropTable(name: string): Promise<void> {
+    return this.inner.dropTable(name);
+  }
+}

nodejs/lancedb/embedding/embedding_function.ts

@@ -0,0 +1,78 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { type Float } from "apache-arrow";
+
+/**
+ * An embedding function that automatically creates vector representation for a given column.
+ */
+export interface EmbeddingFunction<T> {
+  /**
+   * The name of the column that will be used as input for the Embedding Function.
+   */
+  sourceColumn: string;
+
+  /**
+   * The data type of the embedding
+   *
+   * The embedding function should return `number`.  This will be converted into
+   * an Arrow float array.  By default this will be Float32 but this property can
+   * be used to control the conversion.
+   */
+  embeddingDataType?: Float;
+
+  /**
+   * The dimension of the embedding
+   *
+   * This is optional, normally this can be determined by looking at the results of
+   * `embed`.  If this is not specified, and there is an attempt to apply the embedding
+   * to an empty table, then that process will fail.
+   */
+  embeddingDimension?: number;
+
+  /**
+   * The name of the column that will contain the embedding
+   *
+   * By default this is "vector"
+   */
+  destColumn?: string;
+
+  /**
+   * Should the source column be excluded from the resulting table
+   *
+   * By default the source column is included.  Set this to true and
+   * only the embedding will be stored.
+   */
+  excludeSource?: boolean;
+
+  /**
+   * Creates a vector representation for the given values.
+   */
+  embed: (data: T[]) => Promise<number[][]>;
+}
+
+/** Test if the input seems to be an embedding function */
+export function isEmbeddingFunction<T>(
+  value: unknown,
+): value is EmbeddingFunction<T> {
+  if (typeof value !== "object" || value === null) {
+    return false;
+  }
+  if (!("sourceColumn" in value) || !("embed" in value)) {
+    return false;
+  }
+  return (
+    typeof value.sourceColumn === "string" && typeof value.embed === "function"
+  );
+}

nodejs/lancedb/embedding/openai.ts

@@ -0,0 +1,62 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { type EmbeddingFunction } from "./embedding_function";
+import type OpenAI from "openai";
+
+export class OpenAIEmbeddingFunction implements EmbeddingFunction<string> {
+  private readonly _openai: OpenAI;
+  private readonly _modelName: string;
+
+  constructor(
+    sourceColumn: string,
+    openAIKey: string,
+    modelName: string = "text-embedding-ada-002",
+  ) {
+    /**
+     * @type {import("openai").default}
+     */
+    // eslint-disable-next-line @typescript-eslint/naming-convention
+    let Openai;
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      Openai = require("openai");
+    } catch {
+      throw new Error("please install openai@^4.24.1 using npm install openai");
+    }
+
+    this.sourceColumn = sourceColumn;
+    const configuration = {
+      apiKey: openAIKey,
+    };
+
+    this._openai = new Openai(configuration);
+    this._modelName = modelName;
+  }
+
+  async embed(data: string[]): Promise<number[][]> {
+    const response = await this._openai.embeddings.create({
+      model: this._modelName,
+      input: data,
+    });
+
+    const embeddings: number[][] = [];
+    for (let i = 0; i < response.data.length; i++) {
+      embeddings.push(response.data[i].embedding);
+    }
+    return embeddings;
+  }
+
+  sourceColumn: string;
+}

nodejs/lancedb/index.ts

@@ -13,18 +13,14 @@
 // limitations under the License.
 
 import { Connection } from "./connection";
-import { Connection as NativeConnection, ConnectionOptions } from "./native.js";
-
-export {
+import {
+  Connection as LanceDbConnection,
   ConnectionOptions,
-  WriteOptions,
-  Query,
-  MetricType,
 } from "./native.js";
-export { Connection } from "./connection";
-export { Table } from "./table";
-export { Data } from "./arrow";
-export { IvfPQOptions, IndexBuilder } from "./indexer";
+
+export { ConnectionOptions, WriteOptions, Query } from "./native.js";
+export { Connection, CreateTableOptions } from "./connection";
+export { Table, AddDataOptions } from "./table";
 
 /**
  * Connect to a LanceDB instance at the given URI.
@@ -34,31 +30,15 @@ export { IvfPQOptions, IndexBuilder } from "./indexer";
  * - `/path/to/database` - local database
  * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
  * - `db://host:port` - remote database (LanceDB cloud)
- *
- * @param uri The uri of the database. If the database uri starts with `db://` then it connects to a remote database.
- *
+ * @param {string} uri - The uri of the database. If the database uri starts
+ * with `db://` then it connects to a remote database.
  * @see {@link ConnectionOptions} for more details on the URI format.
  */
-export async function connect(uri: string): Promise<Connection>;
 export async function connect(
-  opts: Partial<ConnectionOptions>
-): Promise<Connection>;
-export async function connect(
-  args: string | Partial<ConnectionOptions>
+  uri: string,
+  opts?: Partial<ConnectionOptions>,
 ): Promise<Connection> {
-  let opts: ConnectionOptions;
-  if (typeof args === "string") {
-    opts = { uri: args };
-  } else {
-    opts = Object.assign(
-      {
-        uri: "",
-        apiKey: "",
-        hostOverride: "",
-      },
-      args
-    );
-  }
-  const nativeConn = await NativeConnection.new(opts.uri);
+  opts = opts ?? {};
+  const nativeConn = await LanceDbConnection.new(uri, opts);
   return new Connection(nativeConn);
 }

nodejs/lancedb/indices.ts

@@ -0,0 +1,203 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { Index as LanceDbIndex } from "./native";
+
+/**
+ * Options to create an `IVF_PQ` index
+ */
+export interface IvfPqOptions {
+  /**
+   * The number of IVF partitions to create.
+   *
+   * This value should generally scale with the number of rows in the dataset.
+   * By default the number of partitions is the square root of the number of
+   * rows.
+   *
+   * If this value is too large then the first part of the search (picking the
+   * right partition) will be slow.  If this value is too small then the second
+   * part of the search (searching within a partition) will be slow.
+   */
+  numPartitions?: number;
+
+  /**
+   * Number of sub-vectors of PQ.
+   *
+   * This value controls how much the vector is compressed during the quantization step.
+   * The more sub vectors there are the less the vector is compressed.  The default is
+   * the dimension of the vector divided by 16.  If the dimension is not evenly divisible
+   * by 16 we use the dimension divded by 8.
+   *
+   * The above two cases are highly preferred.  Having 8 or 16 values per subvector allows
+   * us to use efficient SIMD instructions.
+   *
+   * If the dimension is not visible by 8 then we use 1 subvector.  This is not ideal and
+   * will likely result in poor performance.
+   */
+  numSubVectors?: number;
+
+  /**
+   * Distance type to use to build the index.
+   *
+   * Default value is "l2".
+   *
+   * This is used when training the index to calculate the IVF partitions
+   * (vectors are grouped in partitions with similar vectors according to this
+   * distance type) and to calculate a subvector's code during quantization.
+   *
+   * The distance type used to train an index MUST match the distance type used
+   * to search the index.  Failure to do so will yield inaccurate results.
+   *
+   * The following distance types are available:
+   *
+   * "l2" - Euclidean distance. This is a very common distance metric that
+   * accounts for both magnitude and direction when determining the distance
+   * between vectors. L2 distance has a range of [0, ∞).
+   *
+   * "cosine" - Cosine distance.  Cosine distance is a distance metric
+   * calculated from the cosine similarity between two vectors. Cosine
+   * similarity is a measure of similarity between two non-zero vectors of an
+   * inner product space. It is defined to equal the cosine of the angle
+   * between them.  Unlike L2, the cosine distance is not affected by the
+   * magnitude of the vectors.  Cosine distance has a range of [0, 2].
+   *
+   * Note: the cosine distance is undefined when one (or both) of the vectors
+   * are all zeros (there is no direction).  These vectors are invalid and may
+   * never be returned from a vector search.
+   *
+   * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
+   * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
+   * L2 norm is 1), then dot distance is equivalent to the cosine distance.
+   */
+  distanceType?: "l2" | "cosine" | "dot";
+
+  /**
+   * Max iteration to train IVF kmeans.
+   *
+   * When training an IVF PQ index we use kmeans to calculate the partitions.  This parameter
+   * controls how many iterations of kmeans to run.
+   *
+   * Increasing this might improve the quality of the index but in most cases these extra
+   * iterations have diminishing returns.
+   *
+   * The default value is 50.
+   */
+  maxIterations?: number;
+
+  /**
+   * The number of vectors, per partition, to sample when training IVF kmeans.
+   *
+   * When an IVF PQ index is trained, we need to calculate partitions.  These are groups
+   * of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
+   *
+   * Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
+   * random sample of the data.  This parameter controls the size of the sample.  The total
+   * number of vectors used to train the index is `sample_rate * num_partitions`.
+   *
+   * Increasing this value might improve the quality of the index but in most cases the
+   * default should be sufficient.
+   *
+   * The default value is 256.
+   */
+  sampleRate?: number;
+}
+
+export class Index {
+  private readonly inner: LanceDbIndex;
+  private constructor(inner: LanceDbIndex) {
+    this.inner = inner;
+  }
+
+  /**
+   * Create an IvfPq index
+   *
+   * This index stores a compressed (quantized) copy of every vector.  These vectors
+   * are grouped into partitions of similar vectors.  Each partition keeps track of
+   * a centroid which is the average value of all vectors in the group.
+   *
+   * During a query the centroids are compared with the query vector to find the closest
+   * partitions.  The compressed vectors in these partitions are then searched to find
+   * the closest vectors.
+   *
+   * The compression scheme is called product quantization.  Each vector is divided into
+   * subvectors and then each subvector is quantized into a small number of bits.  the
+   * parameters `num_bits` and `num_subvectors` control this process, providing a tradeoff
+   * between index size (and thus search speed) and index accuracy.
+   *
+   * The partitioning process is called IVF and the `num_partitions` parameter controls how
+   * many groups to create.
+   *
+   * Note that training an IVF PQ index on a large dataset is a slow operation and
+   * currently is also a memory intensive operation.
+   */
+  static ivfPq(options?: Partial<IvfPqOptions>) {
+    return new Index(
+      LanceDbIndex.ivfPq(
+        options?.distanceType,
+        options?.numPartitions,
+        options?.numSubVectors,
+        options?.maxIterations,
+        options?.sampleRate,
+      ),
+    );
+  }
+
+  /**
+   * Create a btree index
+   *
+   * A btree index is an index on a scalar columns.  The index stores a copy of the column
+   * in sorted order.  A header entry is created for each block of rows (currently the
+   * block size is fixed at 4096).  These header entries are stored in a separate
+   * cacheable structure (a btree).  To search for data the header is used to determine
+   * which blocks need to be read from disk.
+   *
+   * For example, a btree index in a table with 1Bi rows requires sizeof(Scalar) * 256Ki
+   * bytes of memory and will generally need to read sizeof(Scalar) * 4096 bytes to find
+   * the correct row ids.
+   *
+   * This index is good for scalar columns with mostly distinct values and does best when
+   * the query is highly selective.
+   *
+   * The btree index does not currently have any parameters though parameters such as the
+   * block size may be added in the future.
+   */
+  static btree() {
+    return new Index(LanceDbIndex.btree());
+  }
+}
+
+export interface IndexOptions {
+  /**
+   * Advanced index configuration
+   *
+   * This option allows you to specify a specfic index to create and also
+   * allows you to pass in configuration for training the index.
+   *
+   * See the static methods on Index for details on the various index types.
+   *
+   * If this is not supplied then column data type(s) and column statistics
+   * will be used to determine the most useful kind of index to create.
+   */
+  config?: Index;
+  /**
+   * Whether to replace the existing index
+   *
+   * If this is false, and another index already exists on the same columns
+   * and the same name, then an error will be returned.  This is true even if
+   * that index is out of date.
+   *
+   * The default is true
+   */
+  replace?: boolean;
+}

nodejs/lancedb/native.d.ts

@@ -0,0 +1,147 @@
+/* tslint:disable */
+/* eslint-disable */
+
+/* auto-generated by NAPI-RS */
+
+/** A description of an index currently configured on a column */
+export interface IndexConfig {
+  /** The type of the index */
+  indexType: string
+  /**
+   * The columns in the index
+   *
+   * Currently this is always an array of size 1.  In the future there may
+   * be more columns to represent composite indices.
+   */
+  columns: Array<string>
+}
+/**
+ *  A definition of a column alteration. The alteration changes the column at
+ * `path` to have the new name `name`, to be nullable if `nullable` is true,
+ * and to have the data type `data_type`. At least one of `rename` or `nullable`
+ * must be provided.
+ */
+export interface ColumnAlteration {
+  /**
+   * The path to the column to alter. This is a dot-separated path to the column.
+   * If it is a top-level column then it is just the name of the column. If it is
+   * a nested column then it is the path to the column, e.g. "a.b.c" for a column
+   * `c` nested inside a column `b` nested inside a column `a`.
+   */
+  path: string
+  /**
+   * The new name of the column. If not provided then the name will not be changed.
+   * This must be distinct from the names of all other columns in the table.
+   */
+  rename?: string
+  /** Set the new nullability. Note that a nullable column cannot be made non-nullable. */
+  nullable?: boolean
+}
+/** A definition of a new column to add to a table. */
+export interface AddColumnsSql {
+  /** The name of the new column. */
+  name: string
+  /**
+   * The values to populate the new column with, as a SQL expression.
+   * The expression can reference other columns in the table.
+   */
+  valueSql: string
+}
+export interface ConnectionOptions {
+  apiKey?: string
+  hostOverride?: string
+  /**
+   * (For LanceDB OSS only): The interval, in seconds, at which to check for
+   * updates to the table from other processes. If None, then consistency is not
+   * checked. For performance reasons, this is the default. For strong
+   * consistency, set this to zero seconds. Then every read will check for
+   * updates from other processes. As a compromise, you can set this to a
+   * non-zero value for eventual consistency. If more than that interval
+   * has passed since the last check, then the table will be checked for updates.
+   * Note: this consistency only applies to read operations. Write operations are
+   * always consistent.
+   */
+  readConsistencyInterval?: number
+}
+/** Write mode for writing a table. */
+export const enum WriteMode {
+  Create = 'Create',
+  Append = 'Append',
+  Overwrite = 'Overwrite'
+}
+/** Write options when creating a Table. */
+export interface WriteOptions {
+  mode?: WriteMode
+}
+export function connect(uri: string, options: ConnectionOptions): Promise<Connection>
+export class Connection {
+  /** Create a new Connection instance from the given URI. */
+  static new(uri: string, options: ConnectionOptions): Promise<Connection>
+  display(): string
+  isOpen(): boolean
+  close(): void
+  /** List all tables in the dataset. */
+  tableNames(startAfter?: string | undefined | null, limit?: number | undefined | null): Promise<Array<string>>
+  /**
+   * Create table from a Apache Arrow IPC (file) buffer.
+   *
+   * Parameters:
+   * - name: The name of the table.
+   * - buf: The buffer containing the IPC file.
+   *
+   */
+  createTable(name: string, buf: Buffer, mode: string): Promise<Table>
+  createEmptyTable(name: string, schemaBuf: Buffer, mode: string): Promise<Table>
+  openTable(name: string): Promise<Table>
+  /** Drop table with the name. Or raise an error if the table does not exist. */
+  dropTable(name: string): Promise<void>
+}
+export class Index {
+  static ivfPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
+  static btree(): Index
+}
+/** Typescript-style Async Iterator over RecordBatches  */
+export class RecordBatchIterator {
+  next(): Promise<Buffer | null>
+}
+export class Query {
+  onlyIf(predicate: string): void
+  select(columns: Array<[string, string]>): void
+  limit(limit: number): void
+  nearestTo(vector: Float32Array): VectorQuery
+  execute(): Promise<RecordBatchIterator>
+}
+export class VectorQuery {
+  column(column: string): void
+  distanceType(distanceType: string): void
+  postfilter(): void
+  refineFactor(refineFactor: number): void
+  nprobes(nprobe: number): void
+  bypassVectorIndex(): void
+  onlyIf(predicate: string): void
+  select(columns: Array<[string, string]>): void
+  limit(limit: number): void
+  execute(): Promise<RecordBatchIterator>
+}
+export class Table {
+  display(): string
+  isOpen(): boolean
+  close(): void
+  /** Return Schema as empty Arrow IPC file. */
+  schema(): Promise<Buffer>
+  add(buf: Buffer, mode: string): Promise<void>
+  countRows(filter?: string | undefined | null): Promise<number>
+  delete(predicate: string): Promise<void>
+  createIndex(index: Index | undefined | null, column: string, replace?: boolean | undefined | null): Promise<void>
+  update(onlyIf: string | undefined | null, columns: Array<[string, string]>): Promise<void>
+  query(): Query
+  vectorSearch(vector: Float32Array): VectorQuery
+  addColumns(transforms: Array<AddColumnsSql>): Promise<void>
+  alterColumns(alterations: Array<ColumnAlteration>): Promise<void>
+  dropColumns(columns: Array<string>): Promise<void>
+  version(): Promise<number>
+  checkout(version: number): Promise<void>
+  checkoutLatest(): Promise<void>
+  restore(): Promise<void>
+  listIndices(): Promise<Array<IndexConfig>>
+}

nodejs/lancedb/native.js

@@ -0,0 +1,329 @@
+/* tslint:disable */
+/* eslint-disable */
+/* prettier-ignore */
+
+/* auto-generated by NAPI-RS */
+
+const { existsSync, readFileSync } = require('fs')
+const { join } = require("path");
+
+const { platform, arch } = process;
+
+let nativeBinding = null;
+let localFileExisted = false;
+let loadError = null;
+
+function isMusl() {
+  // For Node 10
+  if (!process.report || typeof process.report.getReport !== "function") {
+    try {
+      const lddPath = require("child_process")
+        .execSync("which ldd")
+        .toString()
+        .trim();
+      return readFileSync(lddPath, "utf8").includes("musl");
+    } catch (e) {
+      return true;
+    }
+  } else {
+    const { glibcVersionRuntime } = process.report.getReport().header;
+    return !glibcVersionRuntime;
+  }
+}
+
+switch (platform) {
+  case "android":
+    switch (arch) {
+      case "arm64":
+        localFileExisted = existsSync(
+          join(__dirname, "lancedb-nodejs.android-arm64.node"),
+        );
+        try {
+          if (localFileExisted) {
+            nativeBinding = require("./lancedb-nodejs.android-arm64.node");
+          } else {
+            nativeBinding = require("lancedb-android-arm64");
+          }
+        } catch (e) {
+          loadError = e;
+        }
+        break;
+      case "arm":
+        localFileExisted = existsSync(
+          join(__dirname, "lancedb-nodejs.android-arm-eabi.node"),
+        );
+        try {
+          if (localFileExisted) {
+            nativeBinding = require("./lancedb-nodejs.android-arm-eabi.node");
+          } else {
+            nativeBinding = require("lancedb-android-arm-eabi");
+          }
+        } catch (e) {
+          loadError = e;
+        }
+        break;
+      default:
+        throw new Error(`Unsupported architecture on Android ${arch}`);
+    }
+    break;
+  case "win32":
+    switch (arch) {
+      case "x64":
+        localFileExisted = existsSync(
+          join(__dirname, "lancedb-nodejs.win32-x64-msvc.node"),
+        );
+        try {
+          if (localFileExisted) {
+            nativeBinding = require("./lancedb-nodejs.win32-x64-msvc.node");
+          } else {
+            nativeBinding = require("lancedb-win32-x64-msvc");
+          }
+        } catch (e) {
+          loadError = e;
+        }
+        break;
+      case "ia32":
+        localFileExisted = existsSync(
+          join(__dirname, "lancedb-nodejs.win32-ia32-msvc.node"),
+        );
+        try {
+          if (localFileExisted) {
+            nativeBinding = require("./lancedb-nodejs.win32-ia32-msvc.node");
+          } else {
+            nativeBinding = require("lancedb-win32-ia32-msvc");
+          }
+        } catch (e) {
+          loadError = e;
+        }
+        break;
+      case "arm64":
+        localFileExisted = existsSync(
+          join(__dirname, "lancedb-nodejs.win32-arm64-msvc.node"),
+        );
+        try {
+          if (localFileExisted) {
+            nativeBinding = require("./lancedb-nodejs.win32-arm64-msvc.node");
+          } else {
+            nativeBinding = require("lancedb-win32-arm64-msvc");
+          }
+        } catch (e) {
+          loadError = e;
+        }
+        break;
+      default:
+        throw new Error(`Unsupported architecture on Windows: ${arch}`);
+    }
+    break;
+  case "darwin":
+    localFileExisted = existsSync(
+      join(__dirname, "lancedb-nodejs.darwin-universal.node"),
+    );
+    try {
+      if (localFileExisted) {
+        nativeBinding = require("./lancedb-nodejs.darwin-universal.node");
+      } else {
+        nativeBinding = require("lancedb-darwin-universal");
+      }
+      break;
+    } catch {}
+    switch (arch) {
+      case "x64":
+        localFileExisted = existsSync(
+          join(__dirname, "lancedb-nodejs.darwin-x64.node"),
+        );
+        try {
+          if (localFileExisted) {
+            nativeBinding = require("./lancedb-nodejs.darwin-x64.node");
+          } else {
+            nativeBinding = require("lancedb-darwin-x64");
+          }
+        } catch (e) {
+          loadError = e;
+        }
+        break;
+      case "arm64":
+        localFileExisted = existsSync(
+          join(__dirname, "lancedb-nodejs.darwin-arm64.node"),
+        );
+        try {
+          if (localFileExisted) {
+            nativeBinding = require("./lancedb-nodejs.darwin-arm64.node");
+          } else {
+            nativeBinding = require("lancedb-darwin-arm64");
+          }
+        } catch (e) {
+          loadError = e;
+        }
+        break;
+      default:
+        throw new Error(`Unsupported architecture on macOS: ${arch}`);
+    }
+    break;
+  case "freebsd":
+    if (arch !== "x64") {
+      throw new Error(`Unsupported architecture on FreeBSD: ${arch}`);
+    }
+    localFileExisted = existsSync(
+      join(__dirname, "lancedb-nodejs.freebsd-x64.node"),
+    );
+    try {
+      if (localFileExisted) {
+        nativeBinding = require("./lancedb-nodejs.freebsd-x64.node");
+      } else {
+        nativeBinding = require("lancedb-freebsd-x64");
+      }
+    } catch (e) {
+      loadError = e;
+    }
+    break;
+  case "linux":
+    switch (arch) {
+      case "x64":
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, "lancedb-nodejs.linux-x64-musl.node"),
+          );
+          try {
+            if (localFileExisted) {
+              nativeBinding = require("./lancedb-nodejs.linux-x64-musl.node");
+            } else {
+              nativeBinding = require("lancedb-linux-x64-musl");
+            }
+          } catch (e) {
+            loadError = e;
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, "lancedb-nodejs.linux-x64-gnu.node"),
+          );
+          try {
+            if (localFileExisted) {
+              nativeBinding = require("./lancedb-nodejs.linux-x64-gnu.node");
+            } else {
+              nativeBinding = require("lancedb-linux-x64-gnu");
+            }
+          } catch (e) {
+            loadError = e;
+          }
+        }
+        break;
+      case "arm64":
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, "lancedb-nodejs.linux-arm64-musl.node"),
+          );
+          try {
+            if (localFileExisted) {
+              nativeBinding = require("./lancedb-nodejs.linux-arm64-musl.node");
+            } else {
+              nativeBinding = require("lancedb-linux-arm64-musl");
+            }
+          } catch (e) {
+            loadError = e;
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, "lancedb-nodejs.linux-arm64-gnu.node"),
+          );
+          try {
+            if (localFileExisted) {
+              nativeBinding = require("./lancedb-nodejs.linux-arm64-gnu.node");
+            } else {
+              nativeBinding = require("lancedb-linux-arm64-gnu");
+            }
+          } catch (e) {
+            loadError = e;
+          }
+        }
+        break;
+      case "arm":
+        localFileExisted = existsSync(
+          join(__dirname, "lancedb-nodejs.linux-arm-gnueabihf.node"),
+        );
+        try {
+          if (localFileExisted) {
+            nativeBinding = require("./lancedb-nodejs.linux-arm-gnueabihf.node");
+          } else {
+            nativeBinding = require("lancedb-linux-arm-gnueabihf");
+          }
+        } catch (e) {
+          loadError = e;
+        }
+        break;
+      case "riscv64":
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, "lancedb-nodejs.linux-riscv64-musl.node"),
+          );
+          try {
+            if (localFileExisted) {
+              nativeBinding = require("./lancedb-nodejs.linux-riscv64-musl.node");
+            } else {
+              nativeBinding = require("lancedb-linux-riscv64-musl");
+            }
+          } catch (e) {
+            loadError = e;
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, "lancedb-nodejs.linux-riscv64-gnu.node"),
+          );
+          try {
+            if (localFileExisted) {
+              nativeBinding = require("./lancedb-nodejs.linux-riscv64-gnu.node");
+            } else {
+              nativeBinding = require("lancedb-linux-riscv64-gnu");
+            }
+          } catch (e) {
+            loadError = e;
+          }
+        }
+        break;
+      case "s390x":
+        localFileExisted = existsSync(
+          join(__dirname, "lancedb-nodejs.linux-s390x-gnu.node"),
+        );
+        try {
+          if (localFileExisted) {
+            nativeBinding = require("./lancedb-nodejs.linux-s390x-gnu.node");
+          } else {
+            nativeBinding = require("lancedb-linux-s390x-gnu");
+          }
+        } catch (e) {
+          loadError = e;
+        }
+        break;
+      default:
+        throw new Error(`Unsupported architecture on Linux: ${arch}`);
+    }
+    break;
+  default:
+    throw new Error(`Unsupported OS: ${platform}, architecture: ${arch}`);
+}
+
+if (!nativeBinding) {
+  if (loadError) {
+    throw loadError;
+  }
+  throw new Error(`Failed to load native binding`);
+}
+
+const {
+  Connection,
+  Index,
+  RecordBatchIterator,
+  Query,
+  VectorQuery,
+  Table,
+  WriteMode,
+  connect,
+} = nativeBinding;
+
+module.exports.Connection = Connection;
+module.exports.Index = Index;
+module.exports.RecordBatchIterator = RecordBatchIterator;
+module.exports.Query = Query;
+module.exports.VectorQuery = VectorQuery;
+module.exports.Table = Table;
+module.exports.WriteMode = WriteMode;
+module.exports.connect = connect;

nodejs/lancedb/query.ts

@@ -0,0 +1,375 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { RecordBatch, tableFromIPC, Table as ArrowTable } from "apache-arrow";
+import {
+  RecordBatchIterator as NativeBatchIterator,
+  Query as NativeQuery,
+  Table as NativeTable,
+  VectorQuery as NativeVectorQuery,
+} from "./native";
+import { type IvfPqOptions } from "./indices";
+class RecordBatchIterator implements AsyncIterator<RecordBatch> {
+  private promisedInner?: Promise<NativeBatchIterator>;
+  private inner?: NativeBatchIterator;
+
+  constructor(promise?: Promise<NativeBatchIterator>) {
+    // TODO: check promise reliably so we dont need to pass two arguments.
+    this.promisedInner = promise;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  async next(): Promise<IteratorResult<RecordBatch<any>>> {
+    if (this.inner === undefined) {
+      this.inner = await this.promisedInner;
+    }
+    if (this.inner === undefined) {
+      throw new Error("Invalid iterator state state");
+    }
+    const n = await this.inner.next();
+    if (n == null) {
+      return Promise.resolve({ done: true, value: null });
+    }
+    const tbl = tableFromIPC(n);
+    if (tbl.batches.length != 1) {
+      throw new Error("Expected only one batch");
+    }
+    return Promise.resolve({ done: false, value: tbl.batches[0] });
+  }
+}
+/* eslint-enable */
+
+/** Common methods supported by all query types */
+export class QueryBase<
+  NativeQueryType extends NativeQuery | NativeVectorQuery,
+  QueryType,
+> implements AsyncIterable<RecordBatch>
+{
+  protected constructor(protected inner: NativeQueryType) {}
+
+  /**
+   * A filter statement to be applied to this query.
+   *
+   * The filter should be supplied as an SQL query string.  For example:
+   * @example
+   * x > 10
+   * y > 0 AND y < 100
+   * x > 5 OR y = 'test'
+   *
+   * Filtering performance can often be improved by creating a scalar index
+   * on the filter column(s).
+   */
+  where(predicate: string): QueryType {
+    this.inner.onlyIf(predicate);
+    return this as unknown as QueryType;
+  }
+
+  /**
+   * Return only the specified columns.
+   *
+   * By default a query will return all columns from the table.  However, this can have
+   * a very significant impact on latency.  LanceDb stores data in a columnar fashion.  This
+   * means we can finely tune our I/O to select exactly the columns we need.
+   *
+   * As a best practice you should always limit queries to the columns that you need.  If you
+   * pass in an array of column names then only those columns will be returned.
+   *
+   * You can also use this method to create new "dynamic" columns based on your existing columns.
+   * For example, you may not care about "a" or "b" but instead simply want "a + b".  This is often
+   * seen in the SELECT clause of an SQL query (e.g. `SELECT a+b FROM my_table`).
+   *
+   * To create dynamic columns you can pass in a Map<string, string>.  A column will be returned
+   * for each entry in the map.  The key provides the name of the column.  The value is
+   * an SQL string used to specify how the column is calculated.
+   *
+   * For example, an SQL query might state `SELECT a + b AS combined, c`.  The equivalent
+   * input to this method would be:
+   * @example
+   * new Map([["combined", "a + b"], ["c", "c"]])
+   *
+   * Columns will always be returned in the order given, even if that order is different than
+   * the order used when adding the data.
+   *
+   * Note that you can pass in a `Record<string, string>` (e.g. an object literal). This method
+   * uses `Object.entries` which should preserve the insertion order of the object.  However,
+   * object insertion order is easy to get wrong and `Map` is more foolproof.
+   */
+  select(
+    columns: string[] | Map<string, string> | Record<string, string>,
+  ): QueryType {
+    let columnTuples: [string, string][];
+    if (Array.isArray(columns)) {
+      columnTuples = columns.map((c) => [c, c]);
+    } else if (columns instanceof Map) {
+      columnTuples = Array.from(columns.entries());
+    } else {
+      columnTuples = Object.entries(columns);
+    }
+    this.inner.select(columnTuples);
+    return this as unknown as QueryType;
+  }
+
+  /**
+   * Set the maximum number of results to return.
+   *
+   * By default, a plain search has no limit.  If this method is not
+   * called then every valid row from the table will be returned.
+   */
+  limit(limit: number): QueryType {
+    this.inner.limit(limit);
+    return this as unknown as QueryType;
+  }
+
+  protected nativeExecute(): Promise<NativeBatchIterator> {
+    return this.inner.execute();
+  }
+
+  /**
+   * Execute the query and return the results as an @see {@link AsyncIterator}
+   * of @see {@link RecordBatch}.
+   *
+   * By default, LanceDb will use many threads to calculate results and, when
+   * the result set is large, multiple batches will be processed at one time.
+   * This readahead is limited however and backpressure will be applied if this
+   * stream is consumed slowly (this constrains the maximum memory used by a
+   * single query)
+   *
+   */
+  protected execute(): RecordBatchIterator {
+    return new RecordBatchIterator(this.nativeExecute());
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  [Symbol.asyncIterator](): AsyncIterator<RecordBatch<any>> {
+    const promise = this.nativeExecute();
+    return new RecordBatchIterator(promise);
+  }
+
+  /** Collect the results as an Arrow @see {@link ArrowTable}. */
+  async toArrow(): Promise<ArrowTable> {
+    const batches = [];
+    for await (const batch of this) {
+      batches.push(batch);
+    }
+    return new ArrowTable(batches);
+  }
+
+  /** Collect the results as an array of objects. */
+  async toArray(): Promise<unknown[]> {
+    const tbl = await this.toArrow();
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+    return tbl.toArray();
+  }
+}
+
+/**
+ * An interface for a query that can be executed
+ *
+ * Supported by all query types
+ */
+export interface ExecutableQuery {}
+
+/**
+ * A builder used to construct a vector search
+ *
+ * This builder can be reused to execute the query many times.
+ */
+export class VectorQuery extends QueryBase<NativeVectorQuery, VectorQuery> {
+  constructor(inner: NativeVectorQuery) {
+    super(inner);
+  }
+
+  /**
+   * Set the number of partitions to search (probe)
+   *
+   * This argument is only used when the vector column has an IVF PQ index.
+   * If there is no index then this value is ignored.
+   *
+   * The IVF stage of IVF PQ divides the input into partitions (clusters) of
+   * related values.
+   *
+   * The partition whose centroids are closest to the query vector will be
+   * exhaustiely searched to find matches.  This parameter controls how many
+   * partitions should be searched.
+   *
+   * Increasing this value will increase the recall of your query but will
+   * also increase the latency of your query.  The default value is 20.  This
+   * default is good for many cases but the best value to use will depend on
+   * your data and the recall that you need to achieve.
+   *
+   * For best results we recommend tuning this parameter with a benchmark against
+   * your actual data to find the smallest possible value that will still give
+   * you the desired recall.
+   */
+  nprobes(nprobes: number): VectorQuery {
+    this.inner.nprobes(nprobes);
+    return this;
+  }
+
+  /**
+   * Set the vector column to query
+   *
+   * This controls which column is compared to the query vector supplied in
+   * the call to @see {@link Query#nearestTo}
+   *
+   * This parameter must be specified if the table has more than one column
+   * whose data type is a fixed-size-list of floats.
+   */
+  column(column: string): VectorQuery {
+    this.inner.column(column);
+    return this;
+  }
+
+  /**
+   * Set the distance metric to use
+   *
+   * When performing a vector search we try and find the "nearest" vectors according
+   * to some kind of distance metric.  This parameter controls which distance metric to
+   * use.  See @see {@link IvfPqOptions.distanceType} for more details on the different
+   * distance metrics available.
+   *
+   * Note: if there is a vector index then the distance type used MUST match the distance
+   * type used to train the vector index.  If this is not done then the results will be
+   * invalid.
+   *
+   * By default "l2" is used.
+   */
+  distanceType(distanceType: string): VectorQuery {
+    this.inner.distanceType(distanceType);
+    return this;
+  }
+
+  /**
+   * A multiplier to control how many additional rows are taken during the refine step
+   *
+   * This argument is only used when the vector column has an IVF PQ index.
+   * If there is no index then this value is ignored.
+   *
+   * An IVF PQ index stores compressed (quantized) values.  They query vector is compared
+   * against these values and, since they are compressed, the comparison is inaccurate.
+   *
+   * This parameter can be used to refine the results.  It can improve both improve recall
+   * and correct the ordering of the nearest results.
+   *
+   * To refine results LanceDb will first perform an ANN search to find the nearest
+   * `limit` * `refine_factor` results.  In other words, if `refine_factor` is 3 and
+   * `limit` is the default (10) then the first 30 results will be selected.  LanceDb
+   * then fetches the full, uncompressed, values for these 30 results.  The results are
+   * then reordered by the true distance and only the nearest 10 are kept.
+   *
+   * Note: there is a difference between calling this method with a value of 1 and never
+   * calling this method at all.  Calling this method with any value will have an impact
+   * on your search latency.  When you call this method with a `refine_factor` of 1 then
+   * LanceDb still needs to fetch the full, uncompressed, values so that it can potentially
+   * reorder the results.
+   *
+   * Note: if this method is NOT called then the distances returned in the _distance column
+   * will be approximate distances based on the comparison of the quantized query vector
+   * and the quantized result vectors.  This can be considerably different than the true
+   * distance between the query vector and the actual uncompressed vector.
+   */
+  refineFactor(refineFactor: number): VectorQuery {
+    this.inner.refineFactor(refineFactor);
+    return this;
+  }
+
+  /**
+   * If this is called then filtering will happen after the vector search instead of
+   * before.
+   *
+   * By default filtering will be performed before the vector search.  This is how
+   * filtering is typically understood to work.  This prefilter step does add some
+   * additional latency.  Creating a scalar index on the filter column(s) can
+   * often improve this latency.  However, sometimes a filter is too complex or scalar
+   * indices cannot be applied to the column.  In these cases postfiltering can be
+   * used instead of prefiltering to improve latency.
+   *
+   * Post filtering applies the filter to the results of the vector search.  This means
+   * we only run the filter on a much smaller set of data.  However, it can cause the
+   * query to return fewer than `limit` results (or even no results) if none of the nearest
+   * results match the filter.
+   *
+   * Post filtering happens during the "refine stage" (described in more detail in
+   * @see {@link VectorQuery#refineFactor}).  This means that setting a higher refine
+   * factor can often help restore some of the results lost by post filtering.
+   */
+  postfilter(): VectorQuery {
+    this.inner.postfilter();
+    return this;
+  }
+
+  /**
+   * If this is called then any vector index is skipped
+   *
+   * An exhaustive (flat) search will be performed.  The query vector will
+   * be compared to every vector in the table.  At high scales this can be
+   * expensive.  However, this is often still useful.  For example, skipping
+   * the vector index can give you ground truth results which you can use to
+   * calculate your recall to select an appropriate value for nprobes.
+   */
+  bypassVectorIndex(): VectorQuery {
+    this.inner.bypassVectorIndex();
+    return this;
+  }
+}
+
+/** A builder for LanceDB queries. */
+export class Query extends QueryBase<NativeQuery, Query> {
+  constructor(tbl: NativeTable) {
+    super(tbl.query());
+  }
+
+  /**
+   * Find the nearest vectors to the given query vector.
+   *
+   * This converts the query from a plain query to a vector query.
+   *
+   * This method will attempt to convert the input to the query vector
+   * expected by the embedding model.  If the input cannot be converted
+   * then an error will be thrown.
+   *
+   * By default, there is no embedding model, and the input should be
+   * an array-like object of numbers (something that can be used as input
+   * to Float32Array.from)
+   *
+   * If there is only one vector column (a column whose data type is a
+   * fixed size list of floats) then the column does not need to be specified.
+   * If there is more than one vector column you must use
+   * @see {@link VectorQuery#column}  to specify which column you would like
+   * to compare with.
+   *
+   * If no index has been created on the vector column then a vector query
+   * will perform a distance comparison between the query vector and every
+   * vector in the database and then sort the results.  This is sometimes
+   * called a "flat search"
+   *
+   * For small databases, with a few hundred thousand vectors or less, this can
+   * be reasonably fast.  In larger databases you should create a vector index
+   * on the column.  If there is a vector index then an "approximate" nearest
+   * neighbor search (frequently called an ANN search) will be performed.  This
+   * search is much faster, but the results will be approximate.
+   *
+   * The query can be further parameterized using the returned builder.  There
+   * are various ANN search parameters that will let you fine tune your recall
+   * accuracy vs search latency.
+   *
+   * Vector searches always have a `limit`.  If `limit` has not been called then
+   * a default `limit` of 10 will be used.  @see {@link Query#limit}
+   */
+  nearestTo(vector: unknown): VectorQuery {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const vectorQuery = this.inner.nearestTo(Float32Array.from(vector as any));
+    return new VectorQuery(vectorQuery);
+  }
+}

nodejs/lancedb/sanitize.ts

@@ -0,0 +1,516 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The utilities in this file help sanitize data from the user's arrow
+// library into the types expected by vectordb's arrow library.  Node
+// generally allows for mulitple versions of the same library (and sometimes
+// even multiple copies of the same version) to be installed at the same
+// time.  However, arrow-js uses instanceof which expected that the input
+// comes from the exact same library instance.  This is not always the case
+// and so we must sanitize the input to ensure that it is compatible.
+
+import {
+  Field,
+  Utf8,
+  FixedSizeBinary,
+  FixedSizeList,
+  Schema,
+  List,
+  Struct,
+  Float,
+  Bool,
+  Date_,
+  Decimal,
+  DataType,
+  Dictionary,
+  Binary,
+  Float32,
+  Interval,
+  Map_,
+  Duration,
+  Union,
+  Time,
+  Timestamp,
+  Type,
+  Null,
+  Int,
+  type Precision,
+  type DateUnit,
+  Int8,
+  Int16,
+  Int32,
+  Int64,
+  Uint8,
+  Uint16,
+  Uint32,
+  Uint64,
+  Float16,
+  Float64,
+  DateDay,
+  DateMillisecond,
+  DenseUnion,
+  SparseUnion,
+  TimeNanosecond,
+  TimeMicrosecond,
+  TimeMillisecond,
+  TimeSecond,
+  TimestampNanosecond,
+  TimestampMicrosecond,
+  TimestampMillisecond,
+  TimestampSecond,
+  IntervalDayTime,
+  IntervalYearMonth,
+  DurationNanosecond,
+  DurationMicrosecond,
+  DurationMillisecond,
+  DurationSecond,
+} from "apache-arrow";
+import type { IntBitWidth, TKeys, TimeBitWidth } from "apache-arrow/type";
+
+function sanitizeMetadata(
+  metadataLike?: unknown,
+): Map<string, string> | undefined {
+  if (metadataLike === undefined || metadataLike === null) {
+    return undefined;
+  }
+  if (!(metadataLike instanceof Map)) {
+    throw Error("Expected metadata, if present, to be a Map<string, string>");
+  }
+  for (const item of metadataLike) {
+    if (!(typeof item[0] === "string" || !(typeof item[1] === "string"))) {
+      throw Error(
+        "Expected metadata, if present, to be a Map<string, string> but it had non-string keys or values",
+      );
+    }
+  }
+  return metadataLike as Map<string, string>;
+}
+
+function sanitizeInt(typeLike: object) {
+  if (
+    !("bitWidth" in typeLike) ||
+    typeof typeLike.bitWidth !== "number" ||
+    !("isSigned" in typeLike) ||
+    typeof typeLike.isSigned !== "boolean"
+  ) {
+    throw Error(
+      "Expected an Int Type to have a `bitWidth` and `isSigned` property",
+    );
+  }
+  return new Int(typeLike.isSigned, typeLike.bitWidth as IntBitWidth);
+}
+
+function sanitizeFloat(typeLike: object) {
+  if (!("precision" in typeLike) || typeof typeLike.precision !== "number") {
+    throw Error("Expected a Float Type to have a `precision` property");
+  }
+  return new Float(typeLike.precision as Precision);
+}
+
+function sanitizeDecimal(typeLike: object) {
+  if (
+    !("scale" in typeLike) ||
+    typeof typeLike.scale !== "number" ||
+    !("precision" in typeLike) ||
+    typeof typeLike.precision !== "number" ||
+    !("bitWidth" in typeLike) ||
+    typeof typeLike.bitWidth !== "number"
+  ) {
+    throw Error(
+      "Expected a Decimal Type to have `scale`, `precision`, and `bitWidth` properties",
+    );
+  }
+  return new Decimal(typeLike.scale, typeLike.precision, typeLike.bitWidth);
+}
+
+function sanitizeDate(typeLike: object) {
+  if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
+    throw Error("Expected a Date type to have a `unit` property");
+  }
+  return new Date_(typeLike.unit as DateUnit);
+}
+
+function sanitizeTime(typeLike: object) {
+  if (
+    !("unit" in typeLike) ||
+    typeof typeLike.unit !== "number" ||
+    !("bitWidth" in typeLike) ||
+    typeof typeLike.bitWidth !== "number"
+  ) {
+    throw Error(
+      "Expected a Time type to have `unit` and `bitWidth` properties",
+    );
+  }
+  return new Time(typeLike.unit, typeLike.bitWidth as TimeBitWidth);
+}
+
+function sanitizeTimestamp(typeLike: object) {
+  if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
+    throw Error("Expected a Timestamp type to have a `unit` property");
+  }
+  let timezone = null;
+  if ("timezone" in typeLike && typeof typeLike.timezone === "string") {
+    timezone = typeLike.timezone;
+  }
+  return new Timestamp(typeLike.unit, timezone);
+}
+
+function sanitizeTypedTimestamp(
+  typeLike: object,
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  Datatype:
+    | typeof TimestampNanosecond
+    | typeof TimestampMicrosecond
+    | typeof TimestampMillisecond
+    | typeof TimestampSecond,
+) {
+  let timezone = null;
+  if ("timezone" in typeLike && typeof typeLike.timezone === "string") {
+    timezone = typeLike.timezone;
+  }
+  return new Datatype(timezone);
+}
+
+function sanitizeInterval(typeLike: object) {
+  if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
+    throw Error("Expected an Interval type to have a `unit` property");
+  }
+  return new Interval(typeLike.unit);
+}
+
+function sanitizeList(typeLike: object) {
+  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
+    throw Error(
+      "Expected a List type to have an array-like `children` property",
+    );
+  }
+  if (typeLike.children.length !== 1) {
+    throw Error("Expected a List type to have exactly one child");
+  }
+  return new List(sanitizeField(typeLike.children[0]));
+}
+
+function sanitizeStruct(typeLike: object) {
+  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
+    throw Error(
+      "Expected a Struct type to have an array-like `children` property",
+    );
+  }
+  return new Struct(typeLike.children.map((child) => sanitizeField(child)));
+}
+
+function sanitizeUnion(typeLike: object) {
+  if (
+    !("typeIds" in typeLike) ||
+    !("mode" in typeLike) ||
+    typeof typeLike.mode !== "number"
+  ) {
+    throw Error(
+      "Expected a Union type to have `typeIds` and `mode` properties",
+    );
+  }
+  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
+    throw Error(
+      "Expected a Union type to have an array-like `children` property",
+    );
+  }
+
+  return new Union(
+    typeLike.mode,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    typeLike.typeIds as any,
+    typeLike.children.map((child) => sanitizeField(child)),
+  );
+}
+
+function sanitizeTypedUnion(
+  typeLike: object,
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  UnionType: typeof DenseUnion | typeof SparseUnion,
+) {
+  if (!("typeIds" in typeLike)) {
+    throw Error(
+      "Expected a DenseUnion/SparseUnion type to have a `typeIds` property",
+    );
+  }
+  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
+    throw Error(
+      "Expected a DenseUnion/SparseUnion type to have an array-like `children` property",
+    );
+  }
+
+  return new UnionType(
+    typeLike.typeIds as Int32Array | number[],
+    typeLike.children.map((child) => sanitizeField(child)),
+  );
+}
+
+function sanitizeFixedSizeBinary(typeLike: object) {
+  if (!("byteWidth" in typeLike) || typeof typeLike.byteWidth !== "number") {
+    throw Error(
+      "Expected a FixedSizeBinary type to have a `byteWidth` property",
+    );
+  }
+  return new FixedSizeBinary(typeLike.byteWidth);
+}
+
+function sanitizeFixedSizeList(typeLike: object) {
+  if (!("listSize" in typeLike) || typeof typeLike.listSize !== "number") {
+    throw Error("Expected a FixedSizeList type to have a `listSize` property");
+  }
+  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
+    throw Error(
+      "Expected a FixedSizeList type to have an array-like `children` property",
+    );
+  }
+  if (typeLike.children.length !== 1) {
+    throw Error("Expected a FixedSizeList type to have exactly one child");
+  }
+  return new FixedSizeList(
+    typeLike.listSize,
+    sanitizeField(typeLike.children[0]),
+  );
+}
+
+function sanitizeMap(typeLike: object) {
+  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
+    throw Error(
+      "Expected a Map type to have an array-like `children` property",
+    );
+  }
+  if (!("keysSorted" in typeLike) || typeof typeLike.keysSorted !== "boolean") {
+    throw Error("Expected a Map type to have a `keysSorted` property");
+  }
+
+  return new Map_(
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    typeLike.children.map((field) => sanitizeField(field)) as any,
+    typeLike.keysSorted,
+  );
+}
+
+function sanitizeDuration(typeLike: object) {
+  if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
+    throw Error("Expected a Duration type to have a `unit` property");
+  }
+  return new Duration(typeLike.unit);
+}
+
+function sanitizeDictionary(typeLike: object) {
+  if (!("id" in typeLike) || typeof typeLike.id !== "number") {
+    throw Error("Expected a Dictionary type to have an `id` property");
+  }
+  if (!("indices" in typeLike) || typeof typeLike.indices !== "object") {
+    throw Error("Expected a Dictionary type to have an `indices` property");
+  }
+  if (!("dictionary" in typeLike) || typeof typeLike.dictionary !== "object") {
+    throw Error("Expected a Dictionary type to have an `dictionary` property");
+  }
+  if (!("isOrdered" in typeLike) || typeof typeLike.isOrdered !== "boolean") {
+    throw Error("Expected a Dictionary type to have an `isOrdered` property");
+  }
+  return new Dictionary(
+    sanitizeType(typeLike.dictionary),
+    sanitizeType(typeLike.indices) as TKeys,
+    typeLike.id,
+    typeLike.isOrdered,
+  );
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function sanitizeType(typeLike: unknown): DataType<any> {
+  if (typeof typeLike !== "object" || typeLike === null) {
+    throw Error("Expected a Type but object was null/undefined");
+  }
+  if (!("typeId" in typeLike) || !(typeof typeLike.typeId !== "function")) {
+    throw Error("Expected a Type to have a typeId function");
+  }
+  let typeId: Type;
+  if (typeof typeLike.typeId === "function") {
+    typeId = (typeLike.typeId as () => unknown)() as Type;
+  } else if (typeof typeLike.typeId === "number") {
+    typeId = typeLike.typeId as Type;
+  } else {
+    throw Error("Type's typeId property was not a function or number");
+  }
+
+  switch (typeId) {
+    case Type.NONE:
+      throw Error("Received a Type with a typeId of NONE");
+    case Type.Null:
+      return new Null();
+    case Type.Int:
+      return sanitizeInt(typeLike);
+    case Type.Float:
+      return sanitizeFloat(typeLike);
+    case Type.Binary:
+      return new Binary();
+    case Type.Utf8:
+      return new Utf8();
+    case Type.Bool:
+      return new Bool();
+    case Type.Decimal:
+      return sanitizeDecimal(typeLike);
+    case Type.Date:
+      return sanitizeDate(typeLike);
+    case Type.Time:
+      return sanitizeTime(typeLike);
+    case Type.Timestamp:
+      return sanitizeTimestamp(typeLike);
+    case Type.Interval:
+      return sanitizeInterval(typeLike);
+    case Type.List:
+      return sanitizeList(typeLike);
+    case Type.Struct:
+      return sanitizeStruct(typeLike);
+    case Type.Union:
+      return sanitizeUnion(typeLike);
+    case Type.FixedSizeBinary:
+      return sanitizeFixedSizeBinary(typeLike);
+    case Type.FixedSizeList:
+      return sanitizeFixedSizeList(typeLike);
+    case Type.Map:
+      return sanitizeMap(typeLike);
+    case Type.Duration:
+      return sanitizeDuration(typeLike);
+    case Type.Dictionary:
+      return sanitizeDictionary(typeLike);
+    case Type.Int8:
+      return new Int8();
+    case Type.Int16:
+      return new Int16();
+    case Type.Int32:
+      return new Int32();
+    case Type.Int64:
+      return new Int64();
+    case Type.Uint8:
+      return new Uint8();
+    case Type.Uint16:
+      return new Uint16();
+    case Type.Uint32:
+      return new Uint32();
+    case Type.Uint64:
+      return new Uint64();
+    case Type.Float16:
+      return new Float16();
+    case Type.Float32:
+      return new Float32();
+    case Type.Float64:
+      return new Float64();
+    case Type.DateMillisecond:
+      return new DateMillisecond();
+    case Type.DateDay:
+      return new DateDay();
+    case Type.TimeNanosecond:
+      return new TimeNanosecond();
+    case Type.TimeMicrosecond:
+      return new TimeMicrosecond();
+    case Type.TimeMillisecond:
+      return new TimeMillisecond();
+    case Type.TimeSecond:
+      return new TimeSecond();
+    case Type.TimestampNanosecond:
+      return sanitizeTypedTimestamp(typeLike, TimestampNanosecond);
+    case Type.TimestampMicrosecond:
+      return sanitizeTypedTimestamp(typeLike, TimestampMicrosecond);
+    case Type.TimestampMillisecond:
+      return sanitizeTypedTimestamp(typeLike, TimestampMillisecond);
+    case Type.TimestampSecond:
+      return sanitizeTypedTimestamp(typeLike, TimestampSecond);
+    case Type.DenseUnion:
+      return sanitizeTypedUnion(typeLike, DenseUnion);
+    case Type.SparseUnion:
+      return sanitizeTypedUnion(typeLike, SparseUnion);
+    case Type.IntervalDayTime:
+      return new IntervalDayTime();
+    case Type.IntervalYearMonth:
+      return new IntervalYearMonth();
+    case Type.DurationNanosecond:
+      return new DurationNanosecond();
+    case Type.DurationMicrosecond:
+      return new DurationMicrosecond();
+    case Type.DurationMillisecond:
+      return new DurationMillisecond();
+    case Type.DurationSecond:
+      return new DurationSecond();
+    default:
+      throw new Error("Unrecoginized type id in schema: " + typeId);
+  }
+}
+
+function sanitizeField(fieldLike: unknown): Field {
+  if (fieldLike instanceof Field) {
+    return fieldLike;
+  }
+  if (typeof fieldLike !== "object" || fieldLike === null) {
+    throw Error("Expected a Field but object was null/undefined");
+  }
+  if (
+    !("type" in fieldLike) ||
+    !("name" in fieldLike) ||
+    !("nullable" in fieldLike)
+  ) {
+    throw Error(
+      "The field passed in is missing a `type`/`name`/`nullable` property",
+    );
+  }
+  const type = sanitizeType(fieldLike.type);
+  const name = fieldLike.name;
+  if (!(typeof name === "string")) {
+    throw Error("The field passed in had a non-string `name` property");
+  }
+  const nullable = fieldLike.nullable;
+  if (!(typeof nullable === "boolean")) {
+    throw Error("The field passed in had a non-boolean `nullable` property");
+  }
+  let metadata;
+  if ("metadata" in fieldLike) {
+    metadata = sanitizeMetadata(fieldLike.metadata);
+  }
+  return new Field(name, type, nullable, metadata);
+}
+
+/**
+ * Convert something schemaLike into a Schema instance
+ *
+ * This method is often needed even when the caller is using a Schema
+ * instance because they might be using a different instance of apache-arrow
+ * than lancedb is using.
+ */
+export function sanitizeSchema(schemaLike: unknown): Schema {
+  if (schemaLike instanceof Schema) {
+    return schemaLike;
+  }
+  if (typeof schemaLike !== "object" || schemaLike === null) {
+    throw Error("Expected a Schema but object was null/undefined");
+  }
+  if (!("fields" in schemaLike)) {
+    throw Error(
+      "The schema passed in does not appear to be a schema (no 'fields' property)",
+    );
+  }
+  let metadata;
+  if ("metadata" in schemaLike) {
+    metadata = sanitizeMetadata(schemaLike.metadata);
+  }
+  if (!Array.isArray(schemaLike.fields)) {
+    throw Error(
+      "The schema passed in had a 'fields' property but it was not an array",
+    );
+  }
+  const sanitizedFields = schemaLike.fields.map((field) =>
+    sanitizeField(field),
+  );
+  return new Schema(sanitizedFields, metadata);
+}

nodejs/lancedb/table.ts

@@ -0,0 +1,353 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { Schema, tableFromIPC } from "apache-arrow";
+import {
+  AddColumnsSql,
+  ColumnAlteration,
+  IndexConfig,
+  Table as _NativeTable,
+} from "./native";
+import { Query, VectorQuery } from "./query";
+import { IndexOptions } from "./indices";
+import { Data, fromDataToBuffer } from "./arrow";
+
+export { IndexConfig } from "./native";
+/**
+ * Options for adding data to a table.
+ */
+export interface AddDataOptions {
+  /**
+   * If "append" (the default) then the new data will be added to the table
+   *
+   * If "overwrite" then the new data will replace the existing data in the table.
+   */
+  mode: "append" | "overwrite";
+}
+
+export interface UpdateOptions {
+  /**
+   * A filter that limits the scope of the update.
+   *
+   * This should be an SQL filter expression.
+   *
+   * Only rows that satisfy the expression will be updated.
+   *
+   * For example, this could be 'my_col == 0' to replace all instances
+   * of 0 in a column with some other default value.
+   */
+  where: string;
+}
+
+/**
+ * A Table is a collection of Records in a LanceDB Database.
+ *
+ * A Table object is expected to be long lived and reused for multiple operations.
+ * Table objects will cache a certain amount of index data in memory.  This cache
+ * will be freed when the Table is garbage collected.  To eagerly free the cache you
+ * can call the `close` method.  Once the Table is closed, it cannot be used for any
+ * further operations.
+ *
+ * Closing a table is optional.  It not closed, it will be closed when it is garbage
+ * collected.
+ */
+export class Table {
+  private readonly inner: _NativeTable;
+
+  /** Construct a Table. Internal use only. */
+  constructor(inner: _NativeTable) {
+    this.inner = inner;
+  }
+
+  /** Return true if the table has not been closed */
+  isOpen(): boolean {
+    return this.inner.isOpen();
+  }
+
+  /**
+   * Close the table, releasing any underlying resources.
+   *
+   * It is safe to call this method multiple times.
+   *
+   * Any attempt to use the table after it is closed will result in an error.
+   */
+  close(): void {
+    this.inner.close();
+  }
+
+  /** Return a brief description of the table */
+  display(): string {
+    return this.inner.display();
+  }
+
+  /** Get the schema of the table. */
+  async schema(): Promise<Schema> {
+    const schemaBuf = await this.inner.schema();
+    const tbl = tableFromIPC(schemaBuf);
+    return tbl.schema;
+  }
+
+  /**
+   * Insert records into this Table.
+   * @param {Data} data Records to be inserted into the Table
+   */
+  async add(data: Data, options?: Partial<AddDataOptions>): Promise<void> {
+    const mode = options?.mode ?? "append";
+
+    const buffer = await fromDataToBuffer(data);
+    await this.inner.add(buffer, mode);
+  }
+
+  /**
+   * Update existing records in the Table
+   *
+   * An update operation can be used to adjust existing values.  Use the
+   * returned builder to specify which columns to update.  The new value
+   * can be a literal value (e.g. replacing nulls with some default value)
+   * or an expression applied to the old value (e.g. incrementing a value)
+   *
+   * An optional condition can be specified (e.g. "only update if the old
+   * value is 0")
+   *
+   * Note: if your condition is something like "some_id_column == 7" and
+   * you are updating many rows (with different ids) then you will get
+   * better performance with a single [`merge_insert`] call instead of
+   * repeatedly calilng this method.
+   * @param {Map<string, string> | Record<string, string>} updates - the
+   * columns to update
+   *
+   * Keys in the map should specify the name of the column to update.
+   * Values in the map provide the new value of the column.  These can
+   * be SQL literal strings (e.g. "7" or "'foo'") or they can be expressions
+   * based on the row being updated (e.g. "my_col + 1")
+   * @param {Partial<UpdateOptions>} options - additional options to control
+   * the update behavior
+   */
+  async update(
+    updates: Map<string, string> | Record<string, string>,
+    options?: Partial<UpdateOptions>,
+  ) {
+    const onlyIf = options?.where;
+    let columns: [string, string][];
+    if (updates instanceof Map) {
+      columns = Array.from(updates.entries());
+    } else {
+      columns = Object.entries(updates);
+    }
+    await this.inner.update(onlyIf, columns);
+  }
+
+  /** Count the total number of rows in the dataset. */
+  async countRows(filter?: string): Promise<number> {
+    return await this.inner.countRows(filter);
+  }
+
+  /** Delete the rows that satisfy the predicate. */
+  async delete(predicate: string): Promise<void> {
+    await this.inner.delete(predicate);
+  }
+
+  /**
+   * Create an index to speed up queries.
+   *
+   * Indices can be created on vector columns or scalar columns.
+   * Indices on vector columns will speed up vector searches.
+   * Indices on scalar columns will speed up filtering (in both
+   * vector and non-vector searches)
+   * @example
+   * // If the column has a vector (fixed size list) data type then
+   * // an IvfPq vector index will be created.
+   * const table = await conn.openTable("my_table");
+   * await table.createIndex(["vector"]);
+   * @example
+   * // For advanced control over vector index creation you can specify
+   * // the index type and options.
+   * const table = await conn.openTable("my_table");
+   * await table.createIndex(["vector"], I)
+   *   .ivf_pq({ num_partitions: 128, num_sub_vectors: 16 })
+   *   .build();
+   * @example
+   * // Or create a Scalar index
+   * await table.createIndex("my_float_col").build();
+   */
+  async createIndex(column: string, options?: Partial<IndexOptions>) {
+    // Bit of a hack to get around the fact that TS has no package-scope.
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const nativeIndex = (options?.config as any)?.inner;
+    await this.inner.createIndex(nativeIndex, column, options?.replace);
+  }
+
+  /**
+   * Create a {@link Query} Builder.
+   *
+   * Queries allow you to search your existing data.  By default the query will
+   * return all the data in the table in no particular order.  The builder
+   * returned by this method can be used to control the query using filtering,
+   * vector similarity, sorting, and more.
+   *
+   * Note: By default, all columns are returned.  For best performance, you should
+   * only fetch the columns you need.  See [`Query::select_with_projection`] for
+   * more details.
+   *
+   * When appropriate, various indices and statistics based pruning will be used to
+   * accelerate the query.
+   * @example
+   * // SQL-style filtering
+   * //
+   * // This query will return up to 1000 rows whose value in the `id` column
+   * // is greater than 5.  LanceDb supports a broad set of filtering functions.
+   * for await (const batch of table.query()
+   *                          .filter("id > 1").select(["id"]).limit(20)) {
+   *  console.log(batch);
+   * }
+   * @example
+   * // Vector Similarity Search
+   * //
+   * // This example will find the 10 rows whose value in the "vector" column are
+   * // closest to the query vector [1.0, 2.0, 3.0].  If an index has been created
+   * // on the "vector" column then this will perform an ANN search.
+   * //
+   * // The `refine_factor` and `nprobes` methods are used to control the recall /
+   * // latency tradeoff of the search.
+   * for await (const batch of table.query()
+   *                    .nearestTo([1, 2, 3])
+   *                    .refineFactor(5).nprobe(10)
+   *                    .limit(10)) {
+   *  console.log(batch);
+   * }
+   * @example
+   * // Scan the full dataset
+   * //
+   * // This query will return everything in the table in no particular order.
+   * for await (const batch of table.query()) {
+   *   console.log(batch);
+   * }
+   * @returns {Query} A builder that can be used to parameterize the query
+   */
+  query(): Query {
+    return new Query(this.inner);
+  }
+
+  /**
+   * Search the table with a given query vector.
+   *
+   * This is a convenience method for preparing a vector query and
+   * is the same thing as calling `nearestTo` on the builder returned
+   * by `query`.  @see {@link Query#nearestTo} for more details.
+   */
+  vectorSearch(vector: unknown): VectorQuery {
+    return this.query().nearestTo(vector);
+  }
+
+  // TODO: Support BatchUDF
+  /**
+   * Add new columns with defined values.
+   * @param {AddColumnsSql[]} newColumnTransforms pairs of column names and
+   * the SQL expression to use to calculate the value of the new column. These
+   * expressions will be evaluated for each row in the table, and can
+   * reference existing columns in the table.
+   */
+  async addColumns(newColumnTransforms: AddColumnsSql[]): Promise<void> {
+    await this.inner.addColumns(newColumnTransforms);
+  }
+
+  /**
+   * Alter the name or nullability of columns.
+   * @param {ColumnAlteration[]} columnAlterations One or more alterations to
+   * apply to columns.
+   */
+  async alterColumns(columnAlterations: ColumnAlteration[]): Promise<void> {
+    await this.inner.alterColumns(columnAlterations);
+  }
+
+  /**
+   * Drop one or more columns from the dataset
+   *
+   * This is a metadata-only operation and does not remove the data from the
+   * underlying storage. In order to remove the data, you must subsequently
+   * call ``compact_files`` to rewrite the data without the removed columns and
+   * then call ``cleanup_files`` to remove the old files.
+   * @param {string[]} columnNames The names of the columns to drop. These can
+   * be nested column references (e.g. "a.b.c") or top-level column names
+   * (e.g. "a").
+   */
+  async dropColumns(columnNames: string[]): Promise<void> {
+    await this.inner.dropColumns(columnNames);
+  }
+
+  /**
+   * Retrieve the version of the table
+   *
+   * LanceDb supports versioning.  Every operation that modifies the table increases
+   * version.  As long as a version hasn't been deleted you can `[Self::checkout]` that
+   * version to view the data at that point.  In addition, you can `[Self::restore]` the
+   * version to replace the current table with a previous version.
+   */
+  async version(): Promise<number> {
+    return await this.inner.version();
+  }
+
+  /**
+   * Checks out a specific version of the Table
+   *
+   * Any read operation on the table will now access the data at the checked out version.
+   * As a consequence, calling this method will disable any read consistency interval
+   * that was previously set.
+   *
+   * This is a read-only operation that turns the table into a sort of "view"
+   * or "detached head".  Other table instances will not be affected.  To make the change
+   * permanent you can use the `[Self::restore]` method.
+   *
+   * Any operation that modifies the table will fail while the table is in a checked
+   * out state.
+   *
+   * To return the table to a normal state use `[Self::checkout_latest]`
+   */
+  async checkout(version: number): Promise<void> {
+    await this.inner.checkout(version);
+  }
+
+  /**
+   * Ensures the table is pointing at the latest version
+   *
+   * This can be used to manually update a table when the read_consistency_interval is None
+   * It can also be used to undo a `[Self::checkout]` operation
+   */
+  async checkoutLatest(): Promise<void> {
+    await this.inner.checkoutLatest();
+  }
+
+  /**
+   * Restore the table to the currently checked out version
+   *
+   * This operation will fail if checkout has not been called previously
+   *
+   * This operation will overwrite the latest version of the table with a
+   * previous version.  Any changes made since the checked out version will
+   * no longer be visible.
+   *
+   * Once the operation concludes the table will no longer be in a checked
+   * out state and the read_consistency_interval, if any, will apply.
+   */
+  async restore(): Promise<void> {
+    await this.inner.restore();
+  }
+
+  /**
+   * List all indices that have been created with Self::create_index
+   */
+  async listIndices(): Promise<IndexConfig[]> {
+    return await this.inner.listIndices();
+  }
+}

nodejs/npm/darwin-arm64/README.md

@@ -1,3 +1,3 @@
-# `vectordb-darwin-arm64`
+# `lancedb-darwin-arm64`
 
-This is the **aarch64-apple-darwin** binary for `vectordb`
+This is the **aarch64-apple-darwin** binary for `lancedb`

nodejs/npm/darwin-arm64/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "vectordb-darwin-arm64",
+  "name": "lancedb-darwin-arm64",
   "version": "0.4.3",
   "os": [
     "darwin"
@@ -7,9 +7,9 @@
   "cpu": [
     "arm64"
   ],
-  "main": "vectordb.darwin-arm64.node",
+  "main": "lancedb.darwin-arm64.node",
   "files": [
-    "vectordb.darwin-arm64.node"
+    "lancedb.darwin-arm64.node"
   ],
   "license": "MIT",
   "engines": {

nodejs/npm/darwin-x64/README.md

@@ -1,3 +1,3 @@
-# `vectordb-darwin-x64`
+# `lancedb-darwin-x64`
 
-This is the **x86_64-apple-darwin** binary for `vectordb`
+This is the **x86_64-apple-darwin** binary for `lancedb`

nodejs/npm/darwin-x64/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "vectordb-darwin-x64",
+  "name": "lancedb-darwin-x64",
   "version": "0.4.3",
   "os": [
     "darwin"
@@ -7,9 +7,9 @@
   "cpu": [
     "x64"
   ],
-  "main": "vectordb.darwin-x64.node",
+  "main": "lancedb.darwin-x64.node",
   "files": [
-    "vectordb.darwin-x64.node"
+    "lancedb.darwin-x64.node"
   ],
   "license": "MIT",
   "engines": {

nodejs/npm/linux-arm64-gnu/README.md

@@ -1,3 +1,3 @@
-# `vectordb-linux-arm64-gnu`
+# `lancedb-linux-arm64-gnu`
 
-This is the **aarch64-unknown-linux-gnu** binary for `vectordb`
+This is the **aarch64-unknown-linux-gnu** binary for `lancedb`

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "vectordb-linux-arm64-gnu",
+  "name": "lancedb-linux-arm64-gnu",
   "version": "0.4.3",
   "os": [
     "linux"
@@ -7,9 +7,9 @@
   "cpu": [
     "arm64"
   ],
-  "main": "vectordb.linux-arm64-gnu.node",
+  "main": "lancedb.linux-arm64-gnu.node",
   "files": [
-    "vectordb.linux-arm64-gnu.node"
+    "lancedb.linux-arm64-gnu.node"
   ],
   "license": "MIT",
   "engines": {

nodejs/npm/linux-x64-gnu/README.md

@@ -1,3 +1,3 @@
-# `vectordb-linux-x64-gnu`
+# `lancedb-linux-x64-gnu`
 
-This is the **x86_64-unknown-linux-gnu** binary for `vectordb`
+This is the **x86_64-unknown-linux-gnu** binary for `lancedb`

nodejs/npm/linux-x64-gnu/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "vectordb-linux-x64-gnu",
+  "name": "lancedb-linux-x64-gnu",
   "version": "0.4.3",
   "os": [
     "linux"
@@ -7,9 +7,9 @@
   "cpu": [
     "x64"
   ],
-  "main": "vectordb.linux-x64-gnu.node",
+  "main": "lancedb.linux-x64-gnu.node",
   "files": [
-    "vectordb.linux-x64-gnu.node"
+    "lancedb.linux-x64-gnu.node"
   ],
   "license": "MIT",
   "engines": {

nodejs/package.json

@@ -1,10 +1,10 @@
 {
-  "name": "vectordb",
+  "name": "lancedb",
   "version": "0.4.3",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "napi": {
-    "name": "vectordb-nodejs",
+    "name": "lancedb-nodejs",
     "triples": {
       "defaults": false,
       "additional": [
@@ -18,15 +18,22 @@
   "license": "Apache 2.0",
   "devDependencies": {
     "@napi-rs/cli": "^2.18.0",
-    "@types/jest": "^29.5.11",
+    "@types/jest": "^29.1.2",
+    "@types/tmp": "^0.2.6",
     "@typescript-eslint/eslint-plugin": "^6.19.0",
     "@typescript-eslint/parser": "^6.19.0",
-    "eslint": "^8.56.0",
+    "apache-arrow-old": "npm:apache-arrow@13.0.0",
+    "eslint": "^8.57.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-plugin-jsdoc": "^48.2.1",
     "jest": "^29.7.0",
+    "prettier": "^3.1.0",
+    "tmp": "^0.2.3",
     "ts-jest": "^29.1.2",
     "typedoc": "^0.25.7",
     "typedoc-plugin-markdown": "^3.17.1",
-    "typescript": "^5.3.3"
+    "typescript": "^5.3.3",
+    "typescript-eslint": "^7.1.0"
   },
   "ava": {
     "timeout": "3m"
@@ -45,23 +52,25 @@
   ],
   "scripts": {
     "artifacts": "napi artifacts",
-    "build:native": "napi build --platform --release --js vectordb/native.js --dts vectordb/native.d.ts dist/",
-    "build:debug": "napi build --platform --dts ../vectordb/native.d.ts --js ../vectordb/native.js dist/",
+    "build:native": "napi build --platform --release --js lancedb/native.js --dts lancedb/native.d.ts dist/",
+    "build:debug": "napi build --platform --dts ../lancedb/native.d.ts --js ../lancedb/native.js dist/",
     "build": "npm run build:debug && tsc -b",
-    "docs": "typedoc --plugin typedoc-plugin-markdown vectordb/index.ts",
-    "lint": "eslint vectordb --ext .js,.ts",
+    "chkformat": "prettier . --check",
+    "docs": "typedoc --plugin typedoc-plugin-markdown lancedb/index.ts",
+    "lint": "eslint lancedb && eslint __test__",
     "prepublishOnly": "napi prepublish -t npm",
-    "test": "npm run build && jest",
+    "test": "npm run build && jest --verbose",
     "universal": "napi universal",
     "version": "napi version"
   },
   "optionalDependencies": {
-    "vectordb-darwin-arm64": "0.4.3",
-    "vectordb-darwin-x64": "0.4.3",
-    "vectordb-linux-arm64-gnu": "0.4.3",
-    "vectordb-linux-x64-gnu": "0.4.3"
+    "lancedb-darwin-arm64": "0.4.3",
+    "lancedb-darwin-x64": "0.4.3",
+    "lancedb-linux-arm64-gnu": "0.4.3",
+    "lancedb-linux-x64-gnu": "0.4.3",
+    "openai": "^4.28.4"
   },
-  "dependencies": {
+  "peerDependencies": {
     "apache-arrow": "^15.0.0"
   }
 }

nodejs/src/connection.rs

@@ -16,31 +16,92 @@ use napi::bindgen_prelude::*;
 use napi_derive::*;
 
 use crate::table::Table;
-use vectordb::connection::Connection as LanceDBConnection;
-use vectordb::ipc::ipc_file_to_batches;
+use crate::ConnectionOptions;
+use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection, CreateTableMode};
+use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
 
 #[napi]
 pub struct Connection {
-    conn: LanceDBConnection,
+    inner: Option<LanceDBConnection>,
+}
+
+impl Connection {
+    pub(crate) fn inner_new(inner: LanceDBConnection) -> Self {
+        Self { inner: Some(inner) }
+    }
+
+    fn get_inner(&self) -> napi::Result<&LanceDBConnection> {
+        self.inner
+            .as_ref()
+            .ok_or_else(|| napi::Error::from_reason("Connection is closed"))
+    }
+}
+
+impl Connection {
+    fn parse_create_mode_str(mode: &str) -> napi::Result<CreateTableMode> {
+        match mode {
+            "create" => Ok(CreateTableMode::Create),
+            "overwrite" => Ok(CreateTableMode::Overwrite),
+            "exist_ok" => Ok(CreateTableMode::exist_ok(|builder| builder)),
+            _ => Err(napi::Error::from_reason(format!("Invalid mode {}", mode))),
+        }
+    }
 }
 
 #[napi]
 impl Connection {
     /// Create a new Connection instance from the given URI.
     #[napi(factory)]
-    pub async fn new(uri: String) -> napi::Result<Self> {
-        Ok(Self {
-            conn: vectordb::connect(&uri).execute().await.map_err(|e| {
-                napi::Error::from_reason(format!("Failed to connect to database: {}", e))
-            })?,
-        })
+    pub async fn new(uri: String, options: ConnectionOptions) -> napi::Result<Self> {
+        let mut builder = ConnectBuilder::new(&uri);
+        if let Some(api_key) = options.api_key {
+            builder = builder.api_key(&api_key);
+        }
+        if let Some(host_override) = options.host_override {
+            builder = builder.host_override(&host_override);
+        }
+        if let Some(interval) = options.read_consistency_interval {
+            builder =
+                builder.read_consistency_interval(std::time::Duration::from_secs_f64(interval));
+        }
+        Ok(Self::inner_new(
+            builder
+                .execute()
+                .await
+                .map_err(|e| napi::Error::from_reason(format!("{}", e)))?,
+        ))
+    }
+
+    #[napi]
+    pub fn display(&self) -> napi::Result<String> {
+        Ok(self.get_inner()?.to_string())
+    }
+
+    #[napi]
+    pub fn is_open(&self) -> bool {
+        self.inner.is_some()
+    }
+
+    #[napi]
+    pub fn close(&mut self) {
+        self.inner.take();
     }
 
     /// List all tables in the dataset.
     #[napi]
-    pub async fn table_names(&self) -> napi::Result<Vec<String>> {
-        self.conn
-            .table_names()
+    pub async fn table_names(
+        &self,
+        start_after: Option<String>,
+        limit: Option<u32>,
+    ) -> napi::Result<Vec<String>> {
+        let mut op = self.get_inner()?.table_names();
+        if let Some(start_after) = start_after {
+            op = op.start_after(start_after);
+        }
+        if let Some(limit) = limit {
+            op = op.limit(limit);
+        }
+        op.execute()
             .await
             .map_err(|e| napi::Error::from_reason(format!("{}", e)))
     }
@@ -52,12 +113,40 @@ impl Connection {
     /// - buf: The buffer containing the IPC file.
     ///
     #[napi]
-    pub async fn create_table(&self, name: String, buf: Buffer) -> napi::Result<Table> {
+    pub async fn create_table(
+        &self,
+        name: String,
+        buf: Buffer,
+        mode: String,
+    ) -> napi::Result<Table> {
         let batches = ipc_file_to_batches(buf.to_vec())
             .map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
+        let mode = Self::parse_create_mode_str(&mode)?;
         let tbl = self
-            .conn
-            .create_table(&name, Box::new(batches))
+            .get_inner()?
+            .create_table(&name, batches)
+            .mode(mode)
+            .execute()
+            .await
+            .map_err(|e| napi::Error::from_reason(format!("{}", e)))?;
+        Ok(Table::new(tbl))
+    }
+
+    #[napi]
+    pub async fn create_empty_table(
+        &self,
+        name: String,
+        schema_buf: Buffer,
+        mode: String,
+    ) -> napi::Result<Table> {
+        let schema = ipc_file_to_schema(schema_buf.to_vec()).map_err(|e| {
+            napi::Error::from_reason(format!("Failed to marshal schema from JS to Rust: {}", e))
+        })?;
+        let mode = Self::parse_create_mode_str(&mode)?;
+        let tbl = self
+            .get_inner()?
+            .create_empty_table(&name, schema)
+            .mode(mode)
             .execute()
             .await
             .map_err(|e| napi::Error::from_reason(format!("{}", e)))?;
@@ -67,7 +156,7 @@ impl Connection {
     #[napi]
     pub async fn open_table(&self, name: String) -> napi::Result<Table> {
         let tbl = self
-            .conn
+            .get_inner()?
             .open_table(&name)
             .execute()
             .await
@@ -78,7 +167,7 @@ impl Connection {
     /// Drop table with the name. Or raise an error if the table does not exist.
     #[napi]
     pub async fn drop_table(&self, name: String) -> napi::Result<()> {
-        self.conn
+        self.get_inner()?
             .drop_table(&name)
             .await
             .map_err(|e| napi::Error::from_reason(format!("{}", e)))

nodejs/src/error.rs

@@ -0,0 +1,12 @@
+pub type Result<T> = napi::Result<T>;
+
+pub trait NapiErrorExt<T> {
+    /// Convert to a napi error using from_reason(err.to_string())
+    fn default_error(self) -> Result<T>;
+}
+
+impl<T> NapiErrorExt<T> for std::result::Result<T, lancedb::Error> {
+    fn default_error(self) -> Result<T> {
+        self.map_err(|err| napi::Error::from_reason(err.to_string()))
+    }
+}

nodejs/src/index.rs

@@ -12,89 +12,68 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use lance_linalg::distance::MetricType as LanceMetricType;
+use std::sync::Mutex;
+
+use lancedb::index::scalar::BTreeIndexBuilder;
+use lancedb::index::vector::IvfPqIndexBuilder;
+use lancedb::index::Index as LanceDbIndex;
 use napi_derive::napi;
 
-#[napi]
-pub enum IndexType {
-    Scalar,
-    IvfPq,
-}
+use crate::util::parse_distance_type;
 
 #[napi]
-pub enum MetricType {
-    L2,
-    Cosine,
-    Dot,
+pub struct Index {
+    inner: Mutex<Option<LanceDbIndex>>,
 }
 
-impl From<MetricType> for LanceMetricType {
-    fn from(metric: MetricType) -> Self {
-        match metric {
-            MetricType::L2 => Self::L2,
-            MetricType::Cosine => Self::Cosine,
-            MetricType::Dot => Self::Dot,
-        }
+impl Index {
+    pub fn consume(&self) -> napi::Result<LanceDbIndex> {
+        self.inner
+            .lock()
+            .unwrap()
+            .take()
+            .ok_or(napi::Error::from_reason(
+                "attempt to use an index more than once",
+            ))
     }
 }
 
 #[napi]
-pub struct IndexBuilder {
-    inner: vectordb::index::IndexBuilder,
-}
-
-#[napi]
-impl IndexBuilder {
-    pub fn new(tbl: &dyn vectordb::Table) -> Self {
-        let inner = tbl.create_index(&[]);
-        Self { inner }
-    }
-
-    #[napi]
-    pub unsafe fn replace(&mut self, v: bool) {
-        self.inner.replace(v);
-    }
-
-    #[napi]
-    pub unsafe fn column(&mut self, c: String) {
-        self.inner.columns(&[c.as_str()]);
-    }
-
-    #[napi]
-    pub unsafe fn name(&mut self, name: String) {
-        self.inner.name(name.as_str());
-    }
-
-    #[napi]
-    pub unsafe fn ivf_pq(
-        &mut self,
-        metric_type: Option<MetricType>,
+impl Index {
+    #[napi(factory)]
+    pub fn ivf_pq(
+        distance_type: Option<String>,
         num_partitions: Option<u32>,
         num_sub_vectors: Option<u32>,
-        num_bits: Option<u32>,
         max_iterations: Option<u32>,
         sample_rate: Option<u32>,
-    ) {
-        self.inner.ivf_pq();
-        metric_type.map(|m| self.inner.metric_type(m.into()));
-        num_partitions.map(|p| self.inner.num_partitions(p));
-        num_sub_vectors.map(|s| self.inner.num_sub_vectors(s));
-        num_bits.map(|b| self.inner.num_bits(b));
-        max_iterations.map(|i| self.inner.max_iterations(i));
-        sample_rate.map(|s| self.inner.sample_rate(s));
+    ) -> napi::Result<Self> {
+        let mut ivf_pq_builder = IvfPqIndexBuilder::default();
+        if let Some(distance_type) = distance_type {
+            let distance_type = parse_distance_type(distance_type)?;
+            ivf_pq_builder = ivf_pq_builder.distance_type(distance_type);
+        }
+        if let Some(num_partitions) = num_partitions {
+            ivf_pq_builder = ivf_pq_builder.num_partitions(num_partitions);
+        }
+        if let Some(num_sub_vectors) = num_sub_vectors {
+            ivf_pq_builder = ivf_pq_builder.num_sub_vectors(num_sub_vectors);
+        }
+        if let Some(max_iterations) = max_iterations {
+            ivf_pq_builder = ivf_pq_builder.max_iterations(max_iterations);
+        }
+        if let Some(sample_rate) = sample_rate {
+            ivf_pq_builder = ivf_pq_builder.sample_rate(sample_rate);
+        }
+        Ok(Self {
+            inner: Mutex::new(Some(LanceDbIndex::IvfPq(ivf_pq_builder))),
+        })
     }
 
-    #[napi]
-    pub unsafe fn scalar(&mut self) {
-        self.inner.scalar();
+    #[napi(factory)]
+    pub fn btree() -> Self {
+        Self {
+            inner: Mutex::new(Some(LanceDbIndex::BTree(BTreeIndexBuilder::default()))),
         }
-
-    #[napi]
-    pub async fn build(&self) -> napi::Result<()> {
-        self.inner
-            .build()
-            .await
-            .map_err(|e| napi::Error::from_reason(format!("Failed to build index: {}", e)))?;
-        Ok(())
     }
 }

nodejs/src/iterator.rs

@@ -13,20 +13,20 @@
 // limitations under the License.
 
 use futures::StreamExt;
-use lance::io::RecordBatchStream;
+use lancedb::arrow::SendableRecordBatchStream;
+use lancedb::ipc::batches_to_ipc_file;
 use napi::bindgen_prelude::*;
 use napi_derive::napi;
-use vectordb::ipc::batches_to_ipc_file;
 
 /** Typescript-style Async Iterator over RecordBatches */
 #[napi]
 pub struct RecordBatchIterator {
-    inner: Box<dyn RecordBatchStream + Unpin>,
+    inner: SendableRecordBatchStream,
 }
 
 #[napi]
 impl RecordBatchIterator {
-    pub(crate) fn new(inner: Box<dyn RecordBatchStream + Unpin>) -> Self {
+    pub(crate) fn new(inner: SendableRecordBatchStream) -> Self {
         Self { inner }
     }
 

nodejs/src/lib.rs

@@ -16,16 +16,28 @@ use connection::Connection;
 use napi_derive::*;
 
 mod connection;
+mod error;
 mod index;
 mod iterator;
 mod query;
 mod table;
+mod util;
 
 #[napi(object)]
+#[derive(Debug)]
 pub struct ConnectionOptions {
-    pub uri: String,
     pub api_key: Option<String>,
     pub host_override: Option<String>,
+    /// (For LanceDB OSS only): The interval, in seconds, at which to check for
+    /// updates to the table from other processes. If None, then consistency is not
+    /// checked. For performance reasons, this is the default. For strong
+    /// consistency, set this to zero seconds. Then every read will check for
+    /// updates from other processes. As a compromise, you can set this to a
+    /// non-zero value for eventual consistency. If more than that interval
+    /// has passed since the last check, then the table will be checked for updates.
+    /// Note: this consistency only applies to read operations. Write operations are
+    /// always consistent.
+    pub read_consistency_interval: Option<f64>,
 }
 
 /// Write mode for writing a table.
@@ -43,6 +55,6 @@ pub struct WriteOptions {
 }
 
 #[napi]
-pub async fn connect(options: ConnectionOptions) -> napi::Result<Connection> {
-    Connection::new(options.uri.clone()).await
+pub async fn connect(uri: String, options: ConnectionOptions) -> napi::Result<Connection> {
+    Connection::new(uri, options).await
 }

nodejs/src/query.rs

@@ -12,38 +12,38 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use lancedb::query::ExecutableQuery;
+use lancedb::query::Query as LanceDbQuery;
+use lancedb::query::QueryBase;
+use lancedb::query::Select;
+use lancedb::query::VectorQuery as LanceDbVectorQuery;
 use napi::bindgen_prelude::*;
 use napi_derive::napi;
-use vectordb::query::Query as LanceDBQuery;
 
-use crate::{iterator::RecordBatchIterator, table::Table};
+use crate::error::NapiErrorExt;
+use crate::iterator::RecordBatchIterator;
+use crate::util::parse_distance_type;
 
 #[napi]
 pub struct Query {
-    inner: LanceDBQuery,
+    inner: LanceDbQuery,
 }
 
 #[napi]
 impl Query {
-    pub fn new(table: &Table) -> Self {
-        Self {
-            inner: table.table.query(),
+    pub fn new(query: LanceDbQuery) -> Self {
+        Self { inner: query }
     }
+
+    // We cannot call this r#where because NAPI gets confused by the r#
+    #[napi]
+    pub fn only_if(&mut self, predicate: String) {
+        self.inner = self.inner.clone().only_if(predicate);
     }
 
     #[napi]
-    pub fn column(&mut self, column: String) {
-        self.inner = self.inner.clone().column(&column);
-    }
-
-    #[napi]
-    pub fn filter(&mut self, filter: String) {
-        self.inner = self.inner.clone().filter(filter);
-    }
-
-    #[napi]
-    pub fn select(&mut self, columns: Vec<String>) {
-        self.inner = self.inner.clone().select(&columns);
+    pub fn select(&mut self, columns: Vec<(String, String)>) {
+        self.inner = self.inner.clone().select(Select::dynamic(&columns));
     }
 
     #[napi]
@@ -52,13 +52,46 @@ impl Query {
     }
 
     #[napi]
-    pub fn prefilter(&mut self, prefilter: bool) {
-        self.inner = self.inner.clone().prefilter(prefilter);
+    pub fn nearest_to(&mut self, vector: Float32Array) -> Result<VectorQuery> {
+        let inner = self
+            .inner
+            .clone()
+            .nearest_to(vector.as_ref())
+            .default_error()?;
+        Ok(VectorQuery { inner })
     }
 
     #[napi]
-    pub fn nearest_to(&mut self, vector: Float32Array) {
-        self.inner = self.inner.clone().nearest_to(&vector);
+    pub async fn execute(&self) -> napi::Result<RecordBatchIterator> {
+        let inner_stream = self.inner.execute().await.map_err(|e| {
+            napi::Error::from_reason(format!("Failed to execute query stream: {}", e))
+        })?;
+        Ok(RecordBatchIterator::new(inner_stream))
+    }
+}
+
+#[napi]
+pub struct VectorQuery {
+    inner: LanceDbVectorQuery,
+}
+
+#[napi]
+impl VectorQuery {
+    #[napi]
+    pub fn column(&mut self, column: String) {
+        self.inner = self.inner.clone().column(&column);
+    }
+
+    #[napi]
+    pub fn distance_type(&mut self, distance_type: String) -> napi::Result<()> {
+        let distance_type = parse_distance_type(distance_type)?;
+        self.inner = self.inner.clone().distance_type(distance_type);
+        Ok(())
+    }
+
+    #[napi]
+    pub fn postfilter(&mut self) {
+        self.inner = self.inner.clone().postfilter();
     }
 
     #[napi]
@@ -72,10 +105,30 @@ impl Query {
     }
 
     #[napi]
-    pub async fn execute_stream(&self) -> napi::Result<RecordBatchIterator> {
-        let inner_stream = self.inner.execute_stream().await.map_err(|e| {
+    pub fn bypass_vector_index(&mut self) {
+        self.inner = self.inner.clone().bypass_vector_index()
+    }
+
+    #[napi]
+    pub fn only_if(&mut self, predicate: String) {
+        self.inner = self.inner.clone().only_if(predicate);
+    }
+
+    #[napi]
+    pub fn select(&mut self, columns: Vec<(String, String)>) {
+        self.inner = self.inner.clone().select(Select::dynamic(&columns));
+    }
+
+    #[napi]
+    pub fn limit(&mut self, limit: u32) {
+        self.inner = self.inner.clone().limit(limit as usize);
+    }
+
+    #[napi]
+    pub async fn execute(&self) -> napi::Result<RecordBatchIterator> {
+        let inner_stream = self.inner.execute().await.map_err(|e| {
             napi::Error::from_reason(format!("Failed to execute query stream: {}", e))
         })?;
-        Ok(RecordBatchIterator::new(Box::new(inner_stream)))
+        Ok(RecordBatchIterator::new(inner_stream))
     }
 }

nodejs/src/table.rs

@@ -13,29 +13,69 @@
 // limitations under the License.
 
 use arrow_ipc::writer::FileWriter;
+use lancedb::ipc::ipc_file_to_batches;
+use lancedb::table::{
+    AddDataMode, ColumnAlteration as LanceColumnAlteration, NewColumnTransform,
+    Table as LanceDbTable,
+};
 use napi::bindgen_prelude::*;
 use napi_derive::napi;
-use vectordb::table::AddDataOptions;
-use vectordb::{ipc::ipc_file_to_batches, table::TableRef};
 
-use crate::index::IndexBuilder;
-use crate::query::Query;
+use crate::error::NapiErrorExt;
+use crate::index::Index;
+use crate::query::{Query, VectorQuery};
 
 #[napi]
 pub struct Table {
-    pub(crate) table: TableRef,
+    // We keep a duplicate of the table name so we can use it for error
+    // messages even if the table has been closed
+    name: String,
+    pub(crate) inner: Option<LanceDbTable>,
+}
+
+impl Table {
+    fn inner_ref(&self) -> napi::Result<&LanceDbTable> {
+        self.inner
+            .as_ref()
+            .ok_or_else(|| napi::Error::from_reason(format!("Table {} is closed", self.name)))
+    }
 }
 
 #[napi]
 impl Table {
-    pub(crate) fn new(table: TableRef) -> Self {
-        Self { table }
+    pub(crate) fn new(table: LanceDbTable) -> Self {
+        Self {
+            name: table.name().to_string(),
+            inner: Some(table),
+        }
+    }
+
+    #[napi]
+    pub fn display(&self) -> String {
+        match &self.inner {
+            None => format!("ClosedTable({})", self.name),
+            Some(inner) => inner.to_string(),
+        }
+    }
+
+    #[napi]
+    pub fn is_open(&self) -> bool {
+        self.inner.is_some()
+    }
+
+    #[napi]
+    pub fn close(&mut self) {
+        self.inner.take();
     }
 
     /// Return Schema as empty Arrow IPC file.
     #[napi]
-    pub fn schema(&self) -> napi::Result<Buffer> {
-        let mut writer = FileWriter::try_new(vec![], &self.table.schema())
+    pub async fn schema(&self) -> napi::Result<Buffer> {
+        let schema =
+            self.inner_ref()?.schema().await.map_err(|e| {
+                napi::Error::from_reason(format!("Failed to create IPC file: {}", e))
+            })?;
+        let mut writer = FileWriter::try_new(vec![], &schema)
             .map_err(|e| napi::Error::from_reason(format!("Failed to create IPC file: {}", e)))?;
         writer
             .finish()
@@ -46,47 +86,259 @@ impl Table {
     }
 
     #[napi]
-    pub async fn add(&self, buf: Buffer) -> napi::Result<()> {
+    pub async fn add(&self, buf: Buffer, mode: String) -> napi::Result<()> {
         let batches = ipc_file_to_batches(buf.to_vec())
             .map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
-        self.table
-            .add(Box::new(batches), AddDataOptions::default())
-            .await
-            .map_err(|e| {
+        let mut op = self.inner_ref()?.add(batches);
+
+        op = if mode == "append" {
+            op.mode(AddDataMode::Append)
+        } else if mode == "overwrite" {
+            op.mode(AddDataMode::Overwrite)
+        } else {
+            return Err(napi::Error::from_reason(format!("Invalid mode: {}", mode)));
+        };
+
+        op.execute().await.map_err(|e| {
             napi::Error::from_reason(format!(
                 "Failed to add batches to table {}: {}",
-                    self.table, e
+                self.name, e
             ))
         })
     }
 
     #[napi]
-    pub async fn count_rows(&self, filter: Option<String>) -> napi::Result<usize> {
-        self.table.count_rows(filter).await.map_err(|e| {
+    pub async fn count_rows(&self, filter: Option<String>) -> napi::Result<i64> {
+        self.inner_ref()?
+            .count_rows(filter)
+            .await
+            .map(|val| val as i64)
+            .map_err(|e| {
                 napi::Error::from_reason(format!(
                     "Failed to count rows in table {}: {}",
-                self.table, e
+                    self.name, e
                 ))
             })
     }
 
     #[napi]
     pub async fn delete(&self, predicate: String) -> napi::Result<()> {
-        self.table.delete(&predicate).await.map_err(|e| {
+        self.inner_ref()?.delete(&predicate).await.map_err(|e| {
             napi::Error::from_reason(format!(
                 "Failed to delete rows in table {}: predicate={}",
-                self.table, e
+                self.name, e
             ))
         })
     }
 
     #[napi]
-    pub fn create_index(&self) -> IndexBuilder {
-        IndexBuilder::new(self.table.as_ref())
+    pub async fn create_index(
+        &self,
+        index: Option<&Index>,
+        column: String,
+        replace: Option<bool>,
+    ) -> napi::Result<()> {
+        let lancedb_index = if let Some(index) = index {
+            index.consume()?
+        } else {
+            lancedb::index::Index::Auto
+        };
+        let mut builder = self.inner_ref()?.create_index(&[column], lancedb_index);
+        if let Some(replace) = replace {
+            builder = builder.replace(replace);
+        }
+        builder.execute().await.default_error()
     }
 
     #[napi]
-    pub fn query(&self) -> Query {
-        Query::new(self)
+    pub async fn update(
+        &self,
+        only_if: Option<String>,
+        columns: Vec<(String, String)>,
+    ) -> napi::Result<()> {
+        let mut op = self.inner_ref()?.update();
+        if let Some(only_if) = only_if {
+            op = op.only_if(only_if);
+        }
+        for (column_name, value) in columns {
+            op = op.column(column_name, value);
+        }
+        op.execute().await.default_error()
+    }
+
+    #[napi]
+    pub fn query(&self) -> napi::Result<Query> {
+        Ok(Query::new(self.inner_ref()?.query()))
+    }
+
+    #[napi]
+    pub fn vector_search(&self, vector: Float32Array) -> napi::Result<VectorQuery> {
+        self.query()?.nearest_to(vector)
+    }
+
+    #[napi]
+    pub async fn add_columns(&self, transforms: Vec<AddColumnsSql>) -> napi::Result<()> {
+        let transforms = transforms
+            .into_iter()
+            .map(|sql| (sql.name, sql.value_sql))
+            .collect::<Vec<_>>();
+        let transforms = NewColumnTransform::SqlExpressions(transforms);
+        self.inner_ref()?
+            .add_columns(transforms, None)
+            .await
+            .map_err(|err| {
+                napi::Error::from_reason(format!(
+                    "Failed to add columns to table {}: {}",
+                    self.name, err
+                ))
+            })?;
+        Ok(())
+    }
+
+    #[napi]
+    pub async fn alter_columns(&self, alterations: Vec<ColumnAlteration>) -> napi::Result<()> {
+        for alteration in &alterations {
+            if alteration.rename.is_none() && alteration.nullable.is_none() {
+                return Err(napi::Error::from_reason(
+                    "Alteration must have a 'rename' or 'nullable' field.",
+                ));
+            }
+        }
+        let alterations = alterations
+            .into_iter()
+            .map(LanceColumnAlteration::from)
+            .collect::<Vec<_>>();
+
+        self.inner_ref()?
+            .alter_columns(&alterations)
+            .await
+            .map_err(|err| {
+                napi::Error::from_reason(format!(
+                    "Failed to alter columns in table {}: {}",
+                    self.name, err
+                ))
+            })?;
+        Ok(())
+    }
+
+    #[napi]
+    pub async fn drop_columns(&self, columns: Vec<String>) -> napi::Result<()> {
+        let col_refs = columns.iter().map(String::as_str).collect::<Vec<_>>();
+        self.inner_ref()?
+            .drop_columns(&col_refs)
+            .await
+            .map_err(|err| {
+                napi::Error::from_reason(format!(
+                    "Failed to drop columns from table {}: {}",
+                    self.name, err
+                ))
+            })?;
+        Ok(())
+    }
+
+    #[napi]
+    pub async fn version(&self) -> napi::Result<i64> {
+        self.inner_ref()?
+            .version()
+            .await
+            .map(|val| val as i64)
+            .default_error()
+    }
+
+    #[napi]
+    pub async fn checkout(&self, version: i64) -> napi::Result<()> {
+        self.inner_ref()?
+            .checkout(version as u64)
+            .await
+            .default_error()
+    }
+
+    #[napi]
+    pub async fn checkout_latest(&self) -> napi::Result<()> {
+        self.inner_ref()?.checkout_latest().await.default_error()
+    }
+
+    #[napi]
+    pub async fn restore(&self) -> napi::Result<()> {
+        self.inner_ref()?.restore().await.default_error()
+    }
+
+    #[napi]
+    pub async fn list_indices(&self) -> napi::Result<Vec<IndexConfig>> {
+        Ok(self
+            .inner_ref()?
+            .list_indices()
+            .await
+            .default_error()?
+            .into_iter()
+            .map(IndexConfig::from)
+            .collect::<Vec<_>>())
     }
 }
+
+#[napi(object)]
+/// A description of an index currently configured on a column
+pub struct IndexConfig {
+    /// The type of the index
+    pub index_type: String,
+    /// The columns in the index
+    ///
+    /// Currently this is always an array of size 1.  In the future there may
+    /// be more columns to represent composite indices.
+    pub columns: Vec<String>,
+}
+
+impl From<lancedb::index::IndexConfig> for IndexConfig {
+    fn from(value: lancedb::index::IndexConfig) -> Self {
+        let index_type = format!("{:?}", value.index_type);
+        Self {
+            index_type,
+            columns: value.columns,
+        }
+    }
+}
+
+///  A definition of a column alteration. The alteration changes the column at
+/// `path` to have the new name `name`, to be nullable if `nullable` is true,
+/// and to have the data type `data_type`. At least one of `rename` or `nullable`
+/// must be provided.
+#[napi(object)]
+pub struct ColumnAlteration {
+    /// The path to the column to alter. This is a dot-separated path to the column.
+    /// If it is a top-level column then it is just the name of the column. If it is
+    /// a nested column then it is the path to the column, e.g. "a.b.c" for a column
+    /// `c` nested inside a column `b` nested inside a column `a`.
+    pub path: String,
+    /// The new name of the column. If not provided then the name will not be changed.
+    /// This must be distinct from the names of all other columns in the table.
+    pub rename: Option<String>,
+    /// Set the new nullability. Note that a nullable column cannot be made non-nullable.
+    pub nullable: Option<bool>,
+}
+
+impl From<ColumnAlteration> for LanceColumnAlteration {
+    fn from(js: ColumnAlteration) -> Self {
+        let ColumnAlteration {
+            path,
+            rename,
+            nullable,
+        } = js;
+        Self {
+            path,
+            rename,
+            nullable,
+            // TODO: wire up this field
+            data_type: None,
+        }
+    }
+}
+
+/// A definition of a new column to add to a table.
+#[napi(object)]
+pub struct AddColumnsSql {
+    /// The name of the new column.
+    pub name: String,
+    /// The values to populate the new column with, as a SQL expression.
+    /// The expression can reference other columns in the table.
+    pub value_sql: String,
+}

nodejs/src/util.rs

@@ -0,0 +1,13 @@
+use lancedb::DistanceType;
+
+pub fn parse_distance_type(distance_type: impl AsRef<str>) -> napi::Result<DistanceType> {
+    match distance_type.as_ref().to_lowercase().as_str() {
+        "l2" => Ok(DistanceType::L2),
+        "cosine" => Ok(DistanceType::Cosine),
+        "dot" => Ok(DistanceType::Dot),
+        _ => Err(napi::Error::from_reason(format!(
+            "Invalid distance type '{}'.  Must be one of l2, cosine, or dot",
+            distance_type.as_ref()
+        ))),
+    }
+}

nodejs/tsconfig.json

@@ -1,9 +1,5 @@
 {
-  "include": [
-    "vectordb/*.ts",
-    "vectordb/**/*.ts",
-    "vectordb/*.js",
-  ],
+  "include": ["lancedb/*.ts", "lancedb/**/*.ts", "lancedb/*.js"],
   "compilerOptions": {
     "target": "es2022",
     "module": "commonjs",
@@ -11,21 +7,17 @@
     "outDir": "./dist",
     "strict": true,
     "allowJs": true,
-    "resolveJsonModule": true,
+    "resolveJsonModule": true
   },
-  "exclude": [
-    "./dist/*",
-  ],
+  "exclude": ["./dist/*"],
   "typedocOptions": {
-    "entryPoints": [
-      "vectordb/index.ts"
-    ],
+    "entryPoints": ["lancedb/index.ts"],
     "out": "../docs/src/javascript/",
     "visibilityFilters": {
       "protected": false,
       "private": false,
       "inherited": true,
-      "external": false,
+      "external": false
     }
   }
 }

nodejs/vectordb/arrow.ts

@@ -1,188 +0,0 @@
-// Copyright 2024 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import {
-  Int64,
-  Field,
-  FixedSizeList,
-  Float,
-  Float32,
-  Schema,
-  Table as ArrowTable,
-  Table,
-  Vector,
-  vectorFromArray,
-  tableToIPC,
-  DataType,
-} from "apache-arrow";
-
-/** Data type accepted by NodeJS SDK */
-export type Data = Record<string, unknown>[] | ArrowTable;
-
-export class VectorColumnOptions {
-  /** Vector column type. */
-  type: Float = new Float32();
-
-  constructor(values?: Partial<VectorColumnOptions>) {
-    Object.assign(this, values);
-  }
-}
-
-/** Options to control the makeArrowTable call. */
-export class MakeArrowTableOptions {
-  /** Provided schema. */
-  schema?: Schema;
-
-  /** Vector columns */
-  vectorColumns: Record<string, VectorColumnOptions> = {
-    vector: new VectorColumnOptions(),
-  };
-
-  constructor(values?: Partial<MakeArrowTableOptions>) {
-    Object.assign(this, values);
-  }
-}
-
-/**
- * An enhanced version of the {@link makeTable} function from Apache Arrow
- * that supports nested fields and embeddings columns.
- *
- * Note that it currently does not support nulls.
- *
- * @param data input data
- * @param options options to control the makeArrowTable call.
- *
- * @example
- *
- * ```ts
- *
- * import { fromTableToBuffer, makeArrowTable } from "../arrow";
- * import { Field, FixedSizeList, Float16, Float32, Int32, Schema } from "apache-arrow";
- *
- * const schema = new Schema([
- *   new Field("a", new Int32()),
- *   new Field("b", new Float32()),
- *   new Field("c", new FixedSizeList(3, new Field("item", new Float16()))),
- *  ]);
- *  const table = makeArrowTable([
- *    { a: 1, b: 2, c: [1, 2, 3] },
- *    { a: 4, b: 5, c: [4, 5, 6] },
- *    { a: 7, b: 8, c: [7, 8, 9] },
- *  ], { schema });
- * ```
- *
- * It guesses the vector columns if the schema is not provided. For example,
- * by default it assumes that the column named `vector` is a vector column.
- *
- * ```ts
- *
- * const schema = new Schema([
- new Field("a", new Float64()),
- new Field("b", new Float64()),
- new Field(
- "vector",
- new FixedSizeList(3, new Field("item", new Float32()))
- ),
- ]);
- const table = makeArrowTable([
- { a: 1, b: 2, vector: [1, 2, 3] },
- { a: 4, b: 5, vector: [4, 5, 6] },
- { a: 7, b: 8, vector: [7, 8, 9] },
- ]);
- assert.deepEqual(table.schema, schema);
- * ```
- *
- * You can specify the vector column types and names using the options as well
- *
- * ```typescript
- *
- * const schema = new Schema([
- new Field('a', new Float64()),
- new Field('b', new Float64()),
- new Field('vec1', new FixedSizeList(3, new Field('item', new Float16()))),
- new Field('vec2', new FixedSizeList(3, new Field('item', new Float16())))
- ]);
- * const table = makeArrowTable([
- { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
- { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
- { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] }
- ], {
- vectorColumns: {
- vec1: { type: new Float16() },
- vec2: { type: new Float16() }
- }
- }
- * assert.deepEqual(table.schema, schema)
- * ```
- */
-export function makeArrowTable(
-  data: Record<string, any>[],
-  options?: Partial<MakeArrowTableOptions>
-): Table {
-  if (data.length === 0) {
-    throw new Error("At least one record needs to be provided");
-  }
-  const opt = new MakeArrowTableOptions(options ?? {});
-  const columns: Record<string, Vector> = {};
-  // TODO: sample dataset to find missing columns
-  const columnNames = Object.keys(data[0]);
-  for (const colName of columnNames) {
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-return
-    let values = data.map((datum) => datum[colName]);
-    let vector: Vector;
-
-    if (opt.schema !== undefined) {
-      // Explicit schema is provided, highest priority
-      const fieldType: DataType | undefined = opt.schema.fields.filter((f) => f.name === colName)[0]?.type as DataType;
-      if (fieldType instanceof Int64) {
-        // wrap in BigInt to avoid bug: https://github.com/apache/arrow/issues/40051
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
-        values = values.map((v) => BigInt(v));
-      }
-      vector = vectorFromArray(values, fieldType);
-    } else {
-      const vectorColumnOptions = opt.vectorColumns[colName];
-      if (vectorColumnOptions !== undefined) {
-        const fslType = new FixedSizeList(
-          (values[0] as any[]).length,
-          new Field("item", vectorColumnOptions.type, false)
-        );
-        vector = vectorFromArray(values, fslType);
-      } else {
-        // Normal case
-        vector = vectorFromArray(values);
-      }
-    }
-    columns[colName] = vector;
-  }
-
-  return new Table(columns);
-}
-
-/**
- * Convert an Arrow Table to a Buffer.
- *
- * @param data Arrow Table
- * @param schema Arrow Schema, optional
- * @returns Buffer node
- */
-export function toBuffer(data: Data, schema?: Schema): Buffer {
-  let tbl: Table;
-  if (data instanceof Table) {
-    tbl = data;
-  } else {
-    tbl = makeArrowTable(data, { schema });
-  }
-  return Buffer.from(tableToIPC(tbl));
-}

nodejs/vectordb/connection.ts

@@ -1,70 +0,0 @@
-// Copyright 2024 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { toBuffer } from "./arrow";
-import { Connection as _NativeConnection } from "./native";
-import { Table } from "./table";
-import { Table as ArrowTable } from "apache-arrow";
-
-/**
- * A LanceDB Connection that allows you to open tables and create new ones.
- *
- * Connection could be local against filesystem or remote against a server.
- */
-export class Connection {
-  readonly inner: _NativeConnection;
-
-  constructor(inner: _NativeConnection) {
-    this.inner = inner;
-  }
-
-  /** List all the table names in this database. */
-  async tableNames(): Promise<string[]> {
-    return this.inner.tableNames();
-  }
-
-  /**
-   * Open a table in the database.
-   *
-   * @param name The name of the table.
-   * @param embeddings An embedding function to use on this table
-   */
-  async openTable(name: string): Promise<Table> {
-    const innerTable = await this.inner.openTable(name);
-    return new Table(innerTable);
-  }
-
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param {string} name - The name of the table.
-   * @param data - Non-empty Array of Records to be inserted into the table
-   */
-  async createTable(
-    name: string,
-    data: Record<string, unknown>[] | ArrowTable
-  ): Promise<Table> {
-    const buf = toBuffer(data);
-    const innerTable = await this.inner.createTable(name, buf);
-    return new Table(innerTable);
-  }
-
-  /**
-   * Drop an existing table.
-   * @param name The name of the table to drop.
-   */
-  async dropTable(name: string): Promise<void> {
-    return this.inner.dropTable(name);
-  }
-}

nodejs/vectordb/indexer.ts

@@ -1,102 +0,0 @@
-// Copyright 2024 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import {
-  MetricType,
-  IndexBuilder as NativeBuilder,
-  Table as NativeTable,
-} from "./native";
-
-/** Options to create `IVF_PQ` index */
-export interface IvfPQOptions {
-  /** Number of IVF partitions. */
-  num_partitions?: number;
-
-  /** Number of sub-vectors in PQ coding. */
-  num_sub_vectors?: number;
-
-  /** Number of bits used for each PQ code.
-   */
-  num_bits?: number;
-
-  /** Metric type to calculate the distance between vectors.
-   *
-   * Supported metrics: `L2`, `Cosine` and `Dot`.
-   */
-  metric_type?: MetricType;
-
-  /** Number of iterations to train K-means.
-   *
-   * Default is 50. The more iterations it usually yield better results,
-   * but it takes longer to train.
-   */
-  max_iterations?: number;
-
-  sample_rate?: number;
-}
-
-/**
- * Building an index on LanceDB {@link Table}
- *
- * @see {@link Table.createIndex} for detailed usage.
- */
-export class IndexBuilder {
-  private inner: NativeBuilder;
-
-  constructor(tbl: NativeTable) {
-    this.inner = tbl.createIndex();
-  }
-
-  /** Instruct the builder to build an `IVF_PQ` index */
-  ivf_pq(options?: IvfPQOptions): IndexBuilder {
-    this.inner.ivfPq(
-      options?.metric_type,
-      options?.num_partitions,
-      options?.num_sub_vectors,
-      options?.num_bits,
-      options?.max_iterations,
-      options?.sample_rate
-    );
-    return this;
-  }
-
-  /** Instruct the builder to build a Scalar index. */
-  scalar(): IndexBuilder {
-    this.scalar();
-    return this;
-  }
-
-  /** Set the column(s) to create index on top of. */
-  column(col: string): IndexBuilder {
-    this.inner.column(col);
-    return this;
-  }
-
-  /** Set to true to replace existing index. */
-  replace(val: boolean): IndexBuilder {
-    this.inner.replace(val);
-    return this;
-  }
-
-  /** Specify the name of the index. Optional */
-  name(n: string): IndexBuilder {
-    this.inner.name(n);
-    return this;
-  }
-
-  /** Building the index. */
-  async build() {
-    await this.inner.build();
-  }
-}

nodejs/vectordb/native.d.ts

@@ -1,80 +0,0 @@
-/* tslint:disable */
-/* eslint-disable */
-
-/* auto-generated by NAPI-RS */
-
-export const enum IndexType {
-  Scalar = 0,
-  IvfPq = 1
-}
-export const enum MetricType {
-  L2 = 0,
-  Cosine = 1,
-  Dot = 2
-}
-export interface ConnectionOptions {
-  uri: string
-  apiKey?: string
-  hostOverride?: string
-}
-/** Write mode for writing a table. */
-export const enum WriteMode {
-  Create = 'Create',
-  Append = 'Append',
-  Overwrite = 'Overwrite'
-}
-/** Write options when creating a Table. */
-export interface WriteOptions {
-  mode?: WriteMode
-}
-export function connect(options: ConnectionOptions): Promise<Connection>
-export class Connection {
-  /** Create a new Connection instance from the given URI. */
-  static new(uri: string): Promise<Connection>
-  /** List all tables in the dataset. */
-  tableNames(): Promise<Array<string>>
-  /**
-   * Create table from a Apache Arrow IPC (file) buffer.
-   *
-   * Parameters:
-   * - name: The name of the table.
-   * - buf: The buffer containing the IPC file.
-   *
-   */
-  createTable(name: string, buf: Buffer): Promise<Table>
-  openTable(name: string): Promise<Table>
-  /** Drop table with the name. Or raise an error if the table does not exist. */
-  dropTable(name: string): Promise<void>
-}
-export class IndexBuilder {
-  replace(v: boolean): void
-  column(c: string): void
-  name(name: string): void
-  ivfPq(metricType?: MetricType | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, numBits?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): void
-  scalar(): void
-  build(): Promise<void>
-}
-/** Typescript-style Async Iterator over RecordBatches  */
-export class RecordBatchIterator {
-  next(): Promise<Buffer | null>
-}
-export class Query {
-  column(column: string): void
-  filter(filter: string): void
-  select(columns: Array<string>): void
-  limit(limit: number): void
-  prefilter(prefilter: boolean): void
-  nearestTo(vector: Float32Array): void
-  refineFactor(refineFactor: number): void
-  nprobes(nprobe: number): void
-  executeStream(): Promise<RecordBatchIterator>
-}
-export class Table {
-  /** Return Schema as empty Arrow IPC file. */
-  schema(): Buffer
-  add(buf: Buffer): Promise<void>
-  countRows(filter?: string | undefined | null): Promise<bigint>
-  delete(predicate: string): Promise<void>
-  createIndex(): IndexBuilder
-  query(): Query
-}

nodejs/vectordb/native.js

@@ -1,308 +0,0 @@
-/* tslint:disable */
-/* eslint-disable */
-/* prettier-ignore */
-
-/* auto-generated by NAPI-RS */
-
-const { existsSync, readFileSync } = require('fs')
-const { join } = require('path')
-
-const { platform, arch } = process
-
-let nativeBinding = null
-let localFileExisted = false
-let loadError = null
-
-function isMusl() {
-  // For Node 10
-  if (!process.report || typeof process.report.getReport !== 'function') {
-    try {
-      const lddPath = require('child_process').execSync('which ldd').toString().trim()
-      return readFileSync(lddPath, 'utf8').includes('musl')
-    } catch (e) {
-      return true
-    }
-  } else {
-    const { glibcVersionRuntime } = process.report.getReport().header
-    return !glibcVersionRuntime
-  }
-}
-
-switch (platform) {
-  case 'android':
-    switch (arch) {
-      case 'arm64':
-        localFileExisted = existsSync(join(__dirname, 'vectordb-nodejs.android-arm64.node'))
-        try {
-          if (localFileExisted) {
-            nativeBinding = require('./vectordb-nodejs.android-arm64.node')
-          } else {
-            nativeBinding = require('vectordb-android-arm64')
-          }
-        } catch (e) {
-          loadError = e
-        }
-        break
-      case 'arm':
-        localFileExisted = existsSync(join(__dirname, 'vectordb-nodejs.android-arm-eabi.node'))
-        try {
-          if (localFileExisted) {
-            nativeBinding = require('./vectordb-nodejs.android-arm-eabi.node')
-          } else {
-            nativeBinding = require('vectordb-android-arm-eabi')
-          }
-        } catch (e) {
-          loadError = e
-        }
-        break
-      default:
-        throw new Error(`Unsupported architecture on Android ${arch}`)
-    }
-    break
-  case 'win32':
-    switch (arch) {
-      case 'x64':
-        localFileExisted = existsSync(
-          join(__dirname, 'vectordb-nodejs.win32-x64-msvc.node')
-        )
-        try {
-          if (localFileExisted) {
-            nativeBinding = require('./vectordb-nodejs.win32-x64-msvc.node')
-          } else {
-            nativeBinding = require('vectordb-win32-x64-msvc')
-          }
-        } catch (e) {
-          loadError = e
-        }
-        break
-      case 'ia32':
-        localFileExisted = existsSync(
-          join(__dirname, 'vectordb-nodejs.win32-ia32-msvc.node')
-        )
-        try {
-          if (localFileExisted) {
-            nativeBinding = require('./vectordb-nodejs.win32-ia32-msvc.node')
-          } else {
-            nativeBinding = require('vectordb-win32-ia32-msvc')
-          }
-        } catch (e) {
-          loadError = e
-        }
-        break
-      case 'arm64':
-        localFileExisted = existsSync(
-          join(__dirname, 'vectordb-nodejs.win32-arm64-msvc.node')
-        )
-        try {
-          if (localFileExisted) {
-            nativeBinding = require('./vectordb-nodejs.win32-arm64-msvc.node')
-          } else {
-            nativeBinding = require('vectordb-win32-arm64-msvc')
-          }
-        } catch (e) {
-          loadError = e
-        }
-        break
-      default:
-        throw new Error(`Unsupported architecture on Windows: ${arch}`)
-    }
-    break
-  case 'darwin':
-    localFileExisted = existsSync(join(__dirname, 'vectordb-nodejs.darwin-universal.node'))
-    try {
-      if (localFileExisted) {
-        nativeBinding = require('./vectordb-nodejs.darwin-universal.node')
-      } else {
-        nativeBinding = require('vectordb-darwin-universal')
-      }
-      break
-    } catch {}
-    switch (arch) {
-      case 'x64':
-        localFileExisted = existsSync(join(__dirname, 'vectordb-nodejs.darwin-x64.node'))
-        try {
-          if (localFileExisted) {
-            nativeBinding = require('./vectordb-nodejs.darwin-x64.node')
-          } else {
-            nativeBinding = require('vectordb-darwin-x64')
-          }
-        } catch (e) {
-          loadError = e
-        }
-        break
-      case 'arm64':
-        localFileExisted = existsSync(
-          join(__dirname, 'vectordb-nodejs.darwin-arm64.node')
-        )
-        try {
-          if (localFileExisted) {
-            nativeBinding = require('./vectordb-nodejs.darwin-arm64.node')
-          } else {
-            nativeBinding = require('vectordb-darwin-arm64')
-          }
-        } catch (e) {
-          loadError = e
-        }
-        break
-      default:
-        throw new Error(`Unsupported architecture on macOS: ${arch}`)
-    }
-    break
-  case 'freebsd':
-    if (arch !== 'x64') {
-      throw new Error(`Unsupported architecture on FreeBSD: ${arch}`)
-    }
-    localFileExisted = existsSync(join(__dirname, 'vectordb-nodejs.freebsd-x64.node'))
-    try {
-      if (localFileExisted) {
-        nativeBinding = require('./vectordb-nodejs.freebsd-x64.node')
-      } else {
-        nativeBinding = require('vectordb-freebsd-x64')
-      }
-    } catch (e) {
-      loadError = e
-    }
-    break
-  case 'linux':
-    switch (arch) {
-      case 'x64':
-        if (isMusl()) {
-          localFileExisted = existsSync(
-            join(__dirname, 'vectordb-nodejs.linux-x64-musl.node')
-          )
-          try {
-            if (localFileExisted) {
-              nativeBinding = require('./vectordb-nodejs.linux-x64-musl.node')
-            } else {
-              nativeBinding = require('vectordb-linux-x64-musl')
-            }
-          } catch (e) {
-            loadError = e
-          }
-        } else {
-          localFileExisted = existsSync(
-            join(__dirname, 'vectordb-nodejs.linux-x64-gnu.node')
-          )
-          try {
-            if (localFileExisted) {
-              nativeBinding = require('./vectordb-nodejs.linux-x64-gnu.node')
-            } else {
-              nativeBinding = require('vectordb-linux-x64-gnu')
-            }
-          } catch (e) {
-            loadError = e
-          }
-        }
-        break
-      case 'arm64':
-        if (isMusl()) {
-          localFileExisted = existsSync(
-            join(__dirname, 'vectordb-nodejs.linux-arm64-musl.node')
-          )
-          try {
-            if (localFileExisted) {
-              nativeBinding = require('./vectordb-nodejs.linux-arm64-musl.node')
-            } else {
-              nativeBinding = require('vectordb-linux-arm64-musl')
-            }
-          } catch (e) {
-            loadError = e
-          }
-        } else {
-          localFileExisted = existsSync(
-            join(__dirname, 'vectordb-nodejs.linux-arm64-gnu.node')
-          )
-          try {
-            if (localFileExisted) {
-              nativeBinding = require('./vectordb-nodejs.linux-arm64-gnu.node')
-            } else {
-              nativeBinding = require('vectordb-linux-arm64-gnu')
-            }
-          } catch (e) {
-            loadError = e
-          }
-        }
-        break
-      case 'arm':
-        localFileExisted = existsSync(
-          join(__dirname, 'vectordb-nodejs.linux-arm-gnueabihf.node')
-        )
-        try {
-          if (localFileExisted) {
-            nativeBinding = require('./vectordb-nodejs.linux-arm-gnueabihf.node')
-          } else {
-            nativeBinding = require('vectordb-linux-arm-gnueabihf')
-          }
-        } catch (e) {
-          loadError = e
-        }
-        break
-      case 'riscv64':
-        if (isMusl()) {
-          localFileExisted = existsSync(
-            join(__dirname, 'vectordb-nodejs.linux-riscv64-musl.node')
-          )
-          try {
-            if (localFileExisted) {
-              nativeBinding = require('./vectordb-nodejs.linux-riscv64-musl.node')
-            } else {
-              nativeBinding = require('vectordb-linux-riscv64-musl')
-            }
-          } catch (e) {
-            loadError = e
-          }
-        } else {
-          localFileExisted = existsSync(
-            join(__dirname, 'vectordb-nodejs.linux-riscv64-gnu.node')
-          )
-          try {
-            if (localFileExisted) {
-              nativeBinding = require('./vectordb-nodejs.linux-riscv64-gnu.node')
-            } else {
-              nativeBinding = require('vectordb-linux-riscv64-gnu')
-            }
-          } catch (e) {
-            loadError = e
-          }
-        }
-        break
-      case 's390x':
-        localFileExisted = existsSync(
-          join(__dirname, 'vectordb-nodejs.linux-s390x-gnu.node')
-        )
-        try {
-          if (localFileExisted) {
-            nativeBinding = require('./vectordb-nodejs.linux-s390x-gnu.node')
-          } else {
-            nativeBinding = require('vectordb-linux-s390x-gnu')
-          }
-        } catch (e) {
-          loadError = e
-        }
-        break
-      default:
-        throw new Error(`Unsupported architecture on Linux: ${arch}`)
-    }
-    break
-  default:
-    throw new Error(`Unsupported OS: ${platform}, architecture: ${arch}`)
-}
-
-if (!nativeBinding) {
-  if (loadError) {
-    throw loadError
-  }
-  throw new Error(`Failed to load native binding`)
-}
-
-const { Connection, IndexType, MetricType, IndexBuilder, RecordBatchIterator, Query, Table, WriteMode, connect } = nativeBinding
-
-module.exports.Connection = Connection
-module.exports.IndexType = IndexType
-module.exports.MetricType = MetricType
-module.exports.IndexBuilder = IndexBuilder
-module.exports.RecordBatchIterator = RecordBatchIterator
-module.exports.Query = Query
-module.exports.Table = Table
-module.exports.WriteMode = WriteMode
-module.exports.connect = connect

nodejs/vectordb/query.ts

@@ -1,152 +0,0 @@
-// Copyright 2024 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { RecordBatch, tableFromIPC, Table as ArrowTable } from "apache-arrow";
-import {
-  RecordBatchIterator as NativeBatchIterator,
-  Query as NativeQuery,
-  Table as NativeTable,
-} from "./native";
-
-class RecordBatchIterator implements AsyncIterator<RecordBatch> {
-  private promised_inner?: Promise<NativeBatchIterator>;
-  private inner?: NativeBatchIterator;
-
-  constructor(
-    inner?: NativeBatchIterator,
-    promise?: Promise<NativeBatchIterator>
-  ) {
-    // TODO: check promise reliably so we dont need to pass two arguments.
-    this.inner = inner;
-    this.promised_inner = promise;
-  }
-
-  async next(): Promise<IteratorResult<RecordBatch<any>, any>> {
-    if (this.inner === undefined) {
-      this.inner = await this.promised_inner;
-    }
-    if (this.inner === undefined) {
-      throw new Error("Invalid iterator state state");
-    }
-    const n = await this.inner.next();
-    if (n == null) {
-      return Promise.resolve({ done: true, value: null });
-    }
-    const tbl = tableFromIPC(n);
-    if (tbl.batches.length != 1) {
-      throw new Error("Expected only one batch");
-    }
-    return Promise.resolve({ done: false, value: tbl.batches[0] });
-  }
-}
-/* eslint-enable */
-
-/** Query executor */
-export class Query implements AsyncIterable<RecordBatch> {
-  private readonly inner: NativeQuery;
-
-  constructor(tbl: NativeTable) {
-    this.inner = tbl.query();
-  }
-
-  /** Set the column to run query. */
-  column(column: string): Query {
-    this.inner.column(column);
-    return this;
-  }
-
-  /** Set the filter predicate, only returns the results that satisfy the filter.
-   *
-   */
-  filter(predicate: string): Query {
-    this.inner.filter(predicate);
-    return this;
-  }
-
-  /**
-   * Select the columns to return. If not set, all columns are returned.
-   */
-  select(columns: string[]): Query {
-    this.inner.select(columns);
-    return this;
-  }
-
-  /**
-   * Set the limit of rows to return.
-   */
-  limit(limit: number): Query {
-    this.inner.limit(limit);
-    return this;
-  }
-
-  prefilter(prefilter: boolean): Query {
-    this.inner.prefilter(prefilter);
-    return this;
-  }
-
-  /**
-   * Set the query vector.
-   */
-  nearestTo(vector: number[]): Query {
-    this.inner.nearestTo(Float32Array.from(vector));
-    return this;
-  }
-
-  /**
-   * Set the number of IVF partitions to use for the query.
-   */
-  nprobes(nprobes: number): Query {
-    this.inner.nprobes(nprobes);
-    return this;
-  }
-
-  /**
-   * Set the refine factor for the query.
-   */
-  refineFactor(refine_factor: number): Query {
-    this.inner.refineFactor(refine_factor);
-    return this;
-  }
-
-  /**
-   * Execute the query and return the results as an AsyncIterator.
-   */
-  async executeStream(): Promise<RecordBatchIterator> {
-    const inner = await this.inner.executeStream();
-    return new RecordBatchIterator(inner);
-  }
-
-  /** Collect the results as an Arrow Table. */
-  async toArrow(): Promise<ArrowTable> {
-    const batches = [];
-    for await (const batch of this) {
-      batches.push(batch);
-    }
-    return new ArrowTable(batches);
-  }
-
-  /** Returns a JSON Array of All results.
-   *
-   */
-  async toArray(): Promise<any[]> {
-    const tbl = await this.toArrow();
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-return
-    return tbl.toArray();
-  }
-
-  [Symbol.asyncIterator](): AsyncIterator<RecordBatch<any>> {
-    const promise = this.inner.executeStream();
-    return new RecordBatchIterator(undefined, promise);
-  }
-}

nodejs/vectordb/table.ts

@@ -1,153 +0,0 @@
-// Copyright 2024 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { Schema, tableFromIPC } from "apache-arrow";
-import { Table as _NativeTable } from "./native";
-import { toBuffer, Data } from "./arrow";
-import { Query } from "./query";
-import { IndexBuilder } from "./indexer";
-
-/**
- * A LanceDB Table is the collection of Records.
- *
- * Each Record has one or more vector fields.
- */
-export class Table {
-  private readonly inner: _NativeTable;
-
-  /** Construct a Table. Internal use only. */
-  constructor(inner: _NativeTable) {
-    this.inner = inner;
-  }
-
-  /** Get the schema of the table. */
-  get schema(): Schema {
-    const schemaBuf = this.inner.schema();
-    const tbl = tableFromIPC(schemaBuf);
-    return tbl.schema;
-  }
-
-  /**
-   * Insert records into this Table.
-   *
-   * @param {Data} data Records to be inserted into the Table
-   * @return The number of rows added to the table
-   */
-  async add(data: Data): Promise<void> {
-    const buffer = toBuffer(data);
-    await this.inner.add(buffer);
-  }
-
-  /** Count the total number of rows in the dataset. */
-  async countRows(filter?: string): Promise<bigint> {
-    return await this.inner.countRows(filter);
-  }
-
-  /** Delete the rows that satisfy the predicate. */
-  async delete(predicate: string): Promise<void> {
-    await this.inner.delete(predicate);
-  }
-
-  /** Create an index over the columns.
-   *
-   * @param {string} column The column to create the index on. If not specified,
-   *                        it will create an index on vector field.
-   *
-   * @example
-   *
-   * By default, it creates vector idnex on one vector column.
-   *
-   * ```typescript
-   * const table = await conn.openTable("my_table");
-   * await table.createIndex().build();
-   * ```
-   *
-   * You can specify `IVF_PQ` parameters via `ivf_pq({})` call.
-   * ```typescript
-   * const table = await conn.openTable("my_table");
-   * await table.createIndex("my_vec_col")
-   *   .ivf_pq({ num_partitions: 128, num_sub_vectors: 16 })
-   *   .build();
-   * ```
-   *
-   * Or create a Scalar index
-   *
-   * ```typescript
-   * await table.createIndex("my_float_col").build();
-   * ```
-   */
-  createIndex(column?: string): IndexBuilder {
-    let builder = new IndexBuilder(this.inner);
-    if (column !== undefined) {
-      builder = builder.column(column);
-    }
-    return builder;
-  }
-
-  /**
-   * Create a generic {@link Query} Builder.
-   *
-   * When appropriate, various indices and statistics based pruning will be used to
-   * accelerate the query.
-   *
-   * @example
-   *
-   * ### Run a SQL-style query
-   * ```typescript
-   * for await (const batch of table.query()
-   *                          .filter("id > 1").select(["id"]).limit(20)) {
-   *  console.log(batch);
-   * }
-   * ```
-   *
-   * ### Run Top-10 vector similarity search
-   * ```typescript
-   * for await (const batch of table.query()
-   *                    .nearestTo([1, 2, 3])
-   *                    .refineFactor(5).nprobe(10)
-   *                    .limit(10)) {
-   *  console.log(batch);
-   * }
-   *```
-   *
-   * ### Scan the full dataset
-   * ```typescript
-   * for await (const batch of table.query()) {
-   *   console.log(batch);
-   * }
-   *
-   * ### Return the full dataset as Arrow Table
-   * ```typescript
-   * let arrowTbl = await table.query().nearestTo([1.0, 2.0, 0.5, 6.7]).toArrow();
-   * ```
-   *
-   * @returns {@link Query}
-   */
-  query(): Query {
-    return new Query(this.inner);
-  }
-
-  /** Search the table with a given query vector.
-   *
-   * This is a convenience method for preparing an ANN {@link Query}.
-   */
-  search(vector: number[], column?: string): Query {
-    const q = this.query();
-    q.nearestTo(vector);
-    if (column !== undefined) {
-      q.column(column);
-    }
-    return q;
-  }
-}

python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.5.7
+current_version = 0.6.5
 commit = True
 message = [python] Bump version: {current_version} → {new_version}
 tag = True

python/ASYNC_MIGRATION.md

@@ -0,0 +1,38 @@
+# Migration from Sync to Async API
+
+A new asynchronous API has been added to LanceDb.  This API is built
+on top of the rust lancedb crate (instead of being built on top of
+pylance).  This will help keep the various language bindings in sync.
+There are some slight changes between the synchronous and the asynchronous
+APIs.  This document will help you migrate.  These changes relate mostly
+to the Connection and Table classes.
+
+## Almost all functions are async
+
+The most important change is that almost all functions are now async.
+This means the functions now return `asyncio` coroutines.  You will
+need to use `await` to call these functions.
+
+## Connection
+
+* The connection now has a `close` method.  You can call this when
+  you are done with the connection to eagerly free resources.  Currently
+  this is limited to freeing/closing the HTTP connection for remote
+  connections.  In the future we may add caching or other resources to
+  native connections so this is probably a good practice even if you aren't using remote connections.
+
+  In addition, the connection can be used as a context manager which may
+  be a more convenient way to ensure the connection is closed.
+
+  It is not mandatory to call the `close` method.  If you don't call it
+  the connection will be closed when the object is garbage collected.
+
+## Table
+
+* The table now has a `close` method, similar to the connection.  This
+  can be used to eagerly free the cache used by a Table object.  Similar
+  to the connection, it can be used as a context manager and it is not
+  mandatory to call the `close` method.
+* Previously `Table.schema` was a property.  Now it is an async method.
+* The method `Table.__len__` was removed and `len(table)` will no longer
+  work.  Use `Table.count_rows` instead.

python/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "lancedb-python"
+version = "0.4.10"
+edition.workspace = true
+description = "Python bindings for LanceDB"
+license.workspace = true
+repository.workspace = true
+keywords.workspace = true
+categories.workspace = true
+rust-version = "1.75.0"
+
+[lib]
+name = "_lancedb"
+crate-type = ["cdylib"]
+
+[dependencies]
+arrow = { version = "50.0.0", features = ["pyarrow"] }
+lancedb = { path = "../rust/lancedb" }
+env_logger = "0.10"
+pyo3 = { version = "0.20", features = ["extension-module", "abi3-py38"] }
+pyo3-asyncio = { version = "0.20", features = ["attributes", "tokio-runtime"] }
+
+# Prevent dynamic linking of lzma, which comes from datafusion
+lzma-sys = { version = "*", features = ["static"] }
+pin-project = "1.1.5"
+futures.workspace = true
+tokio = { version = "1.36.0", features = ["sync"] }
+
+[build-dependencies]
+pyo3-build-config = { version = "0.20.3", features = [
+    "extension-module",
+    "abi3-py38",
+] }

python/README.md

@@ -20,10 +20,10 @@ results = table.search([0.1, 0.3]).limit(20).to_list()
 print(results)
 ```
 
-
 ## Development
 
-Create a virtual environment and activate it:
+LanceDb is based on the rust crate `lancedb` and is built with maturin.  In order to build with maturin
+you will either need a conda environment or a virtual environment (venv).
 
 ```bash
 python -m venv venv
@@ -33,7 +33,15 @@ python -m venv venv
 Install the necessary packages:
 
 ```bash
-python -m pip install .
+python -m pip install .[tests,dev]
+```
+
+To build the python package you can use maturin:
+
+```bash
+# This will build the rust bindings and place them in the appropriate place
+# in your venv or conda environment
+matruin develop
 ```
 
 To run the unit tests:
@@ -45,7 +53,7 @@ pytest
 To run the doc tests:
 
 ```bash
-pytest --doctest-modules lancedb
+pytest --doctest-modules python/lancedb
 ```
 
 To run linter and automatically fix all errors:
@@ -61,31 +69,27 @@ If any packages are missing, install them with:
 pip install <PACKAGE_NAME>
 ```
 
-
 ___
 For **Windows** users, there may be errors when installing packages, so these commands may be helpful:
 
 Activate the virtual environment:
+
 ```bash
 . .\venv\Scripts\activate
 ```
 
 You may need to run the installs separately:
+
 ```bash
 pip install -e .[tests]
 pip install -e .[dev]
 ```
 
-
 `tantivy` requires `rust` to be installed, so install it with `conda`, as it doesn't support windows installation:
+
 ```bash
 pip install wheel
 pip install cargo
 conda install rust
 pip install tantivy
 ```
-
-To run the unit tests:
-```bash
-pytest
-```

python/build.rs

@@ -0,0 +1,3 @@
+fn main() {
+    pyo3_build_config::add_extension_module_link_args();
+}

python/lancedb/common.py

@@ -1,36 +0,0 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-from pathlib import Path
-from typing import Iterable, List, Union
-
-import numpy as np
-import pyarrow as pa
-
-from .util import safe_import_pandas
-
-pd = safe_import_pandas()
-
-DATA = Union[List[dict], dict, "pd.DataFrame", pa.Table, Iterable[pa.RecordBatch]]
-VEC = Union[list, np.ndarray, pa.Array, pa.ChunkedArray]
-URI = Union[str, Path]
-VECTOR_COLUMN_NAME = "vector"
-
-
-class Credential(str):
-    """Credential field"""
-
-    def __repr__(self) -> str:
-        return "********"
-
-    def __str__(self) -> str:
-        return "********"

python/lancedb/db.py

@@ -1,414 +0,0 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-from __future__ import annotations
-
-import os
-from abc import abstractmethod
-from pathlib import Path
-from typing import TYPE_CHECKING, Iterable, List, Optional, Union
-
-import pyarrow as pa
-from overrides import EnforceOverrides, override
-from pyarrow import fs
-
-from .table import LanceTable, Table
-from .util import fs_from_uri, get_uri_location, get_uri_scheme, join_uri
-
-if TYPE_CHECKING:
-    from datetime import timedelta
-
-    from .common import DATA, URI
-    from .embeddings import EmbeddingFunctionConfig
-    from .pydantic import LanceModel
-
-
-class DBConnection(EnforceOverrides):
-    """An active LanceDB connection interface."""
-
-    @abstractmethod
-    def table_names(
-        self, page_token: Optional[str] = None, limit: int = 10
-    ) -> Iterable[str]:
-        """List all table in this database
-
-        Parameters
-        ----------
-        page_token: str, optional
-            The token to use for pagination. If not present, start from the beginning.
-        limit: int, default 10
-            The size of the page to return.
-        """
-        pass
-
-    @abstractmethod
-    def create_table(
-        self,
-        name: str,
-        data: Optional[DATA] = None,
-        schema: Optional[Union[pa.Schema, LanceModel]] = None,
-        mode: str = "create",
-        exist_ok: bool = False,
-        on_bad_vectors: str = "error",
-        fill_value: float = 0.0,
-        embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
-    ) -> Table:
-        """Create a [Table][lancedb.table.Table] in the database.
-
-        Parameters
-        ----------
-        name: str
-            The name of the table.
-        data: The data to initialize the table, *optional*
-            User must provide at least one of `data` or `schema`.
-            Acceptable types are:
-
-            - dict or list-of-dict
-
-            - pandas.DataFrame
-
-            - pyarrow.Table or pyarrow.RecordBatch
-        schema: The schema of the table, *optional*
-            Acceptable types are:
-
-            - pyarrow.Schema
-
-            - [LanceModel][lancedb.pydantic.LanceModel]
-        mode: str; default "create"
-            The mode to use when creating the table.
-            Can be either "create" or "overwrite".
-            By default, if the table already exists, an exception is raised.
-            If you want to overwrite the table, use mode="overwrite".
-        exist_ok: bool, default False
-            If a table by the same name already exists, then raise an exception
-            if exist_ok=False. If exist_ok=True, then open the existing table;
-            it will not add the provided data but will validate against any
-            schema that's specified.
-        on_bad_vectors: str, default "error"
-            What to do if any of the vectors are not the same size or contains NaNs.
-            One of "error", "drop", "fill".
-        fill_value: float
-            The value to use when filling vectors. Only used if on_bad_vectors="fill".
-
-        Returns
-        -------
-        LanceTable
-            A reference to the newly created table.
-
-        !!! note
-
-            The vector index won't be created by default.
-            To create the index, call the `create_index` method on the table.
-
-        Examples
-        --------
-
-        Can create with list of tuples or dictionaries:
-
-        >>> import lancedb
-        >>> db = lancedb.connect("./.lancedb")
-        >>> data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
-        ...         {"vector": [0.2, 1.8], "lat": 40.1, "long":  -74.1}]
-        >>> db.create_table("my_table", data)
-        LanceTable(connection=..., name="my_table")
-        >>> db["my_table"].head()
-        pyarrow.Table
-        vector: fixed_size_list<item: float>[2]
-          child 0, item: float
-        lat: double
-        long: double
-        ----
-        vector: [[[1.1,1.2],[0.2,1.8]]]
-        lat: [[45.5,40.1]]
-        long: [[-122.7,-74.1]]
-
-        You can also pass a pandas DataFrame:
-
-        >>> import pandas as pd
-        >>> data = pd.DataFrame({
-        ...    "vector": [[1.1, 1.2], [0.2, 1.8]],
-        ...    "lat": [45.5, 40.1],
-        ...    "long": [-122.7, -74.1]
-        ... })
-        >>> db.create_table("table2", data)
-        LanceTable(connection=..., name="table2")
-        >>> db["table2"].head()
-        pyarrow.Table
-        vector: fixed_size_list<item: float>[2]
-          child 0, item: float
-        lat: double
-        long: double
-        ----
-        vector: [[[1.1,1.2],[0.2,1.8]]]
-        lat: [[45.5,40.1]]
-        long: [[-122.7,-74.1]]
-
-        Data is converted to Arrow before being written to disk. For maximum
-        control over how data is saved, either provide the PyArrow schema to
-        convert to or else provide a [PyArrow Table](pyarrow.Table) directly.
-
-        >>> custom_schema = pa.schema([
-        ...   pa.field("vector", pa.list_(pa.float32(), 2)),
-        ...   pa.field("lat", pa.float32()),
-        ...   pa.field("long", pa.float32())
-        ... ])
-        >>> db.create_table("table3", data, schema = custom_schema)
-        LanceTable(connection=..., name="table3")
-        >>> db["table3"].head()
-        pyarrow.Table
-        vector: fixed_size_list<item: float>[2]
-          child 0, item: float
-        lat: float
-        long: float
-        ----
-        vector: [[[1.1,1.2],[0.2,1.8]]]
-        lat: [[45.5,40.1]]
-        long: [[-122.7,-74.1]]
-
-
-        It is also possible to create an table from `[Iterable[pa.RecordBatch]]`:
-
-
-        >>> import pyarrow as pa
-        >>> def make_batches():
-        ...     for i in range(5):
-        ...         yield pa.RecordBatch.from_arrays(
-        ...             [
-        ...                 pa.array([[3.1, 4.1], [5.9, 26.5]],
-        ...                     pa.list_(pa.float32(), 2)),
-        ...                 pa.array(["foo", "bar"]),
-        ...                 pa.array([10.0, 20.0]),
-        ...             ],
-        ...             ["vector", "item", "price"],
-        ...         )
-        >>> schema=pa.schema([
-        ...     pa.field("vector", pa.list_(pa.float32(), 2)),
-        ...     pa.field("item", pa.utf8()),
-        ...     pa.field("price", pa.float32()),
-        ... ])
-        >>> db.create_table("table4", make_batches(), schema=schema)
-        LanceTable(connection=..., name="table4")
-
-        """
-        raise NotImplementedError
-
-    def __getitem__(self, name: str) -> LanceTable:
-        return self.open_table(name)
-
-    def open_table(self, name: str) -> Table:
-        """Open a Lance Table in the database.
-
-        Parameters
-        ----------
-        name: str
-            The name of the table.
-
-        Returns
-        -------
-        A LanceTable object representing the table.
-        """
-        raise NotImplementedError
-
-    def drop_table(self, name: str):
-        """Drop a table from the database.
-
-        Parameters
-        ----------
-        name: str
-            The name of the table.
-        """
-        raise NotImplementedError
-
-    def drop_database(self):
-        """
-        Drop database
-        This is the same thing as dropping all the tables
-        """
-        raise NotImplementedError
-
-
-class LanceDBConnection(DBConnection):
-    """
-    A connection to a LanceDB database.
-
-    Parameters
-    ----------
-    uri: str or Path
-        The root uri of the database.
-    read_consistency_interval: timedelta, default None
-        The interval at which to check for updates to the table from other
-        processes. If None, then consistency is not checked. For performance
-        reasons, this is the default. For strong consistency, set this to
-        zero seconds. Then every read will check for updates from other
-        processes. As a compromise, you can set this to a non-zero timedelta
-        for eventual consistency. If more than that interval has passed since
-        the last check, then the table will be checked for updates. Note: this
-        consistency only applies to read operations. Write operations are
-        always consistent.
-
-    Examples
-    --------
-    >>> import lancedb
-    >>> db = lancedb.connect("./.lancedb")
-    >>> db.create_table("my_table", data=[{"vector": [1.1, 1.2], "b": 2},
-    ...                                   {"vector": [0.5, 1.3], "b": 4}])
-    LanceTable(connection=..., name="my_table")
-    >>> db.create_table("another_table", data=[{"vector": [0.4, 0.4], "b": 6}])
-    LanceTable(connection=..., name="another_table")
-    >>> sorted(db.table_names())
-    ['another_table', 'my_table']
-    >>> len(db)
-    2
-    >>> db["my_table"]
-    LanceTable(connection=..., name="my_table")
-    >>> "my_table" in db
-    True
-    >>> db.drop_table("my_table")
-    >>> db.drop_table("another_table")
-    """
-
-    def __init__(
-        self, uri: URI, *, read_consistency_interval: Optional[timedelta] = None
-    ):
-        if not isinstance(uri, Path):
-            scheme = get_uri_scheme(uri)
-        is_local = isinstance(uri, Path) or scheme == "file"
-        if is_local:
-            if isinstance(uri, str):
-                uri = Path(uri)
-            uri = uri.expanduser().absolute()
-            Path(uri).mkdir(parents=True, exist_ok=True)
-        self._uri = str(uri)
-
-        self._entered = False
-        self.read_consistency_interval = read_consistency_interval
-
-    def __repr__(self) -> str:
-        val = f"{self.__class__.__name__}({self._uri}"
-        if self.read_consistency_interval is not None:
-            val += f", read_consistency_interval={repr(self.read_consistency_interval)}"
-        val += ")"
-        return val
-
-    @property
-    def uri(self) -> str:
-        return self._uri
-
-    @override
-    def table_names(
-        self, page_token: Optional[str] = None, limit: int = 10
-    ) -> Iterable[str]:
-        """Get the names of all tables in the database. The names are sorted.
-
-        Returns
-        -------
-        Iterator of str.
-            A list of table names.
-        """
-        try:
-            filesystem = fs_from_uri(self.uri)[0]
-        except pa.ArrowInvalid:
-            raise NotImplementedError("Unsupported scheme: " + self.uri)
-
-        try:
-            loc = get_uri_location(self.uri)
-            paths = filesystem.get_file_info(fs.FileSelector(loc))
-        except FileNotFoundError:
-            # It is ok if the file does not exist since it will be created
-            paths = []
-        tables = [
-            os.path.splitext(file_info.base_name)[0]
-            for file_info in paths
-            if file_info.extension == "lance"
-        ]
-        tables.sort()
-        return tables
-
-    def __len__(self) -> int:
-        return len(self.table_names())
-
-    def __contains__(self, name: str) -> bool:
-        return name in self.table_names()
-
-    @override
-    def create_table(
-        self,
-        name: str,
-        data: Optional[DATA] = None,
-        schema: Optional[Union[pa.Schema, LanceModel]] = None,
-        mode: str = "create",
-        exist_ok: bool = False,
-        on_bad_vectors: str = "error",
-        fill_value: float = 0.0,
-        embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
-    ) -> LanceTable:
-        """Create a table in the database.
-
-        See
-        ---
-        DBConnection.create_table
-        """
-        if mode.lower() not in ["create", "overwrite"]:
-            raise ValueError("mode must be either 'create' or 'overwrite'")
-
-        tbl = LanceTable.create(
-            self,
-            name,
-            data,
-            schema,
-            mode=mode,
-            exist_ok=exist_ok,
-            on_bad_vectors=on_bad_vectors,
-            fill_value=fill_value,
-            embedding_functions=embedding_functions,
-        )
-        return tbl
-
-    @override
-    def open_table(self, name: str) -> LanceTable:
-        """Open a table in the database.
-
-        Parameters
-        ----------
-        name: str
-            The name of the table.
-
-        Returns
-        -------
-        A LanceTable object representing the table.
-        """
-        return LanceTable.open(self, name)
-
-    @override
-    def drop_table(self, name: str, ignore_missing: bool = False):
-        """Drop a table from the database.
-
-        Parameters
-        ----------
-        name: str
-            The name of the table.
-        ignore_missing: bool, default False
-            If True, ignore if the table does not exist.
-        """
-        try:
-            filesystem, path = fs_from_uri(self.uri)
-            table_path = join_uri(path, name + ".lance")
-            filesystem.delete_dir(table_path)
-        except FileNotFoundError:
-            if not ignore_missing:
-                raise
-
-    @override
-    def drop_database(self):
-        filesystem, path = fs_from_uri(self.uri)
-        filesystem.delete_dir(path)