Merge branch 'main' of https://github.com/lancedb/lancedb into hybrid_query

I've also started `ASYNC_MIGRATION.MD` to keep track of the breaking
changes from sync to async python.

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.10
+current_version = 0.4.11
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True
@@ -9,4 +9,4 @@ tag_name = v{new_version}
 
 [bumpversion:file:rust/ffi/node/Cargo.toml]
 
-[bumpversion:file:rust/vectordb/Cargo.toml]
+[bumpversion:file:rust/lancedb/Cargo.toml]

.cargo/config.toml

@@ -33,3 +33,8 @@ rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"
 
 [target.aarch64-apple-darwin]
 rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm,+dotprod"]
+
+# Not all Windows systems have the C runtime installed, so this avoids library
+# not found errors on systems that are missing it.
+[target.x86_64-pc-windows-msvc]
+rustflags = ["-Ctarget-feature=+crt-static"]

.github/workflows/build_linux_wheel/action.yml

@@ -0,0 +1,58 @@
+# We create a composite action to be re-used both for testing and for releasing
+name: build-linux-wheel
+description: "Build a manylinux wheel for lance"
+inputs:
+  python-minor-version:
+    description: "8, 9, 10, 11, 12"
+    required: true
+  args:
+    description: "--release"
+    required: false
+    default: ""
+  arm-build:
+    description: "Build for arm64 instead of x86_64"
+    # Note: this does *not* mean the host is arm64, since we might be cross-compiling.
+    required: false
+    default: "false"
+runs:
+  using: "composite"
+  steps:
+    - name: CONFIRM ARM BUILD
+      shell: bash
+      run: |
+        echo "ARM BUILD: ${{ inputs.arm-build }}"
+    - name: Build x86_64 Manylinux wheel
+      if: ${{ inputs.arm-build == 'false' }}
+      uses: PyO3/maturin-action@v1
+      with:
+        command: build
+        working-directory: python
+        target: x86_64-unknown-linux-gnu
+        manylinux: "2_17"
+        args: ${{ inputs.args }}
+        before-script-linux: |
+          set -e
+          yum install -y openssl-devel \
+            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
+            && unzip /tmp/protoc.zip -d /usr/local \
+            && rm /tmp/protoc.zip
+    - name: Build Arm Manylinux Wheel
+      if: ${{ inputs.arm-build == 'true' }}
+      uses: PyO3/maturin-action@v1
+      with:
+        command: build
+        working-directory: python
+        target: aarch64-unknown-linux-gnu
+        manylinux: "2_24"
+        args: ${{ inputs.args }}
+        before-script-linux: |
+          set -e
+          apt install -y unzip
+          if [ $(uname -m) = "x86_64" ]; then
+            PROTOC_ARCH="x86_64"
+          else
+            PROTOC_ARCH="aarch_64"
+          fi
+          curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$PROTOC_ARCH.zip > /tmp/protoc.zip \
+            && unzip /tmp/protoc.zip -d /usr/local \
+            && rm /tmp/protoc.zip

.github/workflows/build_mac_wheel/action.yml

@@ -0,0 +1,25 @@
+# We create a composite action to be re-used both for testing and for releasing
+name: build_wheel
+description: "Build a lance wheel"
+inputs:
+  python-minor-version:
+    description: "8, 9, 10, 11"
+    required: true
+  args:
+    description: "--release"
+    required: false
+    default: ""
+runs:
+  using: "composite"
+  steps:
+    - name: Install macos dependency
+      shell: bash
+      run: |
+        brew install protobuf
+    - name: Build wheel
+      uses: PyO3/maturin-action@v1
+      with:
+        command: build
+        args: ${{ inputs.args }}
+        working-directory: python
+        interpreter: 3.${{ inputs.python-minor-version }}

.github/workflows/build_windows_wheel/action.yml

@@ -0,0 +1,33 @@
+# We create a composite action to be re-used both for testing and for releasing
+name: build_wheel
+description: "Build a lance wheel"
+inputs:
+  python-minor-version:
+    description: "8, 9, 10, 11"
+    required: true
+  args:
+    description: "--release"
+    required: false
+    default: ""
+runs:
+  using: "composite"
+  steps:
+    - name: Install Protoc v21.12
+      working-directory: C:\
+      run: |
+        New-Item -Path 'C:\protoc' -ItemType Directory
+        Set-Location C:\protoc
+        Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+        7z x protoc.zip
+        Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+      shell: powershell
+    - name: Build wheel
+      uses: PyO3/maturin-action@v1
+      with:
+        command: build
+        args: ${{ inputs.args }}
+        working-directory: python
+    - uses: actions/upload-artifact@v3
+      with:
+        name: windows-wheels
+        path: python\target\wheels

.github/workflows/cargo-publish.yml

@@ -26,4 +26,4 @@ jobs:
           sudo apt install -y protobuf-compiler libssl-dev
       - name: Publish the package
         run: |
-          cargo publish -p vectordb --all-features --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          cargo publish -p lancedb --all-features --token ${{ secrets.CARGO_REGISTRY_TOKEN }}

.github/workflows/pypi-publish.yml

@@ -2,30 +2,91 @@ name: PyPI Publish
 
 on:
   release:
-    types: [ published ]
+    types: [published]
 
 jobs:
-  publish:
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the python-make-release action
-    if: startsWith(github.ref, 'refs/tags/python-v')
-    defaults:
-      run:
-        shell: bash
-        working-directory: python
+  linux:
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        python-minor-version: ["8"]
+        platform:
+          - x86_64
+          - aarch64
+    runs-on: "ubuntu-22.04"
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
-      - name: Build distribution
-        run: |
-          ls -la
-          pip install wheel setuptools --upgrade
-          python setup.py sdist bdist_wheel
-      - name: Publish
-        uses: pypa/gh-action-pypi-publish@v1.8.5
+          python-version: 3.${{ matrix.python-minor-version }}
+      - uses: ./.github/workflows/build_linux_wheel
         with:
-          password: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
-          packages-dir: python/dist
+          python-minor-version: ${{ matrix.python-minor-version }}
+          args: "--release --strip"
+          arm-build: ${{ matrix.platform == 'aarch64' }}
+      - uses: ./.github/workflows/upload_wheel
+        with:
+          token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
+          repo: "pypi"
+  mac:
+    timeout-minutes: 60
+    runs-on: ${{ matrix.config.runner }}
+    strategy:
+      matrix:
+        python-minor-version: ["8"]
+        config:
+          - target: x86_64-apple-darwin
+            runner: macos-13
+          - target: aarch64-apple-darwin
+            runner: macos-14
+    env:
+      MACOSX_DEPLOYMENT_TARGET: 10.15
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.12
+      - uses: ./.github/workflows/build_mac_wheel
+        with:
+          python-minor-version: ${{ matrix.python-minor-version }}
+          args: "--release --strip --target ${{ matrix.config.target }}"
+      - uses: ./.github/workflows/upload_wheel
+        with:
+          python-minor-version: ${{ matrix.python-minor-version }}
+          token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
+          repo: "pypi"
+  windows:
+    timeout-minutes: 60
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        python-minor-version: ["8"]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.${{ matrix.python-minor-version }}
+      - uses: ./.github/workflows/build_windows_wheel
+        with:
+          python-minor-version: ${{ matrix.python-minor-version }}
+          args: "--release --strip"
+          vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }}
+      - uses: ./.github/workflows/upload_wheel
+        with:
+          python-minor-version: ${{ matrix.python-minor-version }}
+          token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
+          repo: "pypi"

.github/workflows/python.yml

@@ -14,49 +14,133 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  linux:
+  lint:
+    name: "Lint"
     timeout-minutes: 30
-    strategy:
-      matrix:
-        python-minor-version: [ "8", "11" ]
     runs-on: "ubuntu-22.04"
     defaults:
       run:
         shell: bash
         working-directory: python
     steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: 3.${{ matrix.python-minor-version }}
-    - name: Install lancedb
-      run: |
-        pip install -e .[tests]
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock ruff
-    - name: Format check
-      run: ruff format --check .
-    - name: Lint
-      run: ruff .
-    - name: Run tests
-      run: pytest -m "not slow" -x -v --durations=30 tests
-    - name: doctest
-      run: pytest --doctest-modules lancedb
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install ruff
+        run: |
+          pip install ruff==0.2.2
+      - name: Format check
+        run: ruff format --check .
+      - name: Lint
+        run: ruff .
+  doctest:
+    name: "Doctest"
+    timeout-minutes: 30
+    runs-on: "ubuntu-22.04"
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: "pip"
+      - name: Install protobuf
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+      - name: Install
+        run: |
+          pip install -e .[tests,dev,embeddings]
+          pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+          pip install mlx
+      - name: Doctest
+        run: pytest --doctest-modules python/lancedb
+  linux:
+    name: "Linux: python-3.${{ matrix.python-minor-version }}"
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        python-minor-version: ["8", "11"]
+    runs-on: "ubuntu-22.04"
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Install protobuf
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.${{ matrix.python-minor-version }}
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+      - uses: ./.github/workflows/build_linux_wheel
+      - uses: ./.github/workflows/run_tests
+      # Make sure wheels are not included in the Rust cache
+      - name: Delete wheels
+        run: rm -rf target/wheels
   platform:
-    name: "Platform: ${{ matrix.config.name }}"
+    name: "Mac: ${{ matrix.config.name }}"
     timeout-minutes: 30
     strategy:
       matrix:
         config:
-          - name: x86 Mac
+          - name: x86
             runner: macos-13
-          - name: Arm Mac
+          - name: Arm
             runner: macos-14
-          - name: x86 Windows
+    runs-on: "${{ matrix.config.runner }}"
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+      - uses: ./.github/workflows/build_mac_wheel
+      - uses: ./.github/workflows/run_tests
+      # Make sure wheels are not included in the Rust cache
+      - name: Delete wheels
+        run: rm -rf target/wheels
+  windows:
+    name: "Windows: ${{ matrix.config.name }}"
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        config:
+          - name: x86
             runner: windows-latest
     runs-on: "${{ matrix.config.runner }}"
     defaults:
@@ -64,21 +148,22 @@ jobs:
         shell: bash
         working-directory: python
     steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: "3.11"
-    - name: Install lancedb
-      run: |
-        pip install -e .[tests]
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock
-    - name: Run tests
-      run: pytest -m "not slow" -x -v --durations=30 tests
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+      - uses: ./.github/workflows/build_windows_wheel
+      - uses: ./.github/workflows/run_tests
+      # Make sure wheels are not included in the Rust cache
+      - name: Delete wheels
+        run: rm -rf target/wheels
   pydantic1x:
     timeout-minutes: 30
     runs-on: "ubuntu-22.04"
@@ -87,21 +172,22 @@ jobs:
         shell: bash
         working-directory: python
     steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        lfs: true
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: 3.9
-    - name: Install lancedb
-      run: |
-        pip install "pydantic<2"
-        pip install -e .[tests]
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock
-    - name: Run tests
-      run: pytest -m "not slow" -x -v --durations=30 tests
-    - name: doctest
-      run: pytest --doctest-modules lancedb
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+      - name: Install lancedb
+        run: |
+          pip install "pydantic<2"
+          pip install -e .[tests]
+          pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+      - name: Run tests
+        run: pytest -m "not slow" -x -v --durations=30 python/tests

.github/workflows/remote-integration.yml

@@ -0,0 +1,37 @@
+name: LanceDb Cloud Integration Test
+
+on:
+  workflow_run:
+    workflows: [Rust]
+    types:
+      - completed
+
+env:
+  LANCEDB_PROJECT: ${{ secrets.LANCEDB_PROJECT }}
+  LANCEDB_API_KEY: ${{ secrets.LANCEDB_API_KEY }}
+  LANCEDB_REGION: ${{ secrets.LANCEDB_REGION }}
+
+jobs:
+  test:
+    timeout-minutes: 30
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+        working-directory: rust
+    steps:
+    - uses: actions/checkout@v4
+      with:
+          fetch-depth: 0
+          lfs: true
+    - uses: Swatinem/rust-cache@v2
+      with:
+          workspaces: rust
+    - name: Install dependencies
+      run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+    - name: Build
+      run: cargo build --all-features
+    - name: Run Integration test
+      run: cargo test --tests -- --ignored

.github/workflows/run_tests/action.yml

@@ -0,0 +1,17 @@
+name: run-tests
+
+description: "Install lance wheel and run unit tests"
+inputs:
+  python-minor-version:
+    required: true
+    description: "8 9 10 11 12"
+runs:
+  using: "composite"
+  steps:
+    - name: Install lancedb
+      shell: bash
+      run: |
+        pip3 install $(ls target/wheels/lancedb-*.whl)[tests,dev]
+    - name: pytest
+      shell: bash
+      run: pytest -m "not slow" -x -v --durations=30 python/python/tests

.github/workflows/rust.yml

@@ -119,3 +119,4 @@ jobs:
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
           cargo build
           cargo test
+  

.github/workflows/upload_wheel/action.yml

@@ -0,0 +1,29 @@
+name: upload-wheel
+
+description: "Upload wheels to Pypi"
+inputs:
+  os:
+    required: true
+    description: "ubuntu-22.04 or macos-13"
+  repo:
+    required: false
+    description: "pypi or testpypi"
+    default: "pypi"
+  token:
+    required: true
+    description: "release token for the repo"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install dependencies
+      shell: bash
+      run: |
+        python -m pip install --upgrade pip
+        pip install twine
+    - name: Publish wheel
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ inputs.token }}
+      shell: bash
+      run: twine upload --repository ${{ inputs.repo }} target/wheels/lancedb-*.whl

.gitignore

@@ -22,6 +22,11 @@ python/dist
 
 **/.hypothesis
 
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
 ## Javascript
 *.node
 **/node_modules
@@ -34,4 +39,6 @@ dist
 ## Rust
 target
 
+**/sccache.log
+
 Cargo.lock

.pre-commit-config.yaml

@@ -5,17 +5,8 @@ repos:
     -   id: check-yaml
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
--   repo: https://github.com/psf/black
-    rev: 22.12.0
-    hooks:
-    -   id: black
 -   repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.0.277
+    rev: v0.2.2
     hooks:
     - id: ruff
--   repo: https://github.com/pycqa/isort
-    rev: 5.12.0
-    hooks:
-    - id: isort
-      name: isort (python)

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["rust/ffi/node", "rust/vectordb", "nodejs"]
+members = ["rust/ffi/node", "rust/lancedb", "nodejs", "python"]
 # Python package needs to be built by maturin.
 exclude = ["python"]
 resolver = "2"
@@ -14,10 +14,10 @@ keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
 
 [workspace.dependencies]
-lance = { "version" = "=0.9.16", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.9.16" }
-lance-linalg = { "version" = "=0.9.16" }
-lance-testing = { "version" = "=0.9.16" }
+lance = { "version" = "=0.10.1", "features" = ["dynamodb"] }
+lance-index = { "version" = "=0.10.1" }
+lance-linalg = { "version" = "=0.10.1" }
+lance-testing = { "version" = "=0.10.1" }
 # Note that this one does not include pyarrow
 arrow = { version = "50.0", optional = false }
 arrow-array = "50.0"

dockerfiles/Dockerfile

@@ -0,0 +1,27 @@
+#Simple base dockerfile that supports basic dependencies required to run lance with FTS and Hybrid Search
+#Usage docker build -t lancedb:latest -f Dockerfile .
+FROM python:3.10-slim-buster
+
+# Install Rust
+RUN apt-get update && apt-get install -y curl build-essential && \
+  curl https://sh.rustup.rs -sSf | sh -s -- -y
+
+# Set the environment variable for Rust
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install protobuf compiler
+RUN apt-get install -y protobuf-compiler && \
+  apt-get clean && \
+  rm -rf /var/lib/apt/lists/*
+
+RUN apt-get -y update &&\
+  apt-get -y upgrade && \
+  apt-get -y install git
+
+
+# Verify installations
+RUN python --version && \
+  rustc --version && \
+  protoc --version
+
+RUN pip install tantivy lancedb

docs/mkdocs.yml

@@ -57,6 +57,16 @@ plugins:
           - https://arrow.apache.org/docs/objects.inv
           - https://pandas.pydata.org/docs/objects.inv
 - mkdocs-jupyter
+- ultralytics:
+    verbose: True
+    enabled: True
+    default_image: "assets/lancedb_and_lance.png" # Default image for all pages
+    add_image: True # Automatically add meta image
+    add_keywords: True # Add page keywords in the header tag
+    add_share_buttons: True # Add social share buttons
+    add_authors: False # Display page authors 
+    add_desc: False
+    add_dates: False
 
 markdown_extensions:
 - admonition
@@ -92,16 +102,16 @@ nav:
     - Full-text search: fts.md
     - Hybrid search:
       - Overview: hybrid_search/hybrid_search.md
+      - Comparing Rerankers: hybrid_search/eval.md
       - Airbnb financial data example: notebooks/hybrid_search.ipynb
     - Filtering: sql.md
     - Versioning & Reproducibility: notebooks/reproducibility.ipynb
     - Configuring Storage: guides/storage.md
   - 🧬 Managing embeddings:
     - Overview: embeddings/index.md
-    - Explicit management: embeddings/embedding_explicit.md
-    - Implicit management: embeddings/embedding_functions.md
-    - Available Functions: embeddings/default_embedding_functions.md
-    - Custom Embedding Functions: embeddings/api.md
+    - Embedding functions: embeddings/embedding_functions.md
+    - Available models: embeddings/default_embedding_functions.md
+    - User-defined embedding functions: embeddings/custom_embedding_function.md
     - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
     - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
   - 🔌 Integrations:
@@ -156,16 +166,16 @@ nav:
     - Full-text search: fts.md
     - Hybrid search:
       - Overview: hybrid_search/hybrid_search.md
+      - Comparing Rerankers: hybrid_search/eval.md
       - Airbnb financial data example: notebooks/hybrid_search.ipynb
     - Filtering: sql.md
     - Versioning & Reproducibility: notebooks/reproducibility.ipynb
     - Configuring Storage: guides/storage.md
 - Managing Embeddings:
   - Overview: embeddings/index.md
-  - Explicit management: embeddings/embedding_explicit.md
-  - Implicit management: embeddings/embedding_functions.md
-  - Available Functions: embeddings/default_embedding_functions.md
-  - Custom Embedding Functions: embeddings/api.md
+  - Embedding functions: embeddings/embedding_functions.md
+  - Available models: embeddings/default_embedding_functions.md
+  - User-defined embedding functions: embeddings/custom_embedding_function.md
   - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
   - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
 - Integrations:

docs/requirements.txt

@@ -2,4 +2,5 @@ mkdocs==1.5.3
 mkdocs-jupyter==0.24.1
 mkdocs-material==9.5.3
 mkdocstrings[python]==0.20.0
-pydantic
+pydantic
+mkdocs-ultralytics-plugin==0.0.44

docs/src/embeddings/embedding_explicit.md

@@ -1,141 +0,0 @@
-In this workflow, you define your own embedding function and pass it as a callable to LanceDB, invoking it in your code to generate the embeddings. Let's look at some examples.
-
-### Hugging Face
-
-!!! note
-    Currently, the Hugging Face method is only supported in the Python SDK.
-
-=== "Python"
-    The most popular open source option is to use the [sentence-transformers](https://www.sbert.net/) 
-    library, which can be installed via pip.
-
-    ```bash
-    pip install sentence-transformers
-    ```
-
-    The example below shows how to use the `paraphrase-albert-small-v2` model to generate embeddings 
-    for a given document.
-
-    ```python
-    from sentence_transformers import SentenceTransformer
-
-    name="paraphrase-albert-small-v2"
-    model = SentenceTransformer(name)
-
-    # used for both training and querying
-    def embed_func(batch):
-        return [model.encode(sentence) for sentence in batch]
-    ```
-
-### OpenAI
-
-Another popular alternative is to use an external API like OpenAI's [embeddings API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings).
-
-=== "Python"
-      ```python
-        import openai
-        import os
-
-        # Configuring the environment variable OPENAI_API_KEY
-        if "OPENAI_API_KEY" not in os.environ:
-        # OR set the key here as a variable
-        openai.api_key = "sk-..."
-
-        # verify that the API key is working
-        assert len(openai.Model.list()["data"]) > 0
-
-        def embed_func(c):
-            rs = openai.Embedding.create(input=c, engine="text-embedding-ada-002")
-            return [record["embedding"] for record in rs["data"]]
-      ```
-
-=== "JavaScript"
-      ```javascript
-        const lancedb = require("vectordb");
-
-        // You need to provide an OpenAI API key
-        const apiKey = "sk-..."
-        // The embedding function will create embeddings for the 'text' column
-        const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
-      ```
-
-## Applying an embedding function to data
-
-=== "Python"
-    Using an embedding function, you can apply it to raw data
-    to generate embeddings for each record.
-
-    Say you have a pandas DataFrame with a `text` column that you want embedded,
-    you can use the `with_embeddings` function to generate embeddings and add them to 
-    an existing table.
-
-    ```python
-     import pandas as pd
-     from lancedb.embeddings import with_embeddings
-
-     df = pd.DataFrame(
-        [
-            {"text": "pepperoni"},
-            {"text": "pineapple"}
-        ]
-    )
-     data = with_embeddings(embed_func, df)
-
-     # The output is used to create / append to a table
-     # db.create_table("my_table", data=data)
-    ```
-
-    If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
-
-    By default, LanceDB calls the function with batches of 1000 rows. This can be configured
-    using the `batch_size` parameter to `with_embeddings`.
-
-    LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
-    API call is reliable.
-
-=== "JavaScript"
-    Using an embedding function, you can apply it to raw data
-    to generate embeddings for each record.
-
-    Simply pass the embedding function created above and LanceDB will use it to generate
-    embeddings for your data.
-
-    ```javascript
-    const db = await lancedb.connect("data/sample-lancedb");
-    const data = [
-    { text: "pepperoni"},
-    { text: "pineapple"}
-    ]
-
-    const table = await db.createTable("vectors", data, embedding)
-    ```
-
-## Querying using an embedding function
-
-!!! warning
-    At query time, you **must** use the same embedding function you used to vectorize your data.
-    If you use a different embedding function, the embeddings will not reside in the same vector
-    space and the results will be nonsensical.
-
-=== "Python"
-     ```python
-     query = "What's the best pizza topping?"
-     query_vector = embed_func([query])[0]
-     results = (
-        tbl.search(query_vector)
-        .limit(10)
-        .to_pandas()
-     )
-     ```
-
-     The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
-
-=== "JavaScript"
-     ```javascript
-      const results = await table
-        .search("What's the best pizza topping?")
-        .limit(10)
-        .execute()
-     ```
-
-     The above snippet returns an array of records with the top 10 nearest neighbors to the query.

docs/src/embeddings/embedding_functions.md

@@ -3,61 +3,126 @@ Representing multi-modal data as vector embeddings is becoming a standard practi
 For this purpose, LanceDB introduces an **embedding functions API**, that allow you simply set up once, during the configuration stage of your project. After this, the table remembers it, effectively making the embedding functions *disappear in the background* so you don't have to worry about manually passing callables, and instead, simply focus on the rest of your data engineering pipeline.
 
 !!! warning
-    Using the implicit embeddings management approach means that you can forget about the manually passing around embedding
-    functions in your code, as long as you don't intend to change it at a later time. If your embedding function changes,
-    you'll have to re-configure your table with the new embedding function and regenerate the embeddings.
+    Using the embedding function registry means that you don't have to explicitly generate the embeddings yourself. 
+    However, if your embedding function changes, you'll have to re-configure your table with the new embedding function 
+    and regenerate the embeddings. In the future, we plan to support the ability to change the embedding function via
+    table metadata and have LanceDB automatically take care of regenerating the embeddings.
+
 
 ## 1. Define the embedding function
-We have some pre-defined embedding functions in the global registry, with more coming soon. Here's let's an implementation of CLIP as example.
-```
-registry = EmbeddingFunctionRegistry.get_instance()
-clip = registry.get("open-clip").create()
 
-```
-You can also define your own embedding function by implementing the `EmbeddingFunction` abstract base interface. It subclasses Pydantic Model which can be utilized to write complex schemas simply as we'll see next!
+=== "Python"
+    In the LanceDB python SDK, we define a global embedding function registry with
+    many different embedding models and even more coming soon. 
+    Here's let's an implementation of CLIP as example.
+
+    ```python
+    from lancedb.embeddings import get_registry
+
+    registry = get_registry()
+    clip = registry.get("open-clip").create()
+    ```
+
+    You can also define your own embedding function by implementing the `EmbeddingFunction` 
+    abstract base interface. It subclasses Pydantic Model which can be utilized to write complex schemas simply as we'll see next!
+
+=== "JavaScript""
+    In the TypeScript SDK, the choices are more limited. For now, only the OpenAI
+    embedding function is available.
+
+    ```javascript
+    const lancedb = require("vectordb");
+
+    // You need to provide an OpenAI API key
+    const apiKey = "sk-..."
+    // The embedding function will create embeddings for the 'text' column
+    const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
+    ```
 
 ## 2. Define the data model or schema
-The embedding function defined above abstracts away all the details about the models and dimensions required to define the schema. You can simply set a field as **source** or **vector** column. Here's how:
 
-```python
-class Pets(LanceModel):
-    vector: Vector(clip.ndims) = clip.VectorField()
-    image_uri: str = clip.SourceField()
-```
+=== "Python"
+    The embedding function defined above abstracts away all the details about the models and dimensions required to define the schema. You can simply set a field as **source** or **vector** column. Here's how:
 
-`VectorField` tells LanceDB to use the clip embedding function to generate query embeddings for the `vector` column and `SourceField` ensures that when adding data, we automatically use the specified embedding function to encode `image_uri`.
+    ```python
+    class Pets(LanceModel):
+        vector: Vector(clip.ndims) = clip.VectorField()
+        image_uri: str = clip.SourceField()
+    ```
 
-## 3. Create LanceDB table
-Now that we have chosen/defined our embedding function and the schema, we can create the table:
+    `VectorField` tells LanceDB to use the clip embedding function to generate query embeddings for the `vector` column and `SourceField` ensures that when adding data, we automatically use the specified embedding function to encode `image_uri`.
 
-```python
-db = lancedb.connect("~/lancedb")
-table = db.create_table("pets", schema=Pets)
+=== "JavaScript"
 
-```
+    For the TypeScript SDK, a schema can be inferred from input data, or an explicit
+    Arrow schema can be provided.
 
-That's it! We've provided all the information needed to embed the source and query inputs. We can now forget about the model and dimension details and start to build our VectorDB pipeline.
+## 3. Create table and add data
 
-## 4. Ingest lots of data and query your table
-Any new or incoming data can just be added and it'll be vectorized automatically.
+Now that we have chosen/defined our embedding function and the schema, 
+we can create the table and ingest data without needing to explicitly generate
+the embeddings at all:
 
-```python
-table.add([{"image_uri": u} for u in uris])
-```
+=== "Python"
+    ```python
+    db = lancedb.connect("~/lancedb")
+    table = db.create_table("pets", schema=Pets)
 
-Our OpenCLIP query embedding function supports querying via both text and images:
+    table.add([{"image_uri": u} for u in uris])
+    ```
 
-```python
-result = table.search("dog")
-```
+=== "JavaScript"
 
-Let's query an image:
+    ```javascript
+    const db = await lancedb.connect("data/sample-lancedb");
+    const data = [
+    { text: "pepperoni"},
+    { text: "pineapple"}
+    ]
 
-```python
-p = Path("path/to/images/samoyed_100.jpg")
-query_image = Image.open(p)
-table.search(query_image)
-```
+    const table = await db.createTable("vectors", data, embedding)
+    ```
+
+## 4. Querying your table
+Not only can you forget about the embeddings during ingestion, you also don't
+need to worry about it when you query the table:
+
+=== "Python"
+
+    Our OpenCLIP query embedding function supports querying via both text and images:
+
+    ```python
+    results = (
+        table.search("dog")
+        .limit(10)
+        .to_pandas()
+    )
+    ```
+
+    Or we can search using an image:
+
+    ```python
+    p = Path("path/to/images/samoyed_100.jpg")
+    query_image = Image.open(p)
+    results = (
+        table.search(query_image)
+        .limit(10)
+        .to_pandas()
+    )
+    ```
+
+    Both of the above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
+
+=== "JavaScript"
+
+    ```javascript
+    const results = await table
+    .search("What's the best pizza topping?")
+    .limit(10)
+    .execute()
+    ```    
+    
+    The above snippet returns an array of records with the top 10 nearest neighbors to the query.
 
 ---
 
@@ -100,4 +165,5 @@ rs[2].image
 
 ![](../assets/dog_clip_output.png)
 
-Now that you have the basic idea about implicit management via embedding functions, let's dive deeper into a [custom API](./api.md) that you can use to implement your own embedding functions.
+Now that you have the basic idea about LanceDB embedding functions and the embedding function registry,
+let's dive deeper into defining your own [custom functions](./custom_embedding_function.md).

docs/src/embeddings/index.md

@@ -1,8 +1,14 @@
-Due to the nature of vector embeddings, they can be used to represent any kind of data, from text to images to audio. This makes them a very powerful tool for machine learning practitioners. However, there's no one-size-fits-all solution for generating embeddings - there are many different libraries and APIs (both commercial and open source) that can be used to generate embeddings from structured/unstructured data.
+Due to the nature of vector embeddings, they can be used to represent any kind of data, from text to images to audio. 
+This makes them a very powerful tool for machine learning practitioners. 
+However, there's no one-size-fits-all solution for generating embeddings - there are many different libraries and APIs 
+(both commercial and open source) that can be used to generate embeddings from structured/unstructured data.
 
-LanceDB supports 2 methods of vectorizing your raw data into embeddings.
+LanceDB supports 3 methods of working with embeddings.
 
-1. **Explicit**: By manually calling LanceDB's `with_embedding` function to vectorize your data via an `embed_func` of your choice
-2. **Implicit**: Allow LanceDB to embed the data and queries in the background as they come in, by using the table's `EmbeddingRegistry` information
+1. You can manually generate embeddings for the data and queries. This is done outside of LanceDB.
+2. You can use the built-in [embedding functions](./embedding_functions.md) to embed the data and queries in the background.
+3. For python users, you can define your own [custom embedding function](./custom_embedding_function.md)
+   that extends the default embedding functions.
 
-See the [explicit](embedding_explicit.md) and [implicit](embedding_functions.md) embedding sections for more details.
+For python users, there is also a legacy [with_embeddings API](./legacy.md).
+It is retained for compatibility and will be removed in a future version.

docs/src/embeddings/legacy.md

@@ -0,0 +1,99 @@
+The legacy `with_embeddings` API is for Python only and is deprecated.
+
+### Hugging Face
+
+The most popular open source option is to use the [sentence-transformers](https://www.sbert.net/) 
+library, which can be installed via pip.
+
+```bash
+pip install sentence-transformers
+```
+
+The example below shows how to use the `paraphrase-albert-small-v2` model to generate embeddings 
+for a given document.
+
+```python
+from sentence_transformers import SentenceTransformer
+
+name="paraphrase-albert-small-v2"
+model = SentenceTransformer(name)
+
+# used for both training and querying
+def embed_func(batch):
+    return [model.encode(sentence) for sentence in batch]
+```
+
+
+### OpenAI
+
+Another popular alternative is to use an external API like OpenAI's [embeddings API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings).
+
+```python
+import openai
+import os
+
+# Configuring the environment variable OPENAI_API_KEY
+if "OPENAI_API_KEY" not in os.environ:
+# OR set the key here as a variable
+openai.api_key = "sk-..."
+
+client = openai.OpenAI()
+
+def embed_func(c):    
+    rs = client.embeddings.create(input=c, model="text-embedding-ada-002")
+    return [record.embedding for record in rs["data"]]
+```
+
+
+## Applying an embedding function to data
+
+Using an embedding function, you can apply it to raw data
+to generate embeddings for each record.
+
+Say you have a pandas DataFrame with a `text` column that you want embedded,
+you can use the `with_embeddings` function to generate embeddings and add them to 
+an existing table.
+
+```python
+    import pandas as pd
+    from lancedb.embeddings import with_embeddings
+
+    df = pd.DataFrame(
+        [
+            {"text": "pepperoni"},
+            {"text": "pineapple"}
+        ]
+    )
+    data = with_embeddings(embed_func, df)
+
+    # The output is used to create / append to a table
+    tbl = db.create_table("my_table", data=data)
+```
+
+If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
+
+By default, LanceDB calls the function with batches of 1000 rows. This can be configured
+using the `batch_size` parameter to `with_embeddings`.
+
+LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
+API call is reliable.
+
+## Querying using an embedding function
+
+!!! warning
+    At query time, you **must** use the same embedding function you used to vectorize your data.
+    If you use a different embedding function, the embeddings will not reside in the same vector
+    space and the results will be nonsensical.
+
+=== "Python"
+     ```python
+     query = "What's the best pizza topping?"
+     query_vector = embed_func([query])[0]
+     results = (
+        tbl.search(query_vector)
+        .limit(10)
+        .to_pandas()
+     )
+     ```
+
+     The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.

docs/src/examples/modal_langchain.py

@@ -1,6 +1,5 @@
 import pickle
 import re
-import sys
 import zipfile
 from pathlib import Path
 

docs/src/guides/tables.md

@@ -636,6 +636,70 @@ The `values` parameter is used to provide the new values for the columns as lite
 
     When rows are updated, they are moved out of the index. The row will still show up in ANN queries, but the query will not be as fast as it would be if the row was in the index. If you update a large proportion of rows, consider rebuilding the index afterwards.
 
+## Consistency
+
+In LanceDB OSS, users can set the `read_consistency_interval` parameter on connections to achieve different levels of read consistency. This parameter determines how frequently the database synchronizes with the underlying storage system to check for updates made by other processes. If another process updates a table, the database will not see the changes until the next synchronization.
+
+There are three possible settings for `read_consistency_interval`:
+
+1. **Unset (default)**: The database does not check for updates to tables made by other processes. This provides the best query performance, but means that clients may not see the most up-to-date data. This setting is suitable for applications where the data does not change during the lifetime of the table reference.
+2. **Zero seconds (Strong consistency)**: The database checks for updates on every read. This provides the strongest consistency guarantees, ensuring that all clients see the latest committed data. However, it has the most overhead. This setting is suitable when consistency matters more than having high QPS.
+3. **Custom interval (Eventual consistency)**: The database checks for updates at a custom interval, such as every 5 seconds. This provides eventual consistency, allowing for some lag between write and read operations. Performance wise, this is a middle ground between strong consistency and no consistency check. This setting is suitable for applications where immediate consistency is not critical, but clients should see updated data eventually.
+
+!!! tip "Consistency in LanceDB Cloud"
+
+    This is only tune-able in LanceDB OSS. In LanceDB Cloud, readers are always eventually consistent.
+
+=== "Python"
+    
+    To set strong consistency, use `timedelta(0)`:
+
+    ```python
+    from datetime import timedelta
+    db = lancedb.connect("./.lancedb",. read_consistency_interval=timedelta(0))
+    table = db.open_table("my_table")
+    ```
+
+    For eventual consistency, use a custom `timedelta`:
+
+    ```python
+    from datetime import timedelta
+    db = lancedb.connect("./.lancedb", read_consistency_interval=timedelta(seconds=5))
+    table = db.open_table("my_table")
+    ```
+
+    By default, a `Table` will never check for updates from other writers. To manually check for updates you can use `checkout_latest`:
+
+    ```python
+    db = lancedb.connect("./.lancedb")
+    table = db.open_table("my_table")
+    
+    # (Other writes happen to my_table from another process)
+
+    # Check for updates
+    table.checkout_latest()
+    ```
+
+=== "JavaScript/Typescript"
+
+    To set strong consistency, use `0`:
+
+    ```javascript
+    const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 0 });
+    const table = await db.openTable("my_table");
+    ```
+
+    For eventual consistency, specify the update interval as seconds:
+
+    ```javascript
+    const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 5 });
+    const table = await db.openTable("my_table");
+    ```
+
+<!-- Node doesn't yet support the version time travel: https://github.com/lancedb/lancedb/issues/1007 
+    Once it does, we can show manual consistency check for Node as well.
+-->
+
 ## What's next?
 
 Learn the best practices on creating an ANN index and getting the most out of it.

docs/src/hybrid_search/eval.md

@@ -0,0 +1,49 @@
+# Hybrid Search
+
+Hybrid Search is a broad (often misused) term. It can mean anything from combining multiple methods for searching, to applying ranking methods to better sort the results. In this blog, we use the definition of "hybrid search" to mean using a combination of keyword-based and vector search.
+
+## The challenge of (re)ranking search results
+Once you have a group of the most relevant search results from multiple search sources, you'd likely standardize the score and rank them accordingly. This process can also be seen as another independent step - reranking.
+There are two approaches for reranking search results from multiple sources.
+* <b>Score-based</b>: Calculate final relevance scores based on a weighted linear combination of individual search algorithm scores. Example - Weighted linear combination of semantic search & keyword-based search results.
+* <b>Relevance-based</b>: Discards the existing scores and calculates the relevance of each search result - query pair. Example - Cross Encoder models
+
+Even though there are many strategies for reranking search results, none works for all cases. Moreover, evaluating them itself is a challenge. Also, reranking can be dataset, application specific so it's hard to generalize.
+
+### Example evaluation of hybrid search with Reranking
+
+Here's some evaluation numbers from experiment comparing these re-rankers on about 800 queries. It is modified version of an evaluation script from [llama-index](https://github.com/run-llama/finetune-embedding/blob/main/evaluate.ipynb) that measures hit-rate at top-k.
+
+<b> With OpenAI ada2 embedding </b>
+
+Vector Search baseline - `0.64`
+
+| Reranker | Top-3 | Top-5 | Top-10 |
+| --- | --- | --- | --- |
+| Linear Combination | `0.73` | `0.74` | `0.85` |
+| Cross Encoder | `0.71` | `0.70` | `0.77` |
+| Cohere | `0.81` | `0.81` | `0.85` |
+| ColBERT | `0.68` | `0.68` | `0.73` |
+
+<p>
+<img src="https://github.com/AyushExel/assets/assets/15766192/d57b1780-ef27-414c-a5c3-73bee7808a45">
+</p>
+
+<b> With OpenAI embedding-v3-small </b>
+
+Vector Search baseline - `0.59`
+
+| Reranker | Top-3 | Top-5 | Top-10 |
+| --- | --- | --- | --- |
+| Linear Combination | `0.68` | `0.70` | `0.84` |
+| Cross Encoder | `0.72` | `0.72` | `0.79` |
+| Cohere | `0.79` | `0.79` | `0.84` |
+| ColBERT | `0.70` | `0.70` | `0.76` |
+
+<p>
+<img src="https://github.com/AyushExel/assets/assets/15766192/259adfd2-6ec6-4df6-a77d-1456598970dd">
+</p>
+
+### Conclusion
+
+The results show that the reranking methods are able to improve the search results. However, the improvement is not consistent across all rerankers. The choice of reranker depends on the dataset and the application. It is also important to note that the reranking methods are not a replacement for the search methods. They are complementary and should be used together to get the best results. The speed to recall tradeoff is also an important factor to consider when choosing the reranker.

docs/src/notebooks/DisappearingEmbeddingFunction.ipynb

@@ -290,7 +290,7 @@
         "from lancedb.pydantic import LanceModel, Vector\n",
         "\n",
         "class Pets(LanceModel):\n",
-        "    vector: Vector(clip.ndims) = clip.VectorField()\n",
+        "    vector: Vector(clip.ndims()) = clip.VectorField()\n",
         "    image_uri: str = clip.SourceField()\n",
         "\n",
         "    @property\n",
@@ -360,7 +360,7 @@
         "    table = db.create_table(\"pets\", schema=Pets)\n",
         "    # use a sampling of 1000 images\n",
         "    p = Path(\"~/Downloads/images\").expanduser()\n",
-        "    uris = [str(f) for f in p.iterdir()]\n",
+        "    uris = [str(f) for f in p.glob(\"*.jpg\")]\n",
         "    uris = sample(uris, 1000)\n",
         "    table.add(pd.DataFrame({\"image_uri\": uris}))"
       ]
@@ -543,7 +543,7 @@
       ],
       "source": [
         "from PIL import Image\n",
-        "p = Path(\"/Users/changshe/Downloads/images/samoyed_100.jpg\")\n",
+        "p = Path(\"~/Downloads/images/samoyed_100.jpg\").expanduser()\n",
         "query_image = Image.open(p)\n",
         "query_image"
       ]

docs/src/notebooks/diffusiondb/datagen.py

@@ -23,10 +23,8 @@ from multiprocessing import Pool
 import lance
 import pyarrow as pa
 from datasets import load_dataset
-from PIL import Image
 from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast
 
-import lancedb
 
 MODEL_ID = "openai/clip-vit-base-patch32"
 

docs/src/python/duckdb.md

@@ -1,6 +1,9 @@
 # DuckDB
 
-LanceDB is very well-integrated with [DuckDB](https://duckdb.org/), an in-process SQL OLAP database. This integration is done via [Arrow](https://duckdb.org/docs/guides/python/sql_on_arrow) .
+In Python, LanceDB tables can also be queried with [DuckDB](https://duckdb.org/), an in-process SQL OLAP database. This means you can write complex SQL queries to analyze your data in LanceDB.
+
+This integration is done via [Apache Arrow](https://duckdb.org/docs/guides/python/sql_on_arrow), which provides zero-copy data sharing between LanceDB and DuckDB. DuckDB is capable of passing down column selections and basic filters to LanceDB, reducing the amount of data that needs to be scanned to perform your query. Finally, the integration allows streaming data from LanceDB tables, allowing you to aggregate tables that won't fit into memory. All of this uses the same mechanism described in DuckDB's blog post *[DuckDB quacks Arrow](https://duckdb.org/2021/12/03/duck-arrow.html)*.
+
 
 We can demonstrate this by first installing `duckdb` and `lancedb`.
 
@@ -19,14 +22,15 @@ data = [
     {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}
 ]
 table = db.create_table("pd_table", data=data)
-arrow_table = table.to_arrow()
 ```
 
-DuckDB can directly query the `pyarrow.Table` object:
+To query the table, first call `to_lance` to convert the table to a "dataset", which is an object that can be queried by DuckDB. Then all you need to do is reference that dataset by the same name in your SQL query.
 
 ```python
 import duckdb
 
+arrow_table = table.to_lance()
+
 duckdb.query("SELECT * FROM arrow_table")
 ```
 

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.4.10",
+  "version": "0.4.11",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.4.10",
+      "version": "0.4.11",
       "cpu": [
         "x64",
         "arm64"
@@ -53,11 +53,11 @@
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.4.10",
-        "@lancedb/vectordb-darwin-x64": "0.4.10",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.4.10",
-        "@lancedb/vectordb-linux-x64-gnu": "0.4.10",
-        "@lancedb/vectordb-win32-x64-msvc": "0.4.10"
+        "@lancedb/vectordb-darwin-arm64": "0.4.11",
+        "@lancedb/vectordb-darwin-x64": "0.4.11",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.4.11",
+        "@lancedb/vectordb-linux-x64-gnu": "0.4.11",
+        "@lancedb/vectordb-win32-x64-msvc": "0.4.11"
       }
     },
     "node_modules/@75lb/deep-merge": {
@@ -329,9 +329,9 @@
       }
     },
     "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.4.10",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.10.tgz",
-      "integrity": "sha512-y/uHOGb0g15pvqv5tdTyZ6oN+0QVpBmZDzKFWW6pPbuSZjB2uPqcs+ti0RB+AUdmS21kavVQqaNsw/HLKEGrHA==",
+      "version": "0.4.11",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.11.tgz",
+      "integrity": "sha512-JDOKmFnuJPFkA7ZmrzBJolROwSjWr7yMvAbi40uLBc25YbbVezodd30u2EFtIwWwtk1GqNYRZ49FZOElKYeC/Q==",
       "cpu": [
         "arm64"
       ],
@@ -341,9 +341,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.4.10",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.4.10.tgz",
-      "integrity": "sha512-XbfR58OkQpAe0xMSTrwJh9ZjGSzG9EZ7zwO6HfYem8PxcLYAcC6eWRWoSG/T0uObyrPTcYYyvHsp0eNQWYBFAQ==",
+      "version": "0.4.11",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.4.11.tgz",
+      "integrity": "sha512-iy6r+8tp2v1EFgJV52jusXtxgO6NY6SkpOdX41xPqN2mQWMkfUAR9Xtks1mgknjPOIKH4MRc8ZS0jcW/UWmilQ==",
       "cpu": [
         "x64"
       ],
@@ -353,9 +353,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.4.10",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.4.10.tgz",
-      "integrity": "sha512-x40WKH9b+KxorRmKr9G7fv8p5mMj8QJQvRMA0v6v+nbZHr2FLlAZV+9mvhHOnm4AGIkPP5335cUgv6Qz6hgwkQ==",
+      "version": "0.4.11",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.4.11.tgz",
+      "integrity": "sha512-5K6IVcTMuH0SZBjlqB5Gg39WC889FpTwIWKufxzQMMXrzxo5J3lKUHVoR28RRlNhDF2d9kZXBEyCpIfDFsV9iQ==",
       "cpu": [
         "arm64"
       ],
@@ -365,9 +365,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.4.10",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.4.10.tgz",
-      "integrity": "sha512-CTGPpuzlqq2nVjUxI9gAJOT1oBANIovtIaFsOmBSnEAHgX7oeAxKy2b6L/kJzsgqSzvR5vfLwYcWFrr6ZmBxSA==",
+      "version": "0.4.11",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.4.11.tgz",
+      "integrity": "sha512-hF9ZChsdqKqqnivOzd9mE7lC3PmhZadXtwThi2RrsPiOLoEaGDfmr6Ni3amVQnB3bR8YEJtTxdQxe0NC4uW/8g==",
       "cpu": [
         "x64"
       ],
@@ -377,9 +377,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.4.10",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.4.10.tgz",
-      "integrity": "sha512-Fd7r74coZyrKzkfXg4WthqOL+uKyJyPTia6imcrMNqKOlTGdKmHf02Qi2QxWZrFaabkRYo4Tpn5FeRJ3yYX8CA==",
+      "version": "0.4.11",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.4.11.tgz",
+      "integrity": "sha512-0+9ut1ccKoqIyGxsVixwx3771Z+DXpl5WfSmOeA8kf3v3jlOg2H+0YUahiXLDid2ju+yeLPrAUYm7A1gKHVhew==",
       "cpu": [
         "x64"
       ],

node/package.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.4.10",
+  "version": "0.4.11",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
     "tsc": "tsc -b",
-    "build": "npm run tsc && cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json",
+    "build": "npm run tsc && cargo-cp-artifact --artifact cdylib lancedb-node index.node -- cargo build --message-format=json",
     "build-release": "npm run build -- --release",
     "test": "npm run tsc && mocha -recursive dist/test",
     "integration-test": "npm run tsc && mocha -recursive dist/integration_test",
@@ -61,11 +61,13 @@
     "uuid": "^9.0.0"
   },
   "dependencies": {
-    "@apache-arrow/ts": "^14.0.2",
     "@neon-rs/load": "^0.0.74",
-    "apache-arrow": "^14.0.2",
     "axios": "^1.4.0"
   },
+  "peerDependencies": {
+    "@apache-arrow/ts": "^14.0.2",
+    "apache-arrow": "^14.0.2"
+  },
   "os": [
     "darwin",
     "linux",
@@ -85,10 +87,10 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.4.10",
-    "@lancedb/vectordb-darwin-x64": "0.4.10",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.4.10",
-    "@lancedb/vectordb-linux-x64-gnu": "0.4.10",
-    "@lancedb/vectordb-win32-x64-msvc": "0.4.10"
+    "@lancedb/vectordb-darwin-arm64": "0.4.11",
+    "@lancedb/vectordb-darwin-x64": "0.4.11",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.4.11",
+    "@lancedb/vectordb-linux-x64-gnu": "0.4.11",
+    "@lancedb/vectordb-win32-x64-msvc": "0.4.11"
   }
-}
+}

node/src/index.ts

@@ -42,7 +42,10 @@ const {
   tableCompactFiles,
   tableListIndices,
   tableIndexStats,
-  tableSchema
+  tableSchema,
+  tableAddColumns,
+  tableAlterColumns,
+  tableDropColumns
   // eslint-disable-next-line @typescript-eslint/no-var-requires
 } = require('../native.js')
 
@@ -96,6 +99,19 @@ export interface ConnectionOptions {
    * This is useful for local testing.
    */
   hostOverride?: string
+
+  /**
+   * (For LanceDB OSS only): The interval, in seconds, at which to check for
+   * updates to the table from other processes. If None, then consistency is not
+   * checked. For performance reasons, this is the default. For strong
+   * consistency, set this to zero seconds. Then every read will check for
+   * updates from other processes. As a compromise, you can set this to a
+   * non-zero value for eventual consistency. If more than that interval
+   * has passed since the last check, then the table will be checked for updates.
+   * Note: this consistency only applies to read operations. Write operations are
+   * always consistent.
+   */
+  readConsistencyInterval?: number
 }
 
 function getAwsArgs (opts: ConnectionOptions): any[] {
@@ -181,7 +197,8 @@ export async function connect (
     opts.awsCredentials?.accessKeyId,
     opts.awsCredentials?.secretKey,
     opts.awsCredentials?.sessionToken,
-    opts.awsRegion
+    opts.awsRegion,
+    opts.readConsistencyInterval
   )
   return new LocalConnection(db, opts)
 }
@@ -324,6 +341,7 @@ export interface Table<T = number[]> {
    *
    * @param column The column to index
    * @param replace If false, fail if an index already exists on the column
+   * it is always set to true for remote connections
    *
    * Scalar indices, like vector indices, can be used to speed up scans.  A scalar
    * index can speed up scans that contain filter expressions on the indexed column.
@@ -367,7 +385,7 @@ export interface Table<T = number[]> {
    * await table.createScalarIndex('my_col')
    * ```
    */
-  createScalarIndex: (column: string, replace: boolean) => Promise<void>
+  createScalarIndex: (column: string, replace?: boolean) => Promise<void>
 
   /**
    * Returns the number of rows in this table.
@@ -486,6 +504,59 @@ export interface Table<T = number[]> {
   filter(value: string): Query<T>
 
   schema: Promise<Schema>
+
+  // TODO: Support BatchUDF
+  /**
+   * Add new columns with defined values.
+   *
+   * @param newColumnTransforms pairs of column names and the SQL expression to use
+   *                            to calculate the value of the new column. These
+   *                            expressions will be evaluated for each row in the
+   *                            table, and can reference existing columns in the table.
+   */
+  addColumns(newColumnTransforms: Array<{ name: string, valueSql: string }>): Promise<void>
+
+  /**
+   * Alter the name or nullability of columns.
+   *
+   * @param columnAlterations One or more alterations to apply to columns.
+   */
+  alterColumns(columnAlterations: ColumnAlteration[]): Promise<void>
+
+  /**
+   * Drop one or more columns from the dataset
+   *
+   * This is a metadata-only operation and does not remove the data from the
+   * underlying storage. In order to remove the data, you must subsequently
+   * call ``compact_files`` to rewrite the data without the removed columns and
+   * then call ``cleanup_files`` to remove the old files.
+   *
+   * @param columnNames The names of the columns to drop. These can be nested
+   *                    column references (e.g. "a.b.c") or top-level column
+   *                    names (e.g. "a").
+   */
+  dropColumns(columnNames: string[]): Promise<void>
+}
+
+/**
+ * A definition of a column alteration. The alteration changes the column at
+ * `path` to have the new name `name`, to be nullable if `nullable` is true,
+ * and to have the data type `data_type`. At least one of `rename` or `nullable`
+ * must be provided.
+ */
+export interface ColumnAlteration {
+  /**
+   * The path to the column to alter. This is a dot-separated path to the column.
+   * If it is a top-level column then it is just the name of the column. If it is
+   * a nested column then it is the path to the column, e.g. "a.b.c" for a column
+   * `c` nested inside a column `b` nested inside a column `a`.
+   */
+  path: string
+  rename?: string
+  /**
+   * Set the new nullability. Note that a nullable column cannot be made non-nullable.
+   */
+  nullable?: boolean
 }
 
 export interface UpdateArgs {
@@ -844,7 +915,10 @@ export class LocalTable<T = number[]> implements Table<T> {
       })
   }
 
-  async createScalarIndex (column: string, replace: boolean): Promise<void> {
+  async createScalarIndex (column: string, replace?: boolean): Promise<void> {
+    if (replace === undefined) {
+      replace = true
+    }
     return tableCreateScalarIndex.call(this._tbl, column, replace)
   }
 
@@ -1014,6 +1088,18 @@ export class LocalTable<T = number[]> implements Table<T> {
       return false
     }
   }
+
+  async addColumns (newColumnTransforms: Array<{ name: string, valueSql: string }>): Promise<void> {
+    return tableAddColumns.call(this._tbl, newColumnTransforms)
+  }
+
+  async alterColumns (columnAlterations: ColumnAlteration[]): Promise<void> {
+    return tableAlterColumns.call(this._tbl, columnAlterations)
+  }
+
+  async dropColumns (columnNames: string[]): Promise<void> {
+    return tableDropColumns.call(this._tbl, columnNames)
+  }
 }
 
 export interface CleanupStats {

node/src/remote/index.ts

@@ -25,7 +25,8 @@ import {
   type UpdateArgs,
   type UpdateSqlArgs,
   makeArrowTable,
-  type MergeInsertArgs
+  type MergeInsertArgs,
+  type ColumnAlteration
 } from '../index'
 import { Query } from '../query'
 
@@ -396,7 +397,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
     }
 
     const column = indexParams.column ?? 'vector'
-    const indexType = 'vector' // only vector index is supported for remote connections
+    const indexType = 'vector'
     const metricType = indexParams.metric_type ?? 'L2'
     const indexCacheSize = indexParams.index_cache_size ?? null
 
@@ -419,8 +420,25 @@ export class RemoteTable<T = number[]> implements Table<T> {
     }
   }
 
-  async createScalarIndex (column: string, replace: boolean): Promise<void> {
-    throw new Error('Not implemented')
+  async createScalarIndex (column: string): Promise<void> {
+    const indexType = 'scalar'
+
+    const data = {
+      column,
+      index_type: indexType,
+      replace: true
+    }
+    const res = await this._client.post(
+      `/v1/table/${this._name}/create_scalar_index/`,
+      data
+    )
+    if (res.status !== 200) {
+      throw new Error(
+        `Server Error, status: ${res.status}, ` +
+          // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+          `message: ${res.statusText}: ${res.data}`
+      )
+    }
   }
 
   async countRows (): Promise<number> {
@@ -474,4 +492,16 @@ export class RemoteTable<T = number[]> implements Table<T> {
       numUnindexedRows: results.data.num_unindexed_rows
     }
   }
+
+  async addColumns (newColumnTransforms: Array<{ name: string, valueSql: string }>): Promise<void> {
+    throw new Error('Add columns is not yet supported in LanceDB Cloud.')
+  }
+
+  async alterColumns (columnAlterations: ColumnAlteration[]): Promise<void> {
+    throw new Error('Alter columns is not yet supported in LanceDB Cloud.')
+  }
+
+  async dropColumns (columnNames: string[]): Promise<void> {
+    throw new Error('Drop columns is not yet supported in LanceDB Cloud.')
+  }
 }

node/src/test/test.ts

@@ -37,8 +37,10 @@ import {
   Utf8,
   Table as ArrowTable,
   vectorFromArray,
+  Float64,
   Float32,
-  Float16
+  Float16,
+  Int64
 } from 'apache-arrow'
 
 const expect = chai.expect
@@ -196,7 +198,7 @@ describe('LanceDB client', function () {
       const table = await con.openTable('vectors')
       const results = await table
         .search([0.1, 0.1])
-        .select(['is_active'])
+        .select(['is_active', 'vector'])
         .execute()
       assert.equal(results.length, 2)
       // vector and _distance are always returned
@@ -1057,3 +1059,63 @@ describe('Compact and cleanup', function () {
     assert.equal(await table.countRows(), 3)
   })
 })
+
+describe('schema evolution', function () {
+  // Create a new sample table
+  it('can add a new column to the schema', async function () {
+    const dir = await track().mkdir('lancejs')
+    const con = await lancedb.connect(dir)
+    const table = await con.createTable('vectors', [
+      { id: 1n, vector: [0.1, 0.2] }
+    ])
+
+    await table.addColumns([{ name: 'price', valueSql: 'cast(10.0 as float)' }])
+
+    const expectedSchema = new Schema([
+      new Field('id', new Int64()),
+      new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true))),
+      new Field('price', new Float32())
+    ])
+    expect(await table.schema).to.deep.equal(expectedSchema)
+  })
+
+  it('can alter the columns in the schema', async function () {
+    const dir = await track().mkdir('lancejs')
+    const con = await lancedb.connect(dir)
+    const schema = new Schema([
+      new Field('id', new Int64(), false),
+      new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true))),
+      new Field('price', new Float64(), false)
+    ])
+    const table = await con.createTable('vectors', [
+      { id: 1n, vector: [0.1, 0.2], price: 10.0 }
+    ])
+    expect(await table.schema).to.deep.equal(schema)
+
+    await table.alterColumns([
+      { path: 'id', rename: 'new_id' },
+      { path: 'price', nullable: true }
+    ])
+
+    const expectedSchema = new Schema([
+      new Field('new_id', new Int64(), false),
+      new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true))),
+      new Field('price', new Float64(), true)
+    ])
+    expect(await table.schema).to.deep.equal(expectedSchema)
+  })
+
+  it('can drop a column from the schema', async function () {
+    const dir = await track().mkdir('lancejs')
+    const con = await lancedb.connect(dir)
+    const table = await con.createTable('vectors', [
+      { id: 1n, vector: [0.1, 0.2] }
+    ])
+    await table.dropColumns(['vector'])
+
+    const expectedSchema = new Schema([
+      new Field('id', new Int64(), false)
+    ])
+    expect(await table.schema).to.deep.equal(expectedSchema)
+  })
+})

nodejs/.eslintrc.js

@@ -18,5 +18,5 @@ module.exports = {
     "@typescript-eslint/method-signature-style": "off",
     "@typescript-eslint/no-explicit-any": "off",
   },
-  ignorePatterns: ["node_modules/", "dist/", "build/", "vectordb/native.*"],
+  ignorePatterns: ["node_modules/", "dist/", "build/", "lancedb/native.*"],
 };

nodejs/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "vectordb-nodejs"
+name = "lancedb-nodejs"
 edition.workspace = true
 version = "0.0.0"
 license.workspace = true
@@ -16,7 +16,7 @@ arrow-ipc.workspace = true
 futures.workspace = true
 lance-linalg.workspace = true
 lance.workspace = true
-vectordb = { path = "../rust/vectordb" }
+lancedb = { path = "../rust/lancedb" }
 napi = { version = "2.15", default-features = false, features = [
     "napi7",
     "async"

nodejs/__test__/arrow.test.ts

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { makeArrowTable, toBuffer } from "../vectordb/arrow";
+import { makeArrowTable, toBuffer } from "../lancedb/arrow";
 import {
   Int64,
   Field,

nodejs/__test__/connection.test.ts

@@ -0,0 +1,34 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import * as os from "os";
+import * as path from "path";
+import * as fs from "fs";
+
+import { connect } from "../dist/index.js";
+
+describe("when working with a connection", () => {
+
+  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "test-connection"));
+
+  it("should fail if creating table twice, unless overwrite is true", async() => {
+    const db = await connect(tmpDir);
+    let tbl = await db.createTable("test", [{ id: 1 }, { id: 2 }]);
+    await expect(tbl.countRows()).resolves.toBe(2);
+    await expect(db.createTable("test", [{ id: 1 }, { id: 2 }])).rejects.toThrow();
+    tbl = await db.createTable("test", [{ id: 3 }], { mode: "overwrite" });
+    await expect(tbl.countRows()).resolves.toBe(1);
+  })
+
+});

nodejs/__test__/index.test.ts

@@ -29,6 +29,6 @@ test("open database", async () => {
   const tbl = await db.createTable("test", [{ id: 1 }, { id: 2 }]);
   expect(await db.tableNames()).toStrictEqual(["test"]);
 
-  const schema = tbl.schema;
+  const schema = await tbl.schema();
   expect(schema).toEqual(new Schema([new Field("id", new Float64(), true)]));
 });

nodejs/__test__/table.test.ts

@@ -17,7 +17,7 @@ import * as path from "path";
 import * as fs from "fs";
 
 import { connect } from "../dist";
-import { Schema, Field, Float32, Int32, FixedSizeList } from "apache-arrow";
+import { Schema, Field, Float32, Int32, FixedSizeList, Int64, Float64 } from "apache-arrow";
 import { makeArrowTable } from "../dist/arrow";
 
 describe("Test creating index", () => {
@@ -181,3 +181,102 @@ describe("Test creating index", () => {
     // TODO: check index type.
   });
 });
+
+describe("Read consistency interval", () => {
+  let tmpDir: string;
+  beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "read-consistency-"));
+  });
+
+  // const intervals = [undefined, 0, 0.1];
+  const intervals = [0];
+  test.each(intervals)("read consistency interval %p", async (interval) => {
+    const db = await connect({ uri: tmpDir });
+    const table = await db.createTable("my_table", [{ id: 1 }]);
+
+    const db2 = await connect({ uri: tmpDir, readConsistencyInterval: interval });
+    const table2 = await db2.openTable("my_table");
+    expect(await table2.countRows()).toEqual(await table.countRows());
+
+    await table.add([{ id: 2 }]);
+
+    if (interval === undefined) {
+      expect(await table2.countRows()).toEqual(1);
+      // TODO: once we implement time travel we can uncomment this part of the test.
+      // await table2.checkout_latest();
+      // expect(await table2.countRows()).toEqual(2);
+    } else if (interval === 0) {
+      expect(await table2.countRows()).toEqual(2);
+    } else {
+      // interval == 0.1
+      expect(await table2.countRows()).toEqual(1);
+      await new Promise(r => setTimeout(r, 100));
+      expect(await table2.countRows()).toEqual(2);
+    }
+  });
+});
+
+
+describe('schema evolution', function () {
+  let tmpDir: string;
+  beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "schema-evolution-"));
+  });
+
+  // Create a new sample table
+  it('can add a new column to the schema', async function () {
+    const con = await connect(tmpDir)
+    const table = await con.createTable('vectors', [
+      { id: 1n, vector: [0.1, 0.2] }
+    ])
+
+    await table.addColumns([{ name: 'price', valueSql: 'cast(10.0 as float)' }])
+
+    const expectedSchema = new Schema([
+      new Field('id', new Int64(), true),
+      new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true)), true),
+      new Field('price', new Float32(), false)
+    ])
+    expect(await table.schema()).toEqual(expectedSchema)
+  });
+
+  it('can alter the columns in the schema', async function () {
+    const con = await connect(tmpDir)
+    const schema = new Schema([
+      new Field('id', new Int64(), true),
+      new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true)), true),
+      new Field('price', new Float64(), false)
+    ])
+    const table = await con.createTable('vectors', [
+      { id: 1n, vector: [0.1, 0.2] }
+    ])
+    // Can create a non-nullable column only through addColumns at the moment.
+    await table.addColumns([{ name: 'price', valueSql: 'cast(10.0 as double)' }])
+    expect(await table.schema()).toEqual(schema)
+
+    await table.alterColumns([
+      { path: 'id', rename: 'new_id' },
+      { path: 'price', nullable: true }
+    ])
+
+    const expectedSchema = new Schema([
+      new Field('new_id', new Int64(), true),
+      new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true)), true),
+      new Field('price', new Float64(), true)
+    ])
+    expect(await table.schema()).toEqual(expectedSchema)
+  });
+
+  it('can drop a column from the schema', async function () {
+    const con = await connect(tmpDir)
+    const table = await con.createTable('vectors', [
+      { id: 1n, vector: [0.1, 0.2] }
+    ])
+    await table.dropColumns(['vector'])
+
+    const expectedSchema = new Schema([
+      new Field('id', new Int64(), true)
+    ])
+    expect(await table.schema()).toEqual(expectedSchema)
+  });
+});

nodejs/__test__/tsconfig.json

@@ -0,0 +1,15 @@
+{
+    "extends": "../tsconfig.json",
+    "compilerOptions": {
+        "outDir": "./dist/spec",
+        "module": "commonjs",
+        "target": "es2022",
+        "types": [
+            "jest",
+            "node"
+        ]
+    },
+    "include": [
+        "**/*",
+    ]
+}

nodejs/lancedb/connection.ts

@@ -17,6 +17,24 @@ import { Connection as _NativeConnection } from "./native";
 import { Table } from "./table";
 import { Table as ArrowTable } from "apache-arrow";
 
+export interface CreateTableOptions {
+  /**
+   * The mode to use when creating the table.
+   *
+   * If this is set to "create" and the table already exists then either
+   * an error will be thrown or, if existOk is true, then nothing will
+   * happen.  Any provided data will be ignored.
+   *
+   * If this is set to "overwrite" then any existing table will be replaced.
+   */
+  mode: "create" | "overwrite";
+  /**
+   * If this is true and the table already exists and the mode is "create"
+   * then no error will be raised.
+   */
+  existOk: boolean;
+}
+
 /**
  * A LanceDB Connection that allows you to open tables and create new ones.
  *
@@ -53,10 +71,18 @@ export class Connection {
    */
   async createTable(
     name: string,
-    data: Record<string, unknown>[] | ArrowTable
+    data: Record<string, unknown>[] | ArrowTable,
+    options?: Partial<CreateTableOptions>
   ): Promise<Table> {
+    let mode: string = options?.mode ?? "create";
+    const existOk = options?.existOk ?? false;
+
+    if (mode === "create" && existOk) {
+      mode = "exist_ok";
+    }
+
     const buf = toBuffer(data);
-    const innerTable = await this.inner.createTable(name, buf);
+    const innerTable = await this.inner.createTable(name, buf, mode);
     return new Table(innerTable);
   }
 

nodejs/lancedb/index.ts

@@ -53,12 +53,12 @@ export async function connect(
     opts = Object.assign(
       {
         uri: "",
-        apiKey: "",
-        hostOverride: "",
+        apiKey: undefined,
+        hostOverride: undefined,
       },
       args
     );
   }
-  const nativeConn = await NativeConnection.new(opts.uri);
+  const nativeConn = await NativeConnection.new(opts);
   return new Connection(nativeConn);
 }

nodejs/lancedb/native.d.ts

@@ -0,0 +1,127 @@
+/* tslint:disable */
+/* eslint-disable */
+
+/* auto-generated by NAPI-RS */
+
+export const enum IndexType {
+  Scalar = 0,
+  IvfPq = 1
+}
+export const enum MetricType {
+  L2 = 0,
+  Cosine = 1,
+  Dot = 2
+}
+/**
+ *  A definition of a column alteration. The alteration changes the column at
+ * `path` to have the new name `name`, to be nullable if `nullable` is true,
+ * and to have the data type `data_type`. At least one of `rename` or `nullable`
+ * must be provided.
+ */
+export interface ColumnAlteration {
+  /**
+   * The path to the column to alter. This is a dot-separated path to the column.
+   * If it is a top-level column then it is just the name of the column. If it is
+   * a nested column then it is the path to the column, e.g. "a.b.c" for a column
+   * `c` nested inside a column `b` nested inside a column `a`.
+   */
+  path: string
+  /**
+   * The new name of the column. If not provided then the name will not be changed.
+   * This must be distinct from the names of all other columns in the table.
+   */
+  rename?: string
+  /** Set the new nullability. Note that a nullable column cannot be made non-nullable. */
+  nullable?: boolean
+}
+/** A definition of a new column to add to a table. */
+export interface AddColumnsSql {
+  /** The name of the new column. */
+  name: string
+  /**
+   * The values to populate the new column with, as a SQL expression.
+   * The expression can reference other columns in the table.
+   */
+  valueSql: string
+}
+export interface ConnectionOptions {
+  uri: string
+  apiKey?: string
+  hostOverride?: string
+  /**
+   * (For LanceDB OSS only): The interval, in seconds, at which to check for
+   * updates to the table from other processes. If None, then consistency is not
+   * checked. For performance reasons, this is the default. For strong
+   * consistency, set this to zero seconds. Then every read will check for
+   * updates from other processes. As a compromise, you can set this to a
+   * non-zero value for eventual consistency. If more than that interval
+   * has passed since the last check, then the table will be checked for updates.
+   * Note: this consistency only applies to read operations. Write operations are
+   * always consistent.
+   */
+  readConsistencyInterval?: number
+}
+/** Write mode for writing a table. */
+export const enum WriteMode {
+  Create = 'Create',
+  Append = 'Append',
+  Overwrite = 'Overwrite'
+}
+/** Write options when creating a Table. */
+export interface WriteOptions {
+  mode?: WriteMode
+}
+export function connect(options: ConnectionOptions): Promise<Connection>
+export class Connection {
+  /** Create a new Connection instance from the given URI. */
+  static new(options: ConnectionOptions): Promise<Connection>
+  /** List all tables in the dataset. */
+  tableNames(): Promise<Array<string>>
+  /**
+   * Create table from a Apache Arrow IPC (file) buffer.
+   *
+   * Parameters:
+   * - name: The name of the table.
+   * - buf: The buffer containing the IPC file.
+   *
+   */
+  createTable(name: string, buf: Buffer, mode: string): Promise<Table>
+  openTable(name: string): Promise<Table>
+  /** Drop table with the name. Or raise an error if the table does not exist. */
+  dropTable(name: string): Promise<void>
+}
+export class IndexBuilder {
+  replace(v: boolean): void
+  column(c: string): void
+  name(name: string): void
+  ivfPq(metricType?: MetricType | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, numBits?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): void
+  scalar(): void
+  build(): Promise<void>
+}
+/** Typescript-style Async Iterator over RecordBatches  */
+export class RecordBatchIterator {
+  next(): Promise<Buffer | null>
+}
+export class Query {
+  column(column: string): void
+  filter(filter: string): void
+  select(columns: Array<string>): void
+  limit(limit: number): void
+  prefilter(prefilter: boolean): void
+  nearestTo(vector: Float32Array): void
+  refineFactor(refineFactor: number): void
+  nprobes(nprobe: number): void
+  executeStream(): Promise<RecordBatchIterator>
+}
+export class Table {
+  /** Return Schema as empty Arrow IPC file. */
+  schema(): Promise<Buffer>
+  add(buf: Buffer): Promise<void>
+  countRows(filter?: string | undefined | null): Promise<number>
+  delete(predicate: string): Promise<void>
+  createIndex(): IndexBuilder
+  query(): Query
+  addColumns(transforms: Array<AddColumnsSql>): Promise<void>
+  alterColumns(alterations: Array<ColumnAlteration>): Promise<void>
+  dropColumns(columns: Array<string>): Promise<void>
+}

nodejs/lancedb/native.js

@@ -32,24 +32,24 @@ switch (platform) {
   case 'android':
     switch (arch) {
       case 'arm64':
-        localFileExisted = existsSync(join(__dirname, 'vectordb-nodejs.android-arm64.node'))
+        localFileExisted = existsSync(join(__dirname, 'lancedb-nodejs.android-arm64.node'))
         try {
           if (localFileExisted) {
-            nativeBinding = require('./vectordb-nodejs.android-arm64.node')
+            nativeBinding = require('./lancedb-nodejs.android-arm64.node')
           } else {
-            nativeBinding = require('vectordb-android-arm64')
+            nativeBinding = require('lancedb-android-arm64')
           }
         } catch (e) {
           loadError = e
         }
         break
       case 'arm':
-        localFileExisted = existsSync(join(__dirname, 'vectordb-nodejs.android-arm-eabi.node'))
+        localFileExisted = existsSync(join(__dirname, 'lancedb-nodejs.android-arm-eabi.node'))
         try {
           if (localFileExisted) {
-            nativeBinding = require('./vectordb-nodejs.android-arm-eabi.node')
+            nativeBinding = require('./lancedb-nodejs.android-arm-eabi.node')
           } else {
-            nativeBinding = require('vectordb-android-arm-eabi')
+            nativeBinding = require('lancedb-android-arm-eabi')
           }
         } catch (e) {
           loadError = e
@@ -63,13 +63,13 @@ switch (platform) {
     switch (arch) {
       case 'x64':
         localFileExisted = existsSync(
-          join(__dirname, 'vectordb-nodejs.win32-x64-msvc.node')
+          join(__dirname, 'lancedb-nodejs.win32-x64-msvc.node')
         )
         try {
           if (localFileExisted) {
-            nativeBinding = require('./vectordb-nodejs.win32-x64-msvc.node')
+            nativeBinding = require('./lancedb-nodejs.win32-x64-msvc.node')
           } else {
-            nativeBinding = require('vectordb-win32-x64-msvc')
+            nativeBinding = require('lancedb-win32-x64-msvc')
           }
         } catch (e) {
           loadError = e
@@ -77,13 +77,13 @@ switch (platform) {
         break
       case 'ia32':
         localFileExisted = existsSync(
-          join(__dirname, 'vectordb-nodejs.win32-ia32-msvc.node')
+          join(__dirname, 'lancedb-nodejs.win32-ia32-msvc.node')
         )
         try {
           if (localFileExisted) {
-            nativeBinding = require('./vectordb-nodejs.win32-ia32-msvc.node')
+            nativeBinding = require('./lancedb-nodejs.win32-ia32-msvc.node')
           } else {
-            nativeBinding = require('vectordb-win32-ia32-msvc')
+            nativeBinding = require('lancedb-win32-ia32-msvc')
           }
         } catch (e) {
           loadError = e
@@ -91,13 +91,13 @@ switch (platform) {
         break
       case 'arm64':
         localFileExisted = existsSync(
-          join(__dirname, 'vectordb-nodejs.win32-arm64-msvc.node')
+          join(__dirname, 'lancedb-nodejs.win32-arm64-msvc.node')
         )
         try {
           if (localFileExisted) {
-            nativeBinding = require('./vectordb-nodejs.win32-arm64-msvc.node')
+            nativeBinding = require('./lancedb-nodejs.win32-arm64-msvc.node')
           } else {
-            nativeBinding = require('vectordb-win32-arm64-msvc')
+            nativeBinding = require('lancedb-win32-arm64-msvc')
           }
         } catch (e) {
           loadError = e
@@ -108,23 +108,23 @@ switch (platform) {
     }
     break
   case 'darwin':
-    localFileExisted = existsSync(join(__dirname, 'vectordb-nodejs.darwin-universal.node'))
+    localFileExisted = existsSync(join(__dirname, 'lancedb-nodejs.darwin-universal.node'))
     try {
       if (localFileExisted) {
-        nativeBinding = require('./vectordb-nodejs.darwin-universal.node')
+        nativeBinding = require('./lancedb-nodejs.darwin-universal.node')
       } else {
-        nativeBinding = require('vectordb-darwin-universal')
+        nativeBinding = require('lancedb-darwin-universal')
       }
       break
     } catch {}
     switch (arch) {
       case 'x64':
-        localFileExisted = existsSync(join(__dirname, 'vectordb-nodejs.darwin-x64.node'))
+        localFileExisted = existsSync(join(__dirname, 'lancedb-nodejs.darwin-x64.node'))
         try {
           if (localFileExisted) {
-            nativeBinding = require('./vectordb-nodejs.darwin-x64.node')
+            nativeBinding = require('./lancedb-nodejs.darwin-x64.node')
           } else {
-            nativeBinding = require('vectordb-darwin-x64')
+            nativeBinding = require('lancedb-darwin-x64')
           }
         } catch (e) {
           loadError = e
@@ -132,13 +132,13 @@ switch (platform) {
         break
       case 'arm64':
         localFileExisted = existsSync(
-          join(__dirname, 'vectordb-nodejs.darwin-arm64.node')
+          join(__dirname, 'lancedb-nodejs.darwin-arm64.node')
         )
         try {
           if (localFileExisted) {
-            nativeBinding = require('./vectordb-nodejs.darwin-arm64.node')
+            nativeBinding = require('./lancedb-nodejs.darwin-arm64.node')
           } else {
-            nativeBinding = require('vectordb-darwin-arm64')
+            nativeBinding = require('lancedb-darwin-arm64')
           }
         } catch (e) {
           loadError = e
@@ -152,12 +152,12 @@ switch (platform) {
     if (arch !== 'x64') {
       throw new Error(`Unsupported architecture on FreeBSD: ${arch}`)
     }
-    localFileExisted = existsSync(join(__dirname, 'vectordb-nodejs.freebsd-x64.node'))
+    localFileExisted = existsSync(join(__dirname, 'lancedb-nodejs.freebsd-x64.node'))
     try {
       if (localFileExisted) {
-        nativeBinding = require('./vectordb-nodejs.freebsd-x64.node')
+        nativeBinding = require('./lancedb-nodejs.freebsd-x64.node')
       } else {
-        nativeBinding = require('vectordb-freebsd-x64')
+        nativeBinding = require('lancedb-freebsd-x64')
       }
     } catch (e) {
       loadError = e
@@ -168,26 +168,26 @@ switch (platform) {
       case 'x64':
         if (isMusl()) {
           localFileExisted = existsSync(
-            join(__dirname, 'vectordb-nodejs.linux-x64-musl.node')
+            join(__dirname, 'lancedb-nodejs.linux-x64-musl.node')
           )
           try {
             if (localFileExisted) {
-              nativeBinding = require('./vectordb-nodejs.linux-x64-musl.node')
+              nativeBinding = require('./lancedb-nodejs.linux-x64-musl.node')
             } else {
-              nativeBinding = require('vectordb-linux-x64-musl')
+              nativeBinding = require('lancedb-linux-x64-musl')
             }
           } catch (e) {
             loadError = e
           }
         } else {
           localFileExisted = existsSync(
-            join(__dirname, 'vectordb-nodejs.linux-x64-gnu.node')
+            join(__dirname, 'lancedb-nodejs.linux-x64-gnu.node')
           )
           try {
             if (localFileExisted) {
-              nativeBinding = require('./vectordb-nodejs.linux-x64-gnu.node')
+              nativeBinding = require('./lancedb-nodejs.linux-x64-gnu.node')
             } else {
-              nativeBinding = require('vectordb-linux-x64-gnu')
+              nativeBinding = require('lancedb-linux-x64-gnu')
             }
           } catch (e) {
             loadError = e
@@ -197,26 +197,26 @@ switch (platform) {
       case 'arm64':
         if (isMusl()) {
           localFileExisted = existsSync(
-            join(__dirname, 'vectordb-nodejs.linux-arm64-musl.node')
+            join(__dirname, 'lancedb-nodejs.linux-arm64-musl.node')
           )
           try {
             if (localFileExisted) {
-              nativeBinding = require('./vectordb-nodejs.linux-arm64-musl.node')
+              nativeBinding = require('./lancedb-nodejs.linux-arm64-musl.node')
             } else {
-              nativeBinding = require('vectordb-linux-arm64-musl')
+              nativeBinding = require('lancedb-linux-arm64-musl')
             }
           } catch (e) {
             loadError = e
           }
         } else {
           localFileExisted = existsSync(
-            join(__dirname, 'vectordb-nodejs.linux-arm64-gnu.node')
+            join(__dirname, 'lancedb-nodejs.linux-arm64-gnu.node')
           )
           try {
             if (localFileExisted) {
-              nativeBinding = require('./vectordb-nodejs.linux-arm64-gnu.node')
+              nativeBinding = require('./lancedb-nodejs.linux-arm64-gnu.node')
             } else {
-              nativeBinding = require('vectordb-linux-arm64-gnu')
+              nativeBinding = require('lancedb-linux-arm64-gnu')
             }
           } catch (e) {
             loadError = e
@@ -225,13 +225,13 @@ switch (platform) {
         break
       case 'arm':
         localFileExisted = existsSync(
-          join(__dirname, 'vectordb-nodejs.linux-arm-gnueabihf.node')
+          join(__dirname, 'lancedb-nodejs.linux-arm-gnueabihf.node')
         )
         try {
           if (localFileExisted) {
-            nativeBinding = require('./vectordb-nodejs.linux-arm-gnueabihf.node')
+            nativeBinding = require('./lancedb-nodejs.linux-arm-gnueabihf.node')
           } else {
-            nativeBinding = require('vectordb-linux-arm-gnueabihf')
+            nativeBinding = require('lancedb-linux-arm-gnueabihf')
           }
         } catch (e) {
           loadError = e
@@ -240,26 +240,26 @@ switch (platform) {
       case 'riscv64':
         if (isMusl()) {
           localFileExisted = existsSync(
-            join(__dirname, 'vectordb-nodejs.linux-riscv64-musl.node')
+            join(__dirname, 'lancedb-nodejs.linux-riscv64-musl.node')
           )
           try {
             if (localFileExisted) {
-              nativeBinding = require('./vectordb-nodejs.linux-riscv64-musl.node')
+              nativeBinding = require('./lancedb-nodejs.linux-riscv64-musl.node')
             } else {
-              nativeBinding = require('vectordb-linux-riscv64-musl')
+              nativeBinding = require('lancedb-linux-riscv64-musl')
             }
           } catch (e) {
             loadError = e
           }
         } else {
           localFileExisted = existsSync(
-            join(__dirname, 'vectordb-nodejs.linux-riscv64-gnu.node')
+            join(__dirname, 'lancedb-nodejs.linux-riscv64-gnu.node')
           )
           try {
             if (localFileExisted) {
-              nativeBinding = require('./vectordb-nodejs.linux-riscv64-gnu.node')
+              nativeBinding = require('./lancedb-nodejs.linux-riscv64-gnu.node')
             } else {
-              nativeBinding = require('vectordb-linux-riscv64-gnu')
+              nativeBinding = require('lancedb-linux-riscv64-gnu')
             }
           } catch (e) {
             loadError = e
@@ -268,13 +268,13 @@ switch (platform) {
         break
       case 's390x':
         localFileExisted = existsSync(
-          join(__dirname, 'vectordb-nodejs.linux-s390x-gnu.node')
+          join(__dirname, 'lancedb-nodejs.linux-s390x-gnu.node')
         )
         try {
           if (localFileExisted) {
-            nativeBinding = require('./vectordb-nodejs.linux-s390x-gnu.node')
+            nativeBinding = require('./lancedb-nodejs.linux-s390x-gnu.node')
           } else {
-            nativeBinding = require('vectordb-linux-s390x-gnu')
+            nativeBinding = require('lancedb-linux-s390x-gnu')
           }
         } catch (e) {
           loadError = e

nodejs/lancedb/table.ts

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 import { Schema, tableFromIPC } from "apache-arrow";
-import { Table as _NativeTable } from "./native";
+import { AddColumnsSql, ColumnAlteration, Table as _NativeTable } from "./native";
 import { toBuffer, Data } from "./arrow";
 import { Query } from "./query";
 import { IndexBuilder } from "./indexer";
@@ -32,8 +32,8 @@ export class Table {
   }
 
   /** Get the schema of the table. */
-  get schema(): Schema {
-    const schemaBuf = this.inner.schema();
+  async schema(): Promise<Schema> {
+    const schemaBuf = await this.inner.schema();
     const tbl = tableFromIPC(schemaBuf);
     return tbl.schema;
   }
@@ -50,7 +50,7 @@ export class Table {
   }
 
   /** Count the total number of rows in the dataset. */
-  async countRows(filter?: string): Promise<bigint> {
+  async countRows(filter?: string): Promise<number> {
     return await this.inner.countRows(filter);
   }
 
@@ -150,4 +150,42 @@ export class Table {
     }
     return q;
   }
+
+  // TODO: Support BatchUDF
+  /**
+   * Add new columns with defined values.
+   *
+   * @param newColumnTransforms pairs of column names and the SQL expression to use
+   *                            to calculate the value of the new column. These
+   *                            expressions will be evaluated for each row in the
+   *                            table, and can reference existing columns in the table.
+   */
+  async addColumns(newColumnTransforms: AddColumnsSql[]): Promise<void> {
+    await this.inner.addColumns(newColumnTransforms);
+  }
+
+  /**
+   * Alter the name or nullability of columns.
+   *
+   * @param columnAlterations One or more alterations to apply to columns.
+   */
+  async alterColumns(columnAlterations: ColumnAlteration[]): Promise<void> {
+    await this.inner.alterColumns(columnAlterations);
+  }
+
+  /**
+   * Drop one or more columns from the dataset
+   *
+   * This is a metadata-only operation and does not remove the data from the
+   * underlying storage. In order to remove the data, you must subsequently
+   * call ``compact_files`` to rewrite the data without the removed columns and
+   * then call ``cleanup_files`` to remove the old files.
+   *
+   * @param columnNames The names of the columns to drop. These can be nested
+   *                    column references (e.g. "a.b.c") or top-level column
+   *                    names (e.g. "a").
+   */
+  async dropColumns(columnNames: string[]): Promise<void> {
+    await this.inner.dropColumns(columnNames);
+  }
 }

nodejs/npm/darwin-arm64/README.md

@@ -1,3 +1,3 @@
-# `vectordb-darwin-arm64`
+# `lancedb-darwin-arm64`
 
-This is the **aarch64-apple-darwin** binary for `vectordb`
+This is the **aarch64-apple-darwin** binary for `lancedb`

nodejs/npm/darwin-arm64/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "vectordb-darwin-arm64",
+  "name": "lancedb-darwin-arm64",
   "version": "0.4.3",
   "os": [
     "darwin"
@@ -7,9 +7,9 @@
   "cpu": [
     "arm64"
   ],
-  "main": "vectordb.darwin-arm64.node",
+  "main": "lancedb.darwin-arm64.node",
   "files": [
-    "vectordb.darwin-arm64.node"
+    "lancedb.darwin-arm64.node"
   ],
   "license": "MIT",
   "engines": {

nodejs/npm/darwin-x64/README.md

@@ -1,3 +1,3 @@
-# `vectordb-darwin-x64`
+# `lancedb-darwin-x64`
 
-This is the **x86_64-apple-darwin** binary for `vectordb`
+This is the **x86_64-apple-darwin** binary for `lancedb`

nodejs/npm/darwin-x64/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "vectordb-darwin-x64",
+  "name": "lancedb-darwin-x64",
   "version": "0.4.3",
   "os": [
     "darwin"
@@ -7,9 +7,9 @@
   "cpu": [
     "x64"
   ],
-  "main": "vectordb.darwin-x64.node",
+  "main": "lancedb.darwin-x64.node",
   "files": [
-    "vectordb.darwin-x64.node"
+    "lancedb.darwin-x64.node"
   ],
   "license": "MIT",
   "engines": {

nodejs/npm/linux-arm64-gnu/README.md

@@ -1,3 +1,3 @@
-# `vectordb-linux-arm64-gnu`
+# `lancedb-linux-arm64-gnu`
 
-This is the **aarch64-unknown-linux-gnu** binary for `vectordb`
+This is the **aarch64-unknown-linux-gnu** binary for `lancedb`

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "vectordb-linux-arm64-gnu",
+  "name": "lancedb-linux-arm64-gnu",
   "version": "0.4.3",
   "os": [
     "linux"
@@ -7,9 +7,9 @@
   "cpu": [
     "arm64"
   ],
-  "main": "vectordb.linux-arm64-gnu.node",
+  "main": "lancedb.linux-arm64-gnu.node",
   "files": [
-    "vectordb.linux-arm64-gnu.node"
+    "lancedb.linux-arm64-gnu.node"
   ],
   "license": "MIT",
   "engines": {

nodejs/npm/linux-x64-gnu/README.md

@@ -1,3 +1,3 @@
-# `vectordb-linux-x64-gnu`
+# `lancedb-linux-x64-gnu`
 
-This is the **x86_64-unknown-linux-gnu** binary for `vectordb`
+This is the **x86_64-unknown-linux-gnu** binary for `lancedb`

nodejs/npm/linux-x64-gnu/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "vectordb-linux-x64-gnu",
+  "name": "lancedb-linux-x64-gnu",
   "version": "0.4.3",
   "os": [
     "linux"
@@ -7,9 +7,9 @@
   "cpu": [
     "x64"
   ],
-  "main": "vectordb.linux-x64-gnu.node",
+  "main": "lancedb.linux-x64-gnu.node",
   "files": [
-    "vectordb.linux-x64-gnu.node"
+    "lancedb.linux-x64-gnu.node"
   ],
   "license": "MIT",
   "engines": {

nodejs/package.json

@@ -1,10 +1,10 @@
 {
-  "name": "vectordb",
+  "name": "lancedb",
   "version": "0.4.3",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "napi": {
-    "name": "vectordb-nodejs",
+    "name": "lancedb-nodejs",
     "triples": {
       "defaults": false,
       "additional": [
@@ -18,7 +18,7 @@
   "license": "Apache 2.0",
   "devDependencies": {
     "@napi-rs/cli": "^2.18.0",
-    "@types/jest": "^29.5.11",
+    "@types/jest": "^29.1.2",
     "@typescript-eslint/eslint-plugin": "^6.19.0",
     "@typescript-eslint/parser": "^6.19.0",
     "eslint": "^8.56.0",
@@ -45,23 +45,23 @@
   ],
   "scripts": {
     "artifacts": "napi artifacts",
-    "build:native": "napi build --platform --release --js vectordb/native.js --dts vectordb/native.d.ts dist/",
-    "build:debug": "napi build --platform --dts ../vectordb/native.d.ts --js ../vectordb/native.js dist/",
+    "build:native": "napi build --platform --release --js lancedb/native.js --dts lancedb/native.d.ts dist/",
+    "build:debug": "napi build --platform --dts ../lancedb/native.d.ts --js ../lancedb/native.js dist/",
     "build": "npm run build:debug && tsc -b",
-    "docs": "typedoc --plugin typedoc-plugin-markdown vectordb/index.ts",
-    "lint": "eslint vectordb --ext .js,.ts",
+    "docs": "typedoc --plugin typedoc-plugin-markdown lancedb/index.ts",
+    "lint": "eslint lancedb --ext .js,.ts",
     "prepublishOnly": "napi prepublish -t npm",
-    "test": "npm run build && jest",
+    "test": "npm run build && jest --verbose",
     "universal": "napi universal",
     "version": "napi version"
   },
   "optionalDependencies": {
-    "vectordb-darwin-arm64": "0.4.3",
-    "vectordb-darwin-x64": "0.4.3",
-    "vectordb-linux-arm64-gnu": "0.4.3",
-    "vectordb-linux-x64-gnu": "0.4.3"
+    "lancedb-darwin-arm64": "0.4.3",
+    "lancedb-darwin-x64": "0.4.3",
+    "lancedb-linux-arm64-gnu": "0.4.3",
+    "lancedb-linux-x64-gnu": "0.4.3"
   },
-  "dependencies": {
+  "peerDependencies": {
     "apache-arrow": "^15.0.0"
   }
 }

nodejs/src/connection.rs

@@ -12,29 +12,51 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::Arc;
-
 use napi::bindgen_prelude::*;
 use napi_derive::*;
 
 use crate::table::Table;
-use vectordb::connection::{Connection as LanceDBConnection, Database};
-use vectordb::ipc::ipc_file_to_batches;
+use crate::ConnectionOptions;
+use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection, CreateTableMode};
+use lancedb::ipc::ipc_file_to_batches;
 
 #[napi]
 pub struct Connection {
-    conn: Arc<dyn LanceDBConnection>,
+    conn: LanceDBConnection,
+}
+
+impl Connection {
+    fn parse_create_mode_str(mode: &str) -> napi::Result<CreateTableMode> {
+        match mode {
+            "create" => Ok(CreateTableMode::Create),
+            "overwrite" => Ok(CreateTableMode::Overwrite),
+            "exist_ok" => Ok(CreateTableMode::exist_ok(|builder| builder)),
+            _ => Err(napi::Error::from_reason(format!("Invalid mode {}", mode))),
+        }
+    }
 }
 
 #[napi]
 impl Connection {
     /// Create a new Connection instance from the given URI.
     #[napi(factory)]
-    pub async fn new(uri: String) -> napi::Result<Self> {
+    pub async fn new(options: ConnectionOptions) -> napi::Result<Self> {
+        let mut builder = ConnectBuilder::new(&options.uri);
+        if let Some(api_key) = options.api_key {
+            builder = builder.api_key(&api_key);
+        }
+        if let Some(host_override) = options.host_override {
+            builder = builder.host_override(&host_override);
+        }
+        if let Some(interval) = options.read_consistency_interval {
+            builder =
+                builder.read_consistency_interval(std::time::Duration::from_secs_f64(interval));
+        }
         Ok(Self {
-            conn: Arc::new(Database::connect(&uri).await.map_err(|e| {
-                napi::Error::from_reason(format!("Failed to connect to database: {}", e))
-            })?),
+            conn: builder
+                .execute()
+                .await
+                .map_err(|e| napi::Error::from_reason(format!("{}", e)))?,
         })
     }
 
@@ -54,12 +76,20 @@ impl Connection {
     /// - buf: The buffer containing the IPC file.
     ///
     #[napi]
-    pub async fn create_table(&self, name: String, buf: Buffer) -> napi::Result<Table> {
+    pub async fn create_table(
+        &self,
+        name: String,
+        buf: Buffer,
+        mode: String,
+    ) -> napi::Result<Table> {
         let batches = ipc_file_to_batches(buf.to_vec())
             .map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
+        let mode = Self::parse_create_mode_str(&mode)?;
         let tbl = self
             .conn
-            .create_table(&name, Box::new(batches), None)
+            .create_table(&name, Box::new(batches))
+            .mode(mode)
+            .execute()
             .await
             .map_err(|e| napi::Error::from_reason(format!("{}", e)))?;
         Ok(Table::new(tbl))
@@ -70,6 +100,7 @@ impl Connection {
         let tbl = self
             .conn
             .open_table(&name)
+            .execute()
             .await
             .map_err(|e| napi::Error::from_reason(format!("{}", e)))?;
         Ok(Table::new(tbl))

nodejs/src/index.rs

@@ -40,12 +40,12 @@ impl From<MetricType> for LanceMetricType {
 
 #[napi]
 pub struct IndexBuilder {
-    inner: vectordb::index::IndexBuilder,
+    inner: lancedb::index::IndexBuilder,
 }
 
 #[napi]
 impl IndexBuilder {
-    pub fn new(tbl: &dyn vectordb::Table) -> Self {
+    pub fn new(tbl: &dyn lancedb::Table) -> Self {
         let inner = tbl.create_index(&[]);
         Self { inner }
     }

nodejs/src/iterator.rs

@@ -14,9 +14,9 @@
 
 use futures::StreamExt;
 use lance::io::RecordBatchStream;
+use lancedb::ipc::batches_to_ipc_file;
 use napi::bindgen_prelude::*;
 use napi_derive::napi;
-use vectordb::ipc::batches_to_ipc_file;
 
 /** Typescript-style Async Iterator over RecordBatches */
 #[napi]

nodejs/src/lib.rs

@@ -22,10 +22,21 @@ mod query;
 mod table;
 
 #[napi(object)]
+#[derive(Debug)]
 pub struct ConnectionOptions {
     pub uri: String,
     pub api_key: Option<String>,
     pub host_override: Option<String>,
+    /// (For LanceDB OSS only): The interval, in seconds, at which to check for
+    /// updates to the table from other processes. If None, then consistency is not
+    /// checked. For performance reasons, this is the default. For strong
+    /// consistency, set this to zero seconds. Then every read will check for
+    /// updates from other processes. As a compromise, you can set this to a
+    /// non-zero value for eventual consistency. If more than that interval
+    /// has passed since the last check, then the table will be checked for updates.
+    /// Note: this consistency only applies to read operations. Write operations are
+    /// always consistent.
+    pub read_consistency_interval: Option<f64>,
 }
 
 /// Write mode for writing a table.
@@ -44,5 +55,5 @@ pub struct WriteOptions {
 
 #[napi]
 pub async fn connect(options: ConnectionOptions) -> napi::Result<Connection> {
-    Connection::new(options.uri.clone()).await
+    Connection::new(options).await
 }

nodejs/src/query.rs

@@ -12,9 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use lancedb::query::Query as LanceDBQuery;
 use napi::bindgen_prelude::*;
 use napi_derive::napi;
-use vectordb::query::Query as LanceDBQuery;
 
 use crate::{iterator::RecordBatchIterator, table::Table};
 

nodejs/src/table.rs

@@ -13,9 +13,13 @@
 // limitations under the License.
 
 use arrow_ipc::writer::FileWriter;
+use lance::dataset::ColumnAlteration as LanceColumnAlteration;
+use lancedb::{
+    ipc::ipc_file_to_batches,
+    table::{AddDataOptions, TableRef},
+};
 use napi::bindgen_prelude::*;
 use napi_derive::napi;
-use vectordb::{ipc::ipc_file_to_batches, table::TableRef};
 
 use crate::index::IndexBuilder;
 use crate::query::Query;
@@ -33,8 +37,12 @@ impl Table {
 
     /// Return Schema as empty Arrow IPC file.
     #[napi]
-    pub fn schema(&self) -> napi::Result<Buffer> {
-        let mut writer = FileWriter::try_new(vec![], &self.table.schema())
+    pub async fn schema(&self) -> napi::Result<Buffer> {
+        let schema =
+            self.table.schema().await.map_err(|e| {
+                napi::Error::from_reason(format!("Failed to create IPC file: {}", e))
+            })?;
+        let mut writer = FileWriter::try_new(vec![], &schema)
             .map_err(|e| napi::Error::from_reason(format!("Failed to create IPC file: {}", e)))?;
         writer
             .finish()
@@ -48,22 +56,29 @@ impl Table {
     pub async fn add(&self, buf: Buffer) -> napi::Result<()> {
         let batches = ipc_file_to_batches(buf.to_vec())
             .map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
-        self.table.add(Box::new(batches), None).await.map_err(|e| {
-            napi::Error::from_reason(format!(
-                "Failed to add batches to table {}: {}",
-                self.table, e
-            ))
-        })
+        self.table
+            .add(Box::new(batches), AddDataOptions::default())
+            .await
+            .map_err(|e| {
+                napi::Error::from_reason(format!(
+                    "Failed to add batches to table {}: {}",
+                    self.table, e
+                ))
+            })
     }
 
     #[napi]
-    pub async fn count_rows(&self, filter: Option<String>) -> napi::Result<usize> {
-        self.table.count_rows(filter).await.map_err(|e| {
-            napi::Error::from_reason(format!(
-                "Failed to count rows in table {}: {}",
-                self.table, e
-            ))
-        })
+    pub async fn count_rows(&self, filter: Option<String>) -> napi::Result<i64> {
+        self.table
+            .count_rows(filter)
+            .await
+            .map(|val| val as i64)
+            .map_err(|e| {
+                napi::Error::from_reason(format!(
+                    "Failed to count rows in table {}: {}",
+                    self.table, e
+                ))
+            })
     }
 
     #[napi]
@@ -85,4 +100,106 @@ impl Table {
     pub fn query(&self) -> Query {
         Query::new(self)
     }
+
+    #[napi]
+    pub async fn add_columns(&self, transforms: Vec<AddColumnsSql>) -> napi::Result<()> {
+        let transforms = transforms
+            .into_iter()
+            .map(|sql| (sql.name, sql.value_sql))
+            .collect::<Vec<_>>();
+        let transforms = lance::dataset::NewColumnTransform::SqlExpressions(transforms);
+        self.table
+            .add_columns(transforms, None)
+            .await
+            .map_err(|err| {
+                napi::Error::from_reason(format!(
+                    "Failed to add columns to table {}: {}",
+                    self.table, err
+                ))
+            })?;
+        Ok(())
+    }
+
+    #[napi]
+    pub async fn alter_columns(&self, alterations: Vec<ColumnAlteration>) -> napi::Result<()> {
+        for alteration in &alterations {
+            if alteration.rename.is_none() && alteration.nullable.is_none() {
+                return Err(napi::Error::from_reason(
+                    "Alteration must have a 'rename' or 'nullable' field.",
+                ));
+            }
+        }
+        let alterations = alterations
+            .into_iter()
+            .map(LanceColumnAlteration::from)
+            .collect::<Vec<_>>();
+
+        self.table
+            .alter_columns(&alterations)
+            .await
+            .map_err(|err| {
+                napi::Error::from_reason(format!(
+                    "Failed to alter columns in table {}: {}",
+                    self.table, err
+                ))
+            })?;
+        Ok(())
+    }
+
+    #[napi]
+    pub async fn drop_columns(&self, columns: Vec<String>) -> napi::Result<()> {
+        let col_refs = columns.iter().map(String::as_str).collect::<Vec<_>>();
+        self.table.drop_columns(&col_refs).await.map_err(|err| {
+            napi::Error::from_reason(format!(
+                "Failed to drop columns from table {}: {}",
+                self.table, err
+            ))
+        })?;
+        Ok(())
+    }
+}
+
+///  A definition of a column alteration. The alteration changes the column at
+/// `path` to have the new name `name`, to be nullable if `nullable` is true,
+/// and to have the data type `data_type`. At least one of `rename` or `nullable`
+/// must be provided.
+#[napi(object)]
+pub struct ColumnAlteration {
+    /// The path to the column to alter. This is a dot-separated path to the column.
+    /// If it is a top-level column then it is just the name of the column. If it is
+    /// a nested column then it is the path to the column, e.g. "a.b.c" for a column
+    /// `c` nested inside a column `b` nested inside a column `a`.
+    pub path: String,
+    /// The new name of the column. If not provided then the name will not be changed.
+    /// This must be distinct from the names of all other columns in the table.
+    pub rename: Option<String>,
+    /// Set the new nullability. Note that a nullable column cannot be made non-nullable.
+    pub nullable: Option<bool>,
+}
+
+impl From<ColumnAlteration> for LanceColumnAlteration {
+    fn from(js: ColumnAlteration) -> Self {
+        let ColumnAlteration {
+            path,
+            rename,
+            nullable,
+        } = js;
+        Self {
+            path,
+            rename,
+            nullable,
+            // TODO: wire up this field
+            data_type: None,
+        }
+    }
+}
+
+/// A definition of a new column to add to a table.
+#[napi(object)]
+pub struct AddColumnsSql {
+    /// The name of the new column.
+    pub name: String,
+    /// The values to populate the new column with, as a SQL expression.
+    /// The expression can reference other columns in the table.
+    pub value_sql: String,
 }

nodejs/tsconfig.json

@@ -1,8 +1,8 @@
 {
   "include": [
-    "vectordb/*.ts",
-    "vectordb/**/*.ts",
-    "vectordb/*.js",
+    "lancedb/*.ts",
+    "lancedb/**/*.ts",
+    "lancedb/*.js",
   ],
   "compilerOptions": {
     "target": "es2022",
@@ -18,7 +18,7 @@
   ],
   "typedocOptions": {
     "entryPoints": [
-      "vectordb/index.ts"
+      "lancedb/index.ts"
     ],
     "out": "../docs/src/javascript/",
     "visibilityFilters": {

nodejs/vectordb/native.d.ts

@@ -1,80 +0,0 @@
-/* tslint:disable */
-/* eslint-disable */
-
-/* auto-generated by NAPI-RS */
-
-export const enum IndexType {
-  Scalar = 0,
-  IvfPq = 1
-}
-export const enum MetricType {
-  L2 = 0,
-  Cosine = 1,
-  Dot = 2
-}
-export interface ConnectionOptions {
-  uri: string
-  apiKey?: string
-  hostOverride?: string
-}
-/** Write mode for writing a table. */
-export const enum WriteMode {
-  Create = 'Create',
-  Append = 'Append',
-  Overwrite = 'Overwrite'
-}
-/** Write options when creating a Table. */
-export interface WriteOptions {
-  mode?: WriteMode
-}
-export function connect(options: ConnectionOptions): Promise<Connection>
-export class Connection {
-  /** Create a new Connection instance from the given URI. */
-  static new(uri: string): Promise<Connection>
-  /** List all tables in the dataset. */
-  tableNames(): Promise<Array<string>>
-  /**
-   * Create table from a Apache Arrow IPC (file) buffer.
-   *
-   * Parameters:
-   * - name: The name of the table.
-   * - buf: The buffer containing the IPC file.
-   *
-   */
-  createTable(name: string, buf: Buffer): Promise<Table>
-  openTable(name: string): Promise<Table>
-  /** Drop table with the name. Or raise an error if the table does not exist. */
-  dropTable(name: string): Promise<void>
-}
-export class IndexBuilder {
-  replace(v: boolean): void
-  column(c: string): void
-  name(name: string): void
-  ivfPq(metricType?: MetricType | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, numBits?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): void
-  scalar(): void
-  build(): Promise<void>
-}
-/** Typescript-style Async Iterator over RecordBatches  */
-export class RecordBatchIterator {
-  next(): Promise<Buffer | null>
-}
-export class Query {
-  column(column: string): void
-  filter(filter: string): void
-  select(columns: Array<string>): void
-  limit(limit: number): void
-  prefilter(prefilter: boolean): void
-  nearestTo(vector: Float32Array): void
-  refineFactor(refineFactor: number): void
-  nprobes(nprobe: number): void
-  executeStream(): Promise<RecordBatchIterator>
-}
-export class Table {
-  /** Return Schema as empty Arrow IPC file. */
-  schema(): Buffer
-  add(buf: Buffer): Promise<void>
-  countRows(filter?: string | undefined | null): Promise<bigint>
-  delete(predicate: string): Promise<void>
-  createIndex(): IndexBuilder
-  query(): Query
-}

python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.5.5
+current_version = 0.6.1
 commit = True
 message = [python] Bump version: {current_version} → {new_version}
 tag = True

python/ASYNC_MIGRATION.md

@@ -0,0 +1,24 @@
+# Migration from Sync to Async API
+
+A new asynchronous API has been added to LanceDb.  This API is built
+on top of the rust lancedb crate (instead of being built on top of
+pylance).  This will help keep the various language bindings in sync.
+There are some slight changes between the synchronous and the asynchronous
+APIs.  This document will help you migrate.  These changes relate mostly
+to the Connection and Table classes.
+
+## Almost all functions are async
+
+The most important change is that almost all functions are now async.
+This means the functions now return `asyncio` coroutines.  You will
+need to use `await` to call these functions.
+
+## Connection
+
+No changes yet.
+
+## Table
+
+* Previously `Table.schema` was a property.  Now it is an async method.
+* The method `Table.__len__` was removed and `len(table)` will no longer
+  work.  Use `Table.count_rows` instead.

python/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "lancedb-python"
+version = "0.4.10"
+edition.workspace = true
+description = "Python bindings for LanceDB"
+license.workspace = true
+repository.workspace = true
+keywords.workspace = true
+categories.workspace = true
+
+
+[lib]
+name = "_lancedb"
+crate-type = ["cdylib"]
+
+[dependencies]
+arrow = { version = "50.0.0", features = ["pyarrow"] }
+lancedb = { path = "../rust/lancedb" }
+env_logger = "0.10"
+pyo3 = { version = "0.20", features = ["extension-module", "abi3-py38"] }
+pyo3-asyncio = { version = "0.20", features = ["attributes", "tokio-runtime"] }
+
+# Prevent dynamic linking of lzma, which comes from datafusion
+lzma-sys = { version = "*", features = ["static"] }
+
+[build-dependencies]
+pyo3-build-config = { version = "0.20.3", features = [
+    "extension-module",
+    "abi3-py38",
+] }

python/README.md

@@ -20,10 +20,10 @@ results = table.search([0.1, 0.3]).limit(20).to_list()
 print(results)
 ```
 
-
 ## Development
 
-Create a virtual environment and activate it:
+LanceDb is based on the rust crate `lancedb` and is built with maturin.  In order to build with maturin
+you will either need a conda environment or a virtual environment (venv).
 
 ```bash
 python -m venv venv
@@ -33,7 +33,15 @@ python -m venv venv
 Install the necessary packages:
 
 ```bash
-python -m pip install .
+python -m pip install .[tests,dev]
+```
+
+To build the python package you can use maturin:
+
+```bash
+# This will build the rust bindings and place them in the appropriate place
+# in your venv or conda environment
+matruin develop
 ```
 
 To run the unit tests:
@@ -45,7 +53,7 @@ pytest
 To run the doc tests:
 
 ```bash
-pytest --doctest-modules lancedb
+pytest --doctest-modules python/lancedb
 ```
 
 To run linter and automatically fix all errors:
@@ -61,31 +69,27 @@ If any packages are missing, install them with:
 pip install <PACKAGE_NAME>
 ```
 
-
 ___
 For **Windows** users, there may be errors when installing packages, so these commands may be helpful:
 
 Activate the virtual environment:
+
 ```bash
 . .\venv\Scripts\activate
 ```
 
 You may need to run the installs separately:
+
 ```bash
 pip install -e .[tests]
 pip install -e .[dev]
 ```
 
-
 `tantivy` requires `rust` to be installed, so install it with `conda`, as it doesn't support windows installation:
+
 ```bash
 pip install wheel
 pip install cargo
 conda install rust
 pip install tantivy
 ```
-
-To run the unit tests:
-```bash
-pytest
-```

python/build.rs

@@ -0,0 +1,3 @@
+fn main() {
+    pyo3_build_config::add_extension_module_link_args();
+}

python/lancedb/__init__.py

@@ -1,90 +0,0 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-import importlib.metadata
-import os
-from datetime import timedelta
-from typing import Optional
-
-__version__ = importlib.metadata.version("lancedb")
-
-from .common import URI
-from .db import DBConnection, LanceDBConnection
-from .remote.db import RemoteDBConnection
-from .schema import vector  # noqa: F401
-from .utils import sentry_log  # noqa: F401
-
-
-def connect(
-    uri: URI,
-    *,
-    api_key: Optional[str] = None,
-    region: str = "us-east-1",
-    host_override: Optional[str] = None,
-    read_consistency_interval: Optional[timedelta] = None,
-) -> DBConnection:
-    """Connect to a LanceDB database.
-
-    Parameters
-    ----------
-    uri: str or Path
-        The uri of the database.
-    api_key: str, optional
-        If presented, connect to LanceDB cloud.
-        Otherwise, connect to a database on file system or cloud storage.
-        Can be set via environment variable `LANCEDB_API_KEY`.
-    region: str, default "us-east-1"
-        The region to use for LanceDB Cloud.
-    host_override: str, optional
-        The override url for LanceDB Cloud.
-    read_consistency_interval: timedelta, default None
-        (For LanceDB OSS only)
-        The interval at which to check for updates to the table from other
-        processes. If None, then consistency is not checked. For performance
-        reasons, this is the default. For strong consistency, set this to
-        zero seconds. Then every read will check for updates from other
-        processes. As a compromise, you can set this to a non-zero timedelta
-        for eventual consistency. If more than that interval has passed since
-        the last check, then the table will be checked for updates. Note: this
-        consistency only applies to read operations. Write operations are
-        always consistent.
-
-
-    Examples
-    --------
-
-    For a local directory, provide a path for the database:
-
-    >>> import lancedb
-    >>> db = lancedb.connect("~/.lancedb")
-
-    For object storage, use a URI prefix:
-
-    >>> db = lancedb.connect("s3://my-bucket/lancedb")
-
-    Connect to LancdDB cloud:
-
-    >>> db = lancedb.connect("db://my_database", api_key="ldb_...")
-
-    Returns
-    -------
-    conn : DBConnection
-        A connection to a LanceDB database.
-    """
-    if isinstance(uri, str) and uri.startswith("db://"):
-        if api_key is None:
-            api_key = os.environ.get("LANCEDB_API_KEY")
-        if api_key is None:
-            raise ValueError(f"api_key is required to connected LanceDB cloud: {uri}")
-        return RemoteDBConnection(uri, api_key, region, host_override)
-    return LanceDBConnection(uri, read_consistency_interval=read_consistency_interval)

python/lancedb/common.py

@@ -1,36 +0,0 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-from pathlib import Path
-from typing import Iterable, List, Union
-
-import numpy as np
-import pyarrow as pa
-
-from .util import safe_import_pandas
-
-pd = safe_import_pandas()
-
-DATA = Union[List[dict], dict, "pd.DataFrame", pa.Table, Iterable[pa.RecordBatch]]
-VEC = Union[list, np.ndarray, pa.Array, pa.ChunkedArray]
-URI = Union[str, Path]
-VECTOR_COLUMN_NAME = "vector"
-
-
-class Credential(str):
-    """Credential field"""
-
-    def __repr__(self) -> str:
-        return "********"
-
-    def __str__(self) -> str:
-        return "********"

python/pyproject.toml

@@ -1,9 +1,9 @@
 [project]
 name = "lancedb"
-version = "0.5.5"
+version = "0.6.1"
 dependencies = [
     "deprecation",
-    "pylance==0.9.16",
+    "pylance==0.10.1",
     "ratelimiter~=1.0",
     "retry>=0.9.2",
     "tqdm>=4.27.0",
@@ -14,7 +14,7 @@ dependencies = [
     "pyyaml>=6.0",
     "click>=8.1.7",
     "requests>=2.31.0",
-    "overrides>=0.7"
+    "overrides>=0.7",
 ]
 description = "lancedb"
 authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }]
@@ -26,7 +26,7 @@ keywords = [
     "data-science",
     "machine-learning",
     "arrow",
-    "data-analytics"
+    "data-analytics",
 ]
 classifiers = [
     "Development Status :: 3 - Alpha",
@@ -48,21 +48,53 @@ classifiers = [
 repository = "https://github.com/lancedb/lancedb"
 
 [project.optional-dependencies]
-tests = ["aiohttp", "pandas>=1.4", "pytest", "pytest-mock", "pytest-asyncio", "duckdb", "pytz", "polars>=0.19"]
+tests = [
+    "aiohttp",
+    "pandas>=1.4",
+    "pytest",
+    "pytest-mock",
+    "pytest-asyncio",
+    "duckdb",
+    "pytz",
+    "polars>=0.19",
+]
 dev = ["ruff", "pre-commit"]
-docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings[python]"]
+docs = [
+    "mkdocs",
+    "mkdocs-jupyter",
+    "mkdocs-material",
+    "mkdocstrings[python]",
+    "mkdocs-ultralytics-plugin==0.0.44",
+]
 clip = ["torch", "pillow", "open-clip"]
-embeddings = ["openai>=1.6.1", "sentence-transformers", "torch", "pillow", "open-clip-torch", "cohere", "huggingface_hub",
-                "InstructorEmbedding", "google.generativeai", "boto3>=1.28.57", "awscli>=1.29.57", "botocore>=1.31.57"]
+embeddings = [
+    "openai>=1.6.1",
+    "sentence-transformers",
+    "torch",
+    "pillow",
+    "open-clip-torch",
+    "cohere",
+    "huggingface_hub",
+    "InstructorEmbedding",
+    "google.generativeai",
+    "boto3>=1.28.57",
+    "awscli>=1.29.57",
+    "botocore>=1.31.57",
+]
+
+[tool.maturin]
+python-source = "python"
+module-name = "lancedb._lancedb"
 
 [project.scripts]
 lancedb = "lancedb.cli.cli:cli"
 
 [build-system]
-requires = ["setuptools", "wheel"]
-build-backend = "setuptools.build_meta"
+requires = ["maturin>=1.4"]
+build-backend = "maturin"
 
-[tool.ruff]
+
+[tool.ruff.lint]
 select = ["F", "E", "W", "I", "G", "TCH", "PERF"]
 
 [tool.pytest.ini_options]
@@ -70,5 +102,5 @@ addopts = "--strict-markers --ignore-glob=lancedb/embeddings/*.py"
 
 markers = [
     "slow: marks tests as slow (deselect with '-m \"not slow\"')",
-    "asyncio"
+    "asyncio",
 ]

python/python/lancedb/__init__.py

@@ -0,0 +1,175 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import importlib.metadata
+import os
+from concurrent.futures import ThreadPoolExecutor
+from datetime import timedelta
+from typing import Optional, Union
+
+__version__ = importlib.metadata.version("lancedb")
+
+from ._lancedb import connect as lancedb_connect
+from .common import URI, sanitize_uri
+from .db import AsyncConnection, AsyncLanceDBConnection, DBConnection, LanceDBConnection
+from .remote.db import RemoteDBConnection
+from .schema import vector  # noqa: F401
+from .utils import sentry_log  # noqa: F401
+
+
+def connect(
+    uri: URI,
+    *,
+    api_key: Optional[str] = None,
+    region: str = "us-east-1",
+    host_override: Optional[str] = None,
+    read_consistency_interval: Optional[timedelta] = None,
+    request_thread_pool: Optional[Union[int, ThreadPoolExecutor]] = None,
+) -> DBConnection:
+    """Connect to a LanceDB database.
+
+    Parameters
+    ----------
+    uri: str or Path
+        The uri of the database.
+    api_key: str, optional
+        If presented, connect to LanceDB cloud.
+        Otherwise, connect to a database on file system or cloud storage.
+        Can be set via environment variable `LANCEDB_API_KEY`.
+    region: str, default "us-east-1"
+        The region to use for LanceDB Cloud.
+    host_override: str, optional
+        The override url for LanceDB Cloud.
+    read_consistency_interval: timedelta, default None
+        (For LanceDB OSS only)
+        The interval at which to check for updates to the table from other
+        processes. If None, then consistency is not checked. For performance
+        reasons, this is the default. For strong consistency, set this to
+        zero seconds. Then every read will check for updates from other
+        processes. As a compromise, you can set this to a non-zero timedelta
+        for eventual consistency. If more than that interval has passed since
+        the last check, then the table will be checked for updates. Note: this
+        consistency only applies to read operations. Write operations are
+        always consistent.
+    request_thread_pool: int or ThreadPoolExecutor, optional
+        The thread pool to use for making batch requests to the LanceDB Cloud API.
+        If an integer, then a ThreadPoolExecutor will be created with that
+        number of threads. If None, then a ThreadPoolExecutor will be created
+        with the default number of threads. If a ThreadPoolExecutor, then that
+        executor will be used for making requests. This is for LanceDB Cloud
+        only and is only used when making batch requests (i.e., passing in
+        multiple queries to the search method at once).
+
+    Examples
+    --------
+
+    For a local directory, provide a path for the database:
+
+    >>> import lancedb
+    >>> db = lancedb.connect("~/.lancedb")
+
+    For object storage, use a URI prefix:
+
+    >>> db = lancedb.connect("s3://my-bucket/lancedb")
+
+    Connect to LancdDB cloud:
+
+    >>> db = lancedb.connect("db://my_database", api_key="ldb_...")
+
+    Returns
+    -------
+    conn : DBConnection
+        A connection to a LanceDB database.
+    """
+    if isinstance(uri, str) and uri.startswith("db://"):
+        if api_key is None:
+            api_key = os.environ.get("LANCEDB_API_KEY")
+        if api_key is None:
+            raise ValueError(f"api_key is required to connected LanceDB cloud: {uri}")
+        if isinstance(request_thread_pool, int):
+            request_thread_pool = ThreadPoolExecutor(request_thread_pool)
+        return RemoteDBConnection(
+            uri, api_key, region, host_override, request_thread_pool=request_thread_pool
+        )
+    return LanceDBConnection(uri, read_consistency_interval=read_consistency_interval)
+
+
+async def connect_async(
+    uri: URI,
+    *,
+    api_key: Optional[str] = None,
+    region: str = "us-east-1",
+    host_override: Optional[str] = None,
+    read_consistency_interval: Optional[timedelta] = None,
+    request_thread_pool: Optional[Union[int, ThreadPoolExecutor]] = None,
+) -> AsyncConnection:
+    """Connect to a LanceDB database.
+
+    Parameters
+    ----------
+    uri: str or Path
+        The uri of the database.
+    api_key: str, optional
+        If present, connect to LanceDB cloud.
+        Otherwise, connect to a database on file system or cloud storage.
+        Can be set via environment variable `LANCEDB_API_KEY`.
+    region: str, default "us-east-1"
+        The region to use for LanceDB Cloud.
+    host_override: str, optional
+        The override url for LanceDB Cloud.
+    read_consistency_interval: timedelta, default None
+        (For LanceDB OSS only)
+        The interval at which to check for updates to the table from other
+        processes. If None, then consistency is not checked. For performance
+        reasons, this is the default. For strong consistency, set this to
+        zero seconds. Then every read will check for updates from other
+        processes. As a compromise, you can set this to a non-zero timedelta
+        for eventual consistency. If more than that interval has passed since
+        the last check, then the table will be checked for updates. Note: this
+        consistency only applies to read operations. Write operations are
+        always consistent.
+    request_thread_pool: int or ThreadPoolExecutor, optional
+        The thread pool to use for making batch requests to the LanceDB Cloud API.
+        If an integer, then a ThreadPoolExecutor will be created with that
+        number of threads. If None, then a ThreadPoolExecutor will be created
+        with the default number of threads. If a ThreadPoolExecutor, then that
+        executor will be used for making requests. This is for LanceDB Cloud
+        only and is only used when making batch requests (i.e., passing in
+        multiple queries to the search method at once).
+
+    Examples
+    --------
+
+    For a local directory, provide a path for the database:
+
+    >>> import lancedb
+    >>> db = lancedb.connect("~/.lancedb")
+
+    For object storage, use a URI prefix:
+
+    >>> db = lancedb.connect("s3://my-bucket/lancedb")
+
+    Connect to LancdDB cloud:
+
+    >>> db = lancedb.connect("db://my_database", api_key="ldb_...")
+
+    Returns
+    -------
+    conn : DBConnection
+        A connection to a LanceDB database.
+    """
+    return AsyncLanceDBConnection(
+        await lancedb_connect(
+            sanitize_uri(uri), api_key, region, host_override, read_consistency_interval
+        )
+    )

python/python/lancedb/_lancedb.pyi

@@ -0,0 +1,24 @@
+from typing import Optional
+
+import pyarrow as pa
+
+class Connection(object):
+    async def table_names(self) -> list[str]: ...
+    async def create_table(
+        self, name: str, mode: str, data: pa.RecordBatchReader
+    ) -> Table: ...
+    async def create_empty_table(
+        self, name: str, mode: str, schema: pa.Schema
+    ) -> Table: ...
+
+class Table(object):
+    def name(self) -> str: ...
+    async def schema(self) -> pa.Schema: ...
+
+async def connect(
+    uri: str,
+    api_key: Optional[str],
+    region: Optional[str],
+    host_override: Optional[str],
+    read_consistency_interval: Optional[float],
+) -> Connection: ...

python/python/lancedb/common.py

@@ -0,0 +1,136 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from pathlib import Path
+from typing import Iterable, List, Optional, Union
+
+import numpy as np
+import pyarrow as pa
+
+from .util import safe_import_pandas
+
+pd = safe_import_pandas()
+
+DATA = Union[List[dict], dict, "pd.DataFrame", pa.Table, Iterable[pa.RecordBatch]]
+VEC = Union[list, np.ndarray, pa.Array, pa.ChunkedArray]
+URI = Union[str, Path]
+VECTOR_COLUMN_NAME = "vector"
+
+
+class Credential(str):
+    """Credential field"""
+
+    def __repr__(self) -> str:
+        return "********"
+
+    def __str__(self) -> str:
+        return "********"
+
+
+def sanitize_uri(uri: URI) -> str:
+    return str(uri)
+
+
+def _casting_recordbatch_iter(
+    input_iter: Iterable[pa.RecordBatch], schema: pa.Schema
+) -> Iterable[pa.RecordBatch]:
+    """
+    Wrapper around an iterator of record batches. If the batches don't match the
+    schema, try to cast them to the schema. If that fails, raise an error.
+
+    This is helpful for users who might have written the iterator with default
+    data types in PyArrow, but specified more specific types in the schema. For
+    example, PyArrow defaults to float64 for floating point types, but Lance
+    uses float32 for vectors.
+    """
+    for batch in input_iter:
+        if not isinstance(batch, pa.RecordBatch):
+            raise TypeError(f"Expected RecordBatch, got {type(batch)}")
+        if batch.schema != schema:
+            try:
+                # RecordBatch doesn't have a cast method, but table does.
+                batch = pa.Table.from_batches([batch]).cast(schema).to_batches()[0]
+            except pa.lib.ArrowInvalid:
+                raise ValueError(
+                    f"Input RecordBatch iterator yielded a batch with schema that "
+                    f"does not match the expected schema.\nExpected:\n{schema}\n"
+                    f"Got:\n{batch.schema}"
+                )
+        yield batch
+
+
+def data_to_reader(
+    data: DATA, schema: Optional[pa.Schema] = None
+) -> pa.RecordBatchReader:
+    """Convert various types of input into a RecordBatchReader"""
+    if pd is not None and isinstance(data, pd.DataFrame):
+        return pa.Table.from_pandas(data, schema=schema).to_reader()
+    elif isinstance(data, pa.Table):
+        return data.to_reader()
+    elif isinstance(data, pa.RecordBatch):
+        return pa.Table.from_batches([data]).to_reader()
+    # elif isinstance(data, LanceDataset):
+    #     return data_obj.scanner().to_reader()
+    elif isinstance(data, pa.dataset.Dataset):
+        return pa.dataset.Scanner.from_dataset(data).to_reader()
+    elif isinstance(data, pa.dataset.Scanner):
+        return data.to_reader()
+    elif isinstance(data, pa.RecordBatchReader):
+        return data
+    elif (
+        type(data).__module__.startswith("polars")
+        and data.__class__.__name__ == "DataFrame"
+    ):
+        return data.to_arrow().to_reader()
+    # for other iterables, assume they are of type Iterable[RecordBatch]
+    elif isinstance(data, Iterable):
+        if schema is not None:
+            data = _casting_recordbatch_iter(data, schema)
+            return pa.RecordBatchReader.from_batches(schema, data)
+        else:
+            raise ValueError(
+                "Must provide schema to write dataset from RecordBatch iterable"
+            )
+    else:
+        raise TypeError(
+            f"Unknown data type {type(data)}. "
+            "Please check "
+            "https://lancedb.github.io/lance/read_and_write.html "
+            "to see supported types."
+        )
+
+
+def validate_schema(schema: pa.Schema):
+    """
+    Make sure the metadata is valid utf8
+    """
+    if schema.metadata is not None:
+        _validate_metadata(schema.metadata)
+
+
+def _validate_metadata(metadata: dict):
+    """
+    Make sure the metadata values are valid utf8 (can be nested)
+
+    Raises ValueError if not valid utf8
+    """
+    for k, v in metadata.items():
+        if isinstance(v, bytes):
+            try:
+                v.decode("utf8")
+            except UnicodeDecodeError:
+                raise ValueError(
+                    f"Metadata key {k} is not valid utf8. "
+                    "Consider base64 encode for generic binary metadata."
+                )
+        elif isinstance(v, dict):
+            _validate_metadata(v)

python/python/lancedb/db.py

@@ -13,6 +13,7 @@
 
 from __future__ import annotations
 
+import inspect
 import os
 from abc import abstractmethod
 from pathlib import Path
@@ -22,15 +23,20 @@ import pyarrow as pa
 from overrides import EnforceOverrides, override
 from pyarrow import fs
 
-from .table import LanceTable, Table
+from lancedb.common import data_to_reader, validate_schema
+from lancedb.embeddings.registry import EmbeddingFunctionRegistry
+from lancedb.utils.events import register_event
+
+from .pydantic import LanceModel
+from .table import AsyncLanceTable, LanceTable, Table, _sanitize_data
 from .util import fs_from_uri, get_uri_location, get_uri_scheme, join_uri
 
 if TYPE_CHECKING:
     from datetime import timedelta
 
+    from ._lancedb import Connection as LanceDbConnection
     from .common import DATA, URI
     from .embeddings import EmbeddingFunctionConfig
-    from .pydantic import LanceModel
 
 
 class DBConnection(EnforceOverrides):
@@ -40,14 +46,21 @@ class DBConnection(EnforceOverrides):
     def table_names(
         self, page_token: Optional[str] = None, limit: int = 10
     ) -> Iterable[str]:
-        """List all table in this database
+        """List all tables in this database, in sorted order
 
         Parameters
         ----------
         page_token: str, optional
             The token to use for pagination. If not present, start from the beginning.
+            Typically, this token is last table name from the previous page.
+            Only supported by LanceDb Cloud.
         limit: int, default 10
             The size of the page to return.
+            Only supported by LanceDb Cloud.
+
+        Returns
+        -------
+        Iterable of str
         """
         pass
 
@@ -412,3 +425,313 @@ class LanceDBConnection(DBConnection):
     def drop_database(self):
         filesystem, path = fs_from_uri(self.uri)
         filesystem.delete_dir(path)
+
+
+class AsyncConnection(EnforceOverrides):
+    """An active LanceDB connection interface."""
+
+    @abstractmethod
+    async def table_names(
+        self, *, page_token: Optional[str] = None, limit: int = 10
+    ) -> Iterable[str]:
+        """List all tables in this database, in sorted order
+
+        Parameters
+        ----------
+        page_token: str, optional
+            The token to use for pagination. If not present, start from the beginning.
+            Typically, this token is last table name from the previous page.
+            Only supported by LanceDb Cloud.
+        limit: int, default 10
+            The size of the page to return.
+            Only supported by LanceDb Cloud.
+
+        Returns
+        -------
+        Iterable of str
+        """
+        pass
+
+    @abstractmethod
+    async def create_table(
+        self,
+        name: str,
+        data: Optional[DATA] = None,
+        schema: Optional[Union[pa.Schema, LanceModel]] = None,
+        mode: str = "create",
+        exist_ok: bool = False,
+        on_bad_vectors: str = "error",
+        fill_value: float = 0.0,
+        embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
+    ) -> Table:
+        """Create a [Table][lancedb.table.Table] in the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+        data: The data to initialize the table, *optional*
+            User must provide at least one of `data` or `schema`.
+            Acceptable types are:
+
+            - dict or list-of-dict
+
+            - pandas.DataFrame
+
+            - pyarrow.Table or pyarrow.RecordBatch
+        schema: The schema of the table, *optional*
+            Acceptable types are:
+
+            - pyarrow.Schema
+
+            - [LanceModel][lancedb.pydantic.LanceModel]
+        mode: str; default "create"
+            The mode to use when creating the table.
+            Can be either "create" or "overwrite".
+            By default, if the table already exists, an exception is raised.
+            If you want to overwrite the table, use mode="overwrite".
+        exist_ok: bool, default False
+            If a table by the same name already exists, then raise an exception
+            if exist_ok=False. If exist_ok=True, then open the existing table;
+            it will not add the provided data but will validate against any
+            schema that's specified.
+        on_bad_vectors: str, default "error"
+            What to do if any of the vectors are not the same size or contains NaNs.
+            One of "error", "drop", "fill".
+        fill_value: float
+            The value to use when filling vectors. Only used if on_bad_vectors="fill".
+
+        Returns
+        -------
+        LanceTable
+            A reference to the newly created table.
+
+        !!! note
+
+            The vector index won't be created by default.
+            To create the index, call the `create_index` method on the table.
+
+        Examples
+        --------
+
+        Can create with list of tuples or dictionaries:
+
+        >>> import lancedb
+        >>> db = lancedb.connect("./.lancedb")
+        >>> data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
+        ...         {"vector": [0.2, 1.8], "lat": 40.1, "long":  -74.1}]
+        >>> db.create_table("my_table", data)
+        LanceTable(connection=..., name="my_table")
+        >>> db["my_table"].head()
+        pyarrow.Table
+        vector: fixed_size_list<item: float>[2]
+          child 0, item: float
+        lat: double
+        long: double
+        ----
+        vector: [[[1.1,1.2],[0.2,1.8]]]
+        lat: [[45.5,40.1]]
+        long: [[-122.7,-74.1]]
+
+        You can also pass a pandas DataFrame:
+
+        >>> import pandas as pd
+        >>> data = pd.DataFrame({
+        ...    "vector": [[1.1, 1.2], [0.2, 1.8]],
+        ...    "lat": [45.5, 40.1],
+        ...    "long": [-122.7, -74.1]
+        ... })
+        >>> db.create_table("table2", data)
+        LanceTable(connection=..., name="table2")
+        >>> db["table2"].head()
+        pyarrow.Table
+        vector: fixed_size_list<item: float>[2]
+          child 0, item: float
+        lat: double
+        long: double
+        ----
+        vector: [[[1.1,1.2],[0.2,1.8]]]
+        lat: [[45.5,40.1]]
+        long: [[-122.7,-74.1]]
+
+        Data is converted to Arrow before being written to disk. For maximum
+        control over how data is saved, either provide the PyArrow schema to
+        convert to or else provide a [PyArrow Table](pyarrow.Table) directly.
+
+        >>> custom_schema = pa.schema([
+        ...   pa.field("vector", pa.list_(pa.float32(), 2)),
+        ...   pa.field("lat", pa.float32()),
+        ...   pa.field("long", pa.float32())
+        ... ])
+        >>> db.create_table("table3", data, schema = custom_schema)
+        LanceTable(connection=..., name="table3")
+        >>> db["table3"].head()
+        pyarrow.Table
+        vector: fixed_size_list<item: float>[2]
+          child 0, item: float
+        lat: float
+        long: float
+        ----
+        vector: [[[1.1,1.2],[0.2,1.8]]]
+        lat: [[45.5,40.1]]
+        long: [[-122.7,-74.1]]
+
+
+        It is also possible to create an table from `[Iterable[pa.RecordBatch]]`:
+
+
+        >>> import pyarrow as pa
+        >>> def make_batches():
+        ...     for i in range(5):
+        ...         yield pa.RecordBatch.from_arrays(
+        ...             [
+        ...                 pa.array([[3.1, 4.1], [5.9, 26.5]],
+        ...                     pa.list_(pa.float32(), 2)),
+        ...                 pa.array(["foo", "bar"]),
+        ...                 pa.array([10.0, 20.0]),
+        ...             ],
+        ...             ["vector", "item", "price"],
+        ...         )
+        >>> schema=pa.schema([
+        ...     pa.field("vector", pa.list_(pa.float32(), 2)),
+        ...     pa.field("item", pa.utf8()),
+        ...     pa.field("price", pa.float32()),
+        ... ])
+        >>> db.create_table("table4", make_batches(), schema=schema)
+        LanceTable(connection=..., name="table4")
+
+        """
+        raise NotImplementedError
+
+    async def open_table(self, name: str) -> Table:
+        """Open a Lance Table in the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+
+        Returns
+        -------
+        A LanceTable object representing the table.
+        """
+        raise NotImplementedError
+
+    async def drop_table(self, name: str):
+        """Drop a table from the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+        """
+        raise NotImplementedError
+
+    async def drop_database(self):
+        """
+        Drop database
+        This is the same thing as dropping all the tables
+        """
+        raise NotImplementedError
+
+
+class AsyncLanceDBConnection(AsyncConnection):
+    def __init__(self, connection: LanceDbConnection):
+        self._inner = connection
+
+    async def __repr__(self) -> str:
+        pass
+
+    @override
+    async def table_names(
+        self,
+        *,
+        page_token=None,
+        limit=None,
+    ) -> Iterable[str]:
+        # TODO: hook in page_token and limit
+        return await self._inner.table_names()
+
+    @override
+    async def create_table(
+        self,
+        name: str,
+        data: Optional[DATA] = None,
+        schema: Optional[Union[pa.Schema, LanceModel]] = None,
+        mode: str = "create",
+        exist_ok: bool = False,
+        on_bad_vectors: str = "error",
+        fill_value: float = 0.0,
+        embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
+    ) -> Table:
+        if mode.lower() not in ["create", "overwrite"]:
+            raise ValueError("mode must be either 'create' or 'overwrite'")
+
+        if inspect.isclass(schema) and issubclass(schema, LanceModel):
+            # convert LanceModel to pyarrow schema
+            # note that it's possible this contains
+            # embedding function metadata already
+            schema = schema.to_arrow_schema()
+
+        metadata = None
+        if embedding_functions is not None:
+            # If we passed in embedding functions explicitly
+            # then we'll override any schema metadata that
+            # may was implicitly specified by the LanceModel schema
+            registry = EmbeddingFunctionRegistry.get_instance()
+            metadata = registry.get_table_metadata(embedding_functions)
+
+        if data is not None:
+            data = _sanitize_data(
+                data,
+                schema,
+                metadata=metadata,
+                on_bad_vectors=on_bad_vectors,
+                fill_value=fill_value,
+            )
+
+        if schema is None:
+            if data is None:
+                raise ValueError("Either data or schema must be provided")
+            elif hasattr(data, "schema"):
+                schema = data.schema
+            elif isinstance(data, Iterable):
+                if metadata:
+                    raise TypeError(
+                        (
+                            "Persistent embedding functions not yet "
+                            "supported for generator data input"
+                        )
+                    )
+
+        if metadata:
+            schema = schema.with_metadata(metadata)
+        validate_schema(schema)
+
+        if mode == "create" and exist_ok:
+            mode = "exist_ok"
+
+        if data is None:
+            new_table = await self._inner.create_empty_table(name, mode, schema)
+        else:
+            data = data_to_reader(data, schema)
+            new_table = await self._inner.create_table(
+                name,
+                mode,
+                data,
+            )
+
+        register_event("create_table")
+        return AsyncLanceTable(new_table)
+
+    @override
+    async def open_table(self, name: str) -> LanceTable:
+        raise NotImplementedError
+
+    @override
+    async def drop_table(self, name: str, ignore_missing: bool = False):
+        raise NotImplementedError
+
+    @override
+    async def drop_database(self):
+        raise NotImplementedError

python/python/lancedb/embeddings/imagebind.py

@@ -0,0 +1,172 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from functools import cached_property
+from typing import List, Union
+
+import numpy as np
+import pyarrow as pa
+
+from ..util import attempt_import_or_raise
+from .base import EmbeddingFunction
+from .registry import register
+from .utils import AUDIO, IMAGES, TEXT
+
+
+@register("imagebind")
+class ImageBindEmbeddings(EmbeddingFunction):
+    """
+    An embedding function that uses the ImageBind API
+    For generating multi-modal embeddings across
+    six different modalities: images, text, audio, depth, thermal, and IMU data
+
+    to download package, run :
+        `pip install imagebind@git+https://github.com/raghavdixit99/ImageBind`
+    """
+
+    name: str = "imagebind_huge"
+    device: str = "cpu"
+    normalize: bool = False
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._ndims = 1024
+        self._audio_extensions = (".mp3", ".wav", ".flac", ".ogg", ".aac")
+        self._image_extensions = (".jpg", ".jpeg", ".png", ".gif", ".bmp")
+
+    @cached_property
+    def embedding_model(self):
+        """
+        Get the embedding model. This is cached so that the model is only loaded
+        once per process.
+        """
+        return self.get_embedding_model()
+
+    @cached_property
+    def _data(self):
+        """
+        Get the data module from imagebind
+        """
+        data = attempt_import_or_raise("imagebind.data", "imagebind")
+        return data
+
+    @cached_property
+    def _ModalityType(self):
+        """
+        Get the ModalityType from imagebind
+        """
+        imagebind = attempt_import_or_raise("imagebind", "imagebind")
+        return imagebind.imagebind_model.ModalityType
+
+    def ndims(self):
+        return self._ndims
+
+    def compute_query_embeddings(
+        self, query: Union[str], *args, **kwargs
+    ) -> List[np.ndarray]:
+        """
+        Compute the embeddings for a given user query
+
+        Parameters
+        ----------
+        query : Union[str]
+            The query to embed. A query can be either text, image paths or audio paths.
+        """
+        query = self.sanitize_input(query)
+        if query[0].endswith(self._audio_extensions):
+            return [self.generate_audio_embeddings(query)]
+        elif query[0].endswith(self._image_extensions):
+            return [self.generate_image_embeddings(query)]
+        else:
+            return [self.generate_text_embeddings(query)]
+
+    def generate_image_embeddings(self, image: IMAGES) -> np.ndarray:
+        torch = attempt_import_or_raise("torch")
+        inputs = {
+            self._ModalityType.VISION: self._data.load_and_transform_vision_data(
+                image, self.device
+            )
+        }
+        with torch.no_grad():
+            image_features = self.embedding_model(inputs)[self._ModalityType.VISION]
+            if self.normalize:
+                image_features /= image_features.norm(dim=-1, keepdim=True)
+            return image_features.cpu().numpy().squeeze()
+
+    def generate_audio_embeddings(self, audio: AUDIO) -> np.ndarray:
+        torch = attempt_import_or_raise("torch")
+        inputs = {
+            self._ModalityType.AUDIO: self._data.load_and_transform_audio_data(
+                audio, self.device
+            )
+        }
+        with torch.no_grad():
+            audio_features = self.embedding_model(inputs)[self._ModalityType.AUDIO]
+            if self.normalize:
+                audio_features /= audio_features.norm(dim=-1, keepdim=True)
+            return audio_features.cpu().numpy().squeeze()
+
+    def generate_text_embeddings(self, text: TEXT) -> np.ndarray:
+        torch = attempt_import_or_raise("torch")
+        inputs = {
+            self._ModalityType.TEXT: self._data.load_and_transform_text(
+                text, self.device
+            )
+        }
+        with torch.no_grad():
+            text_features = self.embedding_model(inputs)[self._ModalityType.TEXT]
+            if self.normalize:
+                text_features /= text_features.norm(dim=-1, keepdim=True)
+            return text_features.cpu().numpy().squeeze()
+
+    def compute_source_embeddings(
+        self, source: Union[IMAGES, AUDIO], *args, **kwargs
+    ) -> List[np.array]:
+        """
+        Get the embeddings for the given sourcefield column in the pydantic model.
+        """
+        source = self.sanitize_input(source)
+        embeddings = []
+        if source[0].endswith(self._audio_extensions):
+            embeddings.extend(self.generate_audio_embeddings(source))
+            return embeddings
+        elif source[0].endswith(self._image_extensions):
+            embeddings.extend(self.generate_image_embeddings(source))
+            return embeddings
+        else:
+            embeddings.extend(self.generate_text_embeddings(source))
+            return embeddings
+
+    def sanitize_input(
+        self, input: Union[IMAGES, AUDIO]
+    ) -> Union[List[bytes], np.ndarray]:
+        """
+        Sanitize the input to the embedding function.
+        """
+        if isinstance(input, (str, bytes)):
+            input = [input]
+        elif isinstance(input, pa.Array):
+            input = input.to_pylist()
+        elif isinstance(input, pa.ChunkedArray):
+            input = input.combine_chunks().to_pylist()
+        return input
+
+    def get_embedding_model(self):
+        """
+        fetches the imagebind embedding model
+        """
+        imagebind = attempt_import_or_raise("imagebind", "imagebind")
+        model = imagebind.imagebind_model.imagebind_huge(pretrained=True)
+        model.eval()
+        model.to(self.device)
+        return model

python/python/lancedb/embeddings/instructor.py

@@ -103,9 +103,9 @@ class InstructorEmbeddingFunction(TextEmbeddingFunction):
     # convert_to_numpy: bool = True # Hardcoding this as numpy can be ingested directly
 
     source_instruction: str = "represent the document for retrieval"
-    query_instruction: (
-        str
-    ) = "represent the document for retrieving the most similar documents"
+    query_instruction: str = (
+        "represent the document for retrieving the most similar documents"
+    )
 
     @weak_lru(maxsize=1)
     def ndims(self):

python/python/lancedb/embeddings/utils.py

@@ -26,7 +26,7 @@ import pyarrow as pa
 from lance.vector import vec_to_table
 from retry import retry
 
-from ..util import safe_import_pandas
+from ..util import deprecated, safe_import_pandas
 from ..utils.general import LOGGER
 
 pd = safe_import_pandas()
@@ -36,8 +36,10 @@ TEXT = Union[str, List[str], pa.Array, pa.ChunkedArray, np.ndarray]
 IMAGES = Union[
     str, bytes, List[str], List[bytes], pa.Array, pa.ChunkedArray, np.ndarray
 ]
+AUDIO = Union[str, bytes, List[str], List[bytes], pa.Array, pa.ChunkedArray, np.ndarray]
 
 
+@deprecated
 def with_embeddings(
     func: Callable,
     data: DATA,

python/python/lancedb/fts.py

@@ -12,6 +12,7 @@
 #  limitations under the License.
 
 """Full text search index using tantivy-py"""
+
 import os
 from typing import List, Tuple