Bump to v0.1.10-python

Support connect with remote database, and implement Search API

.bumpversion.cfg

@@ -0,0 +1,12 @@
+[bumpversion]
+current_version = 0.1.10
+commit = True
+message = Bump version: {current_version} → {new_version}
+tag = True
+tag_name = v{new_version}
+
+[bumpversion:file:node/package.json]
+
+[bumpversion:file:rust/ffi/node/Cargo.toml]
+
+[bumpversion:file:rust/vectordb/Cargo.toml]

.github/workflows/cargo-publish.yml

@@ -0,0 +1,29 @@
+name: Cargo Publish
+
+on:
+  release:
+    types: [ published ]
+
+env:
+  # This env var is used by Swatinem/rust-cache@v2 for the cache
+  # key, so we set it to make sure it is always consistent.
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - uses: actions/checkout@v3
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Publish the package
+        run: |
+          cargo publish -p vectordb --all-features --token ${{ secrets.CARGO_REGISTRY_TOKEN }}

.github/workflows/docs.yml

@@ -39,6 +39,28 @@ jobs:
         run: |
           python -m pip install -e .
           python -m pip install -r ../docs/requirements.txt
+      - name: Set up node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'npm'
+          cache-dependency-path: node/package-lock.json
+      - uses: Swatinem/rust-cache@v2
+      - name: Install node dependencies
+        working-directory: node
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Build node
+        working-directory: node
+        run: |
+          npm ci
+          npm run build
+          npm run tsc
+      - name: Create markdown files
+        working-directory: node
+        run: |
+          npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
       - name: Build docs
         run: |
           PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
@@ -50,4 +72,4 @@ jobs:
           path: "docs/site"
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v1
+        uses: actions/deploy-pages@v1

.github/workflows/docs_test.yml

@@ -0,0 +1,93 @@
+name: Documentation Code Testing
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - docs/**
+      - .github/workflows/docs_test.yml
+  pull_request:
+    paths:
+      - docs/**
+      - .github/workflows/docs_test.yml
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+env:
+  # Disable full debug symbol generation to speed up CI build and keep memory down
+  # "1" means line tables only, which is useful for panic tracebacks.
+  RUSTFLAGS: "-C debuginfo=1"
+  RUST_BACKTRACE: "1"
+
+jobs:
+  test-python:
+    name: Test doc python code
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-minor-version: [ "11" ]
+        os: ["ubuntu-22.04"]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.${{ matrix.python-minor-version }}
+        cache: "pip"
+        cache-dependency-path: "docs/test/requirements.txt"
+    - name: Build Python
+      working-directory: docs/test
+      run:
+        python -m pip install -r requirements.txt
+    - name: Create test files
+      run: |
+        cd docs/test
+        python md_testing.py
+    - name: Test
+      run: |
+        cd docs/test/python
+        for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
+  test-node:
+    name: Test doc nodejs code
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        node-version: [ "18" ]
+        os: ["ubuntu-22.04"]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - name: Set up Node
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+    - name: Install dependecies needed for ubuntu
+      if: ${{ matrix.os == 'ubuntu-22.04' }}
+      run: |
+        sudo apt install -y protobuf-compiler libssl-dev
+    - name: Install node dependencies
+      run: |
+        cd docs/test
+        npm install 
+    - name: Rust cache
+      uses: swatinem/rust-cache@v2
+    - name: Install LanceDB
+      run: |
+        cd docs/test/node_modules/vectordb 
+        npm ci
+        npm run build
+        npm run tsc
+    - name: Create test files
+      run: |
+        cd docs/test
+        node md_testing.js
+    - name: Test
+      run: |
+        cd docs/test/node
+        for d in *; do cd "$d"; echo "$d".js; node "$d".js; cd ..; done

.github/workflows/make-release-commit.yml

@@ -0,0 +1,55 @@
+name: Create release commit
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: 'Dry run (create the local commit/tags but do not push it)'
+        required: true
+        default: "false"
+        type: choice
+        options:
+          - "true"
+          - "false"
+      part:
+        description: 'What kind of release is this?'
+        required: true
+        default: 'patch'
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  bump-version:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out main
+      uses: actions/checkout@v3
+      with:
+        ref: main
+        persist-credentials: false
+        fetch-depth: 0
+        lfs: true
+    - name: Set git configs for bumpversion
+      shell: bash
+      run: |
+        git config user.name 'Lance Release'
+        git config user.email 'lance-dev@lancedb.com'
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+    - name: Bump version, create tag and commit
+      run: |
+        pip install bump2version
+        bumpversion --verbose ${{ inputs.part }}
+    - name: Push new version and tag
+      if: ${{ inputs.dry_run }} == "false"
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+        branch: main
+        tags: true
+

.github/workflows/pypi-publish.yml

@@ -0,0 +1,31 @@
+name: PyPI Publish
+
+on:
+  release:
+    types: [ published ]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    # Only runs on tags that matches the python-make-release action
+    if: startsWith(github.ref, 'refs/tags/python-v')
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+      - name: Build distribution
+        run: |
+          ls -la
+          pip install wheel setuptools --upgrade
+          python setup.py sdist bdist_wheel
+      - name: Publish
+        uses: pypa/gh-action-pypi-publish@v1.8.5
+        with:
+          password: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
+          packages-dir: python/dist

.github/workflows/python-make-release-commit.yml

@@ -0,0 +1,56 @@
+name: Python - Create release commit
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: 'Dry run (create the local commit/tags but do not push it)'
+        required: true
+        default: "false"
+        type: choice
+        options:
+          - "true"
+          - "false"
+      part:
+        description: 'What kind of release is this?'
+        required: true
+        default: 'patch'
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  bump-version:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out main
+      uses: actions/checkout@v3
+      with:
+        ref: main
+        persist-credentials: false
+        fetch-depth: 0
+        lfs: true
+    - name: Set git configs for bumpversion
+      shell: bash
+      run: |
+        git config user.name 'Lance Release'
+        git config user.email 'lance-dev@lancedb.com'
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+    - name: Bump version, create tag and commit
+      working-directory: python
+      run: |
+        pip install bump2version
+        bumpversion --verbose ${{ inputs.part }}
+    - name: Push new version and tag
+      if: ${{ inputs.dry_run }} == "false"
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+        branch: main
+        tags: true
+

.github/workflows/python.yml

@@ -31,9 +31,16 @@ jobs:
     - name: Install lancedb
       run: |
         pip install -e .
-        pip install pytest
+        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+        pip install pytest pytest-mock black isort
+    - name: Black
+      run: black --check --diff --no-color --quiet .
+    - name: isort
+      run: isort --check --diff --quiet .
     - name: Run tests
       run: pytest -x -v --durations=30 tests
+    - name: doctest
+      run: pytest --doctest-modules lancedb
   mac:
     timeout-minutes: 30
     runs-on: "macos-12"
@@ -49,10 +56,13 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
+        python-version: "3.11"
     - name: Install lancedb
       run: |
         pip install -e .
-        pip install pytest
+        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+        pip install pytest pytest-mock black
+    - name: Black
+      run: black --check --diff --no-color --quiet .
     - name: Run tests
-      run: pytest -x -v --durations=30 tests
+      run: pytest -x -v --durations=30 tests

.github/workflows/rust.yml

@@ -0,0 +1,67 @@
+name: Rust
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - rust/**
+      - .github/workflows/rust.yml
+
+env:
+  # This env var is used by Swatinem/rust-cache@v2 for the cache
+  # key, so we set it to make sure it is always consistent.
+  CARGO_TERM_COLOR: always
+  # Disable full debug symbol generation to speed up CI build and keep memory down
+  # "1" means line tables only, which is useful for panic tracebacks.
+  RUSTFLAGS: "-C debuginfo=1"
+  RUST_BACKTRACE: "1"
+
+jobs:
+  linux:
+    timeout-minutes: 30
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+        working-directory: rust
+    steps:
+    - uses: actions/checkout@v3
+      with:
+          fetch-depth: 0
+          lfs: true
+    - uses: Swatinem/rust-cache@v2
+      with:
+          workspaces: rust
+    - name: Install dependencies
+      run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+    - name: Build
+      run: cargo build --all-features
+    - name: Run tests
+      run: cargo test --all-features
+  macos:
+    runs-on: macos-12
+    timeout-minutes: 30
+    defaults:
+      run:
+        shell: bash
+        working-directory: rust
+    steps:
+      - uses: actions/checkout@v3
+        with:
+            fetch-depth: 0
+            lfs: true
+      - name: CPU features
+        run: sysctl -a | grep cpu
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust
+      - name: Install dependencies
+        run: brew install protobuf
+      - name: Build
+        run: cargo build --all-features
+      - name: Run tests
+        run: cargo test --all-features

.gitignore

@@ -3,6 +3,7 @@
 *.egg-info
 **/__pycache__
 .DS_Store
+venv
 
 rust/target
 rust/Cargo.lock
@@ -15,7 +16,7 @@ site
 python/build
 python/dist
 
-notebooks/.ipynb_checkpoints
+**/.ipynb_checkpoints
 
 **/.hypothesis
 
@@ -30,3 +31,4 @@ node/examples/**/dist
 ## Rust
 target
 
+Cargo.lock

.pre-commit-config.yaml

@@ -8,4 +8,14 @@ repos:
 -   repo: https://github.com/psf/black
     rev: 22.12.0
     hooks:
-    -   id: black
+    -   id: black
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.0.277
+    hooks:
+    - id: ruff
+-   repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+    - id: isort
+      name: isort (python)

Cargo.toml

@@ -4,3 +4,11 @@ members = [
     "rust/ffi/node"
 ]
 resolver = "2"
+
+[workspace.dependencies]
+lance = "0.5.3"
+arrow-array = "40.0"
+arrow-data = "40.0"
+arrow-schema = "40.0"
+arrow-ipc = "40.0"
+object_store = "0.6.1"

README.md

@@ -10,6 +10,10 @@
 <a href="https://discord.gg/zMM32dvNtd">Discord</a> •
 <a href="https://twitter.com/lancedb">Twitter</a>
 
+</p>
+
+<img max-width="750px" alt="LanceDB Multimodal Search" src="https://github.com/lancedb/lancedb/assets/917119/09c5afc5-7816-4687-bae4-f2ca194426ec">
+
 </p>
 </div>
 
@@ -23,13 +27,15 @@ The key features of LanceDB include:
 
 * Store, query and filter vectors, metadata and multi-modal data (text, images, videos, point clouds, and more).
 
+* Support for vector similarity search, full-text search and SQL.
+
 * Native Python and Javascript/Typescript support.
 
 * Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure.
 
 * Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lanecdb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
 
-LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/eto-ai/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
+LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
 
 ## Quick Start
 
@@ -59,7 +65,7 @@ pip install lancedb
 ```python
 import lancedb
 
-uri = "/tmp/lancedb"
+uri = "data/sample-lancedb"
 db = lancedb.connect(uri)
 table = db.create_table("my_table",
                          data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
@@ -69,4 +75,4 @@ result = table.search([100, 100]).limit(2).to_df()
 
 ## Blogs, Tutorials & Videos
 * 📈 <a href="https://blog.eto.ai/benchmarking-random-access-in-lance-ed690757a826">2000x better performance with Lance over Parquet</a>
-* 🤖 <a href="https://github.com/lancedb/lancedb/blob/main/notebooks/youtube_transcript_search.ipynb">Build a question and answer bot with LanceDB</a>
+* 🤖 <a href="https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb">Build a question and answer bot with LanceDB</a>

docs/mkdocs.yml

@@ -1,32 +1,77 @@
-site_name: LanceDB Documentation
+site_name: LanceDB Docs
+repo_url: https://github.com/lancedb/lancedb
+repo_name: lancedb/lancedb
 docs_dir: src
 
 theme:
   name: "material"
+  logo: assets/logo.png
+  favicon: assets/logo.png
   features:
     - content.code.copy
+    - content.tabs.link
+  icon:
+    repo: fontawesome/brands/github
+  custom_dir: overrides
 
 plugins:
 - search
+- autorefs
 - mkdocstrings:
     handlers:
       python:
         paths: [../python]
+        selection:
+          docstring_style: numpy
+        rendering:
+          heading_level: 4
+          show_source: false
+          show_symbol_type_in_heading: true
+          show_signature_annotations: true
+          show_root_heading: true
+          members_order: source
+        import:
+          # for cross references
+          - https://arrow.apache.org/docs/objects.inv
+          - https://pandas.pydata.org/docs/objects.inv
 - mkdocs-jupyter
 
-nav:
-- Home: index.md
-- Basics: basic.md
-- Embeddings: embedding.md
-- Indexing: ann_indexes.md
-- Integrations: integrations.md
-- Python API: python.md
-
 markdown_extensions:
+- admonition
+- footnotes
+- pymdownx.superfences
+- pymdownx.details
 - pymdownx.highlight:
     anchor_linenums: true
     line_spans: __span
     pygments_lang_class: true
 - pymdownx.inlinehilite
 - pymdownx.snippets
-- pymdownx.superfences
+- pymdownx.superfences
+- pymdownx.tabbed:
+    alternate_style: true
+
+nav:
+- Home: index.md
+- Basics: basic.md
+- Embeddings: embedding.md
+- Python full-text search: fts.md
+- Python integrations: integrations.md
+- Python examples:
+  - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
+  - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
+  - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
+  - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
+  - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
+- Javascript examples:
+  - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
+- References:
+  - Vector Search: search.md
+  - SQL filters: sql.md
+  - Indexing: ann_indexes.md
+- API references:
+  - Python API: python/python.md
+  - Javascript API: javascript/modules.md
+
+extra_css:
+  - styles/global.css

docs/overrides/partials/header.html

@@ -0,0 +1,176 @@
+<!--
+  Copyright (c) 2016-2023 Martin Donath <martin.donath@squidfunk.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to
+  deal in the Software without restriction, including without limitation the
+  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+  sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+  IN THE SOFTWARE.
+-->
+
+{% set class = "md-header" %}
+{% if "navigation.tabs.sticky" in features %}
+  {% set class = class ~ " md-header--shadow md-header--lifted" %}
+{% elif "navigation.tabs" not in features %}
+  {% set class = class ~ " md-header--shadow" %}
+{% endif %}
+
+<!-- Header -->
+<header class="{{ class }}" data-md-component="header">
+  <nav
+    class="md-header__inner md-grid"
+    aria-label="{{ lang.t('header') }}"
+  >
+
+    <!-- Link to home -->
+    <a
+      href="{{ config.extra.homepage | d(nav.homepage.url, true) | url }}"
+      title="{{ config.site_name | e }}"
+      class="md-header__button md-logo"
+      aria-label="{{ config.site_name }}"
+      data-md-component="logo"
+    >
+      {% include "partials/logo.html" %}
+    </a>
+
+    <!-- Button to open drawer -->
+    <label class="md-header__button md-icon" for="__drawer">
+      {% include ".icons/material/menu" ~ ".svg" %}
+    </label>
+
+    <!-- Header title -->
+    <div class="md-header__title" style="width: auto !important;" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            {{ config.site_name }}
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+            {% if page.meta and page.meta.title %}
+              {{ page.meta.title }}
+            {% else %}
+              {{ page.title }}
+            {% endif %}
+          </span>
+        </div>
+      </div>
+    </div>
+
+    <!-- Color palette -->
+    {% if config.theme.palette %}
+      {% if not config.theme.palette is mapping %}
+        <form class="md-header__option" data-md-component="palette">
+          {% for option in config.theme.palette %}
+            {% set scheme  = option.scheme  | d("default", true) %}
+            {% set primary = option.primary | d("indigo", true) %}
+            {% set accent  = option.accent  | d("indigo", true) %}
+            <input
+              class="md-option"
+              data-md-color-media="{{ option.media }}"
+              data-md-color-scheme="{{ scheme | replace(' ', '-') }}"
+              data-md-color-primary="{{ primary | replace(' ', '-') }}"
+              data-md-color-accent="{{ accent | replace(' ', '-') }}"
+              {% if option.toggle %}
+                aria-label="{{ option.toggle.name }}"
+              {% else  %}
+                aria-hidden="true"
+              {% endif %}
+              type="radio"
+              name="__palette"
+              id="__palette_{{ loop.index }}"
+            />
+            {% if option.toggle %}
+              <label
+                class="md-header__button md-icon"
+                title="{{ option.toggle.name }}"
+                for="__palette_{{ loop.index0 or loop.length }}"
+                hidden
+              >
+                {% include ".icons/" ~ option.toggle.icon ~ ".svg" %}
+              </label>
+            {% endif %}
+          {% endfor %}
+        </form>
+      {% endif %}
+    {% endif %}
+
+    <!-- Site language selector -->
+    {% if config.extra.alternate %}
+      <div class="md-header__option">
+        <div class="md-select">
+          {% set icon = config.theme.icon.alternate or "material/translate" %}
+          <button
+            class="md-header__button md-icon"
+            aria-label="{{ lang.t('select.language') }}"
+          >
+            {% include ".icons/" ~ icon ~ ".svg" %}
+          </button>
+          <div class="md-select__inner">
+            <ul class="md-select__list">
+              {% for alt in config.extra.alternate %}
+                <li class="md-select__item">
+                  <a
+                    href="{{ alt.link | url }}"
+                    hreflang="{{ alt.lang }}"
+                    class="md-select__link"
+                  >
+                    {{ alt.name }}
+                  </a>
+                </li>
+              {% endfor %}
+            </ul>
+          </div>
+        </div>
+      </div>
+    {% endif %}
+
+    <!-- Button to open search modal -->
+    {% if "material/search" in config.plugins %}
+      <label class="md-header__button md-icon" for="__search">
+        {% include ".icons/material/magnify.svg" %}
+      </label>
+
+      <!-- Search interface -->
+      {% include "partials/search.html" %}
+    {% endif %}
+
+    <div style="margin-left: 10px; margin-right: 5px;">
+        <a href="https://discord.com/invite/zMM32dvNtd" target="_blank" rel="noopener noreferrer">
+            <svg fill="#FFFFFF" xmlns="http://www.w3.org/2000/svg"  viewBox="0 0 50 50" width="25px" height="25px"><path d="M 41.625 10.769531 C 37.644531 7.566406 31.347656 7.023438 31.078125 7.003906 C 30.660156 6.96875 30.261719 7.203125 30.089844 7.589844 C 30.074219 7.613281 29.9375 7.929688 29.785156 8.421875 C 32.417969 8.867188 35.652344 9.761719 38.578125 11.578125 C 39.046875 11.867188 39.191406 12.484375 38.902344 12.953125 C 38.710938 13.261719 38.386719 13.429688 38.050781 13.429688 C 37.871094 13.429688 37.6875 13.378906 37.523438 13.277344 C 32.492188 10.15625 26.210938 10 25 10 C 23.789063 10 17.503906 10.15625 12.476563 13.277344 C 12.007813 13.570313 11.390625 13.425781 11.101563 12.957031 C 10.808594 12.484375 10.953125 11.871094 11.421875 11.578125 C 14.347656 9.765625 17.582031 8.867188 20.214844 8.425781 C 20.0625 7.929688 19.925781 7.617188 19.914063 7.589844 C 19.738281 7.203125 19.34375 6.960938 18.921875 7.003906 C 18.652344 7.023438 12.355469 7.566406 8.320313 10.8125 C 6.214844 12.761719 2 24.152344 2 34 C 2 34.175781 2.046875 34.34375 2.132813 34.496094 C 5.039063 39.605469 12.972656 40.941406 14.78125 41 C 14.789063 41 14.800781 41 14.8125 41 C 15.132813 41 15.433594 40.847656 15.621094 40.589844 L 17.449219 38.074219 C 12.515625 36.800781 9.996094 34.636719 9.851563 34.507813 C 9.4375 34.144531 9.398438 33.511719 9.765625 33.097656 C 10.128906 32.683594 10.761719 32.644531 11.175781 33.007813 C 11.234375 33.0625 15.875 37 25 37 C 34.140625 37 38.78125 33.046875 38.828125 33.007813 C 39.242188 32.648438 39.871094 32.683594 40.238281 33.101563 C 40.601563 33.515625 40.5625 34.144531 40.148438 34.507813 C 40.003906 34.636719 37.484375 36.800781 32.550781 38.074219 L 34.378906 40.589844 C 34.566406 40.847656 34.867188 41 35.1875 41 C 35.199219 41 35.210938 41 35.21875 41 C 37.027344 40.941406 44.960938 39.605469 47.867188 34.496094 C 47.953125 34.34375 48 34.175781 48 34 C 48 24.152344 43.785156 12.761719 41.625 10.769531 Z M 18.5 30 C 16.566406 30 15 28.210938 15 26 C 15 23.789063 16.566406 22 18.5 22 C 20.433594 22 22 23.789063 22 26 C 22 28.210938 20.433594 30 18.5 30 Z M 31.5 30 C 29.566406 30 28 28.210938 28 26 C 28 23.789063 29.566406 22 31.5 22 C 33.433594 22 35 23.789063 35 26 C 35 28.210938 33.433594 30 31.5 30 Z"/></svg>
+        </a>
+    </div>
+    <div style="margin-left: 5px; margin-right: 5px;">
+        <a href="https://twitter.com/lancedb" target="_blank" rel="noopener noreferrer">
+            <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0,0,256,256" width="25px" height="25px" fill-rule="nonzero"><g fill-opacity="0" fill="#ffffff" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><path d="M0,256v-256h256v256z" id="bgRectangle"></path></g><g fill="#ffffff" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(4,4)"><path d="M57,17.114c-1.32,1.973 -2.991,3.707 -4.916,5.097c0.018,0.423 0.028,0.847 0.028,1.274c0,13.013 -9.902,28.018 -28.016,28.018c-5.562,0 -12.81,-1.948 -15.095,-4.423c0.772,0.092 1.556,0.138 2.35,0.138c4.615,0 8.861,-1.575 12.23,-4.216c-4.309,-0.079 -7.946,-2.928 -9.199,-6.84c1.96,0.308 4.447,-0.17 4.447,-0.17c0,0 -7.7,-1.322 -7.899,-9.779c2.226,1.291 4.46,1.231 4.46,1.231c0,0 -4.441,-2.734 -4.379,-8.195c0.037,-3.221 1.331,-4.953 1.331,-4.953c8.414,10.361 20.298,10.29 20.298,10.29c0,0 -0.255,-1.471 -0.255,-2.243c0,-5.437 4.408,-9.847 9.847,-9.847c2.832,0 5.391,1.196 7.187,3.111c2.245,-0.443 4.353,-1.263 6.255,-2.391c-0.859,3.44 -4.329,5.448 -4.329,5.448c0,0 2.969,-0.329 5.655,-1.55z"></path></g></g></svg>
+        </a>
+    </div>
+
+    <!-- Repository information -->
+    {% if config.repo_url %}
+      <div class="md-header__source" style="margin-left: -5px !important;">
+        {% include "partials/source.html" %}
+      </div>
+    {% endif %}
+  </nav>
+
+  <!-- Navigation tabs (sticky) -->
+  {% if "navigation.tabs.sticky" in features %}
+    {% if "navigation.tabs" in features %}
+      {% include "partials/tabs.html" %}
+    {% endif %}
+  {% endif %}
+</header>

docs/src/ann_indexes.md

@@ -12,29 +12,43 @@ In the future we will look to automatically create and configure the ANN index.
 
 ## Creating an ANN Index
 
-Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
+=== "Python"
+     Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
 
-```python
+     ```python
-import lancedb
+     import lancedb
-import numpy as np
+     import numpy as np
-uri = "~/.lancedb"
+     uri = "data/sample-lancedb"
-db = lancedb.connect(uri)
+     db = lancedb.connect(uri)
 
-# Create 10,000 sample vectors
+     # Create 10,000 sample vectors
-data = [{"vector": row, "item": f"item {i}"}
+     data = [{"vector": row, "item": f"item {i}"}
-   for i, row in enumerate(np.random.random((10_000, 768)).astype('float32'))]
+        for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]
 
-# Add the vectors to a table
+     # Add the vectors to a table
-tbl = db.create_table("my_vectors", data=data)
+     tbl = db.create_table("my_vectors", data=data)
 
-# Create and train the index - you need to have enough data in the table for an effective training step
+     # Create and train the index - you need to have enough data in the table for an effective training step
-tbl.create_index(num_partitions=256, num_sub_vectors=96)
+     tbl.create_index(num_partitions=256, num_sub_vectors=96)
-```
+     ```
+
+=== "Javascript"
+     ```javascript
+     const vectordb = require('vectordb')
+     const db = await vectordb.connect('data/sample-lancedb')
+
+     let data = []
+     for (let i = 0; i < 10_000; i++) {
+         data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
+     }
+     const table = await db.createTable('my_vectors', data)
+     await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 256, num_sub_vectors: 96 })
+     ```
 
 Since `create_index` has a training step, it can take a few minutes to finish for large tables. You can control the index
 creation by providing the following parameters:
 
-- **metric** (default: "L2"): The distance metric to use. By default we use euclidean distance. We also support cosine distance.
+- **metric** (default: "L2"): The distance metric to use. By default we use euclidean distance. We also support "cosine" distance.
 - **num_partitions** (default: 256): The number of partitions of the index. The number of partitions should be configured so each partition has 3-5K vectors. For example, a table
 with ~1M vectors should use 256 partitions. You can specify arbitrary number of partitions but powers of 2 is most conventional.
 A higher number leads to faster queries, but it makes index generation slower.
@@ -53,22 +67,33 @@ There are a couple of parameters that can be used to fine-tune the search:
   e.g., for 1M vectors divided up into 256 partitions, nprobes should be set to ~20-40.<br/>
   Note: nprobes is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
 - **refine_factor** (default: None): Refine the results by reading extra elements and re-ranking them in memory.<br/>
-  A higher number makes search more accurate but also slower. If you find the recall is less than idea, try refine_factor=10 to start.<br/>
+  A higher number makes search more accurate but also slower. If you find the recall is less than ideal, try refine_factor=10 to start.<br/>
   e.g., for 1M vectors divided into 256 partitions, if you're looking for top 20, then refine_factor=200 reranks the whole partition.<br/>
   Note: refine_factor is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
 
-
+=== "Python"
-```python
+     ```python
-tbl.search(np.random.random((768))) \
+     tbl.search(np.random.random((1536))) \
-    .limit(2) \
+         .limit(2) \
-    .nprobes(20) \
+         .nprobes(20) \
-    .refine_factor(10) \
+         .refine_factor(10) \
-    .to_df()
+         .to_df()
-
+     ```
+     ```
                                               vector       item       score
-0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
+     0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
-1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
+     1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
-```
+     ```
+
+=== "Javascript"
+     ```javascript
+     const results_1 = await table
+         .search(Array(1536).fill(1.2))
+         .limit(2)
+         .nprobes(20)
+         .refineFactor(10)
+         .execute()
+     ```
 
 The search will return the data requested in addition to the score of each item.
 
@@ -78,18 +103,38 @@ The search will return the data requested in addition to the score of each item.
 
 You can further filter the elements returned by a search using a where clause.
 
-```python
+=== "Python"
-tbl.search(np.random.random((768))).where("item != 'item 1141'").to_df()
+     ```python
-```
+     tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_df()
+     ```
+
+=== "Javascript"
+     ```javascript
+     const results_2 = await table
+         .search(Array(1536).fill(1.2))
+         .where("id != '1141'")
+         .execute()
+     ```
 
 ### Projections (select clause)
 
 You can select the columns returned by the query using a select clause.
 
-```python
+=== "Python"
-tbl.search(np.random.random((768))).select(["vector"]).to_df()
+     ```python
-                                              vector      score
+     tbl.search(np.random.random((1536))).select(["vector"]).to_df()
-0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
+     ```
-1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
+     ```
-...
+     vector      score
-```
+     0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
+     1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
+     ...
+     ```
+
+=== "Javascript"
+     ```javascript
+     const results_3 = await table
+         .search(Array(1536).fill(1.2))
+         .select(["id"])
+         .execute()
+     ```

docs/src/basic.md

@@ -1,74 +1,142 @@
 # Basic LanceDB Functionality
 
+We'll cover the basics of using LanceDB on your local machine in this section.
+
+??? info "LanceDB runs embedded on your backend application, so there is no need to run a separate server."
+
+      <img src="../assets/lancedb_embedded_explanation.png" width="650px" />
+
+## Installation
+
+=== "Python"
+      ```shell
+      pip install lancedb
+      ```
+
+=== "Javascript"
+      ```shell
+      npm install vectordb
+      ```
+
 ## How to connect to a database
 
-In local mode, LanceDB stores data in a directory on your local machine. To connect to a local database, you can use the following code:
+=== "Python"
-```python
+      ```python
-import lancedb
+      import lancedb
-uri = "~/.lancedb"
+      uri = "data/sample-lancedb"
-db = lancedb.connect(uri)
+      db = lancedb.connect(uri)
-```
+      ```
 
-LanceDB will create the directory if it doesn't exist (including parent directories).
+      LanceDB will create the directory if it doesn't exist (including parent directories).
 
-If you need a reminder of the uri, use the `db.uri` property.
+      If you need a reminder of the uri, use the `db.uri` property.
+
+=== "Javascript"
+      ```javascript
+      const lancedb = require("vectordb");
+
+      const uri = "data/sample-lancedb";
+      const db = await lancedb.connect(uri);
+      ```
+      
+      LanceDB will create the directory if it doesn't exist (including parent directories).
+
+      If you need a reminder of the uri, you can call `db.uri()`.
 
 ## How to create a table
 
-To create a table, you can use the following code:
+=== "Python"
-```python
+      ```python
-tbl = db.create_table("my_table",
+      tbl = db.create_table("my_table",
-                      data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                        data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                            {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+                              {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-```
+      ```
 
-Under the hood, LanceDB is converting the input data into an Apache Arrow table
+      If the table already exists, LanceDB will raise an error by default.
-and persisting it to disk in [Lance format](github.com/eto-ai/lance).
+      If you want to overwrite the table, you can pass in `mode="overwrite"`
+      to the `create_table` method.
 
-If the table already exists, LanceDB will raise an error by default.
+      You can also pass in a pandas DataFrame directly:
-If you want to overwrite the table, you can pass in `mode="overwrite"`
+      ```python
-to the `create_table` method.
+      import pandas as pd
+      df = pd.DataFrame([{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+      tbl = db.create_table("table_from_df", data=df)
+      ```
 
-You can also pass in a pandas DataFrame directly:
+=== "Javascript"
-```python
+      ```javascript
-import pandas as pd
+      const tb = await db.createTable("my_table",
-df = pd.DataFrame([{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                        data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                   {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+                              {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-tbl = db.create_table("table_from_df", data=df)
+      ```
-```
+      
+!!! warning
+
+      If the table already exists, LanceDB will raise an error by default.
+      If you want to overwrite the table, you can pass in `mode="overwrite"`
+      to the `createTable` function.
+
+??? info "Under the hood, LanceDB is converting the input data into an Apache Arrow table and persisting it to disk in [Lance format](https://www.github.com/lancedb/lance)."
 
 ## How to open an existing table
 
 Once created, you can open a table using the following code:
-```python
-tbl = db.open_table("my_table")
-```
 
-If you forget the name of your table, you can always get a listing of all table names:
+=== "Python"
+      ```python
+      tbl = db.open_table("my_table")
+      ```
 
-```python
+      If you forget the name of your table, you can always get a listing of all table names:
-db.table_names()
+
-```
+      ```python
+      print(db.table_names())
+      ```
+
+=== "Javascript"
+      ```javascript
+      const tbl = await db.openTable("my_table");
+      ```
+
+      If you forget the name of your table, you can always get a listing of all table names:
+
+      ```javascript
+      console.log(await db.tableNames());
+      ```
 
 ## How to add data to a table
 
 After a table has been created, you can always add more data to it using
 
-```python
+=== "Python"
-df = pd.DataFrame([{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
+      ```python
-                   {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}])
+      df = pd.DataFrame([{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
-tbl.add(df)
+                        {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}])
-```
+      tbl.add(df)
+      ```
+
+=== "Javascript"
+      ```javascript
+      await tbl.add([{vector: [1.3, 1.4], item: "fizz", price: 100.0},
+              {vector: [9.5, 56.2], item: "buzz", price: 200.0}])
+      ```
 
 ## How to search for (approximate) nearest neighbors
 
 Once you've embedded the query, you can find its nearest neighbors using the following code:
 
-```python
+=== "Python"
-tbl.search([100, 100]).limit(2).to_df()
+      ```python
-```
+      tbl.search([100, 100]).limit(2).to_df()
+      ```
 
-This returns a pandas DataFrame with the results.
+      This returns a pandas DataFrame with the results.
+
+=== "Javascript"
+      ```javascript
+      const query = await tbl.search([100, 100]).limit(2).execute();
+      ```
 
 ## What's next
 

docs/src/embedding.md

@@ -25,55 +25,88 @@ def embed_func(batch):
     return [model.encode(sentence) for sentence in batch]
 ```
 
+Please note that currently HuggingFace is only supported in the Python SDK.
+
 ### OpenAI example
 
 You can also use an external API like OpenAI to generate embeddings
 
-```python
+=== "Python"
-import openai
+      ```python
-import os
+        import openai
+        import os
 
-# Configuring the environment variable OPENAI_API_KEY
+        # Configuring the environment variable OPENAI_API_KEY
-if "OPENAI_API_KEY" not in os.environ:
+        if "OPENAI_API_KEY" not in os.environ:
-    # OR set the key here as a variable
+        # OR set the key here as a variable
-    openai.api_key = "sk-..."
+        openai.api_key = "sk-..."
 
-# verify that the API key is working
+        # verify that the API key is working
-assert len(openai.Model.list()["data"]) > 0
+        assert len(openai.Model.list()["data"]) > 0
 
-def embed_func(c):
+        def embed_func(c):
-    rs = openai.Embedding.create(input=c, engine="text-embedding-ada-002")
+            rs = openai.Embedding.create(input=c, engine="text-embedding-ada-002")
-    return [record["embedding"] for record in rs["data"]]
+        return [record["embedding"] for record in rs["data"]]
-```
+      ```
+
+=== "Javascript"
+      ```javascript
+        const lancedb = require("vectordb");
+
+        // You need to provide an OpenAI API key
+        const apiKey = "sk-..."
+        // The embedding function will create embeddings for the 'text' column
+        const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
+      ```
 
 ## Applying an embedding function
 
-Using an embedding function, you can apply it to raw data
+=== "Python"
-to generate embeddings for each row.
+    Using an embedding function, you can apply it to raw data
+    to generate embeddings for each row.
 
-Say if you have a pandas DataFrame with a `text` column that you want to be embedded,
+    Say if you have a pandas DataFrame with a `text` column that you want to be embedded,
-you can use the [with_embeddings](https://lancedb.github.io/lancedb/python/#lancedb.embeddings.with_embeddings)
+    you can use the [with_embeddings](https://lancedb.github.io/lancedb/python/#lancedb.embeddings.with_embeddings)
-function to generate embeddings and add create a combined pyarrow table:
+    function to generate embeddings and add create a combined pyarrow table:
 
-```python
-import pandas as pd
-from lancedb.embeddings import with_embeddings
 
-df = pd.DataFrame([{"text": "pepperoni"},
+     ```python
-                   {"text": "pineapple"}])
+     import pandas as pd
-data = with_embeddings(embed_func, df)
+     from lancedb.embeddings import with_embeddings
 
-# The output is used to create / append to a table
+     df = pd.DataFrame([{"text": "pepperoni"},
-# db.create_table("my_table", data=data)
+                        {"text": "pineapple"}])
-```
+     data = with_embeddings(embed_func, df)
 
-If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
+     # The output is used to create / append to a table
+     # db.create_table("my_table", data=data)
+     ```
 
-By default, LanceDB calls the function with batches of 1000 rows. This can be configured
+     If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
-using the `batch_size` parameter to `with_embeddings`.
+
+     By default, LanceDB calls the function with batches of 1000 rows. This can be configured
+     using the `batch_size` parameter to `with_embeddings`.
+
+     LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
+     API call is reliable.
+
+=== "Javascript"
+     Using an embedding function, you can apply it to raw data
+     to generate embeddings for each row.
+
+     You can just pass the embedding function created previously and LanceDB will automatically generate
+     embededings for your data.
+
+      ```javascript
+      const db = await lancedb.connect("data/sample-lancedb");
+      const data = [
+        { text: 'pepperoni'  },
+        { text: 'pineapple' }
+      ]
+
+      const table = await db.createTable('vectors', data, embedding)
+      ```
 
-LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
-API call is reliable.
 
 ## Searching with an embedding function
 
@@ -81,13 +114,25 @@ At inference time, you also need the same embedding function to embed your query
 It's important that you use the same model / function otherwise the embedding vectors don't
 belong in the same latent space and your results will be nonsensical.
 
-```python
+=== "Python"
-query = "What's the best pizza topping?"
+     ```python
-query_vector = embed_func([query])[0]
+     query = "What's the best pizza topping?"
-tbl.search(query_vector).limit(10).to_df()
+     query_vector = embed_func([query])[0]
-```
+     tbl.search(query_vector).limit(10).to_df()
+     ```
+
+     The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
+
+=== "Javascript"
+     ```javascript
+      const results = await table
+        .search('What's the best pizza topping?')
+        .limit(10)
+        .execute()
+     ```
+
+     The above snippet returns an array of records with the 10 closest vectors to the query.
 
-The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
 
 ## Roadmap
 

docs/src/examples/code_documentation_qa_bot_with_langchain.md

@@ -4,4 +4,4 @@
 
 <img id="splash" width="400" alt="langchain" src="https://user-images.githubusercontent.com/917119/236580868-61a246a9-e587-4c2b-8ae5-6fe5f7b7e81e.png">
 
-This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/notebooks/code_qa_bot.ipynb)
+This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/code_qa_bot.ipynb)

docs/src/examples/modal_langchain.py

@@ -0,0 +1,117 @@
+import pickle
+import re
+import sys
+import zipfile
+from pathlib import Path
+
+import requests
+from langchain.chains import RetrievalQA
+from langchain.document_loaders import UnstructuredHTMLLoader
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.llms import OpenAI
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores import LanceDB
+from modal import Image, Secret, Stub, web_endpoint
+
+import lancedb
+
+lancedb_image = Image.debian_slim().pip_install(
+    "lancedb", "langchain", "openai", "pandas", "tiktoken", "unstructured", "tabulate"
+)
+
+stub = Stub(
+    name="example-langchain-lancedb",
+    image=lancedb_image,
+    secrets=[Secret.from_name("my-openai-secret")],
+)
+
+docsearch = None
+docs_path = Path("docs.pkl")
+db_path = Path("lancedb")
+
+
+def get_document_title(document):
+    m = str(document.metadata["source"])
+    title = re.findall("pandas.documentation(.*).html", m)
+    if title[0] is not None:
+        return title[0]
+    return ""
+
+
+def download_docs():
+    pandas_docs = requests.get(
+        "https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip"
+    )
+    with open(Path("pandas.documentation.zip"), "wb") as f:
+        f.write(pandas_docs.content)
+
+    file = zipfile.ZipFile(Path("pandas.documentation.zip"))
+    file.extractall(path=Path("pandas_docs"))
+
+
+def store_docs():
+    docs = []
+
+    if not docs_path.exists():
+        for p in Path("pandas_docs/pandas.documentation").rglob("*.html"):
+            if p.is_dir():
+                continue
+            loader = UnstructuredHTMLLoader(p)
+            raw_document = loader.load()
+
+            m = {}
+            m["title"] = get_document_title(raw_document[0])
+            m["version"] = "2.0rc0"
+            raw_document[0].metadata = raw_document[0].metadata | m
+            raw_document[0].metadata["source"] = str(raw_document[0].metadata["source"])
+            docs = docs + raw_document
+
+        with docs_path.open("wb") as fh:
+            pickle.dump(docs, fh)
+    else:
+        with docs_path.open("rb") as fh:
+            docs = pickle.load(fh)
+
+    return docs
+
+
+def qanda_langchain(query):
+    download_docs()
+    docs = store_docs()
+
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200,)
+    documents = text_splitter.split_documents(docs)
+    embeddings = OpenAIEmbeddings()
+
+    db = lancedb.connect(db_path)
+    table = db.create_table(
+        "pandas_docs",
+        data=[
+            {
+                "vector": embeddings.embed_query("Hello World"),
+                "text": "Hello World",
+                "id": "1",
+            }
+        ],
+        mode="overwrite",
+    )
+    docsearch = LanceDB.from_documents(documents, embeddings, connection=table)
+    qa = RetrievalQA.from_chain_type(
+        llm=OpenAI(), chain_type="stuff", retriever=docsearch.as_retriever()
+    )
+    return qa.run(query)
+
+
+@stub.function()
+@web_endpoint(method="GET")
+def web(query: str):
+    answer = qanda_langchain(query)
+    return {
+        "answer": answer,
+    }
+
+
+@stub.function()
+def cli(query: str):
+    answer = qanda_langchain(query)
+    print(answer)

docs/src/examples/multimodal_search.md

@@ -0,0 +1,7 @@
+# Image multimodal search
+
+## Search through an image dataset using natural language, full text and SQL
+
+<img id="splash" width="400" alt="multimodal search" src="https://github.com/lancedb/lancedb/assets/917119/993a7c9f-be01-449d-942e-1ce1d4ed63af">
+
+This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/multimodal_search.ipynb)

docs/src/examples/nodejs.md

@@ -1,99 +0,0 @@
-# YouTube transcript QA bot with NodeJS
-
-## use LanceDB's Javascript API and OpenAI to build a QA bot for YouTube transcripts
-
-<img id="splash" width="400" alt="nodejs" src="https://github.com/lancedb/lancedb/assets/917119/3a140e75-bf8e-438a-a1e4-af14a72bcf98">
-
-This Q&A bot will allow you to search through youtube transcripts using natural language! We'll introduce how you can use LanceDB's Javascript API to store and manage your data easily.
-
-For this example we're using a HuggingFace dataset that contains YouTube transcriptions: `jamescalam/youtube-transcriptions`, to make it easier, we've converted it to a LanceDB `db` already, which you can download and put in a working directory:
-
-```wget -c https://eto-public.s3.us-west-2.amazonaws.com/lancedb_demo.tar.gz -O - | tar -xz -C .```
-
-Now, we'll create a simple app that can:
-1. Take a text based query and search for contexts in our corpus, using embeddings generated from the OpenAI Embedding API.
-2. Create a prompt with the contexts, and call the OpenAI Completion API to answer the text based query.
-
-Dependencies and setup of OpenAI API:
-
-```javascript
-const lancedb = require("vectordb");
-const { Configuration, OpenAIApi } = require("openai");
-
-const configuration = new Configuration({
-    apiKey: process.env.OPENAI_API_KEY,
-    });
-const openai = new OpenAIApi(configuration);
-```
-
-First, let's set our question and the context amount. The context amount will be used to query similar documents in our corpus.
-
-```javascript
-const QUESTION = "who was the 12th person on the moon and when did they land?";
-const CONTEXT_AMOUNT = 3;
-```
-
-Now, let's generate an embedding from this question:
-
-```javascript
-const embeddingResponse = await openai.createEmbedding({
-    model: "text-embedding-ada-002",
-    input: QUESTION,
-});
-
-const embedding = embeddingResponse.data["data"][0]["embedding"];
-```
-
-Once we have the embedding, we can connect to LanceDB (using the database we downloaded earlier), and search through the chatbot table.
-We'll extract 3 similar documents found.
-
-```javascript
-const db = await lancedb.connect('./lancedb');
-const tbl = await db.openTable('chatbot');
-const query = tbl.search(embedding);
-query.limit = CONTEXT_AMOUNT;
-const context = await query.execute();
-```
-
-Let's combine the context together so we can pass it into our prompt:
-
-```javascript
-for (let i = 1; i < context.length; i++) {
-    context[0]["text"] += " " + context[i]["text"];
-}
-```
-
-Lastly, let's construct the prompt. You could play around with this to create more accurate/better prompts to yield results.
-
-```javascript
-const prompt = "Answer the question based on the context below.\n\n" +
-    "Context:\n" +
-    `${context[0]["text"]}\n` +
-    `\n\nQuestion: ${QUESTION}\nAnswer:`;
-```
-
-We pass the prompt, along with the context, to the completion API.
-
-```javascript
-const completion = await openai.createCompletion({
-    model: "text-davinci-003",
-    prompt,
-    temperature: 0,
-    max_tokens: 400,
-    top_p: 1,
-    frequency_penalty: 0,
-    presence_penalty: 0,
-});
-```
-
-And that's it!
-
-```javascript
-console.log(completion.data.choices[0].text);
-```
-
-The response is (which is non deterministic):
-
-```
-The 12th person on the moon was Harrison Schmitt and he landed on December 11, 1972.
-```

docs/src/examples/serverless_qa_bot_with_modal_and_langchain.md

@@ -0,0 +1,166 @@
+# Serverless QA Bot with Modal and LangChain
+
+## use LanceDB's LangChain integration with Modal to run a serverless app
+
+<img id="splash" width="400" alt="modal" src="https://github.com/lancedb/lancedb/assets/917119/7d80a40f-60d7-48a6-972f-dab05000eccf">
+
+We're going to build a QA bot for your documentation using LanceDB's LangChain integration and use Modal for deployment.
+
+Modal is an end-to-end compute platform for model inference, batch jobs, task queues, web apps and more. It's a great way to deploy your LanceDB models and apps.
+
+To get started, ensure that you have created an account and logged into [Modal](https://modal.com/). To follow along, the full source code is available on Github [here](https://github.com/lancedb/lancedb/blob/main/docs/src/examples/modal_langchain.py).
+
+### Setting up Modal
+
+We'll start by specifying our dependencies and creating a new Modal `Stub`:
+
+```python
+lancedb_image = Image.debian_slim().pip_install(
+    "lancedb",
+    "langchain",
+    "openai",
+    "pandas",
+    "tiktoken",
+    "unstructured",
+    "tabulate"
+)
+
+stub = Stub(
+    name="example-langchain-lancedb",
+    image=lancedb_image,
+    secrets=[Secret.from_name("my-openai-secret")],
+)
+```
+
+We're using Modal's Secrets injection to secure our OpenAI key. To set your own, you can access the Modal UI and enter your key.
+
+### Setting up caches for LanceDB and LangChain
+
+Next, we can setup some globals to cache our LanceDB database, as well as our LangChain docsource:
+
+```python
+docsearch = None
+docs_path = Path("docs.pkl")
+db_path = Path("lancedb")
+```
+
+### Downloading our dataset
+
+We're going use a pregenerated dataset, which stores HTML files of the Pandas 2.0 documentation. 
+You could switch this out for your own dataset.
+
+```python
+def download_docs():
+    pandas_docs = requests.get("https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip")
+    with open(Path("pandas.documentation.zip"), "wb") as f:
+        f.write(pandas_docs.content)
+
+    file = zipfile.ZipFile(Path("pandas.documentation.zip"))
+    file.extractall(path=Path("pandas_docs"))
+```
+
+### Pre-processing the dataset and generating metadata
+
+Once we've downloaded it, we want to parse and pre-process them using LangChain, and then vectorize them and store it in LanceDB.
+Let's first create a function that uses LangChains `UnstructuredHTMLLoader` to parse them.
+We can then add our own metadata to it and store it alongside the data, we'll later be able to use this for filtering metadata.
+
+```python
+def store_docs():
+    docs = []
+
+    if not docs_path.exists():
+        for p in Path("pandas_docs/pandas.documentation").rglob("*.html"):
+            if p.is_dir():
+                continue
+            loader = UnstructuredHTMLLoader(p)
+            raw_document = loader.load()
+
+            m = {}
+            m["title"] = get_document_title(raw_document[0])
+            m["version"] = "2.0rc0"
+            raw_document[0].metadata = raw_document[0].metadata | m
+            raw_document[0].metadata["source"] = str(raw_document[0].metadata["source"])
+            docs = docs + raw_document
+
+        with docs_path.open("wb") as fh:
+            pickle.dump(docs, fh)
+    else:
+        with docs_path.open("rb") as fh:
+            docs = pickle.load(fh)
+
+    return docs
+```
+
+### Simple LangChain chain for a QA bot
+
+Now we can create a simple LangChain chain for our QA bot. We'll use the `RecursiveCharacterTextSplitter` to split our documents into chunks, and then use the `OpenAIEmbeddings` to vectorize them.
+
+Lastly, we'll create a LanceDB table and store the vectorized documents in it, then create a `RetrievalQA` model from the chain and return it.
+
+```python
+def qanda_langchain(query):
+    download_docs()
+    docs = store_docs()
+
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=1000,
+        chunk_overlap=200,
+    )
+    documents = text_splitter.split_documents(docs)
+    embeddings = OpenAIEmbeddings()
+
+    db = lancedb.connect(db_path) 
+    table = db.create_table("pandas_docs", data=[
+        {"vector": embeddings.embed_query("Hello World"), "text": "Hello World", "id": "1"}
+    ], mode="overwrite")
+    docsearch = LanceDB.from_documents(documents, embeddings, connection=table)
+    qa = RetrievalQA.from_chain_type(llm=OpenAI(), chain_type="stuff", retriever=docsearch.as_retriever())
+    return qa.run(query)
+```
+
+### Creating our Modal entry points
+
+Now we can create our Modal entry points for our CLI and web endpoint:
+
+```python
+@stub.function()
+@web_endpoint(method="GET")
+def web(query: str):
+    answer = qanda_langchain(query)
+    return {
+        "answer": answer,
+    }
+    
+@stub.function()
+def cli(query: str):
+    answer = qanda_langchain(query)
+    print(answer)
+```
+
+# Testing it out!
+
+Testing the CLI:
+
+```bash
+modal run modal_langchain.py --query "What are the major differences in pandas 2.0?"
+```
+
+Testing the web endpoint:
+
+```bash
+modal serve modal_langchain.py
+```
+
+In the CLI, Modal will provide you a web endpoint. Copy this endpoint URI for the next step.
+Once this is served, then we can hit it with `curl`. 
+
+Note, the first time this runs, it will take a few minutes to download the dataset and vectorize it.
+An actual production example would pre-cache/load the dataset and vectorized documents prior
+
+```bash
+curl --get --data-urlencode "query=What are the major differences in pandas 2.0?" https://your-modal-endpoint-app.modal.run
+
+{"answer":" The major differences in pandas 2.0 include the ability to use any numpy numeric dtype in a Index, installing optional dependencies with pip extras, and enhancements, bug fixes, and performance improvements."}
+```
+

docs/src/examples/youtube_transcript_bot.md

@@ -4,4 +4,4 @@
 
 <img id="splash" width="400" alt="youtube transcript search" src="https://user-images.githubusercontent.com/917119/236965568-def7394d-171c-45f2-939d-8edfeaadd88c.png">
 
-This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/notebooks/youtube_transcript_search.ipynb)
+This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb)

docs/src/examples/youtube_transcript_bot_with_nodejs.md

@@ -0,0 +1,139 @@
+# YouTube transcript QA bot with NodeJS
+
+## use LanceDB's Javascript API and OpenAI to build a QA bot for YouTube transcripts
+
+<img id="splash" width="400" alt="nodejs" src="https://github.com/lancedb/lancedb/assets/917119/3a140e75-bf8e-438a-a1e4-af14a72bcf98">
+
+This Q&A bot will allow you to search through youtube transcripts using natural language! We'll introduce how to use LanceDB's Javascript API to store and manage your data easily.
+
+```bash
+npm install vectordb
+```
+
+## Download the data
+
+For this example, we're using a sample of a HuggingFace dataset that contains YouTube transcriptions: `jamescalam/youtube-transcriptions`. Download and extract this file under the `data` folder:
+
+```bash
+wget -c https://eto-public.s3.us-west-2.amazonaws.com/datasets/youtube_transcript/youtube-transcriptions_sample.jsonl
+```
+
+## Prepare Context
+
+Each item in the dataset contains just a short chunk of text. We'll need to merge a bunch of these chunks together on a rolling basis. For this demo, we'll look back 20 records to create a more complete context for each sentence.
+
+First, we need to read and parse the input file.
+
+```javascript
+const lines = (await fs.readFile(INPUT_FILE_NAME, 'utf-8'))
+  .toString()
+  .split('\n')
+  .filter(line => line.length > 0)
+  .map(line => JSON.parse(line))
+
+const data = contextualize(lines, 20, 'video_id')
+```
+
+The contextualize function groups the transcripts by video_id and then creates the expanded context for each item.
+
+```javascript
+function contextualize (rows, contextSize, groupColumn) {
+  const grouped = []
+  rows.forEach(row => {
+    if (!grouped[row[groupColumn]]) {
+      grouped[row[groupColumn]] = []
+    }
+    grouped[row[groupColumn]].push(row)
+  })
+
+  const data = []
+  Object.keys(grouped).forEach(key => {
+    for (let i = 0; i < grouped[key].length; i++) {
+      const start = i - contextSize > 0 ? i - contextSize : 0
+      grouped[key][i].context = grouped[key].slice(start, i + 1).map(r => r.text).join(' ')
+    }
+    data.push(...grouped[key])
+  })
+  return data
+}
+```
+
+## Create the LanceDB Table
+
+To load our data into LanceDB, we need to create embedding (vectors) for each item. For this example, we will use the OpenAI embedding functions, which have a native integration with LanceDB.
+
+```javascript
+// You need to provide an OpenAI API key, here we read it from the OPENAI_API_KEY environment variable
+const apiKey = process.env.OPENAI_API_KEY
+// The embedding function will create embeddings for the 'context' column
+const embedFunction = new lancedb.OpenAIEmbeddingFunction('context', apiKey)
+// Connects to LanceDB
+const db = await lancedb.connect('data/youtube-lancedb')
+const tbl = await db.createTable('vectors', data, embedFunction)
+```
+
+## Create and answer the prompt
+
+We will accept questions in natural language and use our corpus stored in LanceDB to answer them. First, we need to set up the OpenAI client:
+
+```javascript
+const configuration = new Configuration({ apiKey })
+const openai = new OpenAIApi(configuration)
+```
+
+Then we can prompt questions and use LanceDB to retrieve the three most relevant transcripts for this prompt.
+
+```javascript
+const query = await rl.question('Prompt: ')
+const results = await tbl
+  .search(query)
+  .select(['title', 'text', 'context'])
+  .limit(3)
+  .execute()
+```
+
+The query and the transcripts' context are appended together in a single prompt:
+
+```javascript
+function createPrompt (query, context) {
+    let prompt =
+        'Answer the question based on the context below.\n\n' +
+        'Context:\n'
+
+    // need to make sure our prompt is not larger than max size
+    prompt = prompt + context.map(c => c.context).join('\n\n---\n\n').substring(0, 3750)
+    prompt = prompt + `\n\nQuestion: ${query}\nAnswer:`
+    return prompt
+}
+```
+
+We can now use the OpenAI Completion API to process our custom prompt and give us an answer.
+
+```javascript
+const response = await openai.createCompletion({
+  model: 'text-davinci-003',
+  prompt: createPrompt(query, results),
+  max_tokens: 400,
+  temperature: 0,
+  top_p: 1,
+  frequency_penalty: 0,
+  presence_penalty: 0
+})
+console.log(response.data.choices[0].text)
+```
+
+## Let's put it all together now
+
+Now we can provide queries and have them answered based on your local LanceDB data.
+
+```bash
+Prompt: who was the 12th person on the moon and when did they land?
+ The 12th person on the moon was Harrison Schmitt and he landed on December 11, 1972.
+Prompt: Which training method should I use for sentence transformers when I only have pairs of related sentences?
+ NLI with multiple negative ranking loss.
+```
+
+## That's a wrap
+
+In this example, you learned how to use LanceDB to store and query embedding representations of your local data. The complete example code is on [GitHub](https://github.com/lancedb/lancedb/tree/main/node/examples), and you can also download the LanceDB dataset using [this link](https://eto-public.s3.us-west-2.amazonaws.com/datasets/youtube_transcript/youtube-lancedb.zip).
+

docs/src/fts.md

@@ -0,0 +1,65 @@
+# [EXPERIMENTAL] Full text search
+
+LanceDB now provides experimental support for full text search.
+This is currently Python only. We plan to push the integration down to Rust in the future
+to make this available for JS as well.
+
+## Installation
+
+To use full text search, you must install optional dependency tantivy-py:
+
+# tantivy 0.19.2
+pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+
+
+## Quickstart
+
+Assume:
+1. `table` is a LanceDB Table
+2. `text` is the name of the Table column that we want to index
+
+For example,
+
+```python
+import lancedb
+
+uri = "data/sample-lancedb"
+db = lancedb.connect(uri)
+
+table = db.create_table("my_table",
+            data=[{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
+                  {"vector": [5.9, 26.5], "text": "There are several kittens playing"}])
+
+```
+
+To create the index:
+
+```python
+table.create_fts_index("text")
+```
+
+To search:
+
+```python
+df = table.search("puppy").limit(10).select(["text"]).to_df()
+```
+
+LanceDB automatically looks for an FTS index if the input is str.
+
+## Multiple text columns
+
+If you have multiple columns to index, pass them all as a list to `create_fts_index`:
+
+```python
+table.create_fts_index(["text1", "text2"])
+```
+
+Note that the search API call does not change - you can search over all indexed columns at once.
+
+## Current limitations
+
+1. Currently we do not yet support incremental writes.
+If you add data after fts index creation, it won't be reflected
+in search results until you do a full reindex.
+
+2. We currently only support local filesystem paths for the fts index.

docs/src/index.md

@@ -1,6 +1,6 @@
 # Welcome to LanceDB's Documentation
 
-LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrivial, filtering and management of embeddings.
+LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrevial, filtering and management of embeddings.
 
 The key features of LanceDB include:
 
@@ -8,42 +8,65 @@ The key features of LanceDB include:
 
 * Store, query and filter vectors, metadata and multi-modal data (text, images, videos, point clouds, and more).
 
-* Native Python and Javascript/Typescript support (coming soon).
+* Support for vector similarity search, full-text search and SQL.
+
+* Native Python and Javascript/Typescript support.
 
 * Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure.
 
-* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lanecdb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
+* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
 
-LanceDB's core is written in Rust 🦀 and is built using Lance, an open-source columnar format designed for performant ML workloads.
+LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
 
+## Quick Start
 
-## Installation
+=== "Python"
+      ```shell
+      pip install lancedb
+      ```
 
-```shell
+      ```python
-pip install lancedb
+      import lancedb
-```
 
-## Quickstart
+      uri = "data/sample-lancedb"
+      db = lancedb.connect(uri)
+      table = db.create_table("my_table",
+                              data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                                    {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+      result = table.search([100, 100]).limit(2).to_df()
+      ```
 
-```python
+=== "Javascript"
-import lancedb
+      ```shell
+      npm install vectordb
+      ```
 
-db = lancedb.connect(".")
+      ```javascript
-table = db.create_table("my_table",
+      const lancedb = require("vectordb");
-                         data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                               {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-result = table.search([100, 100]).limit(2).to_df()
-```
 
-## Complete Demos
+      const uri = "data/sample-lancedb";
+      const db = await lancedb.connect(uri);
+      const table = await db.createTable("my_table", 
+            [{ id: 1, vector: [3.1, 4.1], item: "foo", price: 10.0 },
+            { id: 2, vector: [5.9, 26.5], item: "bar", price: 20.0 }])
+      const results = await table.search([100, 100]).limit(2).execute();
+      ```
 
-We will be adding completed demo apps built using LanceDB.
+## Complete Demos (Python)
-- [YouTube Transcript Search](../notebooks/youtube_transcript_search.ipynb)
+- [YouTube Transcript Search](notebooks/youtube_transcript_search.ipynb)
+- [Documentation QA Bot using LangChain](notebooks/code_qa_bot.ipynb)
+- [Multimodal search using CLIP](notebooks/multimodal_search.ipynb)
+- [Serverless QA Bot with S3 and Lambda](examples/serverless_lancedb_with_s3_and_lambda.md)
+- [Serverless QA Bot with Modal](examples/serverless_qa_bot_with_modal_and_langchain.md)
 
+## Complete Demos (JavaScript)
+- [YouTube Transcript Search](examples/youtube_transcript_bot_with_nodejs.md)
 
 ## Documentation Quick Links
 * [`Basic Operations`](basic.md) - basic functionality of LanceDB.
 * [`Embedding Functions`](embedding.md) - functions for working with embeddings.
 * [`Indexing`](ann_indexes.md) - create vector indexes to speed up queries.
+* [`Full text search`](fts.md) - [EXPERIMENTAL] full-text search API
 * [`Ecosystem Integrations`](integrations.md) - integrating LanceDB with python data tooling ecosystem.
-* [`API Reference`](python.md) - detailed documentation for the LanceDB Python SDK.
+* [`Python API Reference`](python/python.md) - detailed documentation for the LanceDB Python SDK.
+* [`Node API Reference`](javascript/modules.md) - detailed documentation for the LanceDB Python SDK.

docs/src/integrations.md

@@ -6,11 +6,11 @@ Built on top of Apache Arrow, `LanceDB` is easy to integrate with the Python eco
 
 First, we need to connect to a `LanceDB` database.
 
-``` py
+```py
 
 import lancedb
 
-db = lancedb.connect("/tmp/lancedb")
+db = lancedb.connect("data/sample-lancedb")
 ```
 
 And write a `Pandas DataFrame` to LanceDB directly.
@@ -24,12 +24,9 @@ data = pd.DataFrame({
     "price": [10.0, 20.0]
 })
 table = db.create_table("pd_table", data=data)
-
-# Optionally, create a IVF_PQ index
-table.create_index(num_partitions=256, num_sub_vectors=96)
 ```
 
-You will find detailed instructions of creating dataset and index in [Basic Operations](basic.md) and [Indexing](indexing.md)
+You will find detailed instructions of creating dataset and index in [Basic Operations](basic.md) and [Indexing](ann_indexes.md)
 sections.
 
 
@@ -82,7 +79,7 @@ We will re-use the dataset created previously
 ```python
 import lancedb
 
-db = lancedb.connect("/tmp/lancedb")
+db = lancedb.connect("data/sample-lancedb")
 table = db.open_table("pd_table")
 arrow_table = table.to_arrow()
 ```
@@ -90,8 +87,12 @@ arrow_table = table.to_arrow()
 `DuckDB` can directly query the `arrow_table`:
 
 ```python
-In [15]: duckdb.query("SELECT * FROM t")
+import duckdb 
-Out[15]:
+
+duckdb.query("SELECT * FROM arrow_table")
+```
+
+```
 ┌─────────────┬─────────┬────────┐
 │   vector    │  item   │ price  │
 │   float[]   │ varchar │ double │
@@ -99,8 +100,12 @@ Out[15]:
 │ [3.1, 4.1]  │ foo     │   10.0 │
 │ [5.9, 26.5] │ bar     │   20.0 │
 └─────────────┴─────────┴────────┘
+```
+```python
+duckdb.query("SELECT mean(price) FROM arrow_table")
+```
 
-In [16]: duckdb.query("SELECT mean(price) FROM t")
+```
 Out[16]:
 ┌─────────────┐
 │ mean(price) │

docs/src/javascript/.nojekyll

@@ -0,0 +1 @@
+TypeDoc added this file to prevent GitHub Pages from using Jekyll. You can turn off this behavior by setting the `githubPages` option to false.

docs/src/javascript/README.md

@@ -0,0 +1,47 @@
+vectordb / [Exports](modules.md)
+
+# LanceDB
+
+A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb).
+
+## Installation
+
+```bash
+npm install vectordb
+```
+
+## Usage
+
+### Basic Example
+
+```javascript
+const lancedb = require('vectordb');
+const db = await lancedb.connect('data/sample-lancedb');
+const table = await db.createTable("my_table",
+      [{ id: 1, vector: [0.1, 1.0], item: "foo", price: 10.0 },
+      { id: 2, vector: [3.9, 0.5], item: "bar", price: 20.0 }])
+const results = await table.search([0.1, 0.3]).limit(20).execute();
+console.log(results);
+```
+
+The [examples](./examples) folder contains complete examples.
+
+## Development
+
+Run the tests with
+
+```bash
+npm test
+```
+
+To run the linter and have it automatically fix all errors
+
+```bash
+npm run lint -- --fix
+```
+
+To build documentation
+
+```bash
+npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
+```

docs/src/javascript/classes/LocalConnection.md

@@ -0,0 +1,294 @@
+[vectordb](../README.md) / [Exports](../modules.md) / LocalConnection
+
+# Class: LocalConnection
+
+A connection to a LanceDB database.
+
+## Implements
+
+- [`Connection`](../interfaces/Connection.md)
+
+## Table of contents
+
+### Constructors
+
+- [constructor](LocalConnection.md#constructor)
+
+### Properties
+
+- [\_db](LocalConnection.md#_db)
+- [\_uri](LocalConnection.md#_uri)
+
+### Accessors
+
+- [uri](LocalConnection.md#uri)
+
+### Methods
+
+- [createTable](LocalConnection.md#createtable)
+- [createTableArrow](LocalConnection.md#createtablearrow)
+- [dropTable](LocalConnection.md#droptable)
+- [openTable](LocalConnection.md#opentable)
+- [tableNames](LocalConnection.md#tablenames)
+
+## Constructors
+
+### constructor
+
+• **new LocalConnection**(`db`, `uri`)
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `db` | `any` |
+| `uri` | `string` |
+
+#### Defined in
+
+[index.ts:132](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L132)
+
+## Properties
+
+### \_db
+
+• `Private` `Readonly` **\_db**: `any`
+
+#### Defined in
+
+[index.ts:130](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L130)
+
+___
+
+### \_uri
+
+• `Private` `Readonly` **\_uri**: `string`
+
+#### Defined in
+
+[index.ts:129](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L129)
+
+## Accessors
+
+### uri
+
+• `get` **uri**(): `string`
+
+#### Returns
+
+`string`
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[uri](../interfaces/Connection.md#uri)
+
+#### Defined in
+
+[index.ts:137](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L137)
+
+## Methods
+
+### createTable
+
+▸ **createTable**(`name`, `data`, `mode?`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+Creates a new Table and initialize it with new data.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
+| `mode?` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[createTable](../interfaces/Connection.md#createtable)
+
+#### Defined in
+
+[index.ts:177](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L177)
+
+▸ **createTable**(`name`, `data`, `mode`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `data` | `Record`<`string`, `unknown`\>[] |
+| `mode` | [`WriteMode`](../enums/WriteMode.md) |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Implementation of
+
+Connection.createTable
+
+#### Defined in
+
+[index.ts:178](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L178)
+
+▸ **createTable**<`T`\>(`name`, `data`, `mode`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+Creates a new Table and initialize it with new data.
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
+| `mode` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Implementation of
+
+Connection.createTable
+
+#### Defined in
+
+[index.ts:188](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L188)
+
+___
+
+### createTableArrow
+
+▸ **createTableArrow**(`name`, `table`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `table` | `Table`<`any`\> |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[createTableArrow](../interfaces/Connection.md#createtablearrow)
+
+#### Defined in
+
+[index.ts:201](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L201)
+
+___
+
+### dropTable
+
+▸ **dropTable**(`name`): `Promise`<`void`\>
+
+Drop an existing table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table to drop. |
+
+#### Returns
+
+`Promise`<`void`\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[dropTable](../interfaces/Connection.md#droptable)
+
+#### Defined in
+
+[index.ts:211](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L211)
+
+___
+
+### openTable
+
+▸ **openTable**(`name`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+Open a table in the database.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[openTable](../interfaces/Connection.md#opentable)
+
+#### Defined in
+
+[index.ts:153](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L153)
+
+▸ **openTable**<`T`\>(`name`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+Open a table in the database.
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Implementation of
+
+Connection.openTable
+
+#### Defined in
+
+[index.ts:160](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L160)
+
+___
+
+### tableNames
+
+▸ **tableNames**(): `Promise`<`string`[]\>
+
+Get the names of all tables in the database.
+
+#### Returns
+
+`Promise`<`string`[]\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[tableNames](../interfaces/Connection.md#tablenames)
+
+#### Defined in
+
+[index.ts:144](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L144)

docs/src/javascript/classes/LocalTable.md

@@ -0,0 +1,289 @@
+[vectordb](../README.md) / [Exports](../modules.md) / LocalTable
+
+# Class: LocalTable<T\>
+
+A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
+
+## Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+## Implements
+
+- [`Table`](../interfaces/Table.md)<`T`\>
+
+## Table of contents
+
+### Constructors
+
+- [constructor](LocalTable.md#constructor)
+
+### Properties
+
+- [\_embeddings](LocalTable.md#_embeddings)
+- [\_name](LocalTable.md#_name)
+- [\_tbl](LocalTable.md#_tbl)
+
+### Accessors
+
+- [name](LocalTable.md#name)
+
+### Methods
+
+- [add](LocalTable.md#add)
+- [countRows](LocalTable.md#countrows)
+- [createIndex](LocalTable.md#createindex)
+- [delete](LocalTable.md#delete)
+- [overwrite](LocalTable.md#overwrite)
+- [search](LocalTable.md#search)
+
+## Constructors
+
+### constructor
+
+• **new LocalTable**<`T`\>(`tbl`, `name`)
+
+#### Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `tbl` | `any` |
+| `name` | `string` |
+
+#### Defined in
+
+[index.ts:221](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L221)
+
+• **new LocalTable**<`T`\>(`tbl`, `name`, `embeddings`)
+
+#### Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `tbl` | `any` |  |
+| `name` | `string` |  |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use when interacting with this table |
+
+#### Defined in
+
+[index.ts:227](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L227)
+
+## Properties
+
+### \_embeddings
+
+• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
+
+#### Defined in
+
+[index.ts:219](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L219)
+
+___
+
+### \_name
+
+• `Private` `Readonly` **\_name**: `string`
+
+#### Defined in
+
+[index.ts:218](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L218)
+
+___
+
+### \_tbl
+
+• `Private` `Readonly` **\_tbl**: `any`
+
+#### Defined in
+
+[index.ts:217](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L217)
+
+## Accessors
+
+### name
+
+• `get` **name**(): `string`
+
+#### Returns
+
+`string`
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[name](../interfaces/Table.md#name)
+
+#### Defined in
+
+[index.ts:234](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L234)
+
+## Methods
+
+### add
+
+▸ **add**(`data`): `Promise`<`number`\>
+
+Insert records into this Table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+
+#### Returns
+
+`Promise`<`number`\>
+
+The number of rows added to the table
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[add](../interfaces/Table.md#add)
+
+#### Defined in
+
+[index.ts:252](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L252)
+
+___
+
+### countRows
+
+▸ **countRows**(): `Promise`<`number`\>
+
+Returns the number of rows in this table.
+
+#### Returns
+
+`Promise`<`number`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[countRows](../interfaces/Table.md#countrows)
+
+#### Defined in
+
+[index.ts:278](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L278)
+
+___
+
+### createIndex
+
+▸ **createIndex**(`indexParams`): `Promise`<`any`\>
+
+Create an ANN index on this Table vector index.
+
+**`See`**
+
+VectorIndexParams.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `indexParams` | `IvfPQIndexConfig` | The parameters of this Index, |
+
+#### Returns
+
+`Promise`<`any`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[createIndex](../interfaces/Table.md#createindex)
+
+#### Defined in
+
+[index.ts:271](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L271)
+
+___
+
+### delete
+
+▸ **delete**(`filter`): `Promise`<`void`\>
+
+Delete rows from this table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `filter` | `string` | A filter in the same format used by a sql WHERE clause. |
+
+#### Returns
+
+`Promise`<`void`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[delete](../interfaces/Table.md#delete)
+
+#### Defined in
+
+[index.ts:287](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L287)
+
+___
+
+### overwrite
+
+▸ **overwrite**(`data`): `Promise`<`number`\>
+
+Insert records into this Table, replacing its contents.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+
+#### Returns
+
+`Promise`<`number`\>
+
+The number of rows added to the table
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[overwrite](../interfaces/Table.md#overwrite)
+
+#### Defined in
+
+[index.ts:262](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L262)
+
+___
+
+### search
+
+▸ **search**(`query`): [`Query`](Query.md)<`T`\>
+
+Creates a search query to find the nearest neighbors of the given search term
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `query` | `T` | The query search term |
+
+#### Returns
+
+[`Query`](Query.md)<`T`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[search](../interfaces/Table.md#search)
+
+#### Defined in
+
+[index.ts:242](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L242)

docs/src/javascript/classes/OpenAIEmbeddingFunction.md

@@ -0,0 +1,105 @@
+[vectordb](../README.md) / [Exports](../modules.md) / OpenAIEmbeddingFunction
+
+# Class: OpenAIEmbeddingFunction
+
+An embedding function that automatically creates vector representation for a given column.
+
+## Implements
+
+- [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`string`\>
+
+## Table of contents
+
+### Constructors
+
+- [constructor](OpenAIEmbeddingFunction.md#constructor)
+
+### Properties
+
+- [\_modelName](OpenAIEmbeddingFunction.md#_modelname)
+- [\_openai](OpenAIEmbeddingFunction.md#_openai)
+- [sourceColumn](OpenAIEmbeddingFunction.md#sourcecolumn)
+
+### Methods
+
+- [embed](OpenAIEmbeddingFunction.md#embed)
+
+## Constructors
+
+### constructor
+
+• **new OpenAIEmbeddingFunction**(`sourceColumn`, `openAIKey`, `modelName?`)
+
+#### Parameters
+
+| Name | Type | Default value |
+| :------ | :------ | :------ |
+| `sourceColumn` | `string` | `undefined` |
+| `openAIKey` | `string` | `undefined` |
+| `modelName` | `string` | `'text-embedding-ada-002'` |
+
+#### Defined in
+
+[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L21)
+
+## Properties
+
+### \_modelName
+
+• `Private` `Readonly` **\_modelName**: `string`
+
+#### Defined in
+
+[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L19)
+
+___
+
+### \_openai
+
+• `Private` `Readonly` **\_openai**: `any`
+
+#### Defined in
+
+[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L18)
+
+___
+
+### sourceColumn
+
+• **sourceColumn**: `string`
+
+The name of the column that will be used as input for the Embedding Function.
+
+#### Implementation of
+
+[EmbeddingFunction](../interfaces/EmbeddingFunction.md).[sourceColumn](../interfaces/EmbeddingFunction.md#sourcecolumn)
+
+#### Defined in
+
+[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L50)
+
+## Methods
+
+### embed
+
+▸ **embed**(`data`): `Promise`<`number`[][]\>
+
+Creates a vector representation for the given values.
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `data` | `string`[] |
+
+#### Returns
+
+`Promise`<`number`[][]\>
+
+#### Implementation of
+
+[EmbeddingFunction](../interfaces/EmbeddingFunction.md).[embed](../interfaces/EmbeddingFunction.md#embed)
+
+#### Defined in
+
+[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L38)

docs/src/javascript/classes/Query.md

@@ -0,0 +1,349 @@
+[vectordb](../README.md) / [Exports](../modules.md) / Query
+
+# Class: Query<T\>
+
+A builder for nearest neighbor queries for LanceDB.
+
+## Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+## Table of contents
+
+### Constructors
+
+- [constructor](Query.md#constructor)
+
+### Properties
+
+- [\_embeddings](Query.md#_embeddings)
+- [\_filter](Query.md#_filter)
+- [\_limit](Query.md#_limit)
+- [\_metricType](Query.md#_metrictype)
+- [\_nprobes](Query.md#_nprobes)
+- [\_query](Query.md#_query)
+- [\_queryVector](Query.md#_queryvector)
+- [\_refineFactor](Query.md#_refinefactor)
+- [\_select](Query.md#_select)
+- [\_tbl](Query.md#_tbl)
+- [where](Query.md#where)
+
+### Methods
+
+- [execute](Query.md#execute)
+- [filter](Query.md#filter)
+- [limit](Query.md#limit)
+- [metricType](Query.md#metrictype)
+- [nprobes](Query.md#nprobes)
+- [refineFactor](Query.md#refinefactor)
+- [select](Query.md#select)
+
+## Constructors
+
+### constructor
+
+• **new Query**<`T`\>(`tbl`, `query`, `embeddings?`)
+
+#### Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `tbl` | `any` |
+| `query` | `T` |
+| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
+
+#### Defined in
+
+[index.ts:362](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L362)
+
+## Properties
+
+### \_embeddings
+
+• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
+
+#### Defined in
+
+[index.ts:360](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L360)
+
+___
+
+### \_filter
+
+• `Private` `Optional` **\_filter**: `string`
+
+#### Defined in
+
+[index.ts:358](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L358)
+
+___
+
+### \_limit
+
+• `Private` **\_limit**: `number`
+
+#### Defined in
+
+[index.ts:354](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L354)
+
+___
+
+### \_metricType
+
+• `Private` `Optional` **\_metricType**: [`MetricType`](../enums/MetricType.md)
+
+#### Defined in
+
+[index.ts:359](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L359)
+
+___
+
+### \_nprobes
+
+• `Private` **\_nprobes**: `number`
+
+#### Defined in
+
+[index.ts:356](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L356)
+
+___
+
+### \_query
+
+• `Private` `Readonly` **\_query**: `T`
+
+#### Defined in
+
+[index.ts:352](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L352)
+
+___
+
+### \_queryVector
+
+• `Private` `Optional` **\_queryVector**: `number`[]
+
+#### Defined in
+
+[index.ts:353](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L353)
+
+___
+
+### \_refineFactor
+
+• `Private` `Optional` **\_refineFactor**: `number`
+
+#### Defined in
+
+[index.ts:355](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L355)
+
+___
+
+### \_select
+
+• `Private` `Optional` **\_select**: `string`[]
+
+#### Defined in
+
+[index.ts:357](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L357)
+
+___
+
+### \_tbl
+
+• `Private` `Readonly` **\_tbl**: `any`
+
+#### Defined in
+
+[index.ts:351](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L351)
+
+___
+
+### where
+
+• **where**: (`value`: `string`) => [`Query`](Query.md)<`T`\>
+
+#### Type declaration
+
+▸ (`value`): [`Query`](Query.md)<`T`\>
+
+A filter statement to be applied to this query.
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `value` | `string` | A filter in the same format used by a sql WHERE clause. |
+
+##### Returns
+
+[`Query`](Query.md)<`T`\>
+
+#### Defined in
+
+[index.ts:410](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L410)
+
+## Methods
+
+### execute
+
+▸ **execute**<`T`\>(): `Promise`<`T`[]\>
+
+Execute the query and return the results as an Array of Objects
+
+#### Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `Record`<`string`, `unknown`\> |
+
+#### Returns
+
+`Promise`<`T`[]\>
+
+#### Defined in
+
+[index.ts:433](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L433)
+
+___
+
+### filter
+
+▸ **filter**(`value`): [`Query`](Query.md)<`T`\>
+
+A filter statement to be applied to this query.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `value` | `string` | A filter in the same format used by a sql WHERE clause. |
+
+#### Returns
+
+[`Query`](Query.md)<`T`\>
+
+#### Defined in
+
+[index.ts:405](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L405)
+
+___
+
+### limit
+
+▸ **limit**(`value`): [`Query`](Query.md)<`T`\>
+
+Sets the number of results that will be returned
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `value` | `number` | number of results |
+
+#### Returns
+
+[`Query`](Query.md)<`T`\>
+
+#### Defined in
+
+[index.ts:378](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L378)
+
+___
+
+### metricType
+
+▸ **metricType**(`value`): [`Query`](Query.md)<`T`\>
+
+The MetricType used for this Query.
+
+**`See`**
+
+MetricType for the different options
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `value` | [`MetricType`](../enums/MetricType.md) | The metric to the. |
+
+#### Returns
+
+[`Query`](Query.md)<`T`\>
+
+#### Defined in
+
+[index.ts:425](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L425)
+
+___
+
+### nprobes
+
+▸ **nprobes**(`value`): [`Query`](Query.md)<`T`\>
+
+The number of probes used. A higher number makes search more accurate but also slower.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `value` | `number` | The number of probes used. |
+
+#### Returns
+
+[`Query`](Query.md)<`T`\>
+
+#### Defined in
+
+[index.ts:396](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L396)
+
+___
+
+### refineFactor
+
+▸ **refineFactor**(`value`): [`Query`](Query.md)<`T`\>
+
+Refine the results by reading extra elements and re-ranking them in memory.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `value` | `number` | refine factor to use in this query. |
+
+#### Returns
+
+[`Query`](Query.md)<`T`\>
+
+#### Defined in
+
+[index.ts:387](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L387)
+
+___
+
+### select
+
+▸ **select**(`value`): [`Query`](Query.md)<`T`\>
+
+Return only the specified columns.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `value` | `string`[] | Only select the specified columns. If not specified, all columns will be returned. |
+
+#### Returns
+
+[`Query`](Query.md)<`T`\>
+
+#### Defined in
+
+[index.ts:416](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L416)

docs/src/javascript/enums/MetricType.md

@@ -0,0 +1,49 @@
+[vectordb](../README.md) / [Exports](../modules.md) / MetricType
+
+# Enumeration: MetricType
+
+Distance metrics type.
+
+## Table of contents
+
+### Enumeration Members
+
+- [Cosine](MetricType.md#cosine)
+- [Dot](MetricType.md#dot)
+- [L2](MetricType.md#l2)
+
+## Enumeration Members
+
+### Cosine
+
+• **Cosine** = ``"cosine"``
+
+Cosine distance
+
+#### Defined in
+
+[index.ts:481](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L481)
+
+___
+
+### Dot
+
+• **Dot** = ``"dot"``
+
+Dot product
+
+#### Defined in
+
+[index.ts:486](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L486)
+
+___
+
+### L2
+
+• **L2** = ``"l2"``
+
+Euclidean distance
+
+#### Defined in
+
+[index.ts:476](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L476)

docs/src/javascript/enums/WriteMode.md

@@ -0,0 +1,49 @@
+[vectordb](../README.md) / [Exports](../modules.md) / WriteMode
+
+# Enumeration: WriteMode
+
+Write mode for writing a table.
+
+## Table of contents
+
+### Enumeration Members
+
+- [Append](WriteMode.md#append)
+- [Create](WriteMode.md#create)
+- [Overwrite](WriteMode.md#overwrite)
+
+## Enumeration Members
+
+### Append
+
+• **Append** = ``"append"``
+
+Append new data to the table.
+
+#### Defined in
+
+[index.ts:466](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L466)
+
+___
+
+### Create
+
+• **Create** = ``"create"``
+
+Create a new [Table](../interfaces/Table.md).
+
+#### Defined in
+
+[index.ts:462](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L462)
+
+___
+
+### Overwrite
+
+• **Overwrite** = ``"overwrite"``
+
+Overwrite the existing [Table](../interfaces/Table.md) if presented.
+
+#### Defined in
+
+[index.ts:464](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L464)

docs/src/javascript/interfaces/Connection.md

@@ -0,0 +1,152 @@
+[vectordb](../README.md) / [Exports](../modules.md) / Connection
+
+# Interface: Connection
+
+A LanceDB Connection that allows you to open tables and create new ones.
+
+Connection could be local against filesystem or remote against a server.
+
+## Implemented by
+
+- [`LocalConnection`](../classes/LocalConnection.md)
+
+## Table of contents
+
+### Properties
+
+- [uri](Connection.md#uri)
+
+### Methods
+
+- [createTable](Connection.md#createtable)
+- [createTableArrow](Connection.md#createtablearrow)
+- [dropTable](Connection.md#droptable)
+- [openTable](Connection.md#opentable)
+- [tableNames](Connection.md#tablenames)
+
+## Properties
+
+### uri
+
+• **uri**: `string`
+
+#### Defined in
+
+[index.ts:45](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L45)
+
+## Methods
+
+### createTable
+
+▸ **createTable**<`T`\>(`name`, `data`, `mode?`, `embeddings?`): `Promise`<[`Table`](Table.md)<`T`\>\>
+
+Creates a new Table and initialize it with new data.
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
+| `mode?` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
+| `embeddings?` | [`EmbeddingFunction`](EmbeddingFunction.md)<`T`\> | An embedding function to use on this table |
+
+#### Returns
+
+`Promise`<[`Table`](Table.md)<`T`\>\>
+
+#### Defined in
+
+[index.ts:65](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L65)
+
+___
+
+### createTableArrow
+
+▸ **createTableArrow**(`name`, `table`): `Promise`<[`Table`](Table.md)<`number`[]\>\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `table` | `Table`<`any`\> |
+
+#### Returns
+
+`Promise`<[`Table`](Table.md)<`number`[]\>\>
+
+#### Defined in
+
+[index.ts:67](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L67)
+
+___
+
+### dropTable
+
+▸ **dropTable**(`name`): `Promise`<`void`\>
+
+Drop an existing table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table to drop. |
+
+#### Returns
+
+`Promise`<`void`\>
+
+#### Defined in
+
+[index.ts:73](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L73)
+
+___
+
+### openTable
+
+▸ **openTable**<`T`\>(`name`, `embeddings?`): `Promise`<[`Table`](Table.md)<`T`\>\>
+
+Open a table in the database.
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `embeddings?` | [`EmbeddingFunction`](EmbeddingFunction.md)<`T`\> | An embedding function to use on this table |
+
+#### Returns
+
+`Promise`<[`Table`](Table.md)<`T`\>\>
+
+#### Defined in
+
+[index.ts:55](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L55)
+
+___
+
+### tableNames
+
+▸ **tableNames**(): `Promise`<`string`[]\>
+
+#### Returns
+
+`Promise`<`string`[]\>
+
+#### Defined in
+
+[index.ts:47](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L47)

docs/src/javascript/interfaces/EmbeddingFunction.md

@@ -0,0 +1,60 @@
+[vectordb](../README.md) / [Exports](../modules.md) / EmbeddingFunction
+
+# Interface: EmbeddingFunction<T\>
+
+An embedding function that automatically creates vector representation for a given column.
+
+## Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+## Implemented by
+
+- [`OpenAIEmbeddingFunction`](../classes/OpenAIEmbeddingFunction.md)
+
+## Table of contents
+
+### Properties
+
+- [embed](EmbeddingFunction.md#embed)
+- [sourceColumn](EmbeddingFunction.md#sourcecolumn)
+
+## Properties
+
+### embed
+
+• **embed**: (`data`: `T`[]) => `Promise`<`number`[][]\>
+
+#### Type declaration
+
+▸ (`data`): `Promise`<`number`[][]\>
+
+Creates a vector representation for the given values.
+
+##### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `data` | `T`[] |
+
+##### Returns
+
+`Promise`<`number`[][]\>
+
+#### Defined in
+
+[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/embedding_function.ts#L27)
+
+___
+
+### sourceColumn
+
+• **sourceColumn**: `string`
+
+The name of the column that will be used as input for the Embedding Function.
+
+#### Defined in
+
+[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/embedding_function.ts#L22)

docs/src/javascript/interfaces/Table.md

@@ -0,0 +1,195 @@
+[vectordb](../README.md) / [Exports](../modules.md) / Table
+
+# Interface: Table<T\>
+
+A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
+
+## Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+## Implemented by
+
+- [`LocalTable`](../classes/LocalTable.md)
+
+## Table of contents
+
+### Properties
+
+- [add](Table.md#add)
+- [countRows](Table.md#countrows)
+- [createIndex](Table.md#createindex)
+- [delete](Table.md#delete)
+- [name](Table.md#name)
+- [overwrite](Table.md#overwrite)
+- [search](Table.md#search)
+
+## Properties
+
+### add
+
+• **add**: (`data`: `Record`<`string`, `unknown`\>[]) => `Promise`<`number`\>
+
+#### Type declaration
+
+▸ (`data`): `Promise`<`number`\>
+
+Insert records into this Table.
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+
+##### Returns
+
+`Promise`<`number`\>
+
+The number of rows added to the table
+
+#### Defined in
+
+[index.ts:95](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L95)
+
+___
+
+### countRows
+
+• **countRows**: () => `Promise`<`number`\>
+
+#### Type declaration
+
+▸ (): `Promise`<`number`\>
+
+Returns the number of rows in this table.
+
+##### Returns
+
+`Promise`<`number`\>
+
+#### Defined in
+
+[index.ts:115](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L115)
+
+___
+
+### createIndex
+
+• **createIndex**: (`indexParams`: `IvfPQIndexConfig`) => `Promise`<`any`\>
+
+#### Type declaration
+
+▸ (`indexParams`): `Promise`<`any`\>
+
+Create an ANN index on this Table vector index.
+
+**`See`**
+
+VectorIndexParams.
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `indexParams` | `IvfPQIndexConfig` | The parameters of this Index, |
+
+##### Returns
+
+`Promise`<`any`\>
+
+#### Defined in
+
+[index.ts:110](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L110)
+
+___
+
+### delete
+
+• **delete**: (`filter`: `string`) => `Promise`<`void`\>
+
+#### Type declaration
+
+▸ (`filter`): `Promise`<`void`\>
+
+Delete rows from this table.
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `filter` | `string` | A filter in the same format used by a sql WHERE clause. |
+
+##### Returns
+
+`Promise`<`void`\>
+
+#### Defined in
+
+[index.ts:122](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L122)
+
+___
+
+### name
+
+• **name**: `string`
+
+#### Defined in
+
+[index.ts:81](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L81)
+
+___
+
+### overwrite
+
+• **overwrite**: (`data`: `Record`<`string`, `unknown`\>[]) => `Promise`<`number`\>
+
+#### Type declaration
+
+▸ (`data`): `Promise`<`number`\>
+
+Insert records into this Table, replacing its contents.
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+
+##### Returns
+
+`Promise`<`number`\>
+
+The number of rows added to the table
+
+#### Defined in
+
+[index.ts:103](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L103)
+
+___
+
+### search
+
+• **search**: (`query`: `T`) => [`Query`](../classes/Query.md)<`T`\>
+
+#### Type declaration
+
+▸ (`query`): [`Query`](../classes/Query.md)<`T`\>
+
+Creates a search query to find the nearest neighbors of the given search term
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `query` | `T` | The query search term |
+
+##### Returns
+
+[`Query`](../classes/Query.md)<`T`\>
+
+#### Defined in
+
+[index.ts:87](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L87)

docs/src/javascript/modules.md

@@ -0,0 +1,63 @@
+[vectordb](README.md) / Exports
+
+# vectordb
+
+## Table of contents
+
+### Enumerations
+
+- [MetricType](enums/MetricType.md)
+- [WriteMode](enums/WriteMode.md)
+
+### Classes
+
+- [LocalConnection](classes/LocalConnection.md)
+- [LocalTable](classes/LocalTable.md)
+- [OpenAIEmbeddingFunction](classes/OpenAIEmbeddingFunction.md)
+- [Query](classes/Query.md)
+
+### Interfaces
+
+- [Connection](interfaces/Connection.md)
+- [EmbeddingFunction](interfaces/EmbeddingFunction.md)
+- [Table](interfaces/Table.md)
+
+### Type Aliases
+
+- [VectorIndexParams](modules.md#vectorindexparams)
+
+### Functions
+
+- [connect](modules.md#connect)
+
+## Type Aliases
+
+### VectorIndexParams
+
+Ƭ **VectorIndexParams**: `IvfPQIndexConfig`
+
+#### Defined in
+
+[index.ts:345](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L345)
+
+## Functions
+
+### connect
+
+▸ **connect**(`uri`): `Promise`<[`Connection`](interfaces/Connection.md)\>
+
+Connect to a LanceDB instance at the given URI
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `uri` | `string` | The uri of the database. |
+
+#### Returns
+
+`Promise`<[`Connection`](interfaces/Connection.md)\>
+
+#### Defined in
+
+[index.ts:34](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L34)

docs/src/notebooks/code_qa_bot.ipynb

@@ -72,6 +72,8 @@
     "import lancedb\n",
     "import re\n",
     "import pickle\n",
+    "import requests\n",
+    "import zipfile\n",
     "from pathlib import Path\n",
     "\n",
     "from langchain.document_loaders import UnstructuredHTMLLoader\n",
@@ -85,10 +87,25 @@
   {
    "attachments": {},
    "cell_type": "markdown",
-   "id": "6ccf9b2b",
+   "id": "56cc6d50",
    "metadata": {},
    "source": [
-    "You can download the Pandas documentation from https://pandas.pydata.org/docs/. To make sure we're not littering our repo with docs, we won't include it in the LanceDB repo, so download this and store it locally first."
+    "To make this easier, we've downloaded Pandas documentation and stored the raw HTML files for you to download. We'll download them and then use LangChain's HTML document readers to parse them and store them in LanceDB as a vector store, along with relevant metadata."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7da77e75",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pandas_docs = requests.get(\"https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip\")\n",
+    "with open('/tmp/pandas.documentation.zip', 'wb') as f:\n",
+    "    f.write(pandas_docs.content)\n",
+    "\n",
+    "file = zipfile.ZipFile(\"/tmp/pandas.documentation.zip\")\n",
+    "file.extractall(path=\"/tmp/pandas_docs\")"
    ]
   },
   {
@@ -137,7 +154,8 @@
     "docs = []\n",
     "\n",
     "if not docs_path.exists():\n",
-    "    for p in Path(\"./pandas.documentation\").rglob(\"*.html\"):\n",
+    "    for p in Path(\"/tmp/pandas_docs/pandas.documentation\").rglob(\"*.html\"):\n",
+    "        print(p)\n",
     "        if p.is_dir():\n",
     "            continue\n",
     "        loader = UnstructuredHTMLLoader(p)\n",

docs/src/notebooks/diffusiondb/datagen.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+#
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Dataset hf://poloclub/diffusiondb
+"""
+
+import io
+from argparse import ArgumentParser
+from multiprocessing import Pool
+
+import lance
+import pyarrow as pa
+from datasets import load_dataset
+from PIL import Image
+from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast
+
+import lancedb
+
+MODEL_ID = "openai/clip-vit-base-patch32"
+
+device = "cuda"
+
+tokenizer = CLIPTokenizerFast.from_pretrained(MODEL_ID)
+model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32").to(device)
+processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
+
+schema = pa.schema(
+    [
+        pa.field("prompt", pa.string()),
+        pa.field("seed", pa.uint32()),
+        pa.field("step", pa.uint16()),
+        pa.field("cfg", pa.float32()),
+        pa.field("sampler", pa.string()),
+        pa.field("width", pa.uint16()),
+        pa.field("height", pa.uint16()),
+        pa.field("timestamp", pa.timestamp("s")),
+        pa.field("image_nsfw", pa.float32()),
+        pa.field("prompt_nsfw", pa.float32()),
+        pa.field("vector", pa.list_(pa.float32(), 512)),
+        pa.field("image", pa.binary()),
+    ]
+)
+
+
+def pil_to_bytes(img) -> list[bytes]:
+    buf = io.BytesIO()
+    img.save(buf, format="PNG")
+    return buf.getvalue()
+
+
+def generate_clip_embeddings(batch) -> pa.RecordBatch:
+    image = processor(text=None, images=batch["image"], return_tensors="pt")[
+        "pixel_values"
+    ].to(device)
+    img_emb = model.get_image_features(image)
+    batch["vector"] = img_emb.cpu().tolist()
+
+    with Pool() as p:
+        batch["image_bytes"] = p.map(pil_to_bytes, batch["image"])
+    return batch
+
+
+def datagen(args):
+    """Generate DiffusionDB dataset, and use CLIP model to generate image embeddings."""
+    dataset = load_dataset("poloclub/diffusiondb", args.subset)
+    data = []
+    for b in dataset.map(
+        generate_clip_embeddings, batched=True, batch_size=256, remove_columns=["image"]
+    )["train"]:
+        b["image"] = b["image_bytes"]
+        del b["image_bytes"]
+        data.append(b)
+    tbl = pa.Table.from_pylist(data, schema=schema)
+    return tbl
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument(
+        "-o", "--output", metavar="DIR", help="Output lance directory", required=True
+    )
+    parser.add_argument(
+        "-s",
+        "--subset",
+        choices=["2m_all", "2m_first_10k", "2m_first_100k"],
+        default="2m_first_10k",
+        help="subset of the hg dataset",
+    )
+
+    args = parser.parse_args()
+
+    batches = datagen(args)
+    lance.write_dataset(batches, args.output)
+
+
+if __name__ == "__main__":
+    main()

docs/src/notebooks/diffusiondb/requirements.txt

@@ -0,0 +1,9 @@
+datasets
+Pillow
+lancedb
+isort
+black
+transformers
+--index-url https://download.pytorch.org/whl/cu118
+torch
+torchvision

docs/src/notebooks/multimodal_search.ipynb

@@ -0,0 +1,269 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
+      "\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install --quiet -U lancedb\n",
+    "!pip install --quiet gradio transformers torch torchvision"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import io\n",
+    "import PIL\n",
+    "import duckdb\n",
+    "import lancedb"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## First run setup: Download data and pre-process"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<lance.dataset.LanceDataset at 0x3045db590>"
+      ]
+     },
+     "execution_count": 30,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# remove null prompts\n",
+    "import lance\n",
+    "import pyarrow.compute as pc\n",
+    "\n",
+    "# download s3://eto-public/datasets/diffusiondb/small_10k.lance to this uri\n",
+    "data = lance.dataset(\"~/datasets/rawdata.lance\").to_table()\n",
+    "\n",
+    "# First data processing and full-text-search index\n",
+    "db = lancedb.connect(\"~/datasets/demo\")\n",
+    "tbl = db.create_table(\"diffusiondb\", data.filter(~pc.field(\"prompt\").is_null()))\n",
+    "tbl = tbl.create_fts_index([\"prompt\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create / Open LanceDB Table"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "db = lancedb.connect(\"~/datasets/demo\")\n",
+    "tbl = db.open_table(\"diffusiondb\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create CLIP embedding function for the text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast\n",
+    "\n",
+    "MODEL_ID = \"openai/clip-vit-base-patch32\"\n",
+    "\n",
+    "tokenizer = CLIPTokenizerFast.from_pretrained(MODEL_ID)\n",
+    "model = CLIPModel.from_pretrained(MODEL_ID)\n",
+    "processor = CLIPProcessor.from_pretrained(MODEL_ID)\n",
+    "\n",
+    "def embed_func(query):\n",
+    "    inputs = tokenizer([query], padding=True, return_tensors=\"pt\")\n",
+    "    text_features = model.get_text_features(**inputs)\n",
+    "    return text_features.detach().numpy()[0]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Search functions for Gradio"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def find_image_vectors(query):\n",
+    "    emb = embed_func(query)\n",
+    "    code = (\n",
+    "        \"import lancedb\\n\"\n",
+    "        \"db = lancedb.connect('~/datasets/demo')\\n\"\n",
+    "        \"tbl = db.open_table('diffusiondb')\\n\\n\"\n",
+    "        f\"embedding = embed_func('{query}')\\n\"\n",
+    "        \"tbl.search(embedding).limit(9).to_df()\"\n",
+    "    )\n",
+    "    return (_extract(tbl.search(emb).limit(9).to_df()), code)\n",
+    "\n",
+    "def find_image_keywords(query):\n",
+    "    code = (\n",
+    "        \"import lancedb\\n\"\n",
+    "        \"db = lancedb.connect('~/datasets/demo')\\n\"\n",
+    "        \"tbl = db.open_table('diffusiondb')\\n\\n\"\n",
+    "        f\"tbl.search('{query}').limit(9).to_df()\"\n",
+    "    )\n",
+    "    return (_extract(tbl.search(query).limit(9).to_df()), code)\n",
+    "\n",
+    "def find_image_sql(query):\n",
+    "    code = (\n",
+    "        \"import lancedb\\n\"\n",
+    "        \"import duckdb\\n\"\n",
+    "        \"db = lancedb.connect('~/datasets/demo')\\n\"\n",
+    "        \"tbl = db.open_table('diffusiondb')\\n\\n\"\n",
+    "        \"diffusiondb = tbl.to_lance()\\n\"\n",
+    "        f\"duckdb.sql('{query}').to_df()\"\n",
+    "    )    \n",
+    "    diffusiondb = tbl.to_lance()\n",
+    "    return (_extract(duckdb.sql(query).to_df()), code)\n",
+    "\n",
+    "def _extract(df):\n",
+    "    image_col = \"image\"\n",
+    "    return [(PIL.Image.open(io.BytesIO(row[image_col])), row[\"prompt\"]) for _, row in df.iterrows()]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Setup Gradio interface"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running on local URL:  http://127.0.0.1:7881\n",
+      "\n",
+      "To create a public link, set `share=True` in `launch()`.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"http://127.0.0.1:7881/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": []
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import gradio as gr\n",
+    "\n",
+    "\n",
+    "with gr.Blocks() as demo:\n",
+    "    with gr.Row():\n",
+    "        with gr.Tab(\"Embeddings\"):\n",
+    "            vector_query = gr.Textbox(value=\"portraits of a person\", show_label=False)\n",
+    "            b1 = gr.Button(\"Submit\")\n",
+    "        with gr.Tab(\"Keywords\"):\n",
+    "            keyword_query = gr.Textbox(value=\"ninja turtle\", show_label=False)\n",
+    "            b2 = gr.Button(\"Submit\")\n",
+    "        with gr.Tab(\"SQL\"):\n",
+    "            sql_query = gr.Textbox(value=\"SELECT * from diffusiondb WHERE image_nsfw >= 2 LIMIT 9\", show_label=False)\n",
+    "            b3 = gr.Button(\"Submit\")\n",
+    "    with gr.Row():\n",
+    "        code = gr.Code(label=\"Code\", language=\"python\")\n",
+    "    with gr.Row():\n",
+    "        gallery = gr.Gallery(\n",
+    "                label=\"Found images\", show_label=False, elem_id=\"gallery\"\n",
+    "            ).style(columns=[3], rows=[3], object_fit=\"contain\", height=\"auto\")   \n",
+    "        \n",
+    "    b1.click(find_image_vectors, inputs=vector_query, outputs=[gallery, code])\n",
+    "    b2.click(find_image_keywords, inputs=keyword_query, outputs=[gallery, code])\n",
+    "    b3.click(find_image_sql, inputs=sql_query, outputs=[gallery, code])\n",
+    "    \n",
+    "demo.launch()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}

docs/src/notebooks/youtube_transcript_search.ipynb

@@ -1,11 +1,12 @@
 {
  "cells": [
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "42bf01fb",
    "metadata": {},
    "source": [
-    "# We're going to build question and answer bot\n",
+    "# Youtube Transcript Search QA Bot\n",
     "\n",
     "This Q&A bot will allow you to search through youtube transcripts using natural language! By going through this notebook, we'll introduce how you can use LanceDB to store and manage your data easily."
    ]
@@ -35,6 +36,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "22e570f4",
    "metadata": {},
@@ -87,6 +89,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "5ac2b6a3",
    "metadata": {},
@@ -181,6 +184,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "3044e0b0",
    "metadata": {},
@@ -209,6 +213,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "db586267",
    "metadata": {},
@@ -229,6 +234,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "2106b5bb",
    "metadata": {},
@@ -338,6 +344,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "53e4bff1",
    "metadata": {},
@@ -371,6 +378,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "8ef34fca",
    "metadata": {},
@@ -459,6 +467,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "23afc2f9",
    "metadata": {},
@@ -541,6 +550,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "28705959",
    "metadata": {},
@@ -571,6 +581,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "559a095b",
    "metadata": {},

docs/src/python.md

@@ -1,14 +0,0 @@
-# LanceDB Python API Reference
-
-## Installation
-
-```shell
-pip install lancedb
-```
-
-## ::: lancedb
-## ::: lancedb.db
-## ::: lancedb.table
-## ::: lancedb.query
-## ::: lancedb.embeddings
-## ::: lancedb.context

docs/src/python/python.md

@@ -0,0 +1,45 @@
+# LanceDB Python API Reference
+
+## Installation
+
+```shell
+pip install lancedb
+```
+
+## Connection
+
+::: lancedb.connect
+
+::: lancedb.db.DBConnection
+
+## Table
+
+::: lancedb.table.Table
+
+## Querying
+
+::: lancedb.query.Query
+
+::: lancedb.query.LanceQueryBuilder
+
+::: lancedb.query.LanceFtsQueryBuilder
+
+## Embeddings
+
+::: lancedb.embeddings.with_embeddings
+
+::: lancedb.embeddings.EmbeddingFunction
+
+## Context
+
+::: lancedb.context.contextualize
+
+::: lancedb.context.Contextualizer
+
+## Full text search
+
+::: lancedb.fts.create_index
+
+::: lancedb.fts.populate_index
+
+::: lancedb.fts.search_index

docs/src/search.md

@@ -0,0 +1,117 @@
+# Vector Search
+
+`Vector Search` finds the nearest vectors from the database.
+In a recommendation system or search engine, you can find similar products from
+the one you searched.
+In LLM and other AI applications,
+each data point can be [presented by the embeddings generated from some models](embedding.md),
+it returns the most relevant features.
+
+A search in high-dimensional vector space, is to find `K-Nearest-Neighbors (KNN)` of the query vector.
+
+## Metric
+
+In LanceDB, a `Metric` is the way to describe the distance between a pair of vectors.
+Currently, we support the following metrics:
+
+| Metric      | Description                          |
+| ----------- | ------------------------------------ |
+| `L2`        | [Euclidean / L2 distance](https://en.wikipedia.org/wiki/Euclidean_distance) |
+| `Cosine`    | [Cosine Similarity](https://en.wikipedia.org/wiki/Cosine_similarity)|
+| `Dot`       | [Dot Production](https://en.wikipedia.org/wiki/Dot_product) |
+
+
+## Search
+
+### Flat Search
+
+
+If there is no [vector index is created](ann_indexes.md), LanceDB will just brute-force scan
+the vector column and compute the distance.
+
+<!-- Setup Code
+```python
+import lancedb
+import numpy as np
+uri = "data/sample-lancedb"
+db = lancedb.connect(uri)
+
+data = [{"vector": row, "item": f"item {i}"}
+     for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]
+
+db.create_table("my_vectors", data=data)
+```
+-->
+<!-- Setup Code
+```javascript
+const vectordb_setup = require('vectordb')
+const db_setup = await vectordb_setup.connect('data/sample-lancedb')
+
+let data = []
+for (let i = 0; i < 10_000; i++) {
+     data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
+}
+await db_setup.createTable('my_vectors', data)
+```
+-->
+=== "Python"
+
+
+    ```python
+    import lancedb
+    import numpy as np
+
+    db = lancedb.connect("data/sample-lancedb")
+
+    tbl = db.open_table("my_vectors")
+
+    df = tbl.search(np.random.random((1536))) \
+        .limit(10) \
+        .to_df()
+    ```
+
+=== "JavaScript"
+
+    ```javascript
+    const vectordb = require('vectordb')
+    const db = await vectordb.connect('data/sample-lancedb')
+
+    const tbl = await db.openTable("my_vectors")
+
+    const results_1 = await tbl.search(Array(1536).fill(1.2))
+        .limit(20)
+        .execute()
+    ```
+
+
+<!-- Commenting out for now since metricType fails for JS on Ubuntu 22.04.
+
+By default, `l2` will be used as `Metric` type. You can customize the metric type
+as well.
+-->
+
+<!--
+=== "Python"
+-->
+<!--    ```python
+    df = tbl.search(np.random.random((1536))) \
+        .metric("cosine") \
+        .limit(10) \
+        .to_df()
+    ```
+-->
+<!--
+=== "JavaScript"
+-->
+
+<!--   ```javascript
+    const results_2 = await tbl.search(Array(1536).fill(1.2))
+        .metricType("cosine")
+        .limit(20)
+        .execute()
+    ```
+-->
+
+### Search with Vector Index.
+
+See [ANN Index](ann_indexes.md) for more details.

docs/src/sql.md

@@ -0,0 +1,120 @@
+# SQL filters
+
+LanceDB embraces the utilization of standard SQL expressions as predicates for hybrid
+filters. It can be used during hybrid vector search and deletion operations.
+
+Currently, Lance supports a growing list of expressions.
+
+* ``>``, ``>=``, ``<``, ``<=``, ``=``
+* ``AND``, ``OR``, ``NOT``
+* ``IS NULL``, ``IS NOT NULL``
+* ``IS TRUE``, ``IS NOT TRUE``, ``IS FALSE``, ``IS NOT FALSE``
+* ``IN``
+* ``LIKE``, ``NOT LIKE``
+* ``CAST``
+* ``regexp_match(column, pattern)``
+
+For example, the following filter string is acceptable:
+<!-- Setup Code
+```python 
+import lancedb
+import numpy as np
+uri = "data/sample-lancedb"
+db = lancedb.connect(uri)
+
+data = [{"vector": row, "item": f"item {i}"}
+     for i, row in enumerate(np.random.random((10_000, 2)).astype('int'))]
+
+tbl = db.create_table("my_vectors", data=data)
+```
+-->
+<!-- Setup Code
+```javascript 
+const vectordb = require('vectordb')
+const db = await vectordb.connect('data/sample-lancedb')
+
+let data = []
+for (let i = 0; i < 10_000; i++) {
+     data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
+}
+const tbl = await db.createTable('my_vectors', data)
+```
+-->
+=== "Python"
+
+    ```python
+    tbl.search([100, 102]) \
+       .where("""(
+        (label IN [10, 20])
+        AND
+        (note.email IS NOT NULL)
+        ) OR NOT note.created
+        """)
+
+    ```
+=== "Javascript"
+
+    ```javascript
+    tbl.search([100, 102])
+       .where(`(
+            (label IN [10, 20])
+            AND
+            (note.email IS NOT NULL)
+        ) OR NOT note.created
+       `)
+    ```
+
+
+If your column name contains special characters or is a [SQL Keyword](https://docs.rs/sqlparser/latest/sqlparser/keywords/index.html),
+you can use backtick (`` ` ``) to escape it. For nested fields, each segment of the
+path must be wrapped in backticks.
+
+=== "SQL"
+    ```sql
+    `CUBE` = 10 AND `column name with space` IS NOT NULL
+      AND `nested with space`.`inner with space` < 2
+    ```
+
+!!! warning
+    Field names containing periods (``.``) are not supported.
+
+Literals for dates, timestamps, and decimals can be written by writing the string
+value after the type name. For example
+
+=== "SQL"
+    ```sql
+    date_col = date '2021-01-01'
+    and timestamp_col = timestamp '2021-01-01 00:00:00'
+    and decimal_col = decimal(8,3) '1.000'
+    ```
+
+For timestamp columns, the precision can be specified as a number in the type
+parameter. Microsecond precision (6) is the default.
+
+| SQL              | Time unit    |
+|------------------|--------------|
+| ``timestamp(0)`` | Seconds      |
+| ``timestamp(3)`` | Milliseconds |
+| ``timestamp(6)`` | Microseconds |
+| ``timestamp(9)`` | Nanoseconds  |
+
+LanceDB internally stores data in [Apache Arrow](https://arrow.apache.org/) format.
+The mapping from SQL types to Arrow types is:
+
+| SQL type | Arrow type |
+|----------|------------|
+| ``boolean`` | ``Boolean`` |
+| ``tinyint`` / ``tinyint unsigned`` | ``Int8`` / ``UInt8`` |
+| ``smallint`` / ``smallint unsigned`` | ``Int16`` / ``UInt16`` |
+| ``int`` or ``integer`` / ``int unsigned`` or ``integer unsigned`` | ``Int32`` / ``UInt32`` |
+| ``bigint`` / ``bigint unsigned`` | ``Int64`` / ``UInt64`` |
+| ``float`` | ``Float32`` |
+| ``double`` | ``Float64`` |
+| ``decimal(precision, scale)`` | ``Decimal128`` |
+| ``date`` | ``Date32`` |
+| ``timestamp`` | ``Timestamp`` [^1] |
+| ``string`` | ``Utf8`` |
+| ``binary`` | ``Binary`` |
+
+[^1]: See precision mapping in previous table.
+

docs/src/styles/global.css

@@ -0,0 +1,6 @@
+:root {
+    --md-primary-fg-color:        #625eff;
+    --md-primary-fg-color--dark:  #4338ca;
+    --md-text-font: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
+    --md-code-font: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
+}

docs/test/md_testing.js

@@ -0,0 +1,51 @@
+const glob = require("glob");
+const fs = require("fs");
+const path = require("path");
+
+const excludedFiles = [
+  "../src/fts.md",
+  "../src/embedding.md",
+  "../src/examples/serverless_lancedb_with_s3_and_lambda.md",
+  "../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
+  "../src/examples/youtube_transcript_bot_with_nodejs.md",
+];
+const nodePrefix = "javascript";
+const nodeFile = ".js";
+const nodeFolder = "node";
+const globString = "../src/**/*.md";
+const asyncPrefix = "(async () => {\n";
+const asyncSuffix = "})();";
+
+function* yieldLines(lines, prefix, suffix) {
+  let inCodeBlock = false;
+  for (const line of lines) {
+    if (line.trim().startsWith(prefix + nodePrefix)) {
+      inCodeBlock = true;
+    } else if (inCodeBlock && line.trim().startsWith(suffix)) {
+      inCodeBlock = false;
+      yield "\n";
+    } else if (inCodeBlock) {
+      yield line;
+    }
+  }
+}
+
+const files = glob.sync(globString, { recursive: true });
+
+for (const file of files.filter((file) => !excludedFiles.includes(file))) {
+  const lines = [];
+  const data = fs.readFileSync(file, "utf-8");
+  const fileLines = data.split("\n");
+
+  for (const line of yieldLines(fileLines, "```", "```")) {
+    lines.push(line);
+  }
+
+  if (lines.length > 0) {
+    const fileName = path.basename(file, ".md");
+    const outPath = path.join(nodeFolder, fileName, `${fileName}${nodeFile}`);
+    console.log(outPath)
+    fs.mkdirSync(path.dirname(outPath), { recursive: true });
+    fs.writeFileSync(outPath, asyncPrefix + "\n" + lines.join("\n") + asyncSuffix);
+  }
+}

docs/test/md_testing.py

@@ -0,0 +1,41 @@
+import glob
+from typing import Iterator
+from pathlib import Path
+
+excluded_files = [
+    "../src/fts.md",
+    "../src/embedding.md",
+    "../src/examples/serverless_lancedb_with_s3_and_lambda.md",
+    "../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
+    "../src/examples/youtube_transcript_bot_with_nodejs.md"
+]
+
+python_prefix = "py"
+python_file = ".py"
+python_folder = "python"
+glob_string = "../src/**/*.md"
+
+def yield_lines(lines: Iterator[str], prefix: str, suffix: str):
+    in_code_block = False
+    # Python code has strict indentation
+    strip_length = 0
+    for line in lines:
+        if line.strip().startswith(prefix + python_prefix):
+            in_code_block = True
+            strip_length = len(line) - len(line.lstrip())
+        elif in_code_block and line.strip().startswith(suffix):
+            in_code_block = False
+            yield "\n"
+        elif in_code_block:
+            yield line[strip_length:]
+
+for file in filter(lambda file: file not in excluded_files, glob.glob(glob_string, recursive=True)):
+    with open(file, "r") as f:
+        lines = list(yield_lines(iter(f), "```", "```"))
+
+    if len(lines) > 0:
+        out_path = Path(python_folder) / Path(file).name.strip(".md") / (Path(file).name.strip(".md") + python_file)
+        print(out_path)
+        out_path.parent.mkdir(exist_ok=True, parents=True)
+        with open(out_path, "w") as out:
+            out.writelines(lines)

docs/test/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "lancedb-docs-test",
+  "version": "1.0.0",
+  "description": "",
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "fs": "^0.0.1-security",
+    "glob": "^10.2.7",
+    "path": "^0.12.7",
+    "vectordb": "https://gitpkg.now.sh/lancedb/lancedb/node?main"
+  }
+}

docs/test/requirements.txt

@@ -0,0 +1,5 @@
+lancedb @ git+https://github.com/lancedb/lancedb.git#egg=subdir&subdirectory=python
+numpy
+pandas
+pylance
+duckdb

node/.eslintrc.js

@@ -12,5 +12,6 @@ module.exports = {
     sourceType: 'module'
   },
   rules: {
+    "@typescript-eslint/method-signature-style": "off",
   }
 }

node/CHANGELOG.md

@@ -0,0 +1,64 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.1.5] - 2023-06-00
+
+### Added
+
+- Support for macOS X86
+
+## [0.1.4] - 2023-06-03
+
+### Added
+
+- Select / Project query API
+
+### Changed
+
+-  Deprecated created_index in favor of createIndex
+
+## [0.1.3] - 2023-06-01
+
+### Added
+
+- Support S3 and Google Cloud Storage
+- Embedding functions support
+- OpenAI embedding function
+
+## [0.1.2] - 2023-05-27
+
+### Added
+
+- Append records API
+- Extra query params to to nodejs client
+- Create_index API
+ 
+### Fixed
+
+- bugfix: string columns should be converted to Utf8Array (#94)
+
+## [0.1.1] - 2023-05-16
+
+### Added
+
+- create_table API
+- limit parameter for queries
+- Typescript / JavaScript examples
+- Linux support
+
+## [0.1.0] - 2023-05-16
+
+### Added
+
+- Initial  JavaScript / Node.js library for LanceDB
+- Read-only api to query LanceDB datasets
+- Supports macOS arm only
+
+## [pre-0.1.0]
+
+- Various prototypes / test builds
+

node/README.md

@@ -14,9 +14,11 @@ npm install vectordb
 
 ```javascript
 const lancedb = require('vectordb');
-const db = lancedb.connect('<PATH_TO_LANCEDB_DATASET>');
+const db = await lancedb.connect('data/sample-lancedb');
-const table = await db.openTable('my_table');
+const table = await db.createTable("my_table",
-const query = await table.search([0.1, 0.3]).setLimit(20).execute();
+      [{ id: 1, vector: [0.1, 1.0], item: "foo", price: 10.0 },
+      { id: 2, vector: [3.9, 0.5], item: "bar", price: 20.0 }])
+const results = await table.search([0.1, 0.3]).limit(20).execute();
 console.log(results);
 ```
 
@@ -24,12 +26,6 @@ The [examples](./examples) folder contains complete examples.
 
 ## Development
 
-The LanceDB javascript is built with npm:
-
-```bash
-npm run tsc
-```
-
 Run the tests with
 
 ```bash
@@ -41,3 +37,9 @@ To run the linter and have it automatically fix all errors
 ```bash
 npm run lint -- --fix
 ```
+
+To build documentation
+
+```bash
+npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
+```

node/examples/js-openai/index.js

@@ -0,0 +1,41 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict'
+
+async function example () {
+  const lancedb = require('vectordb')
+  // You need to provide an OpenAI API key, here we read it from the OPENAI_API_KEY environment variable
+  const apiKey = process.env.OPENAI_API_KEY
+  // The embedding function will create embeddings for the 'text' column(text in this case)
+  const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
+
+  const db = await lancedb.connect('data/sample-lancedb')
+
+  const data = [
+    { id: 1, text: 'Black T-Shirt', price: 10 },
+    { id: 2, text: 'Leather Jacket', price: 50 }
+  ]
+
+  const table = await db.createTable('vectors', data, embedding)
+  console.log(await db.tableNames())
+
+  const results = await table
+    .search('keeps me warm')
+    .limit(1)
+    .execute()
+  console.log(results[0].text)
+}
+
+example().then(_ => { console.log('All done!') })

node/examples/js-openai/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "vectordb-example-js-openai",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Lance Devs",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "vectordb": "file:../..",
+    "openai": "^3.2.1"
+  }
+}

node/examples/js-youtube-transcripts/index.js

@@ -0,0 +1,122 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict'
+
+const lancedb = require('vectordb')
+const fs = require('fs/promises')
+const readline = require('readline/promises')
+const { stdin: input, stdout: output } = require('process')
+const { Configuration, OpenAIApi } = require('openai')
+
+// Download file from XYZ
+const INPUT_FILE_NAME = 'data/youtube-transcriptions_sample.jsonl';
+
+(async () => {
+  // You need to provide an OpenAI API key, here we read it from the OPENAI_API_KEY environment variable
+  const apiKey = process.env.OPENAI_API_KEY
+  // The embedding function will create embeddings for the 'context' column
+  const embedFunction = new lancedb.OpenAIEmbeddingFunction('context', apiKey)
+
+  // Connects to LanceDB
+  const db = await lancedb.connect('data/youtube-lancedb')
+
+  // Open the vectors table or create one if it does not exist
+  let tbl
+  if ((await db.tableNames()).includes('vectors')) {
+    tbl = await db.openTable('vectors', embedFunction)
+  } else {
+    tbl = await createEmbeddingsTable(db, embedFunction)
+  }
+
+  // Use OpenAI Completion API to generate and answer based on the context that LanceDB provides
+  const configuration = new Configuration({ apiKey })
+  const openai = new OpenAIApi(configuration)
+  const rl = readline.createInterface({ input, output })
+  try {
+    while (true) {
+      const query = await rl.question('Prompt: ')
+      const results = await tbl
+        .search(query)
+        .select(['title', 'text', 'context'])
+        .limit(3)
+        .execute()
+
+      // console.table(results)
+
+      const response = await openai.createCompletion({
+        model: 'text-davinci-003',
+        prompt: createPrompt(query, results),
+        max_tokens: 400,
+        temperature: 0,
+        top_p: 1,
+        frequency_penalty: 0,
+        presence_penalty: 0
+      })
+      console.log(response.data.choices[0].text)
+    }
+  } catch (err) {
+    console.log('Error: ', err)
+  } finally {
+    rl.close()
+  }
+  process.exit(1)
+})()
+
+async function createEmbeddingsTable (db, embedFunction) {
+  console.log(`Creating embeddings from ${INPUT_FILE_NAME}`)
+  // read the input file into a JSON array, skipping empty lines
+  const lines = (await fs.readFile(INPUT_FILE_NAME, 'utf-8'))
+    .toString()
+    .split('\n')
+    .filter(line => line.length > 0)
+    .map(line => JSON.parse(line))
+
+  const data = contextualize(lines, 20, 'video_id')
+  return await db.createTable('vectors', data, embedFunction)
+}
+
+// Each transcript has a small text column, we include previous transcripts in order to
+// have more context information when creating embeddings
+function contextualize (rows, contextSize, groupColumn) {
+  const grouped = []
+  rows.forEach(row => {
+    if (!grouped[row[groupColumn]]) {
+      grouped[row[groupColumn]] = []
+    }
+    grouped[row[groupColumn]].push(row)
+  })
+
+  const data = []
+  Object.keys(grouped).forEach(key => {
+    for (let i = 0; i < grouped[key].length; i++) {
+      const start = i - contextSize > 0 ? i - contextSize : 0
+      grouped[key][i].context = grouped[key].slice(start, i + 1).map(r => r.text).join(' ')
+    }
+    data.push(...grouped[key])
+  })
+  return data
+}
+
+// Creates a prompt by aggregating all relevant contexts
+function createPrompt (query, context) {
+  let prompt =
+      'Answer the question based on the context below.\n\n' +
+      'Context:\n'
+
+  // need to make sure our prompt is not larger than max size
+  prompt = prompt + context.map(c => c.context).join('\n\n---\n\n').substring(0, 3750)
+  prompt = prompt + `\n\nQuestion: ${query}\nAnswer:`
+  return prompt
+}

node/examples/js-youtube-transcripts/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "vectordb-example-js-openai",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Lance Devs",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "vectordb": "file:../..",
+    "openai": "^3.2.1"
+  }
+}

node/examples/js/package.json

@@ -9,6 +9,6 @@
   "author": "Lance Devs",
   "license": "Apache-2.0",
   "dependencies": {
-    "vectordb": "^0.1.0"
+    "vectordb": "file:../.."
   }
 }

node/examples/ts/package.json

@@ -17,6 +17,6 @@
     "typescript": "*"
   },
   "dependencies": {
-    "vectordb": "^0.1.0"
+    "vectordb": "file:../.."
   }
 }

node/gen_test_data.py

@@ -1,8 +0,0 @@
-import lancedb
-
-uri = "sample-lancedb"
-db = lancedb.connect(uri)
-table = db.create_table("my_table",
-                         data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                               {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.1.1",
+  "version": "0.1.10",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -8,8 +8,9 @@
     "tsc": "tsc -b",
     "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json-render-diagnostics",
     "build-release": "npm run build -- --release",
-    "test": "mocha -recursive dist/test",
+    "test": "npm run tsc; mocha -recursive dist/test",
-    "lint": "eslint src --ext .js,.ts"
+    "lint": "eslint src --ext .js,.ts",
+    "clean": "rm -rf node_modules *.node dist/"
   },
   "repository": {
     "type": "git",
@@ -25,21 +26,28 @@
   "license": "Apache-2.0",
   "devDependencies": {
     "@types/chai": "^4.3.4",
+    "@types/chai-as-promised": "^7.1.5",
     "@types/mocha": "^10.0.1",
     "@types/node": "^18.16.2",
+    "@types/sinon": "^10.0.15",
     "@types/temp": "^0.9.1",
     "@typescript-eslint/eslint-plugin": "^5.59.1",
     "cargo-cp-artifact": "^0.1",
     "chai": "^4.3.7",
+    "chai-as-promised": "^7.1.1",
     "eslint": "^8.39.0",
     "eslint-config-standard-with-typescript": "^34.0.1",
-    "eslint-plugin-import": "^2.27.5",
+    "eslint-plugin-import": "^2.26.0",
     "eslint-plugin-n": "^15.7.0",
     "eslint-plugin-promise": "^6.1.1",
     "mocha": "^10.2.0",
+    "openai": "^3.2.1",
+    "sinon": "^15.1.0",
     "temp": "^0.9.4",
     "ts-node": "^10.9.1",
     "ts-node-dev": "^2.0.0",
+    "typedoc": "^0.24.7",
+    "typedoc-plugin-markdown": "^3.15.3",
     "typescript": "*"
   },
   "dependencies": {

node/src/arrow.ts

@@ -15,15 +15,16 @@
 import {
   Field,
   Float32,
-  List,
+  List, type ListBuilder,
   makeBuilder,
   RecordBatchFileWriter,
-  Table,
+  Table, Utf8,
   type Vector,
   vectorFromArray
 } from 'apache-arrow'
+import { type EmbeddingFunction } from './index'
 
-export function convertToTable (data: Array<Record<string, unknown>>): Table {
+export async function convertToTable<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Table> {
   if (data.length === 0) {
     throw new Error('At least one record needs to be provided')
   }
@@ -33,11 +34,7 @@ export function convertToTable (data: Array<Record<string, unknown>>): Table {
 
   for (const columnsKey of columns) {
     if (columnsKey === 'vector') {
-      const children = new Field<Float32>('item', new Float32())
+      const listBuilder = newVectorListBuilder()
-      const list = new List(children)
-      const listBuilder = makeBuilder({
-        type: list
-      })
       const vectorSize = (data[0].vector as any[]).length
       for (const datum of data) {
         if ((datum[columnsKey] as any[]).length !== vectorSize) {
@@ -52,15 +49,37 @@ export function convertToTable (data: Array<Record<string, unknown>>): Table {
       for (const datum of data) {
         values.push(datum[columnsKey])
       }
-      records[columnsKey] = vectorFromArray(values)
+
+      if (columnsKey === embeddings?.sourceColumn) {
+        const vectors = await embeddings.embed(values as T[])
+        const listBuilder = newVectorListBuilder()
+        vectors.map(v => listBuilder.append(v))
+        records.vector = listBuilder.finish().toVector()
+      }
+
+      if (typeof values[0] === 'string') {
+        // `vectorFromArray` converts strings into dictionary vectors, forcing it back to a string column
+        records[columnsKey] = vectorFromArray(values, new Utf8())
+      } else {
+        records[columnsKey] = vectorFromArray(values)
+      }
     }
   }
 
   return new Table(records)
 }
 
-export async function fromRecordsToBuffer (data: Array<Record<string, unknown>>): Promise<Buffer> {
+// Creates a new Arrow ListBuilder that stores a Vector column
-  const table = convertToTable(data)
+function newVectorListBuilder (): ListBuilder<Float32, any> {
+  const children = new Field<Float32>('item', new Float32())
+  const list = new List(children)
+  return makeBuilder({
+    type: list
+  })
+}
+
+export async function fromRecordsToBuffer<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Buffer> {
+  const table = await convertToTable(data, embeddings)
   const writer = RecordBatchFileWriter.writeAll(table)
   return Buffer.from(await writer.toUint8Array())
 }

node/src/embedding/embedding_function.ts

@@ -0,0 +1,28 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * An embedding function that automatically creates vector representation for a given column.
+ */
+export interface EmbeddingFunction<T> {
+  /**
+     * The name of the column that will be used as input for the Embedding Function.
+     */
+  sourceColumn: string
+
+  /**
+     * Creates a vector representation for the given values.
+     */
+  embed: (data: T[]) => Promise<number[][]>
+}

node/src/embedding/openai.ts

@@ -0,0 +1,51 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { type EmbeddingFunction } from '../index'
+
+export class OpenAIEmbeddingFunction implements EmbeddingFunction<string> {
+  private readonly _openai: any
+  private readonly _modelName: string
+
+  constructor (sourceColumn: string, openAIKey: string, modelName: string = 'text-embedding-ada-002') {
+    let openai
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      openai = require('openai')
+    } catch {
+      throw new Error('please install openai using npm install openai')
+    }
+
+    this.sourceColumn = sourceColumn
+    const configuration = new openai.Configuration({
+      apiKey: openAIKey
+    })
+    this._openai = new openai.OpenAIApi(configuration)
+    this._modelName = modelName
+  }
+
+  async embed (data: string[]): Promise<number[][]> {
+    const response = await this._openai.createEmbedding({
+      model: this._modelName,
+      input: data
+    })
+    const embeddings: number[][] = []
+    for (let i = 0; i < response.data.data.length; i++) {
+      embeddings.push(response.data.data[i].embedding as number[])
+    }
+    return embeddings
+  }
+
+  sourceColumn: string
+}

node/src/index.ts

@@ -19,28 +19,119 @@ import {
   Vector
 } from 'apache-arrow'
 import { fromRecordsToBuffer } from './arrow'
+import type { EmbeddingFunction } from './embedding/embedding_function'
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const { databaseNew, databaseTableNames, databaseOpenTable, tableCreate, tableSearch, tableAdd } = require('../native.js')
+const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableSearch, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete } = require('../native.js')
+
+export type { EmbeddingFunction }
+export { OpenAIEmbeddingFunction } from './embedding/openai'
 
 /**
  * Connect to a LanceDB instance at the given URI
  * @param uri The uri of the database.
  */
 export async function connect (uri: string): Promise<Connection> {
-  return new Connection(uri)
+  const db = await databaseNew(uri)
+  return new LocalConnection(db, uri)
+}
+
+/**
+ * A LanceDB Connection that allows you to open tables and create new ones.
+ *
+ * Connection could be local against filesystem or remote against a server.
+ */
+export interface Connection {
+  uri: string
+
+  tableNames(): Promise<string[]>
+
+  /**
+   * Open a table in the database.
+   *
+   * @param name The name of the table.
+   * @param embeddings An embedding function to use on this table
+   */
+  openTable<T>(name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
+
+  /**
+   * Creates a new Table and initialize it with new data.
+   *
+   * @param {string} name - The name of the table.
+   * @param data - Non-empty Array of Records to be inserted into the table
+   * @param {WriteMode} mode - The write mode to use when creating the table.
+   * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
+   */
+  createTable<T>(name: string, data: Array<Record<string, unknown>>, mode?: WriteMode, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
+
+  createTableArrow(name: string, table: ArrowTable): Promise<Table>
+
+  /**
+   * Drop an existing table.
+   * @param name The name of the table to drop.
+   */
+  dropTable(name: string): Promise<void>
+
+}
+
+/**
+ * A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
+ */
+export interface Table<T = number[]> {
+  name: string
+
+  /**
+   * Creates a search query to find the nearest neighbors of the given search term
+   * @param query The query search term
+   */
+  search: (query: T) => Query<T>
+
+  /**
+   * Insert records into this Table.
+   *
+   * @param data Records to be inserted into the Table
+   * @return The number of rows added to the table
+   */
+  add: (data: Array<Record<string, unknown>>) => Promise<number>
+
+  /**
+   * Insert records into this Table, replacing its contents.
+   *
+   * @param data Records to be inserted into the Table
+   * @return The number of rows added to the table
+   */
+  overwrite: (data: Array<Record<string, unknown>>) => Promise<number>
+
+  /**
+   * Create an ANN index on this Table vector index.
+   *
+   * @param indexParams The parameters of this Index, @see VectorIndexParams.
+   */
+  createIndex: (indexParams: VectorIndexParams) => Promise<any>
+
+  /**
+   * Returns the number of rows in this table.
+   */
+  countRows: () => Promise<number>
+
+  /**
+   * Delete rows from this table.
+   *
+   * @param filter  A filter in the same format used by a sql WHERE clause.
+   */
+  delete: (filter: string) => Promise<void>
 }
 
 /**
  * A connection to a LanceDB database.
  */
-export class Connection {
+export class LocalConnection implements Connection {
   private readonly _uri: string
   private readonly _db: any
 
-  constructor (uri: string) {
+  constructor (db: any, uri: string) {
     this._uri = uri
-    this._db = databaseNew(uri)
+    this._db = db
   }
 
   get uri (): string {
@@ -55,17 +146,56 @@ export class Connection {
   }
 
   /**
-     * Open a table in the database.
+   * Open a table in the database.
-     * @param name The name of the table.
+   *
-     */
+   * @param name The name of the table.
-  async openTable (name: string): Promise<Table> {
+   */
+  async openTable (name: string): Promise<Table>
+  /**
+   * Open a table in the database.
+   *
+   * @param name The name of the table.
+   * @param embeddings An embedding function to use on this Table
+   */
+  async openTable<T> (name: string, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
+  async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
     const tbl = await databaseOpenTable.call(this._db, name)
-    return new Table(tbl, name)
+    if (embeddings !== undefined) {
+      return new LocalTable(tbl, name, embeddings)
+    } else {
+      return new LocalTable(tbl, name)
+    }
   }
 
-  async createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table> {
+  /**
-    await tableCreate.call(this._db, name, await fromRecordsToBuffer(data))
+   * Creates a new Table and initialize it with new data.
-    return await this.openTable(name)
+   *
+   * @param name The name of the table.
+   * @param data Non-empty Array of Records to be inserted into the Table
+   * @param mode The write mode to use when creating the table.
+   */
+  async createTable (name: string, data: Array<Record<string, unknown>>, mode?: WriteMode): Promise<Table>
+  async createTable (name: string, data: Array<Record<string, unknown>>, mode: WriteMode): Promise<Table>
+
+  /**
+   * Creates a new Table and initialize it with new data.
+   *
+   * @param name The name of the table.
+   * @param data Non-empty Array of Records to be inserted into the Table
+   * @param mode The write mode to use when creating the table.
+   * @param embeddings An embedding function to use on this Table
+   */
+  async createTable<T> (name: string, data: Array<Record<string, unknown>>, mode: WriteMode, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
+  async createTable<T> (name: string, data: Array<Record<string, unknown>>, mode: WriteMode, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
+    if (mode === undefined) {
+      mode = WriteMode.Create
+    }
+    const tbl = await tableCreate.call(this._db, name, await fromRecordsToBuffer(data, embeddings), mode.toLowerCase())
+    if (embeddings !== undefined) {
+      return new LocalTable(tbl, name, embeddings)
+    } else {
+      return new LocalTable(tbl, name)
+    }
   }
 
   async createTableArrow (name: string, table: ArrowTable): Promise<Table> {
@@ -73,18 +203,32 @@ export class Connection {
     await tableCreate.call(this._db, name, Buffer.from(await writer.toUint8Array()))
     return await this.openTable(name)
   }
+
+  /**
+   * Drop an existing table.
+   * @param name The name of the table to drop.
+   */
+  async dropTable (name: string): Promise<void> {
+    await databaseDropTable.call(this._db, name)
+  }
 }
 
-/**
+export class LocalTable<T = number[]> implements Table<T> {
- * A table in a LanceDB database.
- */
-export class Table {
   private readonly _tbl: any
   private readonly _name: string
+  private readonly _embeddings?: EmbeddingFunction<T>
 
-  constructor (tbl: any, name: string) {
+  constructor (tbl: any, name: string)
+  /**
+   * @param tbl
+   * @param name
+   * @param embeddings An embedding function to use when interacting with this table
+   */
+  constructor (tbl: any, name: string, embeddings: EmbeddingFunction<T>)
+  constructor (tbl: any, name: string, embeddings?: EmbeddingFunction<T>) {
     this._tbl = tbl
     this._name = name
+    this._embeddings = embeddings
   }
 
   get name (): string {
@@ -92,73 +236,212 @@ export class Table {
   }
 
   /**
-     * Create a search query to find the nearest neighbors of the given query vector.
+   * Creates a search query to find the nearest neighbors of the given search term
-     * @param queryVector The query vector.
+   * @param query The query search term
-     */
+   */
-  search (queryVector: number[]): Query {
+  search (query: T): Query<T> {
-    return new Query(this._tbl, queryVector)
+    return new Query(this._tbl, query, this._embeddings)
   }
 
   /**
-   * Insert records into this Table
+   * Insert records into this Table.
-   * @param data Records to be inserted into the Table
    *
-   * @param mode Append / Overwrite existing records. Default: Append
+   * @param data Records to be inserted into the Table
    * @return The number of rows added to the table
    */
   async add (data: Array<Record<string, unknown>>): Promise<number> {
-    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data), WriteMode.Append.toString())
+    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Append.toString())
   }
 
+  /**
+   * Insert records into this Table, replacing its contents.
+   *
+   * @param data Records to be inserted into the Table
+   * @return The number of rows added to the table
+   */
   async overwrite (data: Array<Record<string, unknown>>): Promise<number> {
-    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data), WriteMode.Overwrite.toString())
+    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Overwrite.toString())
+  }
+
+  /**
+   * Create an ANN index on this Table vector index.
+   *
+   * @param indexParams The parameters of this Index, @see VectorIndexParams.
+   */
+  async createIndex (indexParams: VectorIndexParams): Promise<any> {
+    return tableCreateVectorIndex.call(this._tbl, indexParams)
+  }
+
+  /**
+   * Returns the number of rows in this table.
+   */
+  async countRows (): Promise<number> {
+    return tableCountRows.call(this._tbl)
+  }
+
+  /**
+   * Delete rows from this table.
+   *
+   * @param filter A filter in the same format used by a sql WHERE clause.
+   */
+  async delete (filter: string): Promise<void> {
+    return tableDelete.call(this._tbl, filter)
   }
 }
 
+/// Config to build IVF_PQ index.
+///
+export interface IvfPQIndexConfig {
+  /**
+   * The column to be indexed
+   */
+  column?: string
+
+  /**
+   * A unique name for the index
+   */
+  index_name?: string
+
+  /**
+   * Metric type, L2 or Cosine
+   */
+  metric_type?: MetricType
+
+  /**
+   * The number of partitions this index
+   */
+  num_partitions?: number
+
+  /**
+   * The max number of iterations for kmeans training.
+   */
+  max_iters?: number
+
+  /**
+   * Train as optimized product quantization.
+   */
+  use_opq?: boolean
+
+  /**
+   * Number of subvectors to build PQ code
+   */
+  num_sub_vectors?: number
+  /**
+   * The number of bits to present one PQ centroid.
+   */
+  num_bits?: number
+
+  /**
+   * Max number of iterations to train OPQ, if `use_opq` is true.
+   */
+  max_opq_iters?: number
+
+  /**
+   * Replace an existing index with the same name if it exists.
+   */
+  replace?: boolean
+
+  type: 'ivf_pq'
+}
+
+export type VectorIndexParams = IvfPQIndexConfig
+
 /**
  * A builder for nearest neighbor queries for LanceDB.
  */
-export class Query {
+export class Query<T = number[]> {
   private readonly _tbl: any
-  private readonly _query_vector: number[]
+  private readonly _query: T
+  private _queryVector?: number[]
   private _limit: number
-  private readonly _refine_factor?: number
+  private _refineFactor?: number
-  private readonly _nprobes: number
+  private _nprobes: number
-  private readonly _columns?: string[]
+  private _select?: string[]
   private _filter?: string
-  private readonly _metric = 'L2'
+  private _metricType?: MetricType
+  private readonly _embeddings?: EmbeddingFunction<T>
 
-  constructor (tbl: any, queryVector: number[]) {
+  constructor (tbl: any, query: T, embeddings?: EmbeddingFunction<T>) {
     this._tbl = tbl
-    this._query_vector = queryVector
+    this._query = query
     this._limit = 10
     this._nprobes = 20
-    this._refine_factor = undefined
+    this._refineFactor = undefined
-    this._columns = undefined
+    this._select = undefined
     this._filter = undefined
+    this._metricType = undefined
+    this._embeddings = embeddings
   }
 
-  limit (value: number): Query {
+  /***
+   * Sets the number of results that will be returned
+   * @param value number of results
+   */
+  limit (value: number): Query<T> {
     this._limit = value
     return this
   }
 
-  filter (value: string): Query {
+  /**
-    this._filter = value
+   * Refine the results by reading extra elements and re-ranking them in memory.
+   * @param value refine factor to use in this query.
+   */
+  refineFactor (value: number): Query<T> {
+    this._refineFactor = value
     return this
   }
 
   /**
-     * Execute the query and return the results as an Array of Objects
+   * The number of probes used. A higher number makes search more accurate but also slower.
-     */
+   * @param value The number of probes used.
+   */
+  nprobes (value: number): Query<T> {
+    this._nprobes = value
+    return this
+  }
+
+  /**
+   * A filter statement to be applied to this query.
+   * @param value A filter in the same format used by a sql WHERE clause.
+   */
+  filter (value: string): Query<T> {
+    this._filter = value
+    return this
+  }
+
+  where = this.filter
+
+  /** Return only the specified columns.
+   *
+   * @param value Only select the specified columns. If not specified, all columns will be returned.
+   */
+  select (value: string[]): Query<T> {
+    this._select = value
+    return this
+  }
+
+  /**
+   * The MetricType used for this Query.
+   * @param value The metric to the. @see MetricType for the different options
+   */
+  metricType (value: MetricType): Query<T> {
+    this._metricType = value
+    return this
+  }
+
+  /**
+   * Execute the query and return the results as an Array of Objects
+   */
   async execute<T = Record<string, unknown>> (): Promise<T[]> {
-    let buffer
+    if (this._embeddings !== undefined) {
-    if (this._filter != null) {
+      this._queryVector = (await this._embeddings.embed([this._query]))[0]
-      buffer = await tableSearch.call(this._tbl, this._query_vector, this._limit, this._filter)
     } else {
-      buffer = await tableSearch.call(this._tbl, this._query_vector, this._limit)
+      this._queryVector = this._query as number[]
     }
+
+    const buffer = await tableSearch.call(this._tbl, this)
     const data = tableFromIPC(buffer)
+
     return data.toArray().map((entry: Record<string, unknown>) => {
       const newObject: Record<string, unknown> = {}
       Object.keys(entry).forEach((key: string) => {
@@ -173,7 +456,34 @@ export class Query {
   }
 }
 
+/**
+ * Write mode for writing a table.
+ */
 export enum WriteMode {
+  /** Create a new {@link Table}. */
+  Create = 'create',
+  /** Overwrite the existing {@link Table} if presented. */
   Overwrite = 'overwrite',
+  /** Append new data to the table. */
   Append = 'append'
 }
+
+/**
+ * Distance metrics type.
+ */
+export enum MetricType {
+  /**
+   * Euclidean distance
+   */
+  L2 = 'l2',
+
+  /**
+   * Cosine distance
+   */
+  Cosine = 'cosine',
+
+  /**
+   * Dot product
+   */
+  Dot = 'dot'
+}

node/src/test/embedding/openai.ts

@@ -0,0 +1,50 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { describe } from 'mocha'
+import { assert } from 'chai'
+
+import { OpenAIEmbeddingFunction } from '../../embedding/openai'
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const { OpenAIApi } = require('openai')
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const { stub } = require('sinon')
+
+describe('OpenAPIEmbeddings', function () {
+  const stubValue = {
+    data: {
+      data: [
+        {
+          embedding: Array(1536).fill(1.0)
+        },
+        {
+          embedding: Array(1536).fill(2.0)
+        }
+      ]
+    }
+  }
+
+  describe('#embed', function () {
+    it('should create vector embeddings', async function () {
+      const openAIStub = stub(OpenAIApi.prototype, 'createEmbedding').returns(stubValue)
+      const f = new OpenAIEmbeddingFunction('text', 'sk-key')
+      const vectors = await f.embed(['abc', 'def'])
+      assert.isTrue(openAIStub.calledOnce)
+      assert.equal(vectors.length, 2)
+      assert.deepEqual(vectors[0], stubValue.data.data[0].embedding)
+      assert.deepEqual(vectors[1], stubValue.data.data[1].embedding)
+    })
+  })
+})

node/src/test/io.ts

@@ -0,0 +1,52 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// IO tests
+
+import { describe } from 'mocha'
+import { assert } from 'chai'
+
+import * as lancedb from '../index'
+
+describe('LanceDB S3 client', function () {
+  if (process.env.TEST_S3_BASE_URL != null) {
+    const baseUri = process.env.TEST_S3_BASE_URL
+    it('should have a valid url', async function () {
+      const uri = `${baseUri}/valid_url`
+      const table = await createTestDB(uri, 2, 20)
+      const con = await lancedb.connect(uri)
+      assert.equal(con.uri, uri)
+
+      const results = await table.search([0.1, 0.3]).limit(5).execute()
+      assert.equal(results.length, 5)
+    })
+  } else {
+    describe.skip('Skip S3 test', function () {})
+  }
+})
+
+async function createTestDB (uri: string, numDimensions: number = 2, numRows: number = 2): Promise<lancedb.Table> {
+  const con = await lancedb.connect(uri)
+
+  const data = []
+  for (let i = 0; i < numRows; i++) {
+    const vector = []
+    for (let j = 0; j < numDimensions; j++) {
+      vector.push(i + (j * 0.1))
+    }
+    data.push({ id: i + 1, name: `name_${i}`, price: i + 10, is_active: (i % 2 === 0), vector })
+  }
+
+  return await con.createTable('vectors', data)
+}

node/src/test/test.ts

@@ -1,4 +1,4 @@
-// Copyright 2023 Lance Developers.
+// Copyright 2023 LanceDB Developers.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,10 +13,16 @@
 // limitations under the License.
 
 import { describe } from 'mocha'
-import { assert } from 'chai'
 import { track } from 'temp'
+import * as chai from 'chai'
+import * as chaiAsPromised from 'chai-as-promised'
 
 import * as lancedb from '../index'
+import { type EmbeddingFunction, MetricType, Query, WriteMode } from '../index'
+
+const expect = chai.expect
+const assert = chai.assert
+chai.use(chaiAsPromised)
 
 describe('LanceDB client', function () {
   describe('when creating a connection to lancedb', function () {
@@ -63,13 +69,36 @@ describe('LanceDB client', function () {
       assert.equal(results[0].id, 1)
     })
 
-    it('uses a filter', async function () {
+    it('uses a filter / where clause', async function () {
+      // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
+      const assertResults = (results: Array<Record<string, unknown>>) => {
+        assert.equal(results.length, 1)
+        assert.equal(results[0].id, 2)
+      }
+
       const uri = await createTestDB()
       const con = await lancedb.connect(uri)
       const table = await con.openTable('vectors')
-      const results = await table.search([0.1, 0.3]).filter('id == 2').execute()
+      let results = await table.search([0.1, 0.1]).filter('id == 2').execute()
-      assert.equal(results.length, 1)
+      assertResults(results)
-      assert.equal(results[0].id, 2)
+      results = await table.search([0.1, 0.1]).where('id == 2').execute()
+      assertResults(results)
+    })
+
+    it('select only a subset of columns', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      const results = await table.search([0.1, 0.1]).select(['is_active']).execute()
+      assert.equal(results.length, 2)
+      // vector and score are always returned
+      assert.isDefined(results[0].vector)
+      assert.isDefined(results[0].score)
+      assert.isDefined(results[0].is_active)
+
+      assert.isUndefined(results[0].id)
+      assert.isUndefined(results[0].name)
+      assert.isUndefined(results[0].price)
     })
   })
 
@@ -86,12 +115,10 @@ describe('LanceDB client', function () {
       const tableName = `vectors_${Math.floor(Math.random() * 100)}`
       const table = await con.createTable(tableName, data)
       assert.equal(table.name, tableName)
-
+      assert.equal(await table.countRows(), 2)
-      const results = await table.search([0.1, 0.3]).execute()
-      assert.equal(results.length, 2)
     })
 
-    it('appends records to an existing table ', async function () {
+    it('use overwrite flag to overwrite existing table', async function () {
       const dir = await track().mkdir('lancejs')
       const con = await lancedb.connect(dir)
 
@@ -100,17 +127,40 @@ describe('LanceDB client', function () {
         { id: 2, vector: [1.1, 1.2], price: 50 }
       ]
 
+      const tableName = 'overwrite'
+      await con.createTable(tableName, data, WriteMode.Create)
+
+      const newData = [
+        { id: 1, vector: [0.1, 0.2], price: 10 },
+        { id: 2, vector: [1.1, 1.2], price: 50 },
+        { id: 3, vector: [1.1, 1.2], price: 50 }
+      ]
+
+      await expect(con.createTable(tableName, newData)).to.be.rejectedWith(Error, 'already exists')
+
+      const table = await con.createTable(tableName, newData, WriteMode.Overwrite)
+      assert.equal(table.name, tableName)
+      assert.equal(await table.countRows(), 3)
+    })
+
+    it('appends records to an existing table ', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const data = [
+        { id: 1, vector: [0.1, 0.2], price: 10, name: 'a' },
+        { id: 2, vector: [1.1, 1.2], price: 50, name: 'b' }
+      ]
+
       const table = await con.createTable('vectors', data)
-      const results = await table.search([0.1, 0.3]).execute()
+      assert.equal(await table.countRows(), 2)
-      assert.equal(results.length, 2)
 
       const dataAdd = [
-        { id: 3, vector: [2.1, 2.2], price: 10 },
+        { id: 3, vector: [2.1, 2.2], price: 10, name: 'c' },
-        { id: 4, vector: [3.1, 3.2], price: 50 }
+        { id: 4, vector: [3.1, 3.2], price: 50, name: 'd' }
       ]
       await table.add(dataAdd)
-      const resultsAdd = await table.search([0.1, 0.3]).execute()
+      assert.equal(await table.countRows(), 4)
-      assert.equal(resultsAdd.length, 4)
     })
 
     it('overwrite all records in a table', async function () {
@@ -118,29 +168,136 @@ describe('LanceDB client', function () {
       const con = await lancedb.connect(uri)
 
       const table = await con.openTable('vectors')
-      const results = await table.search([0.1, 0.3]).execute()
+      assert.equal(await table.countRows(), 2)
-      assert.equal(results.length, 2)
 
       const dataOver = [
         { vector: [2.1, 2.2], price: 10, name: 'foo' },
         { vector: [3.1, 3.2], price: 50, name: 'bar' }
       ]
       await table.overwrite(dataOver)
-      const resultsAdd = await table.search([0.1, 0.3]).execute()
+      assert.equal(await table.countRows(), 2)
-      assert.equal(resultsAdd.length, 2)
+    })
+
+    it('can delete records from a table', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+
+      const table = await con.openTable('vectors')
+      assert.equal(await table.countRows(), 2)
+
+      await table.delete('price = 10')
+      assert.equal(await table.countRows(), 1)
+    })
+  })
+
+  describe('when creating a vector index', function () {
+    it('overwrite all records in a table', async function () {
+      const uri = await createTestDB(32, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+    }).timeout(10_000) // Timeout is high partially because GH macos runner is pretty slow
+
+    it('replace an existing index', async function () {
+      const uri = await createTestDB(16, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+
+      // Replace should fail if the index already exists
+      await expect(table.createIndex({
+        type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2, replace: false
+      })
+      ).to.be.rejectedWith('LanceError(Index)')
+
+      // Default replace = true
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+    }).timeout(50_000)
+  })
+
+  describe('when using a custom embedding function', function () {
+    class TextEmbedding implements EmbeddingFunction<string> {
+      sourceColumn: string
+
+      constructor (targetColumn: string) {
+        this.sourceColumn = targetColumn
+      }
+
+      _embedding_map = new Map<string, number[]>([
+        ['foo', [2.1, 2.2]],
+        ['bar', [3.1, 3.2]]
+      ])
+
+      async embed (data: string[]): Promise<number[][]> {
+        return data.map(datum => this._embedding_map.get(datum) ?? [0.0, 0.0])
+      }
+    }
+
+    it('should encode the original data into embeddings', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+      const embeddings = new TextEmbedding('name')
+
+      const data = [
+        { price: 10, name: 'foo' },
+        { price: 50, name: 'bar' }
+      ]
+      const table = await con.createTable('vectors', data, WriteMode.Create, embeddings)
+      const results = await table.search('foo').execute()
+      assert.equal(results.length, 2)
     })
   })
 })
 
-async function createTestDB (): Promise<string> {
+describe('Query object', function () {
+  it('sets custom parameters', async function () {
+    const query = new Query(undefined, [0.1, 0.3])
+      .limit(1)
+      .metricType(MetricType.Cosine)
+      .refineFactor(100)
+      .select(['a', 'b'])
+      .nprobes(20) as Record<string, any>
+    assert.equal(query._limit, 1)
+    assert.equal(query._metricType, MetricType.Cosine)
+    assert.equal(query._refineFactor, 100)
+    assert.equal(query._nprobes, 20)
+    assert.deepEqual(query._select, ['a', 'b'])
+  })
+})
+
+async function createTestDB (numDimensions: number = 2, numRows: number = 2): Promise<string> {
   const dir = await track().mkdir('lancejs')
   const con = await lancedb.connect(dir)
 
-  const data = [
+  const data = []
-    { id: 1, vector: [0.1, 0.2], name: 'foo', price: 10, is_active: true },
+  for (let i = 0; i < numRows; i++) {
-    { id: 2, vector: [1.1, 1.2], name: 'bar', price: 50, is_active: false }
+    const vector = []
-  ]
+    for (let j = 0; j < numDimensions; j++) {
+      vector.push(i + (j * 0.1))
+    }
+    data.push({ id: i + 1, name: `name_${i}`, price: i + 10, is_active: (i % 2 === 0), vector })
+  }
 
   await con.createTable('vectors', data)
   return dir
 }
+
+describe('Drop table', function () {
+  it('drop a table', async function () {
+    const dir = await track().mkdir('lancejs')
+    const con = await lancedb.connect(dir)
+
+    const data = [
+      { price: 10, name: 'foo', vector: [1, 2, 3] },
+      { price: 50, name: 'bar', vector: [4, 5, 6] }
+    ]
+    await con.createTable('t1', data)
+    await con.createTable('t2', data)
+
+    assert.deepEqual(await con.tableNames(), ['t1', 't2'])
+
+    await con.dropTable('t1')
+    assert.deepEqual(await con.tableNames(), ['t2'])
+  })
+})

python/.bumpversion.cfg

@@ -0,0 +1,8 @@
+[bumpversion]
+current_version = 0.1.8
+commit = True
+message = [python] Bump version: {current_version} → {new_version}
+tag = True
+tag_name = python-v{new_version}
+
+[bumpversion:file:pyproject.toml]

python/README.md

@@ -0,0 +1,85 @@
+# LanceDB
+
+A Python library for [LanceDB](https://github.com/lancedb/lancedb).
+
+## Installation
+
+```bash
+pip install lancedb
+```
+
+## Usage
+
+### Basic Example
+
+```python
+import lancedb
+db = lancedb.connect('<PATH_TO_LANCEDB_DATASET>')
+table = db.open_table('my_table')
+results = table.search([0.1, 0.3]).limit(20).to_df()
+print(results)
+```
+
+
+## Development
+
+Create a virtual environment and activate it:
+
+```bash
+python -m venv venv
+. ./venv/bin/activate
+```
+
+Install the necessary packages:
+
+```bash
+python -m pip install .
+```
+
+To run the unit tests:
+
+```bash
+pytest
+```
+
+To run linter and automatically fix all errors:
+
+```bash
+black .
+isort .
+```
+
+If any packages are missing, install them with:
+
+```bash
+pip install <PACKAGE_NAME>
+```
+
+
+___
+For **Windows** users, there may be errors when installing packages, so these commands may be helpful:
+
+Activate the virtual environment:
+```bash
+. .\venv\Scripts\activate
+```
+
+You may need to run the installs separately:
+```bash
+pip install -e .[tests]
+pip install -e .[dev]
+```
+
+
+`tantivy` requires `rust` to be installed, so install it with `conda`, as it doesn't support windows installation:
+```bash
+pip install wheel
+pip install cargo
+conda install rust
+pip install tantivy
+```
+
+To run the unit tests:
+```bash
+pytest
+```

python/lancedb/__init__.py

@@ -11,19 +11,48 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from .db import URI, LanceDBConnection
+from typing import Optional
+
+from .db import URI, DBConnection, LanceDBConnection
+from .remote.db import RemoteDBConnection
 
 
-def connect(uri: URI) -> LanceDBConnection:
+def connect(
-    """Connect to a LanceDB instance at the given URI
+    uri: URI, *, api_key: Optional[str] = None, region: str = "us-west-2"
+) -> DBConnection:
+    """Connect to a LanceDB database.
 
     Parameters
     ----------
     uri: str or Path
         The uri of the database.
+    api_token: str, optional
+        If presented, connect to LanceDB cloud.
+        Otherwise, connect to a database on file system or cloud storage.
+
+    Examples
+    --------
+
+    For a local directory, provide a path for the database:
+
+    >>> import lancedb
+    >>> db = lancedb.connect("~/.lancedb")
+
+    For object storage, use a URI prefix:
+
+    >>> db = lancedb.connect("s3://my-bucket/lancedb")
+
+    Connect to LancdDB cloud:
+
+    >>> db = lancedb.connect("db://my_database", api_key="ldb_...")
 
     Returns
     -------
-    A connection to a LanceDB database.
+    conn : DBConnection
+        A connection to a LanceDB database.
     """
+    if isinstance(uri, str) and uri.startswith("db://"):
+        if api_key is None:
+            raise ValueError(f"api_key is required to connected LanceDB cloud: {uri}")
+        return RemoteDBConnection(uri, api_key, region)
     return LanceDBConnection(uri)

python/lancedb/common.py

@@ -23,3 +23,13 @@ URI = Union[str, Path]
 # TODO support generator
 DATA = Union[List[dict], dict, pd.DataFrame]
 VECTOR_COLUMN_NAME = "vector"
+
+
+class Credential(str):
+    """Credential field"""
+
+    def __repr__(self) -> str:
+        return "********"
+
+    def __str__(self) -> str:
+        return "********"

python/lancedb/conftest.py

@@ -0,0 +1,16 @@
+import os
+
+import pytest
+
+# import lancedb so we don't have to in every example
+
+
+@pytest.fixture(autouse=True)
+def doctest_setup(monkeypatch, tmpdir):
+    # disable color for doctests so we don't have to include
+    # escape codes in docstrings
+    monkeypatch.setitem(os.environ, "NO_COLOR", "1")
+    # Explicitly set the column width
+    monkeypatch.setitem(os.environ, "COLUMNS", "80")
+    # Work in a temporary directory
+    monkeypatch.chdir(tmpdir)

python/lancedb/context.py

@@ -14,20 +14,109 @@ from __future__ import annotations
 
 import pandas as pd
 
+from .exceptions import MissingColumnError, MissingValueError
+
 
 def contextualize(raw_df: pd.DataFrame) -> Contextualizer:
     """Create a Contextualizer object for the given DataFrame.
-    Used to create context windows.
+
+    Used to create context windows. Context windows are rolling subsets of text
+    data.
+
+    The input text column should already be separated into rows that will be the
+    unit of the window. So to create a context window over tokens, start with
+    a DataFrame with one token per row. To create a context window over sentences,
+    start with a DataFrame with one sentence per row.
+
+    Examples
+    --------
+    >>> from lancedb.context import contextualize
+    >>> import pandas as pd
+    >>> data = pd.DataFrame({
+    ...    'token': ['The', 'quick', 'brown', 'fox', 'jumped', 'over',
+    ...              'the', 'lazy', 'dog', 'I', 'love', 'sandwiches'],
+    ...    'document_id': [1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2]
+    ... })
+
+    ``window`` determines how many rows to include in each window. In our case
+    this how many tokens, but depending on the input data, it could be sentences,
+    paragraphs, messages, etc.
+
+    >>> contextualize(data).window(3).stride(1).text_col('token').to_df()
+                    token  document_id
+    0     The quick brown            1
+    1     quick brown fox            1
+    2    brown fox jumped            1
+    3     fox jumped over            1
+    4     jumped over the            1
+    5       over the lazy            1
+    6        the lazy dog            1
+    7          lazy dog I            1
+    8          dog I love            1
+    9   I love sandwiches            2
+    10    love sandwiches            2
+    >>> contextualize(data).window(7).stride(1).min_window_size(7).text_col('token').to_df()
+                                      token  document_id
+    0   The quick brown fox jumped over the            1
+    1  quick brown fox jumped over the lazy            1
+    2    brown fox jumped over the lazy dog            1
+    3        fox jumped over the lazy dog I            1
+    4       jumped over the lazy dog I love            1
+    5   over the lazy dog I love sandwiches            1
+
+    ``stride`` determines how many rows to skip between each window start. This can
+    be used to reduce the total number of windows generated.
+
+    >>> contextualize(data).window(4).stride(2).text_col('token').to_df()
+                        token  document_id
+    0     The quick brown fox            1
+    2   brown fox jumped over            1
+    4    jumped over the lazy            1
+    6          the lazy dog I            1
+    8   dog I love sandwiches            1
+    10        love sandwiches            2
+
+    ``groupby`` determines how to group the rows. For example, we would like to have
+    context windows that don't cross document boundaries. In this case, we can
+    pass ``document_id`` as the group by.
+
+    >>> contextualize(data).window(4).stride(2).text_col('token').groupby('document_id').to_df()
+                       token  document_id
+    0    The quick brown fox            1
+    2  brown fox jumped over            1
+    4   jumped over the lazy            1
+    6           the lazy dog            1
+    9      I love sandwiches            2
+
+    ``min_window_size`` determines the minimum size of the  context windows that are generated
+    This can be used to trim the last few context windows which have size less than
+    ``min_window_size``. By default context windows of size 1 are skipped.
+
+    >>> contextualize(data).window(6).stride(3).text_col('token').groupby('document_id').to_df()
+                                 token  document_id
+    0  The quick brown fox jumped over            1
+    3     fox jumped over the lazy dog            1
+    6                     the lazy dog            1
+    9                I love sandwiches            2
+
+    >>> contextualize(data).window(6).stride(3).min_window_size(4).text_col('token').groupby('document_id').to_df()
+                                 token  document_id
+    0  The quick brown fox jumped over            1
+    3     fox jumped over the lazy dog            1
+
     """
     return Contextualizer(raw_df)
 
 
 class Contextualizer:
+    """Create context windows from a DataFrame. See [lancedb.context.contextualize][]."""
+
     def __init__(self, raw_df):
         self._text_col = None
         self._groupby = None
         self._stride = None
         self._window = None
+        self._min_window_size = 2
         self._raw_df = raw_df
 
     def window(self, window: int) -> Contextualizer:
@@ -75,17 +164,50 @@ class Contextualizer:
         self._text_col = text_col
         return self
 
+    def min_window_size(self, min_window_size: int) -> Contextualizer:
+        """Set the (optional) min_window_size size for the context window.
+
+        Parameters
+        ----------
+        min_window_size: int
+            The min_window_size.
+        """
+        self._min_window_size = min_window_size
+        return self
+
     def to_df(self) -> pd.DataFrame:
         """Create the context windows and return a DataFrame."""
 
+        if self._text_col not in self._raw_df.columns.tolist():
+            raise MissingColumnError(self._text_col)
+
+        if self._window is None or self._window < 1:
+            raise MissingValueError(
+                "The value of window is None or less than 1. Specify the "
+                "window size (number of rows to include in each window)"
+            )
+
+        if self._stride is None or self._stride < 1:
+            raise MissingValueError(
+                "The value of stride is None or less than 1. Specify the "
+                "stride (number of rows to skip between each window)"
+            )
+
         def process_group(grp):
             # For each group, create the text rolling window
+            # with values of size >= min_window_size
             text = grp[self._text_col].values
-            contexts = grp.iloc[: -self._window : self._stride, :].copy()
+            contexts = grp.iloc[:: self._stride, :].copy()
-            contexts[self._text_col] = [
+            windows = [
-                " ".join(text[start_i : start_i + self._window])
+                " ".join(text[start_i : min(start_i + self._window, len(grp))])
-                for start_i in range(0, len(grp) - self._window, self._stride)
+                for start_i in range(0, len(grp), self._stride)
+                if start_i + self._window <= len(grp)
+                or len(grp) - start_i >= self._min_window_size
             ]
+            # if last few rows dropped
+            if len(windows) < len(contexts):
+                contexts = contexts.iloc[: len(windows)]
+            contexts[self._text_col] = windows
             return contexts
 
         if self._groupby is None:

python/lancedb/db.py

@@ -13,22 +13,196 @@
 
 from __future__ import annotations
 
+import functools
+import os
+from abc import ABC, abstractmethod
 from pathlib import Path
 
 import pyarrow as pa
+from pyarrow import fs
 
 from .common import DATA, URI
-from .table import LanceTable
+from .table import LanceTable, Table
-from .util import get_uri_scheme
+from .util import get_uri_location, get_uri_scheme
 
 
-class LanceDBConnection:
+class DBConnection(ABC):
+    """An active LanceDB connection interface."""
+
+    @abstractmethod
+    def table_names(self) -> list[str]:
+        """List all table names in the database."""
+        pass
+
+    @abstractmethod
+    def create_table(
+        self,
+        name: str,
+        data: DATA = None,
+        schema: pa.Schema = None,
+        mode: str = "create",
+        on_bad_vectors: str = "error",
+        fill_value: float = 0.0,
+    ) -> Table:
+        """Create a [Table][lancedb.table.Table] in the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+        data: list, tuple, dict, pd.DataFrame; optional
+            The data to insert into the table.
+        schema: pyarrow.Schema; optional
+            The schema of the table.
+        mode: str; default "create"
+            The mode to use when creating the table. Can be either "create" or "overwrite".
+            By default, if the table already exists, an exception is raised.
+            If you want to overwrite the table, use mode="overwrite".
+        on_bad_vectors: str, default "error"
+            What to do if any of the vectors are not the same size or contains NaNs.
+            One of "error", "drop", "fill".
+        fill_value: float
+            The value to use when filling vectors. Only used if on_bad_vectors="fill".
+
+        Note
+        ----
+        The vector index won't be created by default.
+        To create the index, call the `create_index` method on the table.
+
+        Returns
+        -------
+        LanceTable
+            A reference to the newly created table.
+
+        Examples
+        --------
+
+        Can create with list of tuples or dictionaries:
+
+        >>> import lancedb
+        >>> db = lancedb.connect("./.lancedb")
+        >>> data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
+        ...         {"vector": [0.2, 1.8], "lat": 40.1, "long":  -74.1}]
+        >>> db.create_table("my_table", data)
+        LanceTable(my_table)
+        >>> db["my_table"].head()
+        pyarrow.Table
+        vector: fixed_size_list<item: float>[2]
+          child 0, item: float
+        lat: double
+        long: double
+        ----
+        vector: [[[1.1,1.2],[0.2,1.8]]]
+        lat: [[45.5,40.1]]
+        long: [[-122.7,-74.1]]
+
+        You can also pass a pandas DataFrame:
+
+        >>> import pandas as pd
+        >>> data = pd.DataFrame({
+        ...    "vector": [[1.1, 1.2], [0.2, 1.8]],
+        ...    "lat": [45.5, 40.1],
+        ...    "long": [-122.7, -74.1]
+        ... })
+        >>> db.create_table("table2", data)
+        LanceTable(table2)
+        >>> db["table2"].head()
+        pyarrow.Table
+        vector: fixed_size_list<item: float>[2]
+          child 0, item: float
+        lat: double
+        long: double
+        ----
+        vector: [[[1.1,1.2],[0.2,1.8]]]
+        lat: [[45.5,40.1]]
+        long: [[-122.7,-74.1]]
+
+        Data is converted to Arrow before being written to disk. For maximum
+        control over how data is saved, either provide the PyArrow schema to
+        convert to or else provide a PyArrow table directly.
+
+        >>> custom_schema = pa.schema([
+        ...   pa.field("vector", pa.list_(pa.float32(), 2)),
+        ...   pa.field("lat", pa.float32()),
+        ...   pa.field("long", pa.float32())
+        ... ])
+        >>> db.create_table("table3", data, schema = custom_schema)
+        LanceTable(table3)
+        >>> db["table3"].head()
+        pyarrow.Table
+        vector: fixed_size_list<item: float>[2]
+          child 0, item: float
+        lat: float
+        long: float
+        ----
+        vector: [[[1.1,1.2],[0.2,1.8]]]
+        lat: [[45.5,40.1]]
+        long: [[-122.7,-74.1]]
+        """
+        raise NotImplementedError
+
+    def __getitem__(self, name: str) -> LanceTable:
+        return self.open_table(name)
+
+    def open_table(self, name: str) -> Table:
+        """Open a Lance Table in the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+
+        Returns
+        -------
+        A LanceTable object representing the table.
+        """
+        raise NotImplementedError
+
+    def drop_table(self, name: str):
+        """Drop a table from the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+        """
+        raise NotImplementedError
+
+
+class LanceDBConnection(DBConnection):
     """
     A connection to a LanceDB database.
+
+    Parameters
+    ----------
+    uri: str or Path
+        The root uri of the database.
+
+    Examples
+    --------
+    >>> import lancedb
+    >>> db = lancedb.connect("./.lancedb")
+    >>> db.create_table("my_table", data=[{"vector": [1.1, 1.2], "b": 2},
+    ...                                   {"vector": [0.5, 1.3], "b": 4}])
+    LanceTable(my_table)
+    >>> db.create_table("another_table", data=[{"vector": [0.4, 0.4], "b": 6}])
+    LanceTable(another_table)
+    >>> sorted(db.table_names())
+    ['another_table', 'my_table']
+    >>> len(db)
+    2
+    >>> db["my_table"]
+    LanceTable(my_table)
+    >>> "my_table" in db
+    True
+    >>> db.drop_table("my_table")
+    >>> db.drop_table("another_table")
     """
 
     def __init__(self, uri: URI):
-        is_local = isinstance(uri, Path) or get_uri_scheme(uri) == "file"
+        if not isinstance(uri, Path):
+            scheme = get_uri_scheme(uri)
+        is_local = isinstance(uri, Path) or scheme == "file"
         if is_local:
             if isinstance(uri, str):
                 uri = Path(uri)
@@ -36,6 +210,8 @@ class LanceDBConnection:
             Path(uri).mkdir(parents=True, exist_ok=True)
         self._uri = str(uri)
 
+        self._entered = False
+
     @property
     def uri(self) -> str:
         return self._uri
@@ -45,13 +221,27 @@ class LanceDBConnection:
 
         Returns
         -------
-        A list of table names.
+        list of str
+            A list of table names.
         """
-        if get_uri_scheme(self.uri) == "file":
+        try:
-            return [p.stem for p in Path(self.uri).glob("*.lance")]
+            filesystem, path = fs.FileSystem.from_uri(self.uri)
-        raise NotImplementedError(
+        except pa.ArrowInvalid:
-            "List table_names is only supported for local filesystem for now"
+            raise NotImplementedError("Unsupported scheme: " + self.uri)
-        )
+
+        try:
+            paths = filesystem.get_file_info(
+                fs.FileSelector(get_uri_location(self.uri))
+            )
+        except FileNotFoundError:
+            # It is ok if the file does not exist since it will be created
+            paths = []
+        tables = [
+            os.path.splitext(file_info.base_name)[0]
+            for file_info in paths
+            if file_info.extension == "lance"
+        ]
+        return tables
 
     def __len__(self) -> int:
         return len(self.table_names())
@@ -59,15 +249,14 @@ class LanceDBConnection:
     def __contains__(self, name: str) -> bool:
         return name in self.table_names()
 
-    def __getitem__(self, name: str) -> LanceTable:
-        return self.open_table(name)
-
     def create_table(
         self,
         name: str,
         data: DATA = None,
         schema: pa.Schema = None,
         mode: str = "create",
+        on_bad_vectors: str = "error",
+        fill_value: float = 0.0,
     ) -> LanceTable:
         """Create a table in the database.
 
@@ -80,9 +269,14 @@ class LanceDBConnection:
         schema: pyarrow.Schema; optional
             The schema of the table.
         mode: str; default "create"
-            The mode to use when creating the table.
+            The mode to use when creating the table. Can be either "create" or "overwrite".
             By default, if the table already exists, an exception is raised.
             If you want to overwrite the table, use mode="overwrite".
+        on_bad_vectors: str, default "error"
+            What to do if any of the vectors are not the same size or contains NaNs.
+            One of "error", "drop", "fill".
+        fill_value: float
+            The value to use when filling vectors. Only used if on_bad_vectors="fill".
 
         Note
         ----
@@ -91,12 +285,89 @@ class LanceDBConnection:
 
         Returns
         -------
-        A LanceTable object representing the table.
+        LanceTable
+            A reference to the newly created table.
+
+        Examples
+        --------
+
+        Can create with list of tuples or dictionaries:
+
+        >>> import lancedb
+        >>> db = lancedb.connect("./.lancedb")
+        >>> data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
+        ...         {"vector": [0.2, 1.8], "lat": 40.1, "long":  -74.1}]
+        >>> db.create_table("my_table", data)
+        LanceTable(my_table)
+        >>> db["my_table"].head()
+        pyarrow.Table
+        vector: fixed_size_list<item: float>[2]
+          child 0, item: float
+        lat: double
+        long: double
+        ----
+        vector: [[[1.1,1.2],[0.2,1.8]]]
+        lat: [[45.5,40.1]]
+        long: [[-122.7,-74.1]]
+
+        You can also pass a pandas DataFrame:
+
+        >>> import pandas as pd
+        >>> data = pd.DataFrame({
+        ...    "vector": [[1.1, 1.2], [0.2, 1.8]],
+        ...    "lat": [45.5, 40.1],
+        ...    "long": [-122.7, -74.1]
+        ... })
+        >>> db.create_table("table2", data)
+        LanceTable(table2)
+        >>> db["table2"].head()
+        pyarrow.Table
+        vector: fixed_size_list<item: float>[2]
+          child 0, item: float
+        lat: double
+        long: double
+        ----
+        vector: [[[1.1,1.2],[0.2,1.8]]]
+        lat: [[45.5,40.1]]
+        long: [[-122.7,-74.1]]
+
+        Data is converted to Arrow before being written to disk. For maximum
+        control over how data is saved, either provide the PyArrow schema to
+        convert to or else provide a PyArrow table directly.
+
+        >>> custom_schema = pa.schema([
+        ...   pa.field("vector", pa.list_(pa.float32(), 2)),
+        ...   pa.field("lat", pa.float32()),
+        ...   pa.field("long", pa.float32())
+        ... ])
+        >>> db.create_table("table3", data, schema = custom_schema)
+        LanceTable(table3)
+        >>> db["table3"].head()
+        pyarrow.Table
+        vector: fixed_size_list<item: float>[2]
+          child 0, item: float
+        lat: float
+        long: float
+        ----
+        vector: [[[1.1,1.2],[0.2,1.8]]]
+        lat: [[45.5,40.1]]
+        long: [[-122.7,-74.1]]
         """
+        if mode.lower() not in ["create", "overwrite"]:
+            raise ValueError("mode must be either 'create' or 'overwrite'")
+
         if data is not None:
-            tbl = LanceTable.create(self, name, data, schema, mode=mode)
+            tbl = LanceTable.create(
+                self,
+                name,
+                data,
+                schema,
+                mode=mode,
+                on_bad_vectors=on_bad_vectors,
+                fill_value=fill_value,
+            )
         else:
-            tbl = LanceTable(self, name)
+            tbl = LanceTable.open(self, name)
         return tbl
 
     def open_table(self, name: str) -> LanceTable:
@@ -111,4 +382,16 @@ class LanceDBConnection:
         -------
         A LanceTable object representing the table.
         """
-        return LanceTable(self, name)
+        return LanceTable.open(self, name)
+
+    def drop_table(self, name: str):
+        """Drop a table from the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+        """
+        filesystem, path = pa.fs.FileSystem.from_uri(self.uri)
+        table_path = os.path.join(path, name + ".lance")
+        filesystem.delete_dir(table_path)

python/lancedb/embeddings.py

@@ -29,7 +29,31 @@ def with_embeddings(
     wrap_api: bool = True,
     show_progress: bool = False,
     batch_size: int = 1000,
-):
+) -> pa.Table:
+    """Add a vector column to a table using the given embedding function.
+
+    The new columns will be called "vector".
+
+    Parameters
+    ----------
+    func : Callable
+        A function that takes a list of strings and returns a list of vectors.
+    data : pa.Table or pd.DataFrame
+        The data to add an embedding column to.
+    column : str, default "text"
+        The name of the column to use as input to the embedding function.
+    wrap_api : bool, default True
+        Whether to wrap the embedding function in a retry and rate limiter.
+    show_progress : bool, default False
+        Whether to show a progress bar.
+    batch_size : int, default 1000
+        The number of row values to pass to each call of the embedding function.
+
+    Returns
+    -------
+    pa.Table
+        The input table with a new column called "vector" containing the embeddings.
+    """
     func = EmbeddingFunction(func)
     if wrap_api:
         func = func.retry().rate_limit()

python/lancedb/exceptions.py

@@ -0,0 +1,22 @@
+"""Custom exception handling"""
+
+
+class MissingValueError(ValueError):
+    """Exception raised when a required value is missing."""
+
+    pass
+
+
+class MissingColumnError(KeyError):
+    """
+    Exception raised when a column name specified is not in
+    the  DataFrame object
+    """
+
+    def __init__(self, column_name):
+        self.column_name = column_name
+
+    def __str__(self):
+        return (
+            f"Error: Column '{self.column_name}' does not exist in the DataFrame object"
+        )

python/lancedb/fts.py

@@ -0,0 +1,135 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Full text search index using tantivy-py"""
+import os
+from typing import List, Tuple
+
+import pyarrow as pa
+
+try:
+    import tantivy
+except ImportError:
+    raise ImportError(
+        "Please install tantivy-py `pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985` to use the full text search feature."
+    )
+
+from .table import LanceTable
+
+
+def create_index(index_path: str, text_fields: List[str]) -> tantivy.Index:
+    """
+    Create a new Index (not populated)
+
+    Parameters
+    ----------
+    index_path : str
+        Path to the index directory
+    text_fields : List[str]
+        List of text fields to index
+
+    Returns
+    -------
+    index : tantivy.Index
+        The index object (not yet populated)
+    """
+    # Declaring our schema.
+    schema_builder = tantivy.SchemaBuilder()
+    # special field that we'll populate with row_id
+    schema_builder.add_integer_field("doc_id", stored=True)
+    # data fields
+    for name in text_fields:
+        schema_builder.add_text_field(name, stored=True)
+    schema = schema_builder.build()
+    os.makedirs(index_path, exist_ok=True)
+    index = tantivy.Index(schema, path=index_path)
+    return index
+
+
+def populate_index(index: tantivy.Index, table: LanceTable, fields: List[str]) -> int:
+    """
+    Populate an index with data from a LanceTable
+
+    Parameters
+    ----------
+    index : tantivy.Index
+        The index object
+    table : LanceTable
+        The table to index
+    fields : List[str]
+        List of fields to index
+
+    Returns
+    -------
+    int
+        The number of rows indexed
+    """
+    # first check the fields exist and are string or large string type
+    for name in fields:
+        f = table.schema.field(name)  # raises KeyError if not found
+        if not pa.types.is_string(f.type) and not pa.types.is_large_string(f.type):
+            raise TypeError(f"Field {name} is not a string type")
+
+    # create a tantivy writer
+    writer = index.writer()
+    # write data into index
+    dataset = table.to_lance()
+    row_id = 0
+    for b in dataset.to_batches(columns=fields):
+        for i in range(b.num_rows):
+            doc = tantivy.Document()
+            doc.add_integer("doc_id", row_id)
+            for name in fields:
+                doc.add_text(name, b[name][i].as_py())
+            writer.add_document(doc)
+            row_id += 1
+    # commit changes
+    writer.commit()
+    return row_id
+
+
+def search_index(
+    index: tantivy.Index, query: str, limit: int = 10
+) -> Tuple[Tuple[int], Tuple[float]]:
+    """
+    Search an index for a query
+
+    Parameters
+    ----------
+    index : tantivy.Index
+        The index object
+    query : str
+        The query string
+    limit : int
+        The maximum number of results to return
+
+    Returns
+    -------
+    ids_and_score: list[tuple[int], tuple[float]]
+        A tuple of two tuples, the first containing the document ids
+        and the second containing the scores
+    """
+    searcher = index.searcher()
+    query = index.parse_query(query)
+    # get top results
+    results = searcher.search(query, limit)
+    if results.count == 0:
+        return tuple(), tuple()
+    return tuple(
+        zip(
+            *[
+                (searcher.doc(doc_address)["doc_id"][0], score)
+                for score, doc_address in results.hits
+            ]
+        )
+    )

python/lancedb/query.py

@@ -10,20 +10,76 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
+
 from __future__ import annotations
 
+from typing import List, Literal, Optional, Union
+
 import numpy as np
 import pandas as pd
+import pyarrow as pa
+from pydantic import BaseModel
 
 from .common import VECTOR_COLUMN_NAME
 
 
+class Query(BaseModel):
+    """A Query"""
+
+    vector_column: str = VECTOR_COLUMN_NAME
+
+    # vector to search for
+    vector: List[float]
+
+    # sql filter to refine the query with
+    filter: Optional[str] = None
+
+    # top k results to return
+    k: int
+
+    # # metrics
+    metric: str = "L2"
+
+    # which columns to return in the results
+    columns: Optional[List[str]] = None
+
+    # optional query parameters for tuning the results,
+    # e.g. `{"nprobes": "10", "refine_factor": "10"}`
+    nprobes: int = 10
+
+    # Refine factor.
+    refine_factor: Optional[int] = None
+
+
 class LanceQueryBuilder:
     """
     A builder for nearest neighbor queries for LanceDB.
+
+    Examples
+    --------
+    >>> import lancedb
+    >>> data = [{"vector": [1.1, 1.2], "b": 2},
+    ...         {"vector": [0.5, 1.3], "b": 4},
+    ...         {"vector": [0.4, 0.4], "b": 6},
+    ...         {"vector": [0.4, 0.4], "b": 10}]
+    >>> db = lancedb.connect("./.lancedb")
+    >>> table = db.create_table("my_table", data=data)
+    >>> (table.search([0.4, 0.4])
+    ...       .metric("cosine")
+    ...       .where("b < 10")
+    ...       .select(["b"])
+    ...       .limit(2)
+    ...       .to_df())
+       b      vector  score
+    0  6  [0.4, 0.4]    0.0
     """
 
-    def __init__(self, table: "lancedb.table.LanceTable", query: np.ndarray):
+    def __init__(
+        self,
+        table: "lancedb.table.Table",
+        query: Union[np.ndarray, str],
+        vector_column: str = VECTOR_COLUMN_NAME,
+    ):
         self._metric = "L2"
         self._nprobes = 20
         self._refine_factor = None
@@ -32,6 +88,7 @@ class LanceQueryBuilder:
         self._limit = 10
         self._columns = None
         self._where = None
+        self._vector_column = vector_column
 
     def limit(self, limit: int) -> LanceQueryBuilder:
         """Set the maximum number of results to return.
@@ -43,7 +100,8 @@ class LanceQueryBuilder:
 
         Returns
         -------
-        The LanceQueryBuilder object.
+        LanceQueryBuilder
+            The LanceQueryBuilder object.
         """
         self._limit = limit
         return self
@@ -58,7 +116,8 @@ class LanceQueryBuilder:
 
         Returns
         -------
-        The LanceQueryBuilder object.
+        LanceQueryBuilder
+            The LanceQueryBuilder object.
         """
         self._columns = columns
         return self
@@ -73,22 +132,24 @@ class LanceQueryBuilder:
 
         Returns
         -------
-        The LanceQueryBuilder object.
+        LanceQueryBuilder
+            The LanceQueryBuilder object.
         """
         self._where = where
         return self
 
-    def metric(self, metric: str) -> LanceQueryBuilder:
+    def metric(self, metric: Literal["L2", "cosine"]) -> LanceQueryBuilder:
         """Set the distance metric to use.
 
         Parameters
         ----------
-        metric: str
+        metric: "L2" or "cosine"
-            The distance metric to use. By default "l2" is used.
+            The distance metric to use. By default "L2" is used.
 
         Returns
         -------
-        The LanceQueryBuilder object.
+        LanceQueryBuilder
+            The LanceQueryBuilder object.
         """
         self._metric = metric
         return self
@@ -96,6 +157,12 @@ class LanceQueryBuilder:
     def nprobes(self, nprobes: int) -> LanceQueryBuilder:
         """Set the number of probes to use.
 
+        Higher values will yield better recall (more likely to find vectors if
+        they exist) at the expense of latency.
+
+        See discussion in [Querying an ANN Index][../querying-an-ann-index] for
+        tuning advice.
+
         Parameters
         ----------
         nprobes: int
@@ -103,13 +170,20 @@ class LanceQueryBuilder:
 
         Returns
         -------
-        The LanceQueryBuilder object.
+        LanceQueryBuilder
+            The LanceQueryBuilder object.
         """
         self._nprobes = nprobes
         return self
 
     def refine_factor(self, refine_factor: int) -> LanceQueryBuilder:
-        """Set the refine factor to use.
+        """Set the refine factor to use, increasing the number of vectors sampled.
+
+        As an example, a refine factor of 2 will sample 2x as many vectors as
+        requested, re-ranks them, and returns the top half most relevant results.
+
+        See discussion in [Querying an ANN Index][querying-an-ann-index] for
+        tuning advice.
 
         Parameters
         ----------
@@ -118,7 +192,8 @@ class LanceQueryBuilder:
 
         Returns
         -------
-        The LanceQueryBuilder object.
+        LanceQueryBuilder
+            The LanceQueryBuilder object.
         """
         self._refine_factor = refine_factor
         return self
@@ -130,18 +205,52 @@ class LanceQueryBuilder:
         and also the "score" column which is the distance between the query
         vector and the returned vector.
         """
-        ds = self._table.to_lance()
+
-        # TODO indexed search
+        return self.to_arrow().to_pandas()
-        tbl = ds.to_table(
+
-            columns=self._columns,
+    def to_arrow(self) -> pa.Table:
+        """
+        Execute the query and return the results as an
+        [Apache Arrow Table](https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table).
+
+        In addition to the selected columns, LanceDB also returns a vector
+        and also the "score" column which is the distance between the query
+        vector and the returned vectors.
+        """
+        vector = self._query if isinstance(self._query, list) else self._query.tolist()
+        query = Query(
+            vector=vector,
             filter=self._where,
-            nearest={
+            k=self._limit,
-                "column": VECTOR_COLUMN_NAME,
+            metric=self._metric,
-                "q": self._query,
+            columns=self._columns,
-                "k": self._limit,
+            nprobes=self._nprobes,
-                "metric": self._metric,
+            refine_factor=self._refine_factor,
-                "nprobes": self._nprobes,
-                "refine_factor": self._refine_factor,
-            },
         )
-        return tbl.to_pandas()
+        return self._table._execute_query(query)
+
+
+class LanceFtsQueryBuilder(LanceQueryBuilder):
+    def to_arrow(self) -> pd.Table:
+        try:
+            import tantivy
+        except ImportError:
+            raise ImportError(
+                "Please install tantivy-py `pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985` to use the full text search feature."
+            )
+
+        from .fts import search_index
+
+        # get the index path
+        index_path = self._table._get_fts_index_path()
+        # open the index
+        index = tantivy.Index.open(index_path)
+        # get the scores and doc ids
+        row_ids, scores = search_index(index, self._query, self._limit)
+        if len(row_ids) == 0:
+            empty_schema = pa.schema([pa.field("score", pa.float32())])
+            return pa.Table.from_pylist([], schema=empty_schema)
+        scores = pa.array(scores)
+        output_tbl = self._table.to_lance().take(row_ids, columns=self._columns)
+        output_tbl = output_tbl.append_column("score", scores)
+        return output_tbl

python/lancedb/remote/__init__.py

@@ -0,0 +1,60 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import abc
+from typing import List, Optional
+
+import attr
+import pyarrow as pa
+from pydantic import BaseModel
+
+__all__ = ["LanceDBClient", "VectorQuery", "VectorQueryResult"]
+
+
+class VectorQuery(BaseModel):
+    # vector to search for
+    vector: List[float]
+
+    # sql filter to refine the query with
+    filter: Optional[str] = None
+
+    # top k results to return
+    k: int
+
+    # # metrics
+    _metric: str = "L2"
+
+    # which columns to return in the results
+    columns: Optional[List[str]] = None
+
+    # optional query parameters for tuning the results,
+    # e.g. `{"nprobes": "10", "refine_factor": "10"}`
+    nprobes: int = 10
+
+    refine_factor: Optional[int] = None
+
+
+@attr.define
+class VectorQueryResult:
+    # for now the response is directly seralized into a pandas dataframe
+    tbl: pa.Table
+
+    def to_arrow(self) -> pa.Table:
+        return self.tbl
+
+
+class LanceDBClient(abc.ABC):
+    @abc.abstractmethod
+    def query(self, table_name: str, query: VectorQuery) -> VectorQueryResult:
+        """Query the LanceDB server for the given table and query."""
+        pass

python/lancedb/remote/client.py

@@ -0,0 +1,83 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+import functools
+from typing import Dict
+
+import aiohttp
+import attr
+import pyarrow as pa
+
+from lancedb.common import Credential
+from lancedb.remote import VectorQuery, VectorQueryResult
+from lancedb.remote.errors import LanceDBClientError
+
+
+def _check_not_closed(f):
+    @functools.wraps(f)
+    def wrapped(self, *args, **kwargs):
+        if self.closed:
+            raise ValueError("Connection is closed")
+        return f(self, *args, **kwargs)
+
+    return wrapped
+
+
+@attr.define(slots=False)
+class RestfulLanceDBClient:
+    db_name: str
+    region: str
+    api_key: Credential
+    closed: bool = attr.field(default=False, init=False)
+
+    @functools.cached_property
+    def session(self) -> aiohttp.ClientSession:
+        url = f"https://{self.db_name}.{self.region}.api.lancedb.com"
+        return aiohttp.ClientSession(url)
+
+    async def close(self):
+        await self.session.close()
+        self.closed = True
+
+    @functools.cached_property
+    def headers(self) -> Dict[str, str]:
+        return {
+            "x-api-key": self.api_key,
+        }
+
+    @_check_not_closed
+    async def query(self, table_name: str, query: VectorQuery) -> VectorQueryResult:
+        async with self.session.post(
+            f"/1/table/{table_name}/",
+            json=query.dict(exclude_none=True),
+            headers=self.headers,
+        ) as resp:
+            resp: aiohttp.ClientResponse = resp
+            if 400 <= resp.status < 500:
+                raise LanceDBClientError(
+                    f"Bad Request: {resp.status}, error: {await resp.text()}"
+                )
+            if 500 <= resp.status < 600:
+                raise LanceDBClientError(
+                    f"Internal Server Error: {resp.status}, error: {await resp.text()}"
+                )
+            if resp.status != 200:
+                raise LanceDBClientError(
+                    f"Unknown Error: {resp.status}, error: {await resp.text()}"
+                )
+
+            resp_body = await resp.read()
+            with pa.ipc.open_file(pa.BufferReader(resp_body)) as reader:
+                tbl = reader.read_all()
+        return VectorQueryResult(tbl)

python/lancedb/remote/db.py

@@ -0,0 +1,71 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from typing import List
+from urllib.parse import urlparse
+
+import pyarrow as pa
+
+from lancedb.common import DATA
+from lancedb.db import DBConnection
+from lancedb.table import Table
+
+from .client import RestfulLanceDBClient
+
+
+class RemoteDBConnection(DBConnection):
+    """A connection to a remote LanceDB database."""
+
+    def __init__(self, db_url: str, api_key: str, region: str):
+        """Connect to a remote LanceDB database."""
+        parsed = urlparse(db_url)
+        if parsed.scheme != "db":
+            raise ValueError(f"Invalid scheme: {parsed.scheme}, only accepts db://")
+        self.db_name = parsed.netloc
+        self.api_key = api_key
+        self._client = RestfulLanceDBClient(self.db_name, region, api_key)
+
+    def __repr__(self) -> str:
+        return f"RemoveConnect(name={self.db_name})"
+
+    def table_names(self) -> List[str]:
+        raise NotImplementedError
+
+    def open_table(self, name: str) -> Table:
+        """Open a Lance Table in the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+
+        Returns
+        -------
+        A LanceTable object representing the table.
+        """
+        from .table import RemoteTable
+
+        # TODO: check if table exists
+
+        return RemoteTable(self, name)
+
+    def create_table(
+        self,
+        name: str,
+        data: DATA = None,
+        schema: pa.Schema = None,
+        mode: str = "create",
+        on_bad_vectors: str = "error",
+        fill_value: float = 0.0,
+    ) -> Table:
+        raise NotImplementedError

python/lancedb/remote/errors.py

@@ -0,0 +1,16 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+class LanceDBClientError(RuntimeError):
+    pass

python/lancedb/remote/table.py

@@ -0,0 +1,70 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import asyncio
+from typing import Union
+
+import pyarrow as pa
+
+from lancedb.common import DATA, VEC, VECTOR_COLUMN_NAME
+
+from ..query import LanceQueryBuilder, Query
+from ..table import Query, Table
+from .db import RemoteDBConnection
+
+
+class RemoteTable(Table):
+    def __init__(self, conn: RemoteDBConnection, name: str):
+        self._conn = conn
+        self._name = name
+
+    def __repr__(self) -> str:
+        return f"RemoteTable({self._conn.db_name}.{self.name})"
+
+    def schema(self) -> pa.Schema:
+        raise NotImplementedError
+
+    def to_arrow(self) -> pa.Table:
+        raise NotImplementedError
+
+    def create_index(
+        self,
+        metric="L2",
+        num_partitions=256,
+        num_sub_vectors=96,
+        vector_column_name: str = VECTOR_COLUMN_NAME,
+        replace: bool = True,
+    ):
+        raise NotImplementedError
+
+    def add(
+        self,
+        data: DATA,
+        mode: str = "append",
+        on_bad_vectors: str = "error",
+        fill_value: float = 0.0,
+    ) -> int:
+        raise NotImplementedError
+
+    def search(
+        self, query: Union[VEC, str], vector_column: str = VECTOR_COLUMN_NAME
+    ) -> LanceQueryBuilder:
+        return LanceQueryBuilder(self, query, vector_column)
+
+    def _execute_query(self, query: Query) -> pa.Table:
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            loop = asyncio.get_event_loop()
+        result = self._conn._client.query(self._name, query)
+        return loop.run_until_complete(result).to_arrow()

python/lancedb/table.py

@@ -14,34 +14,196 @@
 from __future__ import annotations
 
 import os
+from abc import ABC, abstractmethod
 from functools import cached_property
+from typing import List, Union
 
 import lance
 import numpy as np
 import pandas as pd
 import pyarrow as pa
+import pyarrow.compute as pc
+import pyarrow.fs
 from lance import LanceDataset
 from lance.vector import vec_to_table
 
 from .common import DATA, VEC, VECTOR_COLUMN_NAME
-from .query import LanceQueryBuilder
+from .query import LanceFtsQueryBuilder, LanceQueryBuilder, Query
 
 
-def _sanitize_data(data, schema):
+def _sanitize_data(data, schema, on_bad_vectors, fill_value):
     if isinstance(data, list):
         data = pa.Table.from_pylist(data)
-        data = _sanitize_schema(data, schema=schema)
+        data = _sanitize_schema(
+            data, schema=schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+        )
     if isinstance(data, dict):
         data = vec_to_table(data)
     if isinstance(data, pd.DataFrame):
         data = pa.Table.from_pandas(data)
-        data = _sanitize_schema(data, schema=schema)
+        data = _sanitize_schema(
+            data, schema=schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+        )
     if not isinstance(data, pa.Table):
         raise TypeError(f"Unsupported data type: {type(data)}")
     return data
 
 
-class LanceTable:
+class Table(ABC):
+    """
+    A [Table](Table) is a collection of Records in a LanceDB [Database](Database).
+
+    Examples
+    --------
+
+    Create using [DBConnection.create_table][lancedb.DBConnection.create_table]
+    (more examples in that method's documentation).
+
+    >>> import lancedb
+    >>> db = lancedb.connect("./.lancedb")
+    >>> table = db.create_table("my_table", data=[{"vector": [1.1, 1.2], "b": 2}])
+    >>> table.head()
+    pyarrow.Table
+    vector: fixed_size_list<item: float>[2]
+      child 0, item: float
+    b: int64
+    ----
+    vector: [[[1.1,1.2]]]
+    b: [[2]]
+
+    Can append new data with [Table.add()][lancedb.table.Table.add].
+
+    >>> table.add([{"vector": [0.5, 1.3], "b": 4}])
+    2
+
+    Can query the table with [Table.search][lancedb.table.Table.search].
+
+    >>> table.search([0.4, 0.4]).select(["b"]).to_df()
+       b      vector  score
+    0  4  [0.5, 1.3]   0.82
+    1  2  [1.1, 1.2]   1.13
+
+    Search queries are much faster when an index is created. See
+    [Table.create_index][lancedb.table.Table.create_index].
+    """
+
+    @abstractmethod
+    def schema(self) -> pa.Schema:
+        """Return the [Arrow Schema](https://arrow.apache.org/docs/python/api/datatypes.html#) of
+        this [Table](Table)
+
+        """
+        raise NotImplementedError
+
+    def to_pandas(self) -> pd.DataFrame:
+        """Return the table as a pandas DataFrame.
+
+        Returns
+        -------
+        pd.DataFrame
+        """
+        return self.to_arrow().to_pandas()
+
+    @abstractmethod
+    def to_arrow(self) -> pa.Table:
+        """Return the table as a pyarrow Table.
+
+        Returns
+        -------
+        pa.Table
+        """
+        raise NotImplementedError
+
+    def create_index(
+        self,
+        metric="L2",
+        num_partitions=256,
+        num_sub_vectors=96,
+        vector_column_name: str = VECTOR_COLUMN_NAME,
+        replace: bool = True,
+    ):
+        """Create an index on the table.
+
+        Parameters
+        ----------
+        metric: str, default "L2"
+            The distance metric to use when creating the index.
+            Valid values are "L2", "cosine", or "dot".
+            L2 is euclidean distance.
+        num_partitions: int
+            The number of IVF partitions to use when creating the index.
+            Default is 256.
+        num_sub_vectors: int
+            The number of PQ sub-vectors to use when creating the index.
+            Default is 96.
+        vector_column_name: str, default "vector"
+            The vector column name to create the index.
+        replace: bool, default True
+            If True, replace the existing index if it exists.
+            If False, raise an error if duplicate index exists.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def add(
+        self,
+        data: DATA,
+        mode: str = "append",
+        on_bad_vectors: str = "error",
+        fill_value: float = 0.0,
+    ) -> int:
+        """Add more data to the [Table](Table).
+
+        Parameters
+        ----------
+        data: list-of-dict, dict, pd.DataFrame
+            The data to insert into the table.
+        mode: str
+            The mode to use when writing the data. Valid values are
+            "append" and "overwrite".
+        on_bad_vectors: str, default "error"
+            What to do if any of the vectors are not the same size or contains NaNs.
+            One of "error", "drop", "fill".
+        fill_value: float, default 0.
+            The value to use when filling vectors. Only used if on_bad_vectors="fill".
+
+        Returns
+        -------
+        int
+            The number of vectors in the table.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def search(
+        self, query: Union[VEC, str], vector_column: str = VECTOR_COLUMN_NAME
+    ) -> LanceQueryBuilder:
+        """Create a search query to find the nearest neighbors
+        of the given query vector.
+
+        Parameters
+        ----------
+        query: list, np.ndarray
+            The query vector.
+        vector_column: str, default "vector"
+            The name of the vector column to search.
+
+        Returns
+        -------
+        LanceQueryBuilder
+            A query builder object representing the query.
+            Once executed, the query returns selected columns, the vector,
+            and also the "score" column which is the distance between the query
+            vector and the returned vector.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def _execute_query(self, query: Query) -> pa.Table:
+        pass
+
+
+class LanceTable(Table):
     """
     A table in a LanceDB database.
     """
@@ -55,13 +217,19 @@ class LanceTable:
 
     def _reset_dataset(self):
         try:
-            del self.__dict__["_dataset"]
+            if "_dataset" in self.__dict__:
+                del self.__dict__["_dataset"]
         except AttributeError:
             pass
 
     @property
     def schema(self) -> pa.Schema:
-        """Return the schema of the table."""
+        """Return the schema of the table.
+
+        Returns
+        -------
+        pa.Schema
+            A PyArrow schema object."""
         return self._dataset.schema
 
     def list_versions(self):
@@ -69,12 +237,39 @@ class LanceTable:
         return self._dataset.versions()
 
     @property
-    def version(self):
+    def version(self) -> int:
         """Get the current version of the table"""
         return self._dataset.version
 
     def checkout(self, version: int):
-        """Checkout a version of the table"""
+        """Checkout a version of the table. This is an in-place operation.
+
+        This allows viewing previous versions of the table.
+
+        Parameters
+        ----------
+        version : int
+            The version to checkout.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> db = lancedb.connect("./.lancedb")
+        >>> table = db.create_table("my_table", [{"vector": [1.1, 0.9], "type": "vector"}])
+        >>> table.version
+        1
+        >>> table.to_pandas()
+               vector    type
+        0  [1.1, 0.9]  vector
+        >>> table.add([{"vector": [0.5, 0.2], "type": "vector"}])
+        2
+        >>> table.version
+        2
+        >>> table.checkout(1)
+        >>> table.to_pandas()
+               vector    type
+        0  [1.1, 0.9]  vector
+        """
         max_ver = max([v["version"] for v in self._dataset.versions()])
         if version < 1 or version > max_ver:
             raise ValueError(f"Invalid version {version}")
@@ -95,41 +290,66 @@ class LanceTable:
         return self._dataset.head(n)
 
     def to_pandas(self) -> pd.DataFrame:
-        """Return the table as a pandas DataFrame."""
+        """Return the table as a pandas DataFrame.
+
+        Returns
+        -------
+        pd.DataFrame
+        """
         return self.to_arrow().to_pandas()
 
     def to_arrow(self) -> pa.Table:
-        """Return the table as a pyarrow Table."""
+        """Return the table as a pyarrow Table.
+
+        Returns
+        -------
+        pa.Table"""
         return self._dataset.to_table()
 
     @property
     def _dataset_uri(self) -> str:
         return os.path.join(self._conn.uri, f"{self.name}.lance")
 
-    def create_index(self, metric="L2", num_partitions=256, num_sub_vectors=96):
+    def create_index(
-        """Create an index on the table.
+        self,
-
+        metric="L2",
-        Parameters
+        num_partitions=256,
-        ----------
+        num_sub_vectors=96,
-        metric: str, default "L2"
+        vector_column_name=VECTOR_COLUMN_NAME,
-            The distance metric to use when creating the index. Valid values are "L2" or "cosine".
+        replace: bool = True,
-            L2 is euclidean distance.
+    ):
-        num_partitions: int
+        """Create an index on the table."""
-            The number of IVF partitions to use when creating the index.
-            Default is 256.
-        num_sub_vectors: int
-            The number of PQ sub-vectors to use when creating the index.
-            Default is 96.
-        """
         self._dataset.create_index(
-            column=VECTOR_COLUMN_NAME,
+            column=vector_column_name,
             index_type="IVF_PQ",
             metric=metric,
             num_partitions=num_partitions,
             num_sub_vectors=num_sub_vectors,
+            replace=replace,
         )
         self._reset_dataset()
 
+    def create_fts_index(self, field_names: Union[str, List[str]]):
+        """Create a full-text search index on the table.
+
+        Warning - this API is highly experimental and is highly likely to change
+        in the future.
+
+        Parameters
+        ----------
+        field_names: str or list of str
+            The name(s) of the field to index.
+        """
+        from .fts import create_index, populate_index
+
+        if isinstance(field_names, str):
+            field_names = [field_names]
+        index = create_index(self._get_fts_index_path(), field_names)
+        populate_index(index, self, field_names)
+
+    def _get_fts_index_path(self):
+        return os.path.join(self._dataset_uri, "_indices", "tantivy")
+
     @cached_property
     def _dataset(self) -> LanceDataset:
         return lance.dataset(self._dataset_uri, version=self._version)
@@ -138,7 +358,13 @@ class LanceTable:
         """Return the LanceDataset backing this table."""
         return self._dataset
 
-    def add(self, data: DATA, mode: str = "append") -> int:
+    def add(
+        self,
+        data: DATA,
+        mode: str = "append",
+        on_bad_vectors: str = "error",
+        fill_value: float = 0.0,
+    ) -> int:
         """Add data to the table.
 
         Parameters
@@ -148,17 +374,28 @@ class LanceTable:
         mode: str
             The mode to use when writing the data. Valid values are
             "append" and "overwrite".
+        on_bad_vectors: str, default "error"
+            What to do if any of the vectors are not the same size or contains NaNs.
+            One of "error", "drop", "fill".
+        fill_value: float, default 0.
+            The value to use when filling vectors. Only used if on_bad_vectors="fill".
 
         Returns
         -------
-        The number of vectors added to the table.
+        int
+            The number of vectors in the table.
         """
-        data = _sanitize_data(data, self.schema)
+        # TODO: manage table listing and metadata separately
+        data = _sanitize_data(
+            data, self.schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+        )
         lance.write_dataset(data, self._dataset_uri, mode=mode)
         self._reset_dataset()
         return len(self)
 
-    def search(self, query: VEC) -> LanceQueryBuilder:
+    def search(
+        self, query: Union[VEC, str], vector_column_name=VECTOR_COLUMN_NAME
+    ) -> LanceQueryBuilder:
         """Create a search query to find the nearest neighbors
         of the given query vector.
 
@@ -166,31 +403,148 @@ class LanceTable:
         ----------
         query: list, np.ndarray
             The query vector.
+        vector_column_name: str, default "vector"
+            The name of the vector column to search.
 
         Returns
         -------
-        A LanceQueryBuilder object representing the query.
+        LanceQueryBuilder
-        Once executed, the query returns selected columns, the vector,
+            A query builder object representing the query.
-        and also the "score" column which is the distance between the query
+            Once executed, the query returns selected columns, the vector,
-        vector and the returned vector.
+            and also the "score" column which is the distance between the query
+            vector and the returned vector.
         """
+        if isinstance(query, str):
+            # fts
+            return LanceFtsQueryBuilder(self, query, vector_column_name)
+
         if isinstance(query, list):
             query = np.array(query)
         if isinstance(query, np.ndarray):
             query = query.astype(np.float32)
         else:
             raise TypeError(f"Unsupported query type: {type(query)}")
-        return LanceQueryBuilder(self, query)
+        return LanceQueryBuilder(self, query, vector_column_name)
 
     @classmethod
-    def create(cls, db, name, data, schema=None, mode="create"):
+    def create(
+        cls,
+        db,
+        name,
+        data=None,
+        schema=None,
+        mode="create",
+        on_bad_vectors: str = "error",
+        fill_value: float = 0.0,
+    ):
+        """
+        Create a new table.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> import pandas as pd
+        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
+        >>> db = lancedb.connect("./.lancedb")
+        >>> table = db.create_table("my_table", data)
+        >>> table.to_pandas()
+           x      vector
+        0  1  [1.0, 2.0]
+        1  2  [3.0, 4.0]
+        2  3  [5.0, 6.0]
+
+        Parameters
+        ----------
+        db: LanceDB
+            The LanceDB instance to create the table in.
+        name: str
+            The name of the table to create.
+        data: list-of-dict, dict, pd.DataFrame, default None
+            The data to insert into the table.
+            At least one of `data` or `schema` must be provided.
+        schema: dict, optional
+            The schema of the table. If not provided, the schema is inferred from the data.
+            At least one of `data` or `schema` must be provided.
+        mode: str, default "create"
+            The mode to use when writing the data. Valid values are
+            "create", "overwrite", and "append".
+        on_bad_vectors: str, default "error"
+            What to do if any of the vectors are not the same size or contains NaNs.
+            One of "error", "drop", "fill".
+        fill_value: float, default 0.
+            The value to use when filling vectors. Only used if on_bad_vectors="fill".
+        """
         tbl = LanceTable(db, name)
-        data = _sanitize_data(data, schema)
+        if data is not None:
+            data = _sanitize_data(
+                data, schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+            )
+        else:
+            if schema is None:
+                raise ValueError("Either data or schema must be provided")
+            data = pa.Table.from_pylist([], schema=schema)
         lance.write_dataset(data, tbl._dataset_uri, mode=mode)
+        return LanceTable(db, name)
+
+    @classmethod
+    def open(cls, db, name):
+        tbl = cls(db, name)
+        if not os.path.exists(tbl._dataset_uri):
+            raise FileNotFoundError(
+                f"Table {name} does not exist. Please first call db.create_table({name}, data)"
+            )
         return tbl
 
+    def delete(self, where: str):
+        """Delete rows from the table.
 
-def _sanitize_schema(data: pa.Table, schema: pa.Schema = None) -> pa.Table:
+        Parameters
+        ----------
+        where: str
+            The SQL where clause to use when deleting rows.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> import pandas as pd
+        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
+        >>> db = lancedb.connect("./.lancedb")
+        >>> table = db.create_table("my_table", data)
+        >>> table.to_pandas()
+           x      vector
+        0  1  [1.0, 2.0]
+        1  2  [3.0, 4.0]
+        2  3  [5.0, 6.0]
+        >>> table.delete("x = 2")
+        >>> table.to_pandas()
+           x      vector
+        0  1  [1.0, 2.0]
+        1  3  [5.0, 6.0]
+        """
+        self._dataset.delete(where)
+
+    def _execute_query(self, query: Query) -> pa.Table:
+        ds = self.to_lance()
+        return ds.to_table(
+            columns=query.columns,
+            filter=query.filter,
+            nearest={
+                "column": query.vector_column,
+                "q": query.vector,
+                "k": query.k,
+                "metric": query.metric,
+                "nprobes": query.nprobes,
+                "refine_factor": query.refine_factor,
+            },
+        )
+
+
+def _sanitize_schema(
+    data: pa.Table,
+    schema: pa.Schema = None,
+    on_bad_vectors: str = "error",
+    fill_value: float = 0.0,
+) -> pa.Table:
     """Ensure that the table has the expected schema.
 
     Parameters
@@ -200,21 +554,41 @@ def _sanitize_schema(data: pa.Table, schema: pa.Schema = None) -> pa.Table:
     schema: pa.Schema; optional
         The expected schema. If not provided, this just converts the
         vector column to fixed_size_list(float32) if necessary.
+    on_bad_vectors: str, default "error"
+        What to do if any of the vectors are not the same size or contains NaNs.
+        One of "error", "drop", "fill".
+    fill_value: float, default 0.
+        The value to use when filling vectors. Only used if on_bad_vectors="fill".
     """
     if schema is not None:
         if data.schema == schema:
             return data
         # cast the columns to the expected types
         data = data.combine_chunks()
-        data = _sanitize_vector_column(data, vector_column_name=VECTOR_COLUMN_NAME)
+        data = _sanitize_vector_column(
+            data,
+            vector_column_name=VECTOR_COLUMN_NAME,
+            on_bad_vectors=on_bad_vectors,
+            fill_value=fill_value,
+        )
         return pa.Table.from_arrays(
             [data[name] for name in schema.names], schema=schema
         )
     # just check the vector column
-    return _sanitize_vector_column(data, vector_column_name=VECTOR_COLUMN_NAME)
+    return _sanitize_vector_column(
+        data,
+        vector_column_name=VECTOR_COLUMN_NAME,
+        on_bad_vectors=on_bad_vectors,
+        fill_value=fill_value,
+    )
 
 
-def _sanitize_vector_column(data: pa.Table, vector_column_name: str) -> pa.Table:
+def _sanitize_vector_column(
+    data: pa.Table,
+    vector_column_name: str,
+    on_bad_vectors: str = "error",
+    fill_value: float = 0.0,
+) -> pa.Table:
     """
     Ensure that the vector column exists and has type fixed_size_list(float32)
 
@@ -224,18 +598,103 @@ def _sanitize_vector_column(data: pa.Table, vector_column_name: str) -> pa.Table
         The table to sanitize.
     vector_column_name: str
         The name of the vector column.
+    on_bad_vectors: str, default "error"
+        What to do if any of the vectors are not the same size or contains NaNs.
+        One of "error", "drop", "fill".
+    fill_value: float, default 0.0
+        The value to use when filling vectors. Only used if on_bad_vectors="fill".
     """
-    i = data.column_names.index(vector_column_name)
+    if vector_column_name not in data.column_names:
-    if i < 0:
         raise ValueError(f"Missing vector column: {vector_column_name}")
+    # ChunkedArray is annoying to work with, so we combine chunks here
     vec_arr = data[vector_column_name].combine_chunks()
-    if pa.types.is_fixed_size_list(vec_arr.type):
+    if pa.types.is_list(data[vector_column_name].type):
-        return data
+        # if it's a variable size list array we make sure the dimensions are all the same
-    if not pa.types.is_list(vec_arr.type):
+        has_jagged_ndims = len(vec_arr.values) % len(data) != 0
+        if has_jagged_ndims:
+            data = _sanitize_jagged(
+                data, fill_value, on_bad_vectors, vec_arr, vector_column_name
+            )
+            vec_arr = data[vector_column_name].combine_chunks()
+    elif not pa.types.is_fixed_size_list(vec_arr.type):
         raise TypeError(f"Unsupported vector column type: {vec_arr.type}")
+
+    vec_arr = ensure_fixed_size_list_of_f32(vec_arr)
+    data = data.set_column(
+        data.column_names.index(vector_column_name), vector_column_name, vec_arr
+    )
+
+    has_nans = pc.any(pc.is_nan(vec_arr.values)).as_py()
+    if has_nans:
+        data = _sanitize_nans(
+            data, fill_value, on_bad_vectors, vec_arr, vector_column_name
+        )
+
+    return data
+
+
+def ensure_fixed_size_list_of_f32(vec_arr):
     values = vec_arr.values
     if not pa.types.is_float32(values.type):
         values = values.cast(pa.float32())
-    list_size = len(values) / len(data)
+    if pa.types.is_fixed_size_list(vec_arr.type):
+        list_size = vec_arr.type.list_size
+    else:
+        list_size = len(values) / len(vec_arr)
     vec_arr = pa.FixedSizeListArray.from_arrays(values, list_size)
-    return data.set_column(i, vector_column_name, vec_arr)
+    return vec_arr
+
+
+def _sanitize_jagged(data, fill_value, on_bad_vectors, vec_arr, vector_column_name):
+    """Sanitize jagged vectors."""
+    if on_bad_vectors == "error":
+        raise ValueError(
+            f"Vector column {vector_column_name} has variable length vectors "
+            "Set on_bad_vectors='drop' to remove them, or "
+            "set on_bad_vectors='fill' and fill_value=<value> to replace them."
+        )
+
+    lst_lengths = pc.list_value_length(vec_arr)
+    ndims = pc.max(lst_lengths).as_py()
+    correct_ndims = pc.equal(lst_lengths, ndims)
+
+    if on_bad_vectors == "fill":
+        if fill_value is None:
+            raise ValueError(
+                "`fill_value` must not be None if `on_bad_vectors` is 'fill'"
+            )
+        fill_arr = pa.scalar([float(fill_value)] * ndims)
+        vec_arr = pc.if_else(correct_ndims, vec_arr, fill_arr)
+        data = data.set_column(
+            data.column_names.index(vector_column_name), vector_column_name, vec_arr
+        )
+    elif on_bad_vectors == "drop":
+        data = data.filter(correct_ndims)
+    return data
+
+
+def _sanitize_nans(data, fill_value, on_bad_vectors, vec_arr, vector_column_name):
+    """Sanitize NaNs in vectors"""
+    if on_bad_vectors == "error":
+        raise ValueError(
+            f"Vector column {vector_column_name} has NaNs. "
+            "Set on_bad_vectors='drop' to remove them, or "
+            "set on_bad_vectors='fill' and fill_value=<value> to replace them."
+        )
+    elif on_bad_vectors == "fill":
+        if fill_value is None:
+            raise ValueError(
+                "`fill_value` must not be None if `on_bad_vectors` is 'fill'"
+            )
+        fill_value = float(fill_value)
+        values = pc.if_else(pc.is_nan(vec_arr.values), fill_value, vec_arr.values)
+        ndims = len(vec_arr[0])
+        vec_arr = pa.FixedSizeListArray.from_arrays(values, ndims)
+        data = data.set_column(
+            data.column_names.index(vector_column_name), vector_column_name, vec_arr
+        )
+    elif on_bad_vectors == "drop":
+        is_value_nan = pc.is_nan(vec_arr.values).to_numpy(zero_copy_only=False)
+        is_full = np.any(~is_value_nan.reshape(-1, vec_arr.type.list_size), axis=1)
+        data = data.filter(is_full)
+    return data

python/lancedb/util.py

@@ -11,9 +11,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from urllib.parse import ParseResult, urlparse
+from urllib.parse import urlparse
-
-from pyarrow import fs
 
 
 def get_uri_scheme(uri: str) -> str:
@@ -41,3 +39,23 @@ def get_uri_scheme(uri: str) -> str:
         # So we add special handling here for schemes that are a single character
         scheme = "file"
     return scheme
+
+
+def get_uri_location(uri: str) -> str:
+    """
+    Get the location of a URI. If the parameter is not a url, assumes it is just a path
+
+    Parameters
+    ----------
+    uri : str
+        The URI to parse.
+
+    Returns
+    -------
+    str: Location part of the URL, without scheme
+    """
+    parsed = urlparse(uri)
+    if not parsed.netloc:
+        return parsed.path
+    else:
+        return parsed.netloc + parsed.path

python/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "lancedb"
-version = "0.1.2"
+version = "0.1.10"
-dependencies = ["pylance>=0.4.6", "ratelimiter", "retry", "tqdm"]
+dependencies = ["pylance~=0.5.0", "ratelimiter", "retry", "tqdm", "aiohttp", "pydantic", "attr"]
 description = "lancedb"
 authors = [
     { name = "LanceDB Devs", email = "dev@lancedb.com" },
@@ -33,11 +33,11 @@ classifiers = [
 ]
 
 [project.urls]
-repository = "https://github.com/eto-ai/lancedb"
+repository = "https://github.com/lancedb/lancedb"
 
 [project.optional-dependencies]
 tests = [
-    "pytest"
+    "pytest", "pytest-mock", "pytest-asyncio"
 ]
 dev = [
     "ruff", "pre-commit", "black"

python/tests/test_context.py

@@ -0,0 +1,77 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import pandas as pd
+import pytest
+
+from lancedb.context import contextualize
+
+
+@pytest.fixture
+def raw_df() -> pd.DataFrame:
+    return pd.DataFrame(
+        {
+            "token": [
+                "The",
+                "quick",
+                "brown",
+                "fox",
+                "jumped",
+                "over",
+                "the",
+                "lazy",
+                "dog",
+                "I",
+                "love",
+                "sandwiches",
+            ],
+            "document_id": [1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2],
+        }
+    )
+
+
+def test_contextualizer(raw_df: pd.DataFrame):
+    result = (
+        contextualize(raw_df)
+        .window(6)
+        .stride(3)
+        .text_col("token")
+        .groupby("document_id")
+        .to_df()["token"]
+        .to_list()
+    )
+
+    assert result == [
+        "The quick brown fox jumped over",
+        "fox jumped over the lazy dog",
+        "the lazy dog",
+        "I love sandwiches",
+    ]
+
+
+def test_contextualizer_with_threshold(raw_df: pd.DataFrame):
+    result = (
+        contextualize(raw_df)
+        .window(6)
+        .stride(3)
+        .text_col("token")
+        .groupby("document_id")
+        .min_window_size(4)
+        .to_df()["token"]
+        .to_list()
+    )
+
+    assert result == [
+        "The quick brown fox jumped over",
+        "fox jumped over the lazy dog",
+    ]

python/tests/test_db.py

@@ -11,6 +11,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import numpy as np
 import pandas as pd
 import pytest
 
@@ -97,3 +98,63 @@ def test_create_mode(tmp_path):
     )
     tbl = db.create_table("test", data=new_data, mode="overwrite")
     assert tbl.to_pandas().item.tolist() == ["fizz", "buzz"]
+
+
+def test_delete_table(tmp_path):
+    db = lancedb.connect(tmp_path)
+    data = pd.DataFrame(
+        {
+            "vector": [[3.1, 4.1], [5.9, 26.5]],
+            "item": ["foo", "bar"],
+            "price": [10.0, 20.0],
+        }
+    )
+    db.create_table("test", data=data)
+
+    with pytest.raises(Exception):
+        db.create_table("test", data=data)
+
+    assert db.table_names() == ["test"]
+
+    db.drop_table("test")
+    assert db.table_names() == []
+
+    db.create_table("test", data=data)
+    assert db.table_names() == ["test"]
+
+
+def test_empty_or_nonexistent_table(tmp_path):
+    db = lancedb.connect(tmp_path)
+    with pytest.raises(Exception):
+        db.create_table("test_with_no_data")
+
+    with pytest.raises(Exception):
+        db.open_table("does_not_exist")
+
+
+def test_replace_index(tmp_path):
+    db = lancedb.connect(uri=tmp_path)
+    table = db.create_table(
+        "test",
+        [
+            {"vector": np.random.rand(128), "item": "foo", "price": float(i)}
+            for i in range(1000)
+        ],
+    )
+    table.create_index(
+        num_partitions=2,
+        num_sub_vectors=4,
+    )
+
+    with pytest.raises(Exception):
+        table.create_index(
+            num_partitions=2,
+            num_sub_vectors=4,
+            replace=False,
+        )
+
+    table.create_index(
+        num_partitions=2,
+        num_sub_vectors=4,
+        replace=True,
+    )

python/tests/test_e2e_remote_db.py

@@ -0,0 +1,27 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import numpy as np
+import pytest
+
+from lancedb import LanceDBConnection
+
+# TODO: setup integ test mark and script
+
+
+@pytest.mark.skip(reason="Need to set up a local server")
+def test_against_local_server():
+    conn = LanceDBConnection("lancedb+http://localhost:10024")
+    table = conn.open_table("sift1m_ivf1024_pq16")
+    df = table.search(np.random.rand(128)).to_df()
+    assert len(df) == 10