[python] Bump version: 0.4.3 → 0.4.4

We use `ruff` in CI and dev workflow now.

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.2.4
+current_version = 0.4.2
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True

.github/ISSUE_TEMPLATE/bug-node.yml

@@ -0,0 +1,33 @@
+name: Bug Report - Node / Typescript
+description: File a bug report
+title: "bug(node): "
+labels: [bug, typescript]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: input
+    id: version
+    attributes:
+      label: LanceDB version
+      description: What version of LanceDB are you using? `npm list | grep vectordb`.
+      placeholder: v0.3.2
+    validations:
+      required: false
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: Also tell us, what did you expect to happen?
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Are there known steps to reproduce?
+      description: |
+        Let us know how to reproduce the bug and we may be able to fix it more
+        quickly. This is not required, but it is helpful.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/bug-python.yml

@@ -0,0 +1,33 @@
+name: Bug Report - Python
+description: File a bug report
+title: "bug(python): "
+labels: [bug, python]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: input
+    id: version
+    attributes:
+      label: LanceDB version
+      description: What version of LanceDB are you using? `python -c "import lancedb; print(lancedb.__version__)"`.
+      placeholder: v0.3.2
+    validations:
+      required: false
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: Also tell us, what did you expect to happen?
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Are there known steps to reproduce?
+      description: |
+        Let us know how to reproduce the bug and we may be able to fix it more
+        quickly. This is not required, but it is helpful.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Discord Community Support
+    url: https://discord.com/invite/zMM32dvNtd
+    about: Please ask and answer questions here.

.github/ISSUE_TEMPLATE/documentation.yml

@@ -0,0 +1,23 @@
+name: 'Documentation improvement'
+description: Report an issue with the documentation.
+labels: [documentation]
+
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: >
+        Describe the issue with the documentation and how it can be fixed or improved.
+    validations:
+      required: true
+
+  - type: input
+    id: link
+    attributes:
+      label: Link
+      description: >
+        Provide a link to the existing documentation, if applicable.
+      placeholder: ex. https://lancedb.github.io/lancedb/guides/tables/...
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/feature.yml

@@ -0,0 +1,31 @@
+name: Feature suggestion
+description: Suggestion a new feature for LanceDB
+title: "Feature: "
+labels: [enhancement]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Share a new idea for a feature or improvement. Be sure to search existing
+        issues first to avoid duplicates.
+  - type: dropdown
+    id: sdk
+    attributes:
+      label: SDK
+      description: Which SDK are you using? This helps us prioritize.
+      options:
+        - Python
+        - Node
+        - Rust
+      default: 0
+    validations:
+      required: false
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: |
+        Describe the feature and why it would be useful. If applicable, consider
+        providing a code example of what it might be like to use the feature.
+    validations:
+      required: true

.github/workflows/docs_test.yml

@@ -88,6 +88,9 @@ jobs:
         cd docs/test
         node md_testing.js
     - name: Test
+      env:
+        LANCEDB_URI: ${{ secrets.LANCEDB_URI }}
+        LANCEDB_DEV_API_KEY: ${{ secrets.LANCEDB_DEV_API_KEY }}
       run: |
         cd docs/test/node
         for d in *; do cd "$d"; echo "$d".js; node "$d".js; cd ..; done

.github/workflows/node.yml

@@ -9,6 +9,11 @@ on:
       - node/**
       - rust/ffi/node/**
       - .github/workflows/node.yml
+      - docker-compose.yml
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 env:
   # Disable full debug symbol generation to speed up CI build and keep memory down
@@ -107,3 +112,56 @@ jobs:
     - name: Test
       run: |
         npm run test
+  aws-integtest:
+    timeout-minutes: 45
+    runs-on: "ubuntu-22.04"
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
+    env:
+      AWS_ACCESS_KEY_ID: ACCESSKEY
+      AWS_SECRET_ACCESS_KEY: SECRETKEY
+      AWS_DEFAULT_REGION: us-west-2
+      # this one is for s3
+      AWS_ENDPOINT: http://localhost:4566
+      # this one is for dynamodb
+      DYNAMODB_ENDPOINT: http://localhost:4566
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 18
+        cache: 'npm'
+        cache-dependency-path: node/package-lock.json
+    - name: start local stack
+      run: docker compose -f ../docker-compose.yml up -d --wait
+    - name: create s3
+      run: aws s3 mb s3://lancedb-integtest --endpoint $AWS_ENDPOINT
+    - name: create ddb
+      run: |
+        aws dynamodb create-table \
+          --table-name lancedb-integtest \
+          --attribute-definitions '[{"AttributeName": "base_uri", "AttributeType": "S"}, {"AttributeName": "version", "AttributeType": "N"}]' \
+          --key-schema '[{"AttributeName": "base_uri", "KeyType": "HASH"}, {"AttributeName": "version", "KeyType": "RANGE"}]' \
+          --provisioned-throughput '{"ReadCapacityUnits": 10, "WriteCapacityUnits": 10}' \
+          --endpoint-url $DYNAMODB_ENDPOINT
+    - uses: Swatinem/rust-cache@v2
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y protobuf-compiler libssl-dev
+    - name: Build
+      run: |
+        npm ci
+        npm run tsc
+        npm run build
+        npm run pack-build
+        npm install --no-save ./dist/lancedb-vectordb-*.tgz
+        # Remove index.node to test with dependency installed
+        rm index.node
+    - name: Test
+      run: npm run integration-test

.github/workflows/npm-publish.yml

@@ -38,13 +38,17 @@ jobs:
             node/vectordb-*.tgz
 
   node-macos:
-    runs-on: macos-12
+    strategy:
+      matrix:
+        config:
+          - arch: x86_64-apple-darwin
+            runner: macos-13
+          - arch: aarch64-apple-darwin
+            # xlarge is implicitly arm64.
+            runner: macos-13-xlarge
+    runs-on: ${{ matrix.config.runner }}
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64-apple-darwin, aarch64-apple-darwin]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -54,11 +58,8 @@ jobs:
         run: |
           cd node
           npm ci
-      - name: Install rustup target
-        if: ${{ matrix.target == 'aarch64-apple-darwin' }}
-        run: rustup target add aarch64-apple-darwin
       - name: Build MacOS native node modules
-        run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
+        run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }}
       - name: Upload Darwin Artifacts
         uses: actions/upload-artifact@v3
         with:
@@ -66,6 +67,7 @@ jobs:
           path: |
             node/dist/lancedb-vectordb-darwin*.tgz
          
+
   node-linux:
     name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu
     runs-on: ${{ matrix.config.runner }}

.github/workflows/python.yml

@@ -8,6 +8,11 @@ on:
     paths:
       - python/**
       - .github/workflows/python.yml
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   linux:
     timeout-minutes: 30
@@ -32,18 +37,26 @@ jobs:
       run: |
         pip install -e .[tests]
         pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock black isort
-    - name: Black
-      run: black --check --diff --no-color --quiet .
-    - name: isort
-      run: isort --check --diff --quiet .
+        pip install pytest pytest-mock ruff
+    - name: Lint
+      run: ruff format --check .
     - name: Run tests
-      run: pytest -x -v --durations=30 tests
+      run: pytest -m "not slow" -x -v --durations=30 tests
     - name: doctest
       run: pytest --doctest-modules lancedb
-  mac:
+  platform:
+    name: "Platform: ${{ matrix.config.name }}"
     timeout-minutes: 30
-    runs-on: "macos-12"
+    strategy:
+      matrix:
+        config:
+          - name: x86 Mac
+            runner: macos-13
+          - name: Arm Mac
+            runner: macos-13-xlarge
+          - name: x86 Windows
+            runner: windows-latest
+    runs-on: "${{ matrix.config.runner }}"
     defaults:
       run:
         shell: bash
@@ -61,8 +74,32 @@ jobs:
       run: |
         pip install -e .[tests]
         pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock black
-    - name: Black
-      run: black --check --diff --no-color --quiet .
+        pip install pytest pytest-mock
     - name: Run tests
-      run: pytest -x -v --durations=30 tests
+      run: pytest -m "not slow" -x -v --durations=30 tests
+  pydantic1x:
+    timeout-minutes: 30
+    runs-on: "ubuntu-22.04"
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+    - name: Install lancedb
+      run: |
+        pip install "pydantic<2"
+        pip install -e .[tests]
+        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+        pip install pytest pytest-mock
+    - name: Run tests
+      run: pytest -m "not slow" -x -v --durations=30 tests
+    - name: doctest
+      run: pytest --doctest-modules lancedb

.github/workflows/rust.yml

@@ -10,6 +10,10 @@ on:
       - rust/**
       - .github/workflows/rust.yml
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   # This env var is used by Swatinem/rust-cache@v2 for the cache
   # key, so we set it to make sure it is always consistent.
@@ -20,6 +24,29 @@ env:
   RUST_BACKTRACE: "1"
 
 jobs:
+  lint:
+    timeout-minutes: 30
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+        working-directory: rust
+    steps:
+    - uses: actions/checkout@v3
+      with:
+          fetch-depth: 0
+          lfs: true
+    - uses: Swatinem/rust-cache@v2
+      with:
+        workspaces: rust
+    - name: Install dependencies
+      run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+    - name: Run format
+      run: cargo fmt --all -- --check
+    - name: Run clippy
+      run: cargo clippy --all --all-features -- -D warnings
   linux:
     timeout-minutes: 30
     runs-on: ubuntu-22.04
@@ -44,8 +71,11 @@ jobs:
     - name: Run tests
       run: cargo test --all-features
   macos:
-    runs-on: macos-12
     timeout-minutes: 30
+    strategy:
+      matrix:
+        mac-runner: [ "macos-13", "macos-13-xlarge" ]
+    runs-on: "${{ matrix.mac-runner }}"
     defaults:
       run:
         shell: bash

Cargo.toml

@@ -1,16 +1,28 @@
 [workspace]
-members = [
-    "rust/vectordb",
-    "rust/ffi/node"
-]
+members = ["rust/ffi/node", "rust/vectordb"]
+# Python package needs to be built by maturin.
+exclude = ["python"]
 resolver = "2"
 
 [workspace.dependencies]
-lance = "=0.6.5"
-arrow-array = "43.0"
-arrow-data = "43.0"
-arrow-schema = "43.0"
-arrow-ipc = "43.0"
-half = { "version" = "=2.2.1", default-features = false }
-object_store = "0.6.1"
+lance = { "version" = "=0.9.6", "features" = ["dynamodb"] }
+lance-index = { "version" = "=0.9.6" }
+lance-linalg = { "version" = "=0.9.6" }
+lance-testing = { "version" = "=0.9.6" }
+# Note that this one does not include pyarrow
+arrow = { version = "49.0.0", optional = false }
+arrow-array = "49.0"
+arrow-data = "49.0"
+arrow-ipc = "49.0"
+arrow-ord = "49.0"
+arrow-schema = "49.0"
+arrow-arith = "49.0"
+arrow-cast = "49.0"
+chrono = "0.4.23"
+half = { "version" = "=2.3.1", default-features = false, features = [
+    "num-traits",
+] }
+log = "0.4"
+object_store = "0.8.0"
 snafu = "0.7.4"
+url = "2"

README.md

@@ -5,10 +5,11 @@
 
 **Developer-friendly, serverless vector database for AI applications**
 
-<a href="https://lancedb.github.io/lancedb/">Documentation</a> •
-<a href="https://blog.lancedb.com/">Blog</a> •
-<a href="https://discord.gg/zMM32dvNtd">Discord</a> •
-<a href="https://twitter.com/lancedb">Twitter</a>
+<a href='https://github.com/lancedb/vectordb-recipes/tree/main' target="_blank"><img alt='LanceDB' src='https://img.shields.io/badge/VectorDB_Recipes-100000?style=for-the-badge&logo=LanceDB&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
+<a href='https://lancedb.github.io/lancedb/' target="_blank"><img alt='lancdb' src='https://img.shields.io/badge/DOCS-100000?style=for-the-badge&logo=lancdb&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
+[![Medium](https://img.shields.io/badge/Medium-12100E?style=for-the-badge&logo=medium&logoColor=white)](https://blog.lancedb.com/) 
+[![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/zMM32dvNtd) 
+[![Twitter](https://img.shields.io/badge/Twitter-%231DA1F2.svg?style=for-the-badge&logo=Twitter&logoColor=white)](https://twitter.com/lancedb)
 
 </p>
 
@@ -33,6 +34,8 @@ The key features of LanceDB include:
 
 * Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure.
 
+* GPU support in building vector index(*).
+
 * Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lanecdb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
 
 LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
@@ -52,8 +55,7 @@ const table = await db.createTable('vectors',
       [{ id: 1, vector: [0.1, 0.2], item: "foo", price: 10 },
        { id: 2, vector: [1.1, 1.2], item: "bar", price: 50 }])
 
-const query = table.search([0.1, 0.3]);
-query.limit = 20;
+const query = table.search([0.1, 0.3]).limit(2);
 const results = await query.execute();
 ```
 
@@ -70,7 +72,7 @@ db = lancedb.connect(uri)
 table = db.create_table("my_table",
                          data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
                                {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-result = table.search([100, 100]).limit(2).to_df()
+result = table.search([100, 100]).limit(2).to_pandas()
 ```
 
 ## Blogs, Tutorials & Videos

ci/build_macos_artifacts.sh

@@ -1,6 +1,7 @@
 # Builds the macOS artifacts (node binaries).
 # Usage: ./ci/build_macos_artifacts.sh [target]
 # Targets supported: x86_64-apple-darwin aarch64-apple-darwin
+set -e
 
 prebuild_rust() {
     # Building here for the sake of easier debugging.

docker-compose.yml

@@ -0,0 +1,18 @@
+version: "3.9"
+services:
+  localstack:
+    image: localstack/localstack:0.14
+    ports:
+      - 4566:4566
+    environment:
+      - SERVICES=s3,dynamodb
+      - DEBUG=1
+      - LS_LOG=trace
+      - DOCKER_HOST=unix:///var/run/docker.sock
+      - AWS_ACCESS_KEY_ID=ACCESSKEY
+      - AWS_SECRET_ACCESS_KEY=SECRETKEY
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:4566/health" ]
+      interval: 5s
+      retries: 3
+      start_period: 10s

docs/README.md

@@ -0,0 +1,26 @@
+# LanceDB Documentation
+
+LanceDB docs are deployed to https://lancedb.github.io/lancedb/.
+
+Docs is built and deployed automatically by [Github Actions](.github/workflows/docs.yml)
+whenever a commit is pushed to the `main` branch. So it is possible for the docs to show
+unreleased features.
+
+## Building the docs
+
+### Setup
+1. Install LanceDB. From LanceDB repo root: `pip install -e python`
+2. Install dependencies. From LanceDB repo root: `pip install -r docs/requirements.txt`
+3. Make sure you have node and npm setup
+4. Make sure protobuf and libssl are installed
+
+### Building node module and create markdown files
+
+See [Javascript docs README](docs/src/javascript/README.md)
+
+### Build docs
+From LanceDB repo root:
+
+Run: `PYTHONPATH=. mkdocs build -f docs/mkdocs.yml`
+
+If successful, you should see a `docs/site` directory that you can verify locally.

docs/mkdocs.yml

@@ -1,4 +1,5 @@
 site_name: LanceDB Docs
+site_url: https://lancedb.github.io/lancedb/
 repo_url: https://github.com/lancedb/lancedb
 edit_uri: https://github.com/lancedb/lancedb/tree/main/docs/src
 repo_name: lancedb/lancedb
@@ -21,6 +22,7 @@ theme:
     - navigation.tracking
     - navigation.instant
     - navigation.indexes
+    - navigation.expand
   icon:
     repo: fontawesome/brands/github
   custom_dir: overrides
@@ -36,7 +38,7 @@ plugins:
           docstring_style: numpy
         rendering:
           heading_level: 4
-          show_source: false
+          show_source: true
           show_symbol_type_in_heading: true
           show_signature_annotations: true
           show_root_heading: true
@@ -67,7 +69,19 @@ nav:
 - Home:
   - 🏢 Home: index.md
   - 💡 Basics: basic.md
-  - 🧬 Embeddings: embedding.md
+  - 📚 Guides:
+    - Create Ingest Update Delete: guides/tables.md
+    - Vector Search: search.md
+    - SQL filters: sql.md
+    - Indexing: ann_indexes.md
+    - Versioning & Reproducibility: notebooks/reproducibility.ipynb
+  - 🧬 Embeddings:
+    - embeddings/index.md
+    - Ingest Embedding Functions: embeddings/embedding_functions.md
+    - Available Functions: embeddings/default_embedding_functions.md
+    - Create Custom Embedding Functions: embeddings/api.md
+    - Example - Multi-lingual semantic search: notebooks/multi_lingual_example.ipynb
+    - Example - MultiModal CLIP Embeddings: notebooks/DisappearingEmbeddingFunction.ipynb
   - 🔍 Python full-text search: fts.md
   - 🔌 Integrations:
     - integrations/index.md
@@ -84,6 +98,7 @@ nav:
     - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
     - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
     - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
+    - Example - Calculate CLIP Embeddings with Roboflow Inference: examples/image_embeddings_roboflow.md
     - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
     - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
   - 🌐 Javascript examples:
@@ -91,13 +106,22 @@ nav:
     - Serverless Website Chatbot: examples/serverless_website_chatbot.md
     - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
     - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
-  - 📚 Guides:
-    - Tables: guides/tables.md
-    - Vector Search: search.md
-    - SQL filters: sql.md
-    - Indexing: ann_indexes.md
+  - ⚙️ CLI & Config: cli_config.md
+
 - Basics: basic.md
-- Embeddings: embedding.md
+- Guides:
+  - Create Ingest Update Delete: guides/tables.md
+  - Vector Search: search.md
+  - SQL filters: sql.md
+  - Indexing: ann_indexes.md
+  - Versioning & Reproducibility: notebooks/reproducibility.ipynb
+- Embeddings:
+  - embeddings/index.md
+  - Ingest Embedding Functions: embeddings/embedding_functions.md
+  - Available Functions: embeddings/default_embedding_functions.md
+  - Create Custom Embedding Functions: embeddings/api.md
+  - Example - Multi-lingual semantic search: notebooks/multi_lingual_example.ipynb
+  - Example - MultiModal CLIP Embeddings: notebooks/DisappearingEmbeddingFunction.ipynb
 - Python full-text search: fts.md
 - Integrations:
   - integrations/index.md
@@ -121,15 +145,11 @@ nav:
   - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
   - Serverless Chatbot from any website: examples/serverless_website_chatbot.md
   - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
-
-- Guides:
-  - Tables: guides/tables.md
-  - Vector Search: search.md
-  - SQL filters: sql.md
-  - Indexing: ann_indexes.md
 - API references:
-  - Python API: python/python.md
+  - OSS Python API: python/python.md
+  - SaaS Python API: python/saas-python.md
   - Javascript API: javascript/modules.md
+  - SaaS Javascript API: javascript/saas-modules.md
 - LanceDB Cloud↗: https://noteforms.com/forms/lancedb-mailing-list-cloud-kty1o5?notionforms=1&utm_source=notionforms
 
 extra_css:

docs/requirements.txt

@@ -2,3 +2,4 @@ mkdocs==1.4.2
 mkdocs-jupyter==0.24.1
 mkdocs-material==9.1.3
 mkdocstrings[python]==0.20.0
+pydantic

docs/src/ann_indexes.md

@@ -6,7 +6,7 @@ LanceDB provides many parameters to fine-tune the index's size, the speed of que
 
 Currently, LanceDB does *not* automatically create the ANN index.
 LanceDB has optimized code for KNN as well. For many use-cases, datasets under 100K vectors won't require index creation at all.
-If you can live with <100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
+If you can live with < 100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
 
 In the future we will look to automatically create and configure the ANN index.
 
@@ -68,6 +68,44 @@ a single PQ code.
   <figcaption>IVF_PQ index with <code>num_partitions=2, num_sub_vectors=4</code></figcaption>
 </figure>
 
+### Use GPU to build vector index
+
+Lance Python SDK has experimental GPU support for creating IVF index.
+Using GPU for index creation requires [PyTorch>2.0](https://pytorch.org/) being installed.
+
+You can specify the GPU device to train IVF partitions via
+
+- **accelerator**: Specify to ``cuda`` or ``mps`` (on Apple Silicon) to enable GPU training.
+
+=== "Linux"
+
+     <!-- skip-test -->
+     ``` { .python .copy }
+     # Create index using CUDA on Nvidia GPUs.
+     tbl.create_index(
+          num_partitions=256,
+          num_sub_vectors=96,
+          accelerator="cuda"
+     )
+     ```
+
+=== "Macos"
+
+     <!-- skip-test -->
+     ```python
+     # Create index using MPS on Apple Silicon.
+     tbl.create_index(
+          num_partitions=256,
+          num_sub_vectors=96,
+          accelerator="mps"
+     )
+     ```
+
+Trouble shootings:
+
+If you see ``AssertionError: Torch not compiled with CUDA enabled``, you need to [install
+PyTorch with CUDA support](https://pytorch.org/get-started/locally/).
+
 
 ## Querying an ANN Index
 
@@ -91,7 +129,7 @@ There are a couple of parameters that can be used to fine-tune the search:
          .limit(2) \
          .nprobes(20) \
          .refine_factor(10) \
-         .to_df()
+         .to_pandas()
      ```
      ```
                                               vector       item       _distance
@@ -118,7 +156,7 @@ You can further filter the elements returned by a search using a where clause.
 
 === "Python"
      ```python
-     tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_df()
+     tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
      ```
 
 === "Javascript"
@@ -126,6 +164,7 @@ You can further filter the elements returned by a search using a where clause.
      const results_2 = await table
          .search(Array(1536).fill(1.2))
          .where("id != '1141'")
+         .limit(2)
          .execute()
      ```
 
@@ -135,7 +174,7 @@ You can select the columns returned by the query using a select clause.
 
 === "Python"
      ```python
-     tbl.search(np.random.random((1536))).select(["vector"]).to_df()
+     tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
      ```
      ```
         vector                                             _distance
@@ -149,33 +188,34 @@ You can select the columns returned by the query using a select clause.
      const results_3 = await table
          .search(Array(1536).fill(1.2))
          .select(["id"])
+         .limit(2)
          .execute()
      ```
 
 ## FAQ
 
-### When is it necessary to create an ANN vector index.
+### When is it necessary to create an ANN vector index?
 
-`LanceDB` has manually tuned SIMD code for computing vector distances.
-In our benchmarks, computing 100K pairs of 1K dimension vectors only take less than 20ms.
-For small dataset (<100K rows) or the applications which can accept 100ms latency, vector indices are usually not necessary.
+`LanceDB` has manually-tuned SIMD code for computing vector distances.
+In our benchmarks, computing 100K pairs of 1K dimension vectors takes **less than 20ms**.
+For small datasets (< 100K rows) or applications that can accept 100ms latency, vector indices are usually not necessary.
 
 For large-scale or higher dimension vectors, it is beneficial to create vector index.
 
-### How big is my index, and how many memory will it take.
+### How big is my index, and how many memory will it take?
 
-In LanceDB, all vector indices are disk-based, meaning that when responding to a vector query, only the relevant pages from the index file are loaded from disk and cached in memory. Additionally, each sub-vector is usually encoded into 1 byte PQ code.
+In LanceDB, all vector indices are **disk-based**, meaning that when responding to a vector query, only the relevant pages from the index file are loaded from disk and cached in memory. Additionally, each sub-vector is usually encoded into 1 byte PQ code.
 
 For example, with a 1024-dimension dataset, if we choose `num_sub_vectors=64`, each sub-vector has `1024 / 64 = 16` float32 numbers.
 Product quantization can lead to approximately `16 * sizeof(float32) / 1 = 64` times of space reduction.
 
-### How to choose `num_partitions` and `num_sub_vectors` for `IVF_PQ` index.
+### How to choose `num_partitions` and `num_sub_vectors` for `IVF_PQ` index?
 
 `num_partitions` is used to decide how many partitions the first level `IVF` index uses.
 Higher number of partitions could lead to more efficient I/O during queries and better accuracy, but it takes much more time to train.
 On `SIFT-1M` dataset, our benchmark shows that keeping each partition 1K-4K rows lead to a good latency / recall.
 
-`num_sub_vectors` decides how many Product Quantization code to generate on each vector. Because
-Product Quantization is a lossy compression of the original vector, the more `num_sub_vectors` usually results to
-less space distortion, and thus yield better accuracy. However, similarly, more `num_sub_vectors` causes heavier I/O and
-more PQ computation, thus, higher latency. `dimension / num_sub_vectors` should be aligned with 8 for better SIMD efficiency.
+`num_sub_vectors` specifies how many Product Quantization (PQ) short codes to generate on each vector. Because
+PQ is a lossy compression of the original vector, a higher `num_sub_vectors` usually results in
+less space distortion, and thus yields better accuracy. However, a higher `num_sub_vectors` also causes heavier I/O and
+more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.

docs/src/basic.md

@@ -64,18 +64,26 @@ We'll cover the basics of using LanceDB on your local machine in this section.
       tbl = db.create_table("table_from_df", data=df)
       ```
 
+      !!! warning
+
+            If the table already exists, LanceDB will raise an error by default.
+            If you want to overwrite the table, you can pass in `mode="overwrite"`
+            to the `createTable` function.
+
 === "Javascript"
       ```javascript
-      const tb = await db.createTable("my_table",
-                        data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                              {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+      const tb = await db.createTable(
+        "myTable",
+        [{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+         {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
       ```
 
-!!! warning
+      !!! warning
+
+            If the table already exists, LanceDB will raise an error by default.
+            If you want to overwrite the table, you can pass in `"overwrite"`
+            to the `createTable` function like this: `await con.createTable(tableName, data, { writeMode: WriteMode.Overwrite })`
       
-      If the table already exists, LanceDB will raise an error by default.
-      If you want to overwrite the table, you can pass in `mode="overwrite"`
-      to the `createTable` function.
 
 ??? info "Under the hood, LanceDB is converting the input data into an Apache Arrow table and persisting it to disk in [Lance format](https://www.github.com/lancedb/lance)."
 
@@ -108,7 +116,7 @@ Once created, you can open a table using the following code:
 
 === "Javascript"
       ```javascript
-      const tbl = await db.openTable("my_table");
+      const tbl = await db.openTable("myTable");
       ```
 
       If you forget the name of your table, you can always get a listing of all table names:
@@ -123,9 +131,15 @@ After a table has been created, you can always add more data to it using
 
 === "Python"
       ```python
-      df = pd.DataFrame([{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
-                        {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}])
-      tbl.add(df)
+
+      # Option 1: Add a list of dicts to a table
+      data = [{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
+            {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}]
+      tbl.add(data)
+
+      # Option 2: Add a pandas DataFrame to a table
+      df = pd.DataFrame(data)
+      tbl.add(data)
       ```
 
 === "Javascript"
@@ -140,7 +154,7 @@ Once you've embedded the query, you can find its nearest neighbors using the fol
 
 === "Python"
       ```python
-      tbl.search([100, 100]).limit(2).to_df()
+      tbl.search([100, 100]).limit(2).to_pandas()
       ```
 
       This returns a pandas DataFrame with the results.
@@ -188,10 +202,17 @@ Use the `drop_table()` method on the database to remove a table.
       db.drop_table("my_table")
       ```
 
-This permanently removes the table and is not recoverable, unlike deleting rows.
-By default, if the table does not exist an exception is raised. To suppress this,
-you can pass in `ignore_missing=True`.
+      This permanently removes the table and is not recoverable, unlike deleting rows.
+      By default, if the table does not exist an exception is raised. To suppress this,
+      you can pass in `ignore_missing=True`.
 
+=== "JavaScript"
+      ```javascript
+      await db.dropTable('myTable')
+      ```
+
+      This permanently removes the table and is not recoverable, unlike deleting rows.
+      If the table does not exist an exception is raised. 
 
 ## What's next
 

docs/src/cli_config.md

@@ -0,0 +1,37 @@
+
+## LanceDB CLI
+Once lanceDB is installed, you can access the CLI using `lancedb` command on the console
+```
+lancedb
+```
+This lists out all the various command-line options available. You can get the usage or help for a particular command
+```
+lancedb {command} --help
+```
+
+## LanceDB config
+LanceDB uses a global config file to store certain settings. These settings are configurable using the lanceDB cli.
+To view your config settings, you can use:
+```
+lancedb config
+```
+These config parameters can be tuned using the cli.
+```
+lancedb {config_name} --{argument}
+```
+
+## LanceDB Opt-in Diagnostics
+When enabled, LanceDB will send anonymous events to help us improve LanceDB. These diagnostics are used only for error reporting and no data is collected. Error & stats allow us to automate certain aspects of bug reporting, prioritization of fixes and feature requests.
+These diagnostics are opt-in and can be enabled or disabled using the `lancedb diagnostics` command. These are enabled by default.
+Get usage help.
+```
+lancedb diagnostics --help
+```
+Disable diagnostics
+```
+lancedb diagnostics --disabled
+```
+Enable diagnostics
+```
+lancedb diagnostics --enabled
+```

docs/src/embeddings/api.md

@@ -0,0 +1,213 @@
+To use your own custom embedding function, you need to follow these 2 simple steps.
+1. Create your embedding function by implementing the `EmbeddingFunction` interface
+2. Register your embedding function in the global `EmbeddingFunctionRegistry`.
+
+Let us see how this looks like in action.
+
+![](../assets/embeddings_api.png)
+
+
+`EmbeddingFunction` & `EmbeddingFunctionRegistry` handle low-level details for serializing schema and model information as metadata. To build a custom embdding function, you don't need to worry about those details and simply focus on setting up the model.
+
+## `TextEmbeddingFunction` Interface
+
+There is another optional layer of abstraction provided in form of `TextEmbeddingFunction`. You can use this if your model isn't multi-modal in nature and only operates on text. In such case both source and vector fields will have the same pathway for vectorization, so you simply just need to setup the model and rest is handled by `TextEmbeddingFunction`. You can read more about the class and its attributes in the class reference.
+
+
+Let's implement `SentenceTransformerEmbeddings` class. All you need to do is implement the `generate_embeddings()` and `ndims` function to handle the input types you expect and register the class in the global `EmbeddingFunctionRegistry`
+
+```python
+from lancedb.embeddings import register
+
+@register("sentence-transformers")
+class SentenceTransformerEmbeddings(TextEmbeddingFunction):
+    name: str = "all-MiniLM-L6-v2"
+    # set more default instance vars like device, etc.
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._ndims = None
+    
+    def generate_embeddings(self, texts):
+        return self._embedding_model().encode(list(texts), ...).tolist()
+
+    def ndims(self):
+        if self._ndims is None:
+            self._ndims = len(self.generate_embeddings("foo")[0])
+        return self._ndims
+
+    @cached(cache={}) 
+    def _embedding_model(self):
+        return sentence_transformers.SentenceTransformer(name)
+
+```
+
+This is a stripped down version of our implementation of `SentenceTransformerEmbeddings` that removes certain optimizations and defaul settings.
+
+Now you can use this embedding function to create your table schema and that's it! you can then ingest data and run queries without manually vectorizing the inputs.
+
+```python
+from lancedb.pydantic import LanceModel, Vector
+
+registry = EmbeddingFunctionRegistry.get_instance()
+stransformer = registry.get("sentence-transformers").create()
+
+class TextModelSchema(LanceModel):
+    vector: Vector(stransformer.ndims) = stransformer.VectorField()
+    text: str = stransformer.SourceField()
+
+tbl = db.create_table("table", schema=TextModelSchema)
+
+tbl.add(pd.DataFrame({"text": ["halo", "world"]}))
+result = tbl.search("world").limit(5)
+```
+
+NOTE:
+
+You can always implement the `EmbeddingFunction` interface directly if you want or need to, `TextEmbeddingFunction` just makes it much simpler and faster for you to do so, by setting up the boiler plat for text-specific use case
+
+## Multi-modal embedding function example
+You can also use the `EmbeddingFunction` interface to implement more complex workflows such as multi-modal embedding function support. LanceDB implements `OpenClipEmeddingFunction` class that suppports multi-modal seach. Here's the implementation that you can use as a reference to build your own multi-modal embedding functions.
+
+```python
+@register("open-clip")
+class OpenClipEmbeddings(EmbeddingFunction):
+    name: str = "ViT-B-32"
+    pretrained: str = "laion2b_s34b_b79k"
+    device: str = "cpu"
+    batch_size: int = 64
+    normalize: bool = True
+    _model = PrivateAttr()
+    _preprocess = PrivateAttr()
+    _tokenizer = PrivateAttr()
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        open_clip = self.safe_import("open_clip", "open-clip") # EmbeddingFunction util to import external libs and raise if not found
+        model, _, preprocess = open_clip.create_model_and_transforms(
+            self.name, pretrained=self.pretrained
+        )
+        model.to(self.device)
+        self._model, self._preprocess = model, preprocess
+        self._tokenizer = open_clip.get_tokenizer(self.name)
+        self._ndims = None
+
+    def ndims(self):
+        if self._ndims is None:
+            self._ndims = self.generate_text_embeddings("foo").shape[0]
+        return self._ndims
+
+    def compute_query_embeddings(
+        self, query: Union[str, "PIL.Image.Image"], *args, **kwargs
+    ) -> List[np.ndarray]:
+        """
+        Compute the embeddings for a given user query
+
+        Parameters
+        ----------
+        query : Union[str, PIL.Image.Image]
+            The query to embed. A query can be either text or an image.
+        """
+        if isinstance(query, str):
+            return [self.generate_text_embeddings(query)]
+        else:
+            PIL = self.safe_import("PIL", "pillow")
+            if isinstance(query, PIL.Image.Image):
+                return [self.generate_image_embedding(query)]
+            else:
+                raise TypeError("OpenClip supports str or PIL Image as query")
+
+    def generate_text_embeddings(self, text: str) -> np.ndarray:
+        torch = self.safe_import("torch")
+        text = self.sanitize_input(text)
+        text = self._tokenizer(text)
+        text.to(self.device)
+        with torch.no_grad():
+            text_features = self._model.encode_text(text.to(self.device))
+            if self.normalize:
+                text_features /= text_features.norm(dim=-1, keepdim=True)
+            return text_features.cpu().numpy().squeeze()
+
+    def sanitize_input(self, images: IMAGES) -> Union[List[bytes], np.ndarray]:
+        """
+        Sanitize the input to the embedding function.
+        """
+        if isinstance(images, (str, bytes)):
+            images = [images]
+        elif isinstance(images, pa.Array):
+            images = images.to_pylist()
+        elif isinstance(images, pa.ChunkedArray):
+            images = images.combine_chunks().to_pylist()
+        return images
+
+    def compute_source_embeddings(
+        self, images: IMAGES, *args, **kwargs
+    ) -> List[np.array]:
+        """
+        Get the embeddings for the given images
+        """
+        images = self.sanitize_input(images)
+        embeddings = []
+        for i in range(0, len(images), self.batch_size):
+            j = min(i + self.batch_size, len(images))
+            batch = images[i:j]
+            embeddings.extend(self._parallel_get(batch))
+        return embeddings
+
+    def _parallel_get(self, images: Union[List[str], List[bytes]]) -> List[np.ndarray]:
+        """
+        Issue concurrent requests to retrieve the image data
+        """
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [
+                executor.submit(self.generate_image_embedding, image)
+                for image in images
+            ]
+            return [future.result() for future in futures]
+
+    def generate_image_embedding(
+        self, image: Union[str, bytes, "PIL.Image.Image"]
+    ) -> np.ndarray:
+        """
+        Generate the embedding for a single image
+
+        Parameters
+        ----------
+        image : Union[str, bytes, PIL.Image.Image]
+            The image to embed. If the image is a str, it is treated as a uri.
+            If the image is bytes, it is treated as the raw image bytes.
+        """
+        torch = self.safe_import("torch")
+        # TODO handle retry and errors for https
+        image = self._to_pil(image)
+        image = self._preprocess(image).unsqueeze(0)
+        with torch.no_grad():
+            return self._encode_and_normalize_image(image)
+
+    def _to_pil(self, image: Union[str, bytes]):
+        PIL = self.safe_import("PIL", "pillow")
+        if isinstance(image, bytes):
+            return PIL.Image.open(io.BytesIO(image))
+        if isinstance(image, PIL.Image.Image):
+            return image
+        elif isinstance(image, str):
+            parsed = urlparse.urlparse(image)
+            # TODO handle drive letter on windows.
+            if parsed.scheme == "file":
+                return PIL.Image.open(parsed.path)
+            elif parsed.scheme == "":
+                return PIL.Image.open(image if os.name == "nt" else parsed.path)
+            elif parsed.scheme.startswith("http"):
+                return PIL.Image.open(io.BytesIO(url_retrieve(image)))
+            else:
+                raise NotImplementedError("Only local and http(s) urls are supported")
+
+    def _encode_and_normalize_image(self, image_tensor: "torch.Tensor"):
+        """
+        encode a single image tensor and optionally normalize the output
+        """
+        image_features = self._model.encode_image(image_tensor)
+        if self.normalize:
+            image_features /= image_features.norm(dim=-1, keepdim=True)
+        return image_features.cpu().numpy().squeeze()
+```

docs/src/embeddings/default_embedding_functions.md

@@ -0,0 +1,208 @@
+There are various Embedding functions available out of the box with LanceDB. We're working on supporting other popular embedding APIs.
+
+## Text Embedding Functions
+Here are the text embedding functions registered by default.
+Embedding functions have an inbuilt rate limit handler wrapper for source and query embedding function calls that retry with exponential standoff. 
+Each `EmbeddingFunction` implementation automatically takes `max_retries` as an argument which has the default value of 7.
+
+### Sentence Transformers
+Here are the parameters that you can set when registering a `sentence-transformers` object, and their default values:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"all-MiniLM-L6-v2"` | The name of the model. |
+| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
+| `normalize` | `bool` | `True` | Whether to normalize the input text before feeding it to the model. |
+
+
+```python
+db = lancedb.connect("/tmp/db")
+registry = EmbeddingFunctionRegistry.get_instance()
+func = registry.get("sentence-transformers").create(device="cpu")
+
+class Words(LanceModel):
+    text: str = func.SourceField()
+    vector: Vector(func.ndims()) = func.VectorField()
+
+table = db.create_table("words", schema=Words)
+table.add(
+    [
+        {"text": "hello world"}
+        {"text": "goodbye world"}
+    ]
+)
+
+query = "greetings"
+actual = table.search(query).limit(1).to_pydantic(Words)[0]
+print(actual.text)
+```
+
+### OpenAIEmbeddings
+LanceDB has OpenAI embeddings function in the registry by default. It is registered as `openai` and here are the parameters that you can customize when creating the instances
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"text-embedding-ada-002"` | The name of the model. |
+
+
+
+```python
+db = lancedb.connect("/tmp/db")
+registry = EmbeddingFunctionRegistry.get_instance()
+func = registry.get("openai").create()
+
+class Words(LanceModel):
+    text: str = func.SourceField()
+    vector: Vector(func.ndims()) = func.VectorField()
+
+table = db.create_table("words", schema=Words)
+table.add(
+    [
+        {"text": "hello world"}
+        {"text": "goodbye world"}
+    ]
+    )
+
+query = "greetings"
+actual = table.search(query).limit(1).to_pydantic(Words)[0]
+print(actual.text)
+```
+
+### Instructor Embeddings
+Instructor is an instruction-finetuned text embedding model that can generate text embeddings tailored to any task (e.g. classification, retrieval, clustering, text evaluation, etc.) and domains (e.g. science, finance, etc.) by simply providing the task instruction, without any finetuning.
+
+If you want to calculate customized embeddings for specific sentences, you may follow the unified template to write instructions:
+
+                          Represent the `domain` `text_type` for `task_objective`:
+
+* `domain` is optional, and it specifies the domain of the text, e.g. science, finance, medicine, etc.
+* `text_type` is required, and it specifies the encoding unit, e.g. sentence, document, paragraph, etc.
+* `task_objective` is optional, and it specifies the objective of embedding, e.g. retrieve a document, classify the sentence, etc.
+
+More information about the model can be found here - https://github.com/xlang-ai/instructor-embedding
+
+| Argument | Type | Default | Description |
+|---|---|---|---|
+| `name` | `str` | "hkunlp/instructor-base" | The name of the model to use |
+| `batch_size` | `int` | `32` | The batch size to use when generating embeddings |
+| `device` | `str` | `"cpu"` | The device to use when generating embeddings |
+| `show_progress_bar` | `bool` | `True` | Whether to show a progress bar when generating embeddings |
+| `normalize_embeddings` | `bool` | `True` | Whether to normalize the embeddings |
+| `quantize` | `bool` | `False` | Whether to quantize the model |
+| `source_instruction` | `str` | `"represent the docuement for retreival"` | The instruction for the source column |
+| `query_instruction` | `str` | `"represent the document for retreiving the most similar documents"` | The instruction for the query |
+
+
+
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry, InstuctorEmbeddingFunction
+
+instructor = get_registry().get("instructor").create(
+                            source_instruction="represent the docuement for retreival",
+                            query_instruction="represent the document for retreiving the most similar documents"
+                            )
+
+class Schema(LanceModel):
+    vector: Vector(instructor.ndims()) = instructor.VectorField()
+    text: str = instructor.SourceField()
+
+db = lancedb.connect("~/.lancedb")
+tbl = db.create_table("test", schema=Schema, mode="overwrite")
+
+texts = [{"text": "Capitalism has been dominant in the Western world since the end of feudalism, but most feel[who?] that..."},
+        {"text": "The disparate impact theory is especially controversial under the Fair Housing Act because the Act..."},
+        {"text": "Disparate impact in United States labor law refers to practices in employment, housing, and other areas that.."}]
+
+tbl.add(texts)
+```
+
+## Multi-modal embedding functions
+Multi-modal embedding functions allow you to query your table using both images and text.
+
+### OpenClipEmbeddings
+We support CLIP model embeddings using the open source alternative, open-clip which supports various customizations. It is registered as `open-clip` and supports the following customizations:
+
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"ViT-B-32"` | The name of the model. |
+| `pretrained` | `str` | `"laion2b_s34b_b79k"` | The name of the pretrained model to load. |
+| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
+| `batch_size` | `int` | `64` | The number of images to process in a batch. |
+| `normalize` | `bool` | `True` | Whether to normalize the input images before feeding them to the model. |
+
+
+This embedding function supports ingesting images as both bytes and urls. You can query them using both test and other images.
+
+NOTE:
+LanceDB supports ingesting images directly from accessible links.
+
+
+```python
+
+db = lancedb.connect(tmp_path)
+registry = EmbeddingFunctionRegistry.get_instance()
+func = registry.get("open-clip").create()
+
+class Images(LanceModel):
+    label: str
+    image_uri: str = func.SourceField() # image uri as the source
+    image_bytes: bytes = func.SourceField() # image bytes as the source
+    vector: Vector(func.ndims()) = func.VectorField() # vector column 
+    vec_from_bytes: Vector(func.ndims()) = func.VectorField() # Another vector column 
+
+table = db.create_table("images", schema=Images)
+labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
+uris = [
+    "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
+    "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
+    "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
+    "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
+    "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
+    "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
+]
+# get each uri as bytes
+image_bytes = [requests.get(uri).content for uri in uris]
+table.add(
+    [{"label": labels, "image_uri": uris, "image_bytes": image_bytes}]
+)
+```
+Now we can search using text from both the default vector column and the custom vector column
+```python
+
+# text search
+actual = table.search("man's best friend").limit(1).to_pydantic(Images)[0]
+print(actual.label) # prints "dog"
+
+frombytes = (
+    table.search("man's best friend", vector_column_name="vec_from_bytes")
+    .limit(1)
+    .to_pydantic(Images)[0]
+)
+print(frombytes.label)
+
+```
+
+Because we're using a multi-modal embedding function, we can also search using images
+
+```python
+# image search
+query_image_uri = "http://farm1.staticflickr.com/200/467715466_ed4a31801f_z.jpg"
+image_bytes = requests.get(query_image_uri).content
+query_image = Image.open(io.BytesIO(image_bytes))
+actual = table.search(query_image).limit(1).to_pydantic(Images)[0]
+print(actual.label == "dog")
+
+# image search using a custom vector column
+other = (
+    table.search(query_image, vector_column_name="vec_from_bytes")
+    .limit(1)
+    .to_pydantic(Images)[0]
+)
+print(actual.label)
+
+```
+
+If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue.

docs/src/embeddings/embedding_functions.md

@@ -0,0 +1,104 @@
+Representing multi-modal data as vector embeddings is becoming a standard practice. Embedding functions themselves can be thought of as a part of the processing pipeline that each request(input) has to be passed through. After initial setup these components are not expected to change for a particular project. 
+
+Our new embedding functions API allow you simply set it up once and the table remembers it, effectively making the **embedding functions disappear in the background** so you don't have to worry about modelling and can simply focus on the DB aspects of VectorDB.
+
+You can simply follow these steps and forget about the details of your embedding functions as long as you don't intend to change it.
+
+### Step 1 - Define the embedding function
+We have some pre-defined embedding functions in the global registry with more coming soon. Here's let's an implementation of CLIP as example.
+```
+from lancedb.embeddings import EmbeddingFunctionRegistry
+
+registry = EmbeddingFunctionRegistry.get_instance()
+clip = registry.get("open-clip").create()
+
+```
+You can also define your own embedding function by implementing the `EmbeddingFunction` abstract base interface. It subclasses PyDantic Model which can be utilized to write complex schemas simply as we'll see next!
+
+### Step 2 - Define the Data Model or Schema
+Our embedding function from the previous section abstracts away all the details about the models and dimensions required to define the schema. You can simply set a field as **source** or **vector** column. Here's how
+
+```python
+from lancedb.pydantic import LanceModel, Vector
+
+class Pets(LanceModel):
+    vector: Vector(clip.ndims) = clip.VectorField()
+    image_uri: str = clip.SourceField()
+
+```
+`VectorField` tells LanceDB to use the clip embedding function to generate query embeddings for `vector` column & `SourceField` tells that when adding data, automatically use the embedding function to encode `image_uri`.
+
+
+### Step 3 - Create LanceDB Table
+Now that we have chosen/defined our embedding function and the schema, we can create the table
+
+```python
+import lancedb
+
+db = lancedb.connect("~/lancedb")
+table = db.create_table("pets", schema=Pets)
+```
+
+That's it! We have ingested all the information needed to embed source and query inputs. We can now forget about the model and dimension details and start to build our VectorDB.
+
+### Step 4 - Ingest lots of data and run vector search!
+Now you can just add the data and it'll be vectorized automatically
+
+```python
+table.add([{"image_uri": u} for u in uris])
+```
+
+Our OpenCLIP query embedding function support querying via both text and images. 
+
+```python
+result = table.search("dog")
+```
+
+Let's query an image
+
+```python
+from pathlib import Path
+
+p = Path("path/to/images/samoyed_100.jpg")
+query_image = Image.open(p)
+table.search(query_image)
+```
+
+### Rate limit Handling
+`EmbeddingFunction` class wraps the calls for source and query embedding generation inside a rate limit handler that retries the requests with exponential backoff after successive failures. By default the maximum retires is set to 7. You can tune it by setting it to a different number or disable it by setting it to 0. Example:
+
+```python
+clip = registry.get("open-clip").create() # Defaults to 7 max retries
+clip = registry.get("open-clip").create(max_retries=10) # Increase max retries to 10
+clip = registry.get("open-clip").create(max_retries=0) # Retries disabled
+```
+
+NOTE:
+Embedding functions can also fail due to other errors that have nothing to do with rate limits. This is why the errors are also logged.
+
+### A little fun with PyDantic
+LanceDB is integrated with PyDantic. In fact, we've used the integration in the above example to define the schema. It is also being used behind the scene by the embedding function API to ingest useful information as table metadata.
+You can also use it for adding utility operations in the schema. For example, in our multi-modal example, you can search images using text or another image. Let's define a utility function to plot the image.
+
+```python
+from lancedb.pydantic import LanceModel, Vector
+
+class Pets(LanceModel):
+    vector: Vector(clip.ndims) = clip.VectorField()
+    image_uri: str = clip.SourceField()
+
+    @property
+    def image(self):
+        return Image.open(self.image_uri)
+```
+
+Now, you can covert your search results to PyDantic model and use its property.
+
+```python
+rs = table.search(query_image).limit(3).to_pydantic(Pets)
+rs[2].image
+```
+
+![](../assets/dog_clip_output.png)
+
+Now that you have the basic idea about LanceDB embedding function, let us dive deeper into the API that you can use to implement your own embedding functions!

docs/src/embeddings/index.md

@@ -1,13 +1,20 @@
-# Embedding Functions
+# Embedding
 
-Embeddings are high dimensional floating-point vector representations of your data or query.
-Anything can be embedded using some embedding model or function.
-For a given embedding function, the output will always have the same number of dimensions.
+Embeddings are high dimensional floating-point vector representations of your data or query. Anything can be embedded using some embedding model or function. Position of embedding in a high dimensional vector space has semantic significance to a degree that depends on the type of modal and training. These embeddings when projected in a 2-D space generally group similar entities close-by forming groups.
 
-## Creating an embedding function
+![](../assets/embedding_intro.png)
 
-Any function that takes as input a batch (list) of data and outputs a batch (list) of embeddings
-can be used by LanceDB as an embedding function. The input and output batch sizes should be the same.
+# Creating an embedding function
+
+LanceDB supports 2 major ways of vectorizing your data, explicit and implicit.
+
+1. By manually embedding the data before ingesting in the table
+2. By automatically embedding the data and query as they come, by ingesting embedding function information in the table itself! Covered in [Next Section](embedding_functions.md)
+
+Whatever workflow you prefer, we have the tools to support you.
+## Explicit Vectorization
+
+In this workflow, you can create your embedding function and vectorize your data using lancedb's `with_embedding` function. Let's look at some examples.
 
 ### HuggingFace example
 
@@ -118,7 +125,7 @@ belong in the same latent space and your results will be nonsensical.
      ```python
      query = "What's the best pizza topping?"
      query_vector = embed_func([query])[0]
-     tbl.search(query_vector).limit(10).to_df()
+     tbl.search(query_vector).limit(10).to_pandas()
      ```
 
      The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
@@ -134,9 +141,9 @@ belong in the same latent space and your results will be nonsensical.
      The above snippet returns an array of records with the 10 closest vectors to the query.
 
 
-## Roadmap
+## Implicit vectorization / Ingesting embedding functions
+Representing multi-modal data as vector embeddings is becoming a standard practice. Embedding functions themselves be thought of as a part of the processing pipeline that each request(input) has to be passed through. After initial setup these components are not expected to change for a particular project. 
 
-In the near future, we'll be integrating the embedding functions deeper into LanceDB<br/>.
-The goal is that you just have to configure the function once when you create the table,
-and then you'll never have to deal with embeddings / vectors after that unless you want to.
-We'll also integrate more popular models and APIs.
+This is main motivation behind our new embedding functions API, that allow you simply set it up once and the table remembers it, effectively making the **embedding functions disappear in the background** so you don't have to worry about modelling and simply focus on the DB aspects of VectorDB.
+
+Learn more in the Next Section

docs/src/examples/image_embeddings_roboflow.md

@@ -0,0 +1,165 @@
+# How to Load Image Embeddings into LanceDB
+
+With the rise of Large Multimodal Models (LMMs) such as [GPT-4 Vision](https://blog.roboflow.com/gpt-4-vision/), the need for storing image embeddings is growing. The most effective way to store text and image embeddings is in a vector database such as LanceDB. Vector databases are a special kind of data store that enables efficient search over stored embeddings. 
+
+[CLIP](https://blog.roboflow.com/openai-clip/), a multimodal model developed by OpenAI, is commonly used to calculate image embeddings. These embeddings can then be used with a vector database to build a semantic search engine that you can query using images or text. For example, you could use LanceDB and CLIP embeddings to build a search engine for a database of folders.
+
+In this guide, we are going to show you how to use Roboflow Inference to load image embeddings into LanceDB. Without further ado, let’s get started!
+
+## Step #1: Install Roboflow Inference
+
+[Roboflow Inference](https://inference.roboflow.com) enables you to run state-of-the-art computer vision models with minimal configuration. Inference supports a range of models, from fine-tuned object detection, classification, and segmentation models to foundation models like CLIP. We will use Inference to calculate CLIP image embeddings.
+
+Inference provides a HTTP API through which you can run vision models.
+
+Inference powers the Roboflow hosted API, and is available as an open source utility. In this guide, we are going to run Inference locally, which enables you to calculate CLIP embeddings on your own hardware. We will also show you how to use the hosted Roboflow CLIP API, which is ideal if you need to scale and do not want to manage a system for calculating embeddings.
+
+To get started, first install the Inference CLI:
+
+```
+pip install inference-cli
+```
+
+Next, install Docker. Refer to the official Docker installation instructions for your operating system to get Docker set up. Once Docker is ready, you can start Inference using the following command:
+
+```
+inference server start
+```
+
+An Inference server will start running at ‘http://localhost:9001’.
+
+## Step #2: Set Up a LanceDB Vector Database
+
+Now that we have Inference running, we can set up a LanceDB vector database. You can run LanceDB in JavaScript and Python. For this guide, we will use the Python API. But, you can take the HTTP requests we make below and change them to JavaScript if required.
+
+For this guide, we are going to search the [COCO 128 dataset](https://universe.roboflow.com/team-roboflow/coco-128), which contains a wide range of objects. The variability in objects present in this dataset makes it a good dataset to demonstrate the capabilities of vector search. If you want to use this dataset, you can download [COCO 128 from Roboflow Universe](https://universe.roboflow.com/team-roboflow/coco-128). With that said, you can search whatever folder of images you want.
+
+Once you have a dataset ready, install LanceDB with the following command:
+
+```
+pip install lancedb
+```
+
+We also need to install a specific commit of `tantivy`, a dependency of the LanceDB full text search engine we will use later in this guide:
+
+```
+pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+```
+
+Create a new Python file and add the following code:
+
+```python
+import cv2
+import supervision as sv
+import requests
+
+import lancedb
+
+db = lancedb.connect("./embeddings")
+
+IMAGE_DIR = "images/"
+API_KEY = os.environ.get("ROBOFLOW_API_KEY")
+SERVER_URL = "http://localhost:9001"
+
+results = []
+
+for i, image in enumerate(os.listdir(IMAGE_DIR)):
+    infer_clip_payload = {
+        #Images can be provided as urls or as base64 encoded strings
+        "image": {
+            "type": "base64",
+            "value": base64.b64encode(open(IMAGE_DIR + image, "rb").read()).decode("utf-8"),
+        },
+    }
+
+    res = requests.post(
+        f"{SERVER_URL}/clip/embed_image?api_key={API_KEY}",
+        json=infer_clip_payload,
+    )
+
+    embeddings = res.json()['embeddings']
+
+    print("Calculated embedding for image: ", image)
+
+    image = {"vector": embeddings[0], "name": os.path.join(IMAGE_DIR, image)}
+
+    results.append(image)
+
+tbl = db.create_table("images", data=results)
+
+tbl.create_fts_index("name")
+```
+
+To use the code above, you will need a Roboflow API key. [Learn how to retrieve a Roboflow API key](https://docs.roboflow.com/api-reference/authentication#retrieve-an-api-key). Run the following command to set up your API key in your environment:
+
+```
+export ROBOFLOW_API_KEY=""
+```
+
+Replace the `IMAGE_DIR` value with the folder in which you are storing the images for which you want to calculate embeddings. If you want to use the Roboflow CLIP API to calculate embeddings, replace the `SERVER_URL` value with `https://infer.roboflow.com`.
+
+Run the script above to create a new LanceDB database. This database will be stored on your local machine. The database will be called `embeddings` and the table will be called `images`.
+
+The script above calculates all embeddings for a folder then creates a new table. To add additional images, use the following code:
+
+```python
+def make_batches():
+    for i in range(5):
+        yield [
+                {"vector": [3.1, 4.1], "name": "image1.png"},
+                {"vector": [5.9, 26.5], "name": "image2.png"}
+            ]
+
+tbl = db.open_table("images")
+tbl.add(make_batches())
+```
+
+Replacing the `make_batches()` function with code to load embeddings for images.
+
+## Step #3: Run a Search Query
+
+We are now ready to run a search query. To run a search query, we need a text embedding that represents a text query. We can use this embedding to search our LanceDB database for an entry.
+
+Let’s calculate a text embedding for the query “cat”, then run a search query:
+
+```python
+infer_clip_payload = {
+    "text": "cat",
+}
+
+res = requests.post(
+    f"{SERVER_URL}/clip/embed_text?api_key={API_KEY}",
+    json=infer_clip_payload,
+)
+
+embeddings = res.json()['embeddings']
+
+df = tbl.search(embeddings[0]).limit(3).to_list()
+
+print("Results:")
+
+for i in df:
+    print(i["name"])
+```
+
+This code will search for the three images most closely related to the prompt “cat”. The names of the most similar three images will be printed to the console. Here are the three top results:
+
+```
+dataset/images/train/000000000650_jpg.rf.1b74ba165c5a3513a3211d4a80b69e1c.jpg
+dataset/images/train/000000000138_jpg.rf.af439ef1c55dd8a4e4b142d186b9c957.jpg
+dataset/images/train/000000000165_jpg.rf.eae14d5509bf0c9ceccddbb53a5f0c66.jpg
+```
+
+Let’s open the top image:
+
+![Cat](https://media.roboflow.com/cat_lancedb.jpg)
+
+The top image was a cat. Our search was successful.
+
+## Conclusion
+
+LanceDB is a vector database that you can use to store and efficiently search your image embeddings. You can use Roboflow Inference, a scalable computer vision inference server, to calculate CLIP embeddings that you can store in LanceDB.
+
+You can use Inference and LanceDB together to build a range of applications with image embeddings, from a media search engine to a retrieval-augmented generation pipeline for use with LMMs.
+
+To learn more about Inference and its capabilities, refer to the Inference documentation.

docs/src/examples/serverless_lancedb_with_s3_and_lambda.md

@@ -80,14 +80,14 @@ def handler(event, context):
     # Shape of SIFT is (128,1M), d=float32
     query_vector = np.array(event['query_vector'], dtype=np.float32)
 
-    rs = table.search(query_vector).limit(2).to_df()
+    rs = table.search(query_vector).limit(2).to_list()
 
     return {
         "statusCode": status_code,
         "headers": {
             "Content-Type": "application/json"
         },
-        "body": rs.to_json()
+        "body": json.dumps(rs)
     }
 ``` 
 

docs/src/fts.md

@@ -6,17 +6,19 @@ to make this available for JS as well.
 
 ## Installation
 
-To use full text search, you must install optional dependency tantivy-py:
+To use full text search, you must install the dependency `tantivy-py`:
 
-# tantivy 0.19.2
-pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+# tantivy 0.20.1
+```sh
+pip install tantivy==0.20.1
+```
 
 
 ## Quickstart
 
 Assume:
 1. `table` is a LanceDB Table
-2. `text` is the name of the Table column that we want to index
+2. `text` is the name of the `Table` column that we want to index
 
 For example,
 
@@ -27,8 +29,9 @@ uri = "data/sample-lancedb"
 db = lancedb.connect(uri)
 
 table = db.create_table("my_table",
-            data=[{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
-                  {"vector": [5.9, 26.5], "text": "There are several kittens playing"}])
+            data=[{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy", "meta": "foo"},
+                  {"vector": [5.9, 26.5], "text": "Sam was a loyal puppy", "meta": "bar"},
+                  {"vector": [15.9, 6.5], "text": "There are several kittens playing"}])
 
 ```
 
@@ -41,7 +44,13 @@ table.create_fts_index("text")
 To search:
 
 ```python
-df = table.search("puppy").limit(10).select(["text"]).to_df()
+table.search("puppy").limit(10).select(["text"]).to_list()
+```
+
+Which returns a list of dictionaries:
+
+```python
+[{'text': 'Frodo was a happy puppy', 'score': 0.6931471824645996}]
 ```
 
 LanceDB automatically looks for an FTS index if the input is str.
@@ -56,10 +65,51 @@ table.create_fts_index(["text1", "text2"])
 
 Note that the search API call does not change - you can search over all indexed columns at once.
 
+## Filtering
+
+Currently the LanceDB full text search feature supports *post-filtering*, meaning filters are
+applied on top of the full text search results. This can be invoked via the familiar
+`where` syntax:
+
+```python
+table.search("puppy").limit(10).where("meta='foo'").to_list()
+```
+
+## Syntax
+
+For full-text search you can perform either a phrase query like "the old man and the sea", 
+or a structured search query like "(Old AND Man) AND Sea".
+Double quotes are used to disambiguate.
+
+For example: 
+
+If you intended "they could have been dogs OR cats" as a phrase query, this actually
+raises a syntax error since `OR` is a recognized operator. If you make `or` lower case,
+this avoids the syntax error. However, it is cumbersome to have to remember what will
+conflict with the query syntax. Instead, if you search using 
+`table.search('"they could have been dogs OR cats"')`, then the syntax checker avoids
+checking inside the quotes.
+
+
+## Configurations
+
+By default, LanceDB configures a 1GB heap size limit for creating the index. You can 
+reduce this if running on a smaller node, or increase this for faster performance while
+indexing a larger corpus.
+
+```python
+# configure a 512MB heap size
+heap = 1024 * 1024 * 512
+table.create_fts_index(["text1", "text2"], writer_heap_size=heap, replace=True)
+```
+
 ## Current limitations
 
 1. Currently we do not yet support incremental writes.
-If you add data after fts index creation, it won't be reflected
-in search results until you do a full reindex.
+   If you add data after fts index creation, it won't be reflected
+   in search results until you do a full reindex.
 
 2. We currently only support local filesystem paths for the fts index. 
+   This is a tantivy limitation. We've implemented an object store plugin
+   but there's no way in tantivy-py to specify to use it.
+

docs/src/guides/tables.md

@@ -1,5 +1,7 @@
  <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/tables_guide.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
-A Table is a collection of Records in a LanceDB Database. You can follow along on colab!
+A Table is a collection of Records in a LanceDB Database. Tables in Lance have a schema that defines the columns and their types. These schemas can include nested columns and can evolve over time.
+
+This guide will show how to create tables, insert data into them, and update the data. You can follow along on colab!
 
 ## Creating a LanceDB Table
 
@@ -42,7 +44,7 @@ A Table is a collection of Records in a LanceDB Database. You can follow along o
     import pandas as pd
 
     data = pd.DataFrame({
-        "vector": [[1.1, 1.2], [0.2, 1.8]],
+        "vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],
         "lat": [45.5, 40.1],
         "long": [-122.7, -74.1]
     })
@@ -56,7 +58,7 @@ A Table is a collection of Records in a LanceDB Database. You can follow along o
 
     ```python
     custom_schema = pa.schema([
-    pa.field("vector", pa.list_(pa.float32(), 2)),
+    pa.field("vector", pa.list_(pa.float32(), 4)),
     pa.field("lat", pa.float32()),
     pa.field("long", pa.float32())
     ])
@@ -70,8 +72,8 @@ A Table is a collection of Records in a LanceDB Database. You can follow along o
     ```python
     table = pa.Table.from_arrays(
             [
-                pa.array([[3.1, 4.1], [5.9, 26.5]],
-                        pa.list_(pa.float32(), 2)),
+                pa.array([[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],
+                        pa.list_(pa.float32(), 4)),
                 pa.array(["foo", "bar"]),
                 pa.array([10.0, 20.0]),
             ],
@@ -84,14 +86,24 @@ A Table is a collection of Records in a LanceDB Database. You can follow along o
     ```
 
     ### From Pydantic Models
-    LanceDB supports to create Apache Arrow Schema from a Pydantic BaseModel via pydantic_to_schema() method.
+    When you create an empty table without data, you must specify the table schema.
+    LanceDB supports creating tables by specifying a pyarrow schema or a specialized
+    pydantic model called `LanceModel`.
+
+    For example, the following Content model specifies a table with 5 columns:
+    movie_id, vector, genres, title, and imdb_id. When you create a table, you can
+    pass the class as the value of the `schema` parameter to `create_table`.
+    The `vector` column is a `Vector` type, which is a specialized pydantic type that
+    can be configured with the vector dimensions. It is also important to note that
+    LanceDB only understands subclasses of `lancedb.pydantic.LanceModel`
+    (which itself derives from `pydantic.BaseModel`).
 
     ```python
-    from lancedb.pydantic import vector, LanceModel
+    from lancedb.pydantic import Vector, LanceModel
 
     class Content(LanceModel):
         movie_id: int
-        vector: vector(128)
+        vector: Vector(128)
         genres: str
         title: str
         imdb_id: int
@@ -103,9 +115,87 @@ A Table is a collection of Records in a LanceDB Database. You can follow along o
     import pyarrow as pa
     db = lancedb.connect("~/.lancedb")
     table_name = "movielens_small"
-    table = db.create_table(table_name, schema=Content.to_arrow_schema())
+    table = db.create_table(table_name, schema=Content)
     ```
 
+    #### Nested schemas
+
+    Sometimes your data model may contain nested objects.
+    For example, you may want to store the document string
+    and the document soure name as a nested Document object:
+
+    ```python
+    class Document(BaseModel):
+        content: str
+        source: str
+    ```
+
+    This can be used as the type of a LanceDB table column:
+
+    ```python
+    class NestedSchema(LanceModel):
+        id: str
+        vector: Vector(1536)
+        document: Document
+
+    tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite")
+    ```
+
+    This creates a struct column called "document" that has two subfields 
+    called "content" and "source":
+
+    ```
+    In [28]: tbl.schema
+    Out[28]:
+    id: string not null
+    vector: fixed_size_list<item: float>[1536] not null
+        child 0, item: float
+    document: struct<content: string not null, source: string not null> not null
+        child 0, content: string not null
+        child 1, source: string not null    
+    ```
+
+    #### Validators
+
+    Note that neither pydantic nor pyarrow automatically validates that input data
+    is of the *correct* timezone, but this is easy to add as a custom field validator:
+
+    ```python
+    from datetime import datetime
+    from zoneinfo import ZoneInfo
+
+    from lancedb.pydantic import LanceModel
+    from pydantic import Field, field_validator, ValidationError, ValidationInfo
+
+    tzname = "America/New_York"
+    tz = ZoneInfo(tzname)
+
+    class TestModel(LanceModel):
+        dt_with_tz: datetime = Field(json_schema_extra={"tz": tzname})
+
+        @field_validator('dt_with_tz')
+        @classmethod
+        def tz_must_match(cls, dt: datetime) -> datetime:
+            assert dt.tzinfo == tz
+            return dt        
+
+    ok = TestModel(dt_with_tz=datetime.now(tz))
+
+    try:
+        TestModel(dt_with_tz=datetime.now(ZoneInfo("Asia/Shanghai")))
+        assert 0 == 1, "this should raise ValidationError"
+    except ValidationError:
+        print("A ValidationError was raised.")
+        pass
+    ```
+
+    When you run this code it should print "A ValidationError was raised."
+
+    #### Pydantic custom types
+
+    LanceDB does NOT yet support converting pydantic custom types. If this is something you need,
+    please file a feature request on the [LanceDB Github repo](https://github.com/lancedb/lancedb/issues/new).
+
     ### Using Iterators / Writing Large Datasets
 
     It is recommended to use itertators to add large datasets in batches when creating your table in one go. This does not create multiple versions of your dataset unlike manually adding batches using `table.add()`
@@ -121,8 +211,8 @@ A Table is a collection of Records in a LanceDB Database. You can follow along o
         for i in range(5):
             yield pa.RecordBatch.from_arrays(
                 [
-                    pa.array([[3.1, 4.1], [5.9, 26.5]],
-                            pa.list_(pa.float32(), 2)),
+                    pa.array([[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],
+                            pa.list_(pa.float32(), 4)),
                     pa.array(["foo", "bar"]),
                     pa.array([10.0, 20.0]),
                 ],
@@ -130,7 +220,7 @@ A Table is a collection of Records in a LanceDB Database. You can follow along o
             )
 
     schema = pa.schema([
-        pa.field("vector", pa.list_(pa.float32(), 2)),
+        pa.field("vector", pa.list_(pa.float32(), 4)),
         pa.field("item", pa.utf8()),
         pa.field("price", pa.float32()),
     ])
@@ -141,7 +231,7 @@ A Table is a collection of Records in a LanceDB Database. You can follow along o
     You can also use iterators of other types like Pandas dataframe or Pylists directly in the above example.
 
     ## Creating Empty Table
-    You can also create empty tables in python. Initialize it with schema and later ingest data into it.
+    You can create empty tables in python. Initialize it with schema and later ingest data into it.
 
     ```python
     import lancedb
@@ -168,7 +258,7 @@ A Table is a collection of Records in a LanceDB Database. You can follow along o
     from lancedb.pydantic import LanceModel, vector
 
     class Model(LanceModel):
-          vector: vector(2)
+          vector: Vector(2)
 
     tbl = db.create_table("table5", schema=Model.to_arrow_schema())
     ```
@@ -191,8 +281,8 @@ A Table is a collection of Records in a LanceDB Database. You can follow along o
     ```javascript
     data
     const tb = await db.createTable("my_table",
-                  data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+      [{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+       {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
     ```
 
     !!! info "Note"
@@ -241,8 +331,9 @@ After a table has been created, you can always add more data to it using
     ### Adding Pandas DataFrame
 
     ```python
-    df = pd.DataFrame([{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
-                  {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}])
+    df = pd.DataFrame({
+        "vector": [[1.3, 1.4], [9.5, 56.2]], "item": ["fizz", "buzz"], "price": [100.0, 200.0]
+    })
     tbl.add(df)
     ```
 
@@ -251,17 +342,12 @@ After a table has been created, you can always add more data to it using
     ### Adding to table using Iterator
 
     ```python
-    import pandas as pd
-
     def make_batches():
         for i in range(5):
-            yield pd.DataFrame(
-                {
-                    "vector": [[3.1, 4.1], [1, 1]],
-                    "item": ["foo", "bar"],
-                    "price": [10.0, 20.0],
-                })
-    
+            yield [
+                    {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                    {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}
+                ]
     tbl.add(make_batches())
     ```
 
@@ -296,9 +382,10 @@ Use the `delete()` method on tables to delete rows from a table. To choose which
 
     ```python
     import lancedb
-    import pandas as pd
 
-    data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
+    data = [{"x": 1, "vector": [1, 2]},
+            {"x": 2, "vector": [3, 4]},
+            {"x": 3, "vector": [5, 6]}]
     db = lancedb.connect("./.lancedb")
     table = db.create_table("my_table", data)
     table.to_pandas()
@@ -354,6 +441,106 @@ Use the `delete()` method on tables to delete rows from a table. To choose which
     await tbl.countRows() // Returns 1
     ```
 
+## Updating a Table
+
+This can be used to update zero to all rows depending on how many rows match the where clause. The update queries follow the form of a SQL UPDATE statement. The `where` parameter is a SQL filter that matches on the metadata columns. The `values` or `values_sql` parameters are used to provide the new values for the columns.
+
+| Parameter   | Type | Description |
+|---|---|---|
+| `where` | `str` | The SQL where clause to use when updating rows. For example, `'x = 2'` or `'x IN (1, 2, 3)'`. The filter must not be empty, or it will error. |
+| `values` | `dict` | The values to update. The keys are the column names and the values are the values to set. |
+| `values_sql` | `dict` | The values to update. The keys are the column names and the values are the SQL expressions to set. For example, `{'x': 'x + 1'}` will increment the value of the `x` column by 1. |
+
+!!! info "SQL syntax"
+
+    See [SQL filters](sql.md) for more information on the supported SQL syntax.
+
+!!! warning "Warning"
+
+    Updating nested columns is not yet supported.
+
+=== "Python"
+
+    API Reference: [lancedb.table.Table.update][]
+
+    ```python
+    import lancedb
+    import pandas as pd
+
+    # Create a lancedb connection
+    db = lancedb.connect("./.lancedb")
+
+    # Create a table from a pandas DataFrame
+    data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
+    table = db.create_table("my_table", data)
+
+    # Update the table where x = 2
+    table.update(where="x = 2", values={"vector": [10, 10]})
+
+    # Get the updated table as a pandas DataFrame
+    df = table.to_pandas()
+
+    # Print the DataFrame
+    print(df)
+    ```
+
+    Output
+    ```shell
+        x  vector
+    0  1  [1.0, 2.0]
+    1  3  [5.0, 6.0]
+    2  2  [10.0, 10.0]
+    ```
+
+=== "Javascript/Typescript"
+
+    API Reference: [vectordb.Table.update](../../javascript/interfaces/Table/#update)
+
+    ```javascript
+    const lancedb = require("vectordb");
+
+    const db = await lancedb.connect("./.lancedb");
+
+    const data = [
+      {x: 1, vector: [1, 2]},
+      {x: 2, vector: [3, 4]},
+      {x: 3, vector: [5, 6]},
+    ];
+    const tbl = await db.createTable("my_table", data)
+
+    await tbl.update({ where: "x = 2", values: {vector: [10, 10]} })
+    ```
+
+The `values` parameter is used to provide the new values for the columns as literal values. You can also use the `values_sql` / `valuesSql` parameter to provide SQL expressions for the new values. For example, you can use `values_sql="x + 1"` to increment the value of the `x` column by 1.
+
+=== "Python"
+
+    ```python
+    # Update the table where x = 2
+    table.update(valuesSql={"x": "x + 1"})
+
+    print(table.to_pandas())
+    ```
+
+    Output
+    ```shell
+        x  vector
+    0  2  [1.0, 2.0]
+    1  4  [5.0, 6.0]
+    2  3  [10.0, 10.0]
+    ```
+
+=== "Javascript/Typescript"
+
+    ```javascript
+    await tbl.update({ valuesSql: { x: "x + 1" } })
+    ```
+
+!!! info "Note"
+
+    When rows are updated, they are moved out of the index. The row will still show up in ANN queries, but the query will not be as fast as it would be if the row was in the index. If you update a large proportion of rows, consider rebuilding the index afterwards.
+
+
 ## What's Next?
 
 Learn how to Query your tables and create indices

docs/src/index.md

@@ -1,6 +1,6 @@
 # LanceDB
 
-LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrevial, filtering and management of embeddings.
+LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrieval, filtering and management of embeddings.
 
 ![Illustration](/lancedb/assets/ecosystem-illustration.png)
 
@@ -36,7 +36,7 @@ LanceDB's core is written in Rust 🦀 and is built using <a href="https://githu
       table = db.create_table("my_table",
                               data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
                                     {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      result = table.search([100, 100]).limit(2).to_df()
+      result = table.search([100, 100]).limit(2).to_list()
       ```
 
 === "Javascript"
@@ -67,7 +67,7 @@ LanceDB's core is written in Rust 🦀 and is built using <a href="https://githu
 
 ## Documentation Quick Links
 * [`Basic Operations`](basic.md) - basic functionality of LanceDB.
-* [`Embedding Functions`](embedding.md) - functions for working with embeddings.
+* [`Embedding Functions`](embeddings/index.md) - functions for working with embeddings.
 * [`Indexing`](ann_indexes.md) - create vector indexes to speed up queries.
 * [`Full text search`](fts.md) - [EXPERIMENTAL] full-text search API
 * [`Ecosystem Integrations`](python/integration.md) - integrating LanceDB with python data tooling ecosystem.

docs/src/javascript/README.md

@@ -11,8 +11,13 @@ npm install vectordb
 ```
 
 This will download the appropriate native library for your platform. We currently
-support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
-yet support Windows or musl-based Linux (such as Alpine Linux).
+support:
+
+* Linux (x86_64 and aarch64)
+* MacOS (Intel and ARM/M1/M2)
+* Windows (x86_64 only)
+
+We do not yet support musl-based Linux (such as Alpine Linux) or aarch64 Windows.
 
 ## Usage
 

docs/src/javascript/classes/DefaultWriteOptions.md

@@ -0,0 +1,41 @@
+[vectordb](../README.md) / [Exports](../modules.md) / DefaultWriteOptions
+
+# Class: DefaultWriteOptions
+
+Write options when creating a Table.
+
+## Implements
+
+- [`WriteOptions`](../interfaces/WriteOptions.md)
+
+## Table of contents
+
+### Constructors
+
+- [constructor](DefaultWriteOptions.md#constructor)
+
+### Properties
+
+- [writeMode](DefaultWriteOptions.md#writemode)
+
+## Constructors
+
+### constructor
+
+• **new DefaultWriteOptions**()
+
+## Properties
+
+### writeMode
+
+• **writeMode**: [`WriteMode`](../enums/WriteMode.md) = `WriteMode.Create`
+
+A [WriteMode](../enums/WriteMode.md) to use on this operation
+
+#### Implementation of
+
+[WriteOptions](../interfaces/WriteOptions.md).[writeMode](../interfaces/WriteOptions.md#writemode)
+
+#### Defined in
+
+[index.ts:778](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L778)

docs/src/javascript/classes/LocalConnection.md

@@ -26,7 +26,7 @@ A connection to a LanceDB database.
 ### Methods
 
 - [createTable](LocalConnection.md#createtable)
-- [createTableArrow](LocalConnection.md#createtablearrow)
+- [createTableImpl](LocalConnection.md#createtableimpl)
 - [dropTable](LocalConnection.md#droptable)
 - [openTable](LocalConnection.md#opentable)
 - [tableNames](LocalConnection.md#tablenames)
@@ -46,7 +46,7 @@ A connection to a LanceDB database.
 
 #### Defined in
 
-[index.ts:184](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L184)
+[index.ts:355](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L355)
 
 ## Properties
 
@@ -56,17 +56,25 @@ A connection to a LanceDB database.
 
 #### Defined in
 
-[index.ts:182](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L182)
+[index.ts:353](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L353)
 
 ___
 
 ### \_options
 
-• `Private` `Readonly` **\_options**: [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
+• `Private` `Readonly` **\_options**: () => [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
+
+#### Type declaration
+
+▸ (): [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
+
+##### Returns
+
+[`ConnectionOptions`](../interfaces/ConnectionOptions.md)
 
 #### Defined in
 
-[index.ts:181](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L181)
+[index.ts:352](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L352)
 
 ## Accessors
 
@@ -84,27 +92,34 @@ ___
 
 #### Defined in
 
-[index.ts:189](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L189)
+[index.ts:360](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L360)
 
 ## Methods
 
 ### createTable
 
-▸ **createTable**(`name`, `data`, `mode?`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+▸ **createTable**\<`T`\>(`name`, `data?`, `optsOrEmbedding?`, `opt?`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
-Creates a new Table and initialize it with new data.
+Creates a new Table, optionally initializing it with new data.
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
 
 #### Parameters
 
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
-| `mode?` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` \| [`CreateTableOptions`](../interfaces/CreateTableOptions.md)\<`T`\> |
+| `data?` | `Record`\<`string`, `unknown`\>[] |
+| `optsOrEmbedding?` | [`WriteOptions`](../interfaces/WriteOptions.md) \| [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> |
+| `opt?` | [`WriteOptions`](../interfaces/WriteOptions.md) |
 
 #### Returns
 
-`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+`Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
 #### Implementation of
 
@@ -112,120 +127,44 @@ Creates a new Table and initialize it with new data.
 
 #### Defined in
 
-[index.ts:230](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L230)
-
-▸ **createTable**(`name`, `data`, `mode`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` |
-| `data` | `Record`<`string`, `unknown`\>[] |
-| `mode` | [`WriteMode`](../enums/WriteMode.md) |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-#### Implementation of
-
-Connection.createTable
-
-#### Defined in
-
-[index.ts:231](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L231)
-
-▸ **createTable**<`T`\>(`name`, `data`, `mode`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-Creates a new Table and initialize it with new data.
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
-| `mode` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-#### Implementation of
-
-Connection.createTable
-
-#### Defined in
-
-[index.ts:241](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L241)
-
-▸ **createTable**<`T`\>(`name`, `data`, `mode`, `embeddings?`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` |
-| `data` | `Record`<`string`, `unknown`\>[] |
-| `mode` | [`WriteMode`](../enums/WriteMode.md) |
-| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-#### Implementation of
-
-Connection.createTable
-
-#### Defined in
-
-[index.ts:242](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L242)
+[index.ts:395](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L395)
 
 ___
 
-### createTableArrow
+### createTableImpl
 
-▸ **createTableArrow**(`name`, `table`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+▸ `Private` **createTableImpl**\<`T`\>(`«destructured»`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
 
 #### Parameters
 
 | Name | Type |
 | :------ | :------ |
-| `name` | `string` |
-| `table` | `Table`<`any`\> |
+| `«destructured»` | `Object` |
+| › `data?` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] |
+| › `embeddingFunction?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> |
+| › `name` | `string` |
+| › `schema?` | `Schema`\<`any`\> |
+| › `writeOptions?` | [`WriteOptions`](../interfaces/WriteOptions.md) |
 
 #### Returns
 
-`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-#### Implementation of
-
-[Connection](../interfaces/Connection.md).[createTableArrow](../interfaces/Connection.md#createtablearrow)
+`Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
 #### Defined in
 
-[index.ts:266](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L266)
+[index.ts:413](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L413)
 
 ___
 
 ### dropTable
 
-▸ **dropTable**(`name`): `Promise`<`void`\>
+▸ **dropTable**(`name`): `Promise`\<`void`\>
 
 Drop an existing table.
 
@@ -237,7 +176,7 @@ Drop an existing table.
 
 #### Returns
 
-`Promise`<`void`\>
+`Promise`\<`void`\>
 
 #### Implementation of
 
@@ -245,13 +184,13 @@ Drop an existing table.
 
 #### Defined in
 
-[index.ts:276](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L276)
+[index.ts:453](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L453)
 
 ___
 
 ### openTable
 
-▸ **openTable**(`name`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+▸ **openTable**(`name`): `Promise`\<[`Table`](../interfaces/Table.md)\<`number`[]\>\>
 
 Open a table in the database.
 
@@ -263,7 +202,7 @@ Open a table in the database.
 
 #### Returns
 
-`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+`Promise`\<[`Table`](../interfaces/Table.md)\<`number`[]\>\>
 
 #### Implementation of
 
@@ -271,9 +210,9 @@ Open a table in the database.
 
 #### Defined in
 
-[index.ts:205](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L205)
+[index.ts:376](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L376)
 
-▸ **openTable**<`T`\>(`name`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+▸ **openTable**\<`T`\>(`name`, `embeddings`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
 Open a table in the database.
 
@@ -288,11 +227,11 @@ Open a table in the database.
 | Name | Type | Description |
 | :------ | :------ | :------ |
 | `name` | `string` | The name of the table. |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> | An embedding function to use on this Table |
 
 #### Returns
 
-`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+`Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
 #### Implementation of
 
@@ -300,9 +239,9 @@ Connection.openTable
 
 #### Defined in
 
-[index.ts:212](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L212)
+[index.ts:384](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L384)
 
-▸ **openTable**<`T`\>(`name`, `embeddings?`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+▸ **openTable**\<`T`\>(`name`, `embeddings?`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
 #### Type parameters
 
@@ -315,11 +254,11 @@ Connection.openTable
 | Name | Type |
 | :------ | :------ |
 | `name` | `string` |
-| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
+| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> |
 
 #### Returns
 
-`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+`Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
 #### Implementation of
 
@@ -327,19 +266,19 @@ Connection.openTable
 
 #### Defined in
 
-[index.ts:213](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L213)
+[index.ts:385](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L385)
 
 ___
 
 ### tableNames
 
-▸ **tableNames**(): `Promise`<`string`[]\>
+▸ **tableNames**(): `Promise`\<`string`[]\>
 
 Get the names of all tables in the database.
 
 #### Returns
 
-`Promise`<`string`[]\>
+`Promise`\<`string`[]\>
 
 #### Implementation of
 
@@ -347,4 +286,4 @@ Get the names of all tables in the database.
 
 #### Defined in
 
-[index.ts:196](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L196)
+[index.ts:367](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L367)

docs/src/javascript/classes/LocalTable.md

@@ -1,6 +1,6 @@
 [vectordb](../README.md) / [Exports](../modules.md) / LocalTable
 
-# Class: LocalTable<T\>
+# Class: LocalTable\<T\>
 
 A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
 
@@ -12,7 +12,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 ## Implements
 
-- [`Table`](../interfaces/Table.md)<`T`\>
+- [`Table`](../interfaces/Table.md)\<`T`\>
 
 ## Table of contents
 
@@ -26,6 +26,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 - [\_name](LocalTable.md#_name)
 - [\_options](LocalTable.md#_options)
 - [\_tbl](LocalTable.md#_tbl)
+- [where](LocalTable.md#where)
 
 ### Accessors
 
@@ -34,17 +35,23 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 ### Methods
 
 - [add](LocalTable.md#add)
+- [cleanupOldVersions](LocalTable.md#cleanupoldversions)
+- [compactFiles](LocalTable.md#compactfiles)
 - [countRows](LocalTable.md#countrows)
 - [createIndex](LocalTable.md#createindex)
 - [delete](LocalTable.md#delete)
+- [filter](LocalTable.md#filter)
+- [indexStats](LocalTable.md#indexstats)
+- [listIndices](LocalTable.md#listindices)
 - [overwrite](LocalTable.md#overwrite)
 - [search](LocalTable.md#search)
+- [update](LocalTable.md#update)
 
 ## Constructors
 
 ### constructor
 
-• **new LocalTable**<`T`\>(`tbl`, `name`, `options`)
+• **new LocalTable**\<`T`\>(`tbl`, `name`, `options`)
 
 #### Type parameters
 
@@ -62,9 +69,9 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 #### Defined in
 
-[index.ts:287](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L287)
+[index.ts:464](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L464)
 
-• **new LocalTable**<`T`\>(`tbl`, `name`, `options`, `embeddings`)
+• **new LocalTable**\<`T`\>(`tbl`, `name`, `options`, `embeddings`)
 
 #### Type parameters
 
@@ -79,21 +86,21 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 | `tbl` | `any` |  |
 | `name` | `string` |  |
 | `options` | [`ConnectionOptions`](../interfaces/ConnectionOptions.md) |  |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use when interacting with this table |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> | An embedding function to use when interacting with this table |
 
 #### Defined in
 
-[index.ts:294](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L294)
+[index.ts:471](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L471)
 
 ## Properties
 
 ### \_embeddings
 
-• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
+• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\>
 
 #### Defined in
 
-[index.ts:284](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L284)
+[index.ts:461](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L461)
 
 ___
 
@@ -103,27 +110,61 @@ ___
 
 #### Defined in
 
-[index.ts:283](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L283)
+[index.ts:460](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L460)
 
 ___
 
 ### \_options
 
-• `Private` `Readonly` **\_options**: [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
+• `Private` `Readonly` **\_options**: () => [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
+
+#### Type declaration
+
+▸ (): [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
+
+##### Returns
+
+[`ConnectionOptions`](../interfaces/ConnectionOptions.md)
 
 #### Defined in
 
-[index.ts:285](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L285)
+[index.ts:462](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L462)
 
 ___
 
 ### \_tbl
 
-• `Private` `Readonly` **\_tbl**: `any`
+• `Private` **\_tbl**: `any`
 
 #### Defined in
 
-[index.ts:282](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L282)
+[index.ts:459](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L459)
+
+___
+
+### where
+
+• **where**: (`value`: `string`) => [`Query`](Query.md)\<`T`\>
+
+#### Type declaration
+
+▸ (`value`): [`Query`](Query.md)\<`T`\>
+
+Creates a filter query to find all rows matching the specified criteria
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `value` | `string` | The filter criteria (like SQL where clause syntax) |
+
+##### Returns
+
+[`Query`](Query.md)\<`T`\>
+
+#### Defined in
+
+[index.ts:499](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L499)
 
 ## Accessors
 
@@ -141,13 +182,13 @@ ___
 
 #### Defined in
 
-[index.ts:302](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L302)
+[index.ts:479](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L479)
 
 ## Methods
 
 ### add
 
-▸ **add**(`data`): `Promise`<`number`\>
+▸ **add**(`data`): `Promise`\<`number`\>
 
 Insert records into this Table.
 
@@ -155,11 +196,11 @@ Insert records into this Table.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
 
 #### Returns
 
-`Promise`<`number`\>
+`Promise`\<`number`\>
 
 The number of rows added to the table
 
@@ -169,19 +210,69 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:320](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L320)
+[index.ts:507](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L507)
+
+___
+
+### cleanupOldVersions
+
+▸ **cleanupOldVersions**(`olderThan?`, `deleteUnverified?`): `Promise`\<[`CleanupStats`](../interfaces/CleanupStats.md)\>
+
+Clean up old versions of the table, freeing disk space.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `olderThan?` | `number` | The minimum age in minutes of the versions to delete. If not provided, defaults to two weeks. |
+| `deleteUnverified?` | `boolean` | Because they may be part of an in-progress transaction, uncommitted files newer than 7 days old are not deleted by default. This means that failed transactions can leave around data that takes up disk space for up to 7 days. You can override this safety mechanism by setting this option to `true`, only if you promise there are no in progress writes while you run this operation. Failure to uphold this promise can lead to corrupted tables. |
+
+#### Returns
+
+`Promise`\<[`CleanupStats`](../interfaces/CleanupStats.md)\>
+
+#### Defined in
+
+[index.ts:596](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L596)
+
+___
+
+### compactFiles
+
+▸ **compactFiles**(`options?`): `Promise`\<[`CompactionMetrics`](../interfaces/CompactionMetrics.md)\>
+
+Run the compaction process on the table.
+
+This can be run after making several small appends to optimize the table
+for faster reads.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `options?` | [`CompactionOptions`](../interfaces/CompactionOptions.md) | Advanced options configuring compaction. In most cases, you can omit this arguments, as the default options are sensible for most tables. |
+
+#### Returns
+
+`Promise`\<[`CompactionMetrics`](../interfaces/CompactionMetrics.md)\>
+
+Metrics about the compaction operation.
+
+#### Defined in
+
+[index.ts:615](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L615)
 
 ___
 
 ### countRows
 
-▸ **countRows**(): `Promise`<`number`\>
+▸ **countRows**(): `Promise`\<`number`\>
 
 Returns the number of rows in this table.
 
 #### Returns
 
-`Promise`<`number`\>
+`Promise`\<`number`\>
 
 #### Implementation of
 
@@ -189,20 +280,16 @@ Returns the number of rows in this table.
 
 #### Defined in
 
-[index.ts:362](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L362)
+[index.ts:543](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L543)
 
 ___
 
 ### createIndex
 
-▸ **createIndex**(`indexParams`): `Promise`<`any`\>
+▸ **createIndex**(`indexParams`): `Promise`\<`any`\>
 
 Create an ANN index on this Table vector index.
 
-**`See`**
-
-VectorIndexParams.
-
 #### Parameters
 
 | Name | Type | Description |
@@ -211,7 +298,11 @@ VectorIndexParams.
 
 #### Returns
 
-`Promise`<`any`\>
+`Promise`\<`any`\>
+
+**`See`**
+
+VectorIndexParams.
 
 #### Implementation of
 
@@ -219,13 +310,13 @@ VectorIndexParams.
 
 #### Defined in
 
-[index.ts:355](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L355)
+[index.ts:536](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L536)
 
 ___
 
 ### delete
 
-▸ **delete**(`filter`): `Promise`<`void`\>
+▸ **delete**(`filter`): `Promise`\<`void`\>
 
 Delete rows from this table.
 
@@ -237,7 +328,7 @@ Delete rows from this table.
 
 #### Returns
 
-`Promise`<`void`\>
+`Promise`\<`void`\>
 
 #### Implementation of
 
@@ -245,13 +336,81 @@ Delete rows from this table.
 
 #### Defined in
 
-[index.ts:371](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L371)
+[index.ts:552](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L552)
+
+___
+
+### filter
+
+▸ **filter**(`value`): [`Query`](Query.md)\<`T`\>
+
+Creates a filter query to find all rows matching the specified criteria
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `value` | `string` | The filter criteria (like SQL where clause syntax) |
+
+#### Returns
+
+[`Query`](Query.md)\<`T`\>
+
+#### Defined in
+
+[index.ts:495](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L495)
+
+___
+
+### indexStats
+
+▸ **indexStats**(`indexUuid`): `Promise`\<[`IndexStats`](../interfaces/IndexStats.md)\>
+
+Get statistics about an index.
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `indexUuid` | `string` |
+
+#### Returns
+
+`Promise`\<[`IndexStats`](../interfaces/IndexStats.md)\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[indexStats](../interfaces/Table.md#indexstats)
+
+#### Defined in
+
+[index.ts:628](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L628)
+
+___
+
+### listIndices
+
+▸ **listIndices**(): `Promise`\<[`VectorIndex`](../interfaces/VectorIndex.md)[]\>
+
+List the indicies on this table.
+
+#### Returns
+
+`Promise`\<[`VectorIndex`](../interfaces/VectorIndex.md)[]\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[listIndices](../interfaces/Table.md#listindices)
+
+#### Defined in
+
+[index.ts:624](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L624)
 
 ___
 
 ### overwrite
 
-▸ **overwrite**(`data`): `Promise`<`number`\>
+▸ **overwrite**(`data`): `Promise`\<`number`\>
 
 Insert records into this Table, replacing its contents.
 
@@ -259,11 +418,11 @@ Insert records into this Table, replacing its contents.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
 
 #### Returns
 
-`Promise`<`number`\>
+`Promise`\<`number`\>
 
 The number of rows added to the table
 
@@ -273,13 +432,13 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:338](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L338)
+[index.ts:522](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L522)
 
 ___
 
 ### search
 
-▸ **search**(`query`): [`Query`](Query.md)<`T`\>
+▸ **search**(`query`): [`Query`](Query.md)\<`T`\>
 
 Creates a search query to find the nearest neighbors of the given search term
 
@@ -291,7 +450,7 @@ Creates a search query to find the nearest neighbors of the given search term
 
 #### Returns
 
-[`Query`](Query.md)<`T`\>
+[`Query`](Query.md)\<`T`\>
 
 #### Implementation of
 
@@ -299,4 +458,30 @@ Creates a search query to find the nearest neighbors of the given search term
 
 #### Defined in
 
-[index.ts:310](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L310)
+[index.ts:487](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L487)
+
+___
+
+### update
+
+▸ **update**(`args`): `Promise`\<`void`\>
+
+Update rows in this table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `args` | [`UpdateArgs`](../interfaces/UpdateArgs.md) \| [`UpdateSqlArgs`](../interfaces/UpdateSqlArgs.md) | see [UpdateArgs](../interfaces/UpdateArgs.md) and [UpdateSqlArgs](../interfaces/UpdateSqlArgs.md) for more details |
+
+#### Returns
+
+`Promise`\<`void`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[update](../interfaces/Table.md#update)
+
+#### Defined in
+
+[index.ts:563](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L563)

docs/src/javascript/classes/OpenAIEmbeddingFunction.md

@@ -6,7 +6,7 @@ An embedding function that automatically creates vector representation for a giv
 
 ## Implements
 
-- [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`string`\>
+- [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`string`\>
 
 ## Table of contents
 
@@ -40,7 +40,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L21)
+[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L21)
 
 ## Properties
 
@@ -50,7 +50,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L19)
+[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L19)
 
 ___
 
@@ -60,7 +60,7 @@ ___
 
 #### Defined in
 
-[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L18)
+[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L18)
 
 ___
 
@@ -76,13 +76,13 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L50)
+[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L50)
 
 ## Methods
 
 ### embed
 
-▸ **embed**(`data`): `Promise`<`number`[][]\>
+▸ **embed**(`data`): `Promise`\<`number`[][]\>
 
 Creates a vector representation for the given values.
 
@@ -94,7 +94,7 @@ Creates a vector representation for the given values.
 
 #### Returns
 
-`Promise`<`number`[][]\>
+`Promise`\<`number`[][]\>
 
 #### Implementation of
 
@@ -102,4 +102,4 @@ Creates a vector representation for the given values.
 
 #### Defined in
 
-[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L38)
+[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L38)

docs/src/javascript/classes/Query.md

@@ -1,6 +1,6 @@
 [vectordb](../README.md) / [Exports](../modules.md) / Query
 
-# Class: Query<T\>
+# Class: Query\<T\>
 
 A builder for nearest neighbor queries for LanceDB.
 
@@ -23,6 +23,7 @@ A builder for nearest neighbor queries for LanceDB.
 - [\_limit](Query.md#_limit)
 - [\_metricType](Query.md#_metrictype)
 - [\_nprobes](Query.md#_nprobes)
+- [\_prefilter](Query.md#_prefilter)
 - [\_query](Query.md#_query)
 - [\_queryVector](Query.md#_queryvector)
 - [\_refineFactor](Query.md#_refinefactor)
@@ -34,9 +35,11 @@ A builder for nearest neighbor queries for LanceDB.
 
 - [execute](Query.md#execute)
 - [filter](Query.md#filter)
+- [isElectron](Query.md#iselectron)
 - [limit](Query.md#limit)
 - [metricType](Query.md#metrictype)
 - [nprobes](Query.md#nprobes)
+- [prefilter](Query.md#prefilter)
 - [refineFactor](Query.md#refinefactor)
 - [select](Query.md#select)
 
@@ -44,7 +47,7 @@ A builder for nearest neighbor queries for LanceDB.
 
 ### constructor
 
-• **new Query**<`T`\>(`tbl`, `query`, `embeddings?`)
+• **new Query**\<`T`\>(`query?`, `tbl?`, `embeddings?`)
 
 #### Type parameters
 
@@ -56,23 +59,23 @@ A builder for nearest neighbor queries for LanceDB.
 
 | Name | Type |
 | :------ | :------ |
-| `tbl` | `any` |
-| `query` | `T` |
-| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
+| `query?` | `T` |
+| `tbl?` | `any` |
+| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> |
 
 #### Defined in
 
-[index.ts:448](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L448)
+[query.ts:38](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L38)
 
 ## Properties
 
 ### \_embeddings
 
-• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
+• `Protected` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\>
 
 #### Defined in
 
-[index.ts:446](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L446)
+[query.ts:36](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L36)
 
 ___
 
@@ -82,17 +85,17 @@ ___
 
 #### Defined in
 
-[index.ts:444](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L444)
+[query.ts:33](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L33)
 
 ___
 
 ### \_limit
 
-• `Private` **\_limit**: `number`
+• `Private` `Optional` **\_limit**: `number`
 
 #### Defined in
 
-[index.ts:440](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L440)
+[query.ts:29](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L29)
 
 ___
 
@@ -102,7 +105,7 @@ ___
 
 #### Defined in
 
-[index.ts:445](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L445)
+[query.ts:34](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L34)
 
 ___
 
@@ -112,17 +115,27 @@ ___
 
 #### Defined in
 
-[index.ts:442](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L442)
+[query.ts:31](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L31)
+
+___
+
+### \_prefilter
+
+• `Private` **\_prefilter**: `boolean`
+
+#### Defined in
+
+[query.ts:35](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L35)
 
 ___
 
 ### \_query
 
-• `Private` `Readonly` **\_query**: `T`
+• `Private` `Optional` `Readonly` **\_query**: `T`
 
 #### Defined in
 
-[index.ts:438](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L438)
+[query.ts:26](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L26)
 
 ___
 
@@ -132,7 +145,7 @@ ___
 
 #### Defined in
 
-[index.ts:439](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L439)
+[query.ts:28](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L28)
 
 ___
 
@@ -142,7 +155,7 @@ ___
 
 #### Defined in
 
-[index.ts:441](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L441)
+[query.ts:30](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L30)
 
 ___
 
@@ -152,27 +165,27 @@ ___
 
 #### Defined in
 
-[index.ts:443](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L443)
+[query.ts:32](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L32)
 
 ___
 
 ### \_tbl
 
-• `Private` `Readonly` **\_tbl**: `any`
+• `Private` `Optional` `Readonly` **\_tbl**: `any`
 
 #### Defined in
 
-[index.ts:437](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L437)
+[query.ts:27](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L27)
 
 ___
 
 ### where
 
-• **where**: (`value`: `string`) => [`Query`](Query.md)<`T`\>
+• **where**: (`value`: `string`) => [`Query`](Query.md)\<`T`\>
 
 #### Type declaration
 
-▸ (`value`): [`Query`](Query.md)<`T`\>
+▸ (`value`): [`Query`](Query.md)\<`T`\>
 
 A filter statement to be applied to this query.
 
@@ -184,17 +197,17 @@ A filter statement to be applied to this query.
 
 ##### Returns
 
-[`Query`](Query.md)<`T`\>
+[`Query`](Query.md)\<`T`\>
 
 #### Defined in
 
-[index.ts:496](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L496)
+[query.ts:87](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L87)
 
 ## Methods
 
 ### execute
 
-▸ **execute**<`T`\>(): `Promise`<`T`[]\>
+▸ **execute**\<`T`\>(): `Promise`\<`T`[]\>
 
 Execute the query and return the results as an Array of Objects
 
@@ -202,21 +215,21 @@ Execute the query and return the results as an Array of Objects
 
 | Name | Type |
 | :------ | :------ |
-| `T` | `Record`<`string`, `unknown`\> |
+| `T` | `Record`\<`string`, `unknown`\> |
 
 #### Returns
 
-`Promise`<`T`[]\>
+`Promise`\<`T`[]\>
 
 #### Defined in
 
-[index.ts:519](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L519)
+[query.ts:115](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L115)
 
 ___
 
 ### filter
 
-▸ **filter**(`value`): [`Query`](Query.md)<`T`\>
+▸ **filter**(`value`): [`Query`](Query.md)\<`T`\>
 
 A filter statement to be applied to this query.
 
@@ -228,17 +241,31 @@ A filter statement to be applied to this query.
 
 #### Returns
 
-[`Query`](Query.md)<`T`\>
+[`Query`](Query.md)\<`T`\>
 
 #### Defined in
 
-[index.ts:491](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L491)
+[query.ts:82](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L82)
+
+___
+
+### isElectron
+
+▸ `Private` **isElectron**(): `boolean`
+
+#### Returns
+
+`boolean`
+
+#### Defined in
+
+[query.ts:142](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L142)
 
 ___
 
 ### limit
 
-▸ **limit**(`value`): [`Query`](Query.md)<`T`\>
+▸ **limit**(`value`): [`Query`](Query.md)\<`T`\>
 
 Sets the number of results that will be returned
 
@@ -250,24 +277,20 @@ Sets the number of results that will be returned
 
 #### Returns
 
-[`Query`](Query.md)<`T`\>
+[`Query`](Query.md)\<`T`\>
 
 #### Defined in
 
-[index.ts:464](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L464)
+[query.ts:55](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L55)
 
 ___
 
 ### metricType
 
-▸ **metricType**(`value`): [`Query`](Query.md)<`T`\>
+▸ **metricType**(`value`): [`Query`](Query.md)\<`T`\>
 
 The MetricType used for this Query.
 
-**`See`**
-
-MetricType for the different options
-
 #### Parameters
 
 | Name | Type | Description |
@@ -276,17 +299,21 @@ MetricType for the different options
 
 #### Returns
 
-[`Query`](Query.md)<`T`\>
+[`Query`](Query.md)\<`T`\>
+
+**`See`**
+
+MetricType for the different options
 
 #### Defined in
 
-[index.ts:511](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L511)
+[query.ts:102](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L102)
 
 ___
 
 ### nprobes
 
-▸ **nprobes**(`value`): [`Query`](Query.md)<`T`\>
+▸ **nprobes**(`value`): [`Query`](Query.md)\<`T`\>
 
 The number of probes used. A higher number makes search more accurate but also slower.
 
@@ -298,17 +325,37 @@ The number of probes used. A higher number makes search more accurate but also s
 
 #### Returns
 
-[`Query`](Query.md)<`T`\>
+[`Query`](Query.md)\<`T`\>
 
 #### Defined in
 
-[index.ts:482](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L482)
+[query.ts:73](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L73)
+
+___
+
+### prefilter
+
+▸ **prefilter**(`value`): [`Query`](Query.md)\<`T`\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `value` | `boolean` |
+
+#### Returns
+
+[`Query`](Query.md)\<`T`\>
+
+#### Defined in
+
+[query.ts:107](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L107)
 
 ___
 
 ### refineFactor
 
-▸ **refineFactor**(`value`): [`Query`](Query.md)<`T`\>
+▸ **refineFactor**(`value`): [`Query`](Query.md)\<`T`\>
 
 Refine the results by reading extra elements and re-ranking them in memory.
 
@@ -320,17 +367,17 @@ Refine the results by reading extra elements and re-ranking them in memory.
 
 #### Returns
 
-[`Query`](Query.md)<`T`\>
+[`Query`](Query.md)\<`T`\>
 
 #### Defined in
 
-[index.ts:473](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L473)
+[query.ts:64](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L64)
 
 ___
 
 ### select
 
-▸ **select**(`value`): [`Query`](Query.md)<`T`\>
+▸ **select**(`value`): [`Query`](Query.md)\<`T`\>
 
 Return only the specified columns.
 
@@ -342,8 +389,8 @@ Return only the specified columns.
 
 #### Returns
 
-[`Query`](Query.md)<`T`\>
+[`Query`](Query.md)\<`T`\>
 
 #### Defined in
 
-[index.ts:502](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L502)
+[query.ts:93](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L93)

docs/src/javascript/classes/RemoteConnection.md

@@ -0,0 +1,226 @@
+[vectordb](../README.md) / [Exports](../saas-modules.md) / RemoteConnection
+
+# Class: RemoteConnection
+
+A connection to a remote LanceDB database. The class RemoteConnection implements interface Connection
+
+## Implements
+
+- [`Connection`](../interfaces/Connection.md)
+
+## Table of contents
+
+### Constructors
+
+- [constructor](RemoteConnection.md#constructor)
+
+### Methods
+
+- [createTable](RemoteConnection.md#createtable)
+- [tableNames](RemoteConnection.md#tablenames)
+- [openTable](RemoteConnection.md#opentable)
+- [dropTable](RemoteConnection.md#droptable)
+
+
+## Constructors
+
+### constructor
+
+• **new RemoteConnection**(`client`, `dbName`)
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `client` | `HttpLancedbClient` |
+| `dbName` | `string` |
+
+#### Defined in
+
+[remote/index.ts:37](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L37)
+
+## Methods
+
+### createTable
+
+▸ **createTable**(`name`, `data`, `mode?`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+Creates a new Table and initialize it with new data.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
+| `mode?` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[createTable](../interfaces/Connection.md#createtable)
+
+#### Defined in
+
+[remote/index.ts:75](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L75)
+
+▸ **createTable**(`name`, `data`, `mode`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `data` | `Record`<`string`, `unknown`\>[] |
+| `mode` | [`WriteMode`](../enums/WriteMode.md) |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Implementation of
+
+Connection.createTable
+
+#### Defined in
+
+[remote/index.ts:231](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L231)
+
+___
+
+### dropTable
+
+▸ **dropTable**(`name`): `Promise`<`void`\>
+
+Drop an existing table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table to drop. |
+
+#### Returns
+
+`Promise`<`void`\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[dropTable](../interfaces/Connection.md#droptable)
+
+#### Defined in
+
+[remote/index.ts:131](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L131)
+
+___
+
+### openTable
+
+▸ **openTable**(`name`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+Open a table in the database.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[openTable](../interfaces/Connection.md#opentable)
+
+#### Defined in
+
+[remote/index.ts:65](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L65)
+
+▸ **openTable**<`T`\>(`name`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+Open a table in the database.
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Implementation of
+
+Connection.openTable
+
+#### Defined in
+
+[remote/index.ts:66](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L66)
+
+▸ **openTable**<`T`\>(`name`, `embeddings?`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Implementation of
+
+Connection.openTable
+
+#### Defined in
+
+[remote/index.ts:67](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L67)
+
+___
+
+### tableNames
+
+▸ **tableNames**(): `Promise`<`string`[]\>
+
+Get the names of all tables in the database, with pagination.
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `pageToken` | `string` |
+| `limit` | `int` |
+
+#### Returns
+
+`Promise`<`string`[]\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[tableNames](../interfaces/Connection.md#tablenames)
+
+#### Defined in
+
+[remote/index.ts:60](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L60)

docs/src/javascript/classes/RemoteQuery.md

@@ -0,0 +1,76 @@
+[vectordb](../README.md) / [Exports](../saas-modules.md) / RemoteQuery
+
+# Class: Query<T\>
+
+A builder for nearest neighbor queries for LanceDB.
+
+## Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+## Table of contents
+
+### Constructors
+
+- [constructor](RemoteQuery.md#constructor)
+
+### Properties
+
+- [\_embeddings](RemoteQuery.md#_embeddings)
+- [\_query](RemoteQuery.md#_query)
+- [\_name](RemoteQuery.md#_name)
+- [\_client](RemoteQuery.md#_client)
+
+### Methods
+
+- [execute](RemoteQuery.md#execute)
+
+
+## Constructors
+
+### constructor
+
+• **new Query**<`T`\>(`name`, `client`, `query`, `embeddings?`)
+
+#### Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `client` | `HttpLancedbClient` |
+| `query` | `T` |
+| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
+
+#### Defined in
+
+[remote/index.ts:137](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L137)
+
+## Methods
+
+### execute
+
+▸ **execute**<`T`\>(): `Promise`<`T`[]\>
+
+Execute the query and return the results as an Array of Objects
+
+#### Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `Record`<`string`, `unknown`\> |
+
+#### Returns
+
+`Promise`<`T`[]\>
+
+#### Defined in
+
+[remote/index.ts:143](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L143)

docs/src/javascript/classes/RemoteTable.md

@@ -0,0 +1,355 @@
+[vectordb](../README.md) / [Exports](../saas-modules.md) / RemoteTable
+
+# Class: RemoteTable<T\>
+
+A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
+
+## Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+## Implements
+
+- [`Table`](../interfaces/Table.md)<`T`\>
+
+## Table of contents
+
+### Constructors
+
+- [constructor](RemoteTable.md#constructor)
+
+### Properties
+
+- [\_name](RemoteTable.md#_name)
+- [\_client](RemoteTable.md#_client)
+- [\_embeddings](RemoteTable.md#_embeddings)
+
+### Accessors
+
+- [name](RemoteTable.md#name)
+
+### Methods
+
+- [add](RemoteTable.md#add)
+- [countRows](RemoteTable.md#countrows)
+- [createIndex](RemoteTable.md#createindex)
+- [delete](RemoteTable.md#delete)
+- [listIndices](classes/RemoteTable.md#listindices)
+- [indexStats](classes/RemoteTable.md#liststats)
+- [overwrite](RemoteTable.md#overwrite)
+- [search](RemoteTable.md#search)
+- [schema](classes/RemoteTable.md#schema)
+- [update](RemoteTable.md#update)
+
+## Constructors
+
+### constructor
+
+• **new RemoteTable**<`T`\>(`client`, `name`)
+
+#### Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `client` | `HttpLancedbClient` |
+| `name` | `string` |
+
+#### Defined in
+
+[remote/index.ts:186](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L186)
+
+• **new RemoteTable**<`T`\>(`client`, `name`, `embeddings`)
+
+#### Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `client` | `HttpLancedbClient` |  |
+| `name` | `string` |  |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use when interacting with this table |
+
+#### Defined in
+
+[remote/index.ts:187](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L187)
+
+## Accessors
+
+### name
+
+• `get` **name**(): `string`
+
+#### Returns
+
+`string`
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[name](../interfaces/Table.md#name)
+
+#### Defined in
+
+[remote/index.ts:194](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L194)
+
+## Methods
+
+### add
+
+▸ **add**(`data`): `Promise`<`number`\>
+
+Insert records into this Table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+
+#### Returns
+
+`Promise`<`number`\>
+
+The number of rows added to the table
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[add](../interfaces/Table.md#add)
+
+#### Defined in
+
+[remote/index.ts:293](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L293)
+
+___
+
+### countRows
+
+▸ **countRows**(): `Promise`<`number`\>
+
+Returns the number of rows in this table.
+
+#### Returns
+
+`Promise`<`number`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[countRows](../interfaces/Table.md#countrows)
+
+#### Defined in
+
+[remote/index.ts:290](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L290)
+
+___
+
+### createIndex
+
+▸ **createIndex**(`metric_type`, `column`, `index_cache_size`): `Promise`<`any`\>
+
+Create an ANN index on this Table vector index.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `metric_type` | `string` | distance metric type, L2 or cosine or dot |
+| `column` | `string` | the name of the column to be indexed |
+
+#### Returns
+
+`Promise`<`any`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[createIndex](../interfaces/Table.md#createindex)
+
+#### Defined in
+
+[remote/index.ts:249](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L249)
+
+___
+
+### delete
+
+▸ **delete**(`filter`): `Promise`<`void`\>
+
+Delete rows from this table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `filter` | `string` | A filter in the same format used by a sql WHERE clause. |
+
+#### Returns
+
+`Promise`<`void`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[delete](../interfaces/Table.md#delete)
+
+#### Defined in
+
+[remote/index.ts:295](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L295)
+
+___
+
+### overwrite
+
+▸ **overwrite**(`data`): `Promise`<`number`\>
+
+Insert records into this Table, replacing its contents.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+
+#### Returns
+
+`Promise`<`number`\>
+
+The number of rows added to the table
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[overwrite](../interfaces/Table.md#overwrite)
+
+#### Defined in
+
+[remote/index.ts:231](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L231)
+
+___
+
+### search
+
+▸ **search**(`query`): [`Query`](Query.md)<`T`\>
+
+Creates a search query to find the nearest neighbors of the given search term
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `query` | `T` | The query search term |
+
+#### Returns
+
+[`Query`](Query.md)<`T`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[search](../interfaces/Table.md#search)
+
+#### Defined in
+
+[remote/index.ts:209](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L209)
+
+___
+
+### update
+
+▸ **update**(`args`): `Promise`<`void`\>
+
+Update zero to all rows depending on how many rows match the where clause.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `args` | `UpdateArgs` or `UpdateSqlArgs` | The query search arguments |
+
+#### Returns
+
+`Promise`<`any`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[search](../interfaces/Table.md#update)
+
+#### Defined in
+
+[remote/index.ts:299](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L299)
+
+___
+
+### schema
+
+▸ **schema**(): `Promise`<`void`\>
+
+Get the schema of the table
+
+
+#### Returns
+
+`Promise`<`any`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[search](../interfaces/Table.md#schema)
+
+#### Defined in
+
+[remote/index.ts:198](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L198)
+
+___
+
+### listIndices
+
+▸ **listIndices**(): `Promise`<`void`\>
+
+List the indices of the table
+
+
+#### Returns
+
+`Promise`<`any`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[search](../interfaces/Table.md#listIndices)
+
+#### Defined in
+
+[remote/index.ts:319](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L319)
+
+___
+
+### indexStats
+
+▸ **indexStats**(`indexUuid`): `Promise`<`void`\>
+
+Get the indexed/unindexed of rows from the table
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `indexUuid` | `string`  | the uuid of the index |
+
+#### Returns
+
+`Promise`<`numIndexedRows`\>
+`Promise`<`numUnindexedRows`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[search](../interfaces/Table.md#indexStats)
+
+#### Defined in
+
+[remote/index.ts:328](https://github.com/lancedb/lancedb/blob/main/node/src/remote/index.ts#L328)

docs/src/javascript/enums/MetricType.md

@@ -22,7 +22,7 @@ Cosine distance
 
 #### Defined in
 
-[index.ts:567](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L567)
+[index.ts:798](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L798)
 
 ___
 
@@ -34,7 +34,7 @@ Dot product
 
 #### Defined in
 
-[index.ts:572](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L572)
+[index.ts:803](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L803)
 
 ___
 
@@ -46,4 +46,4 @@ Euclidean distance
 
 #### Defined in
 
-[index.ts:562](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L562)
+[index.ts:793](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L793)

docs/src/javascript/enums/WriteMode.md

@@ -22,7 +22,7 @@ Append new data to the table.
 
 #### Defined in
 
-[index.ts:552](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L552)
+[index.ts:766](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L766)
 
 ___
 
@@ -34,7 +34,7 @@ Create a new [Table](../interfaces/Table.md).
 
 #### Defined in
 
-[index.ts:548](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L548)
+[index.ts:762](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L762)
 
 ___
 
@@ -46,4 +46,4 @@ Overwrite the existing [Table](../interfaces/Table.md) if presented.
 
 #### Defined in
 
-[index.ts:550](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L550)
+[index.ts:764](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L764)

docs/src/javascript/interfaces/AwsCredentials.md

@@ -18,7 +18,7 @@
 
 #### Defined in
 
-[index.ts:31](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L31)
+[index.ts:34](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L34)
 
 ___
 
@@ -28,7 +28,7 @@ ___
 
 #### Defined in
 
-[index.ts:33](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L33)
+[index.ts:36](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L36)
 
 ___
 
@@ -38,4 +38,4 @@ ___
 
 #### Defined in
 
-[index.ts:35](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L35)
+[index.ts:38](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L38)

docs/src/javascript/interfaces/CleanupStats.md

@@ -0,0 +1,34 @@
+[vectordb](../README.md) / [Exports](../modules.md) / CleanupStats
+
+# Interface: CleanupStats
+
+## Table of contents
+
+### Properties
+
+- [bytesRemoved](CleanupStats.md#bytesremoved)
+- [oldVersions](CleanupStats.md#oldversions)
+
+## Properties
+
+### bytesRemoved
+
+• **bytesRemoved**: `number`
+
+The number of bytes removed from disk.
+
+#### Defined in
+
+[index.ts:637](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L637)
+
+___
+
+### oldVersions
+
+• **oldVersions**: `number`
+
+The number of old table versions removed.
+
+#### Defined in
+
+[index.ts:641](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L641)

docs/src/javascript/interfaces/CompactionMetrics.md

@@ -0,0 +1,62 @@
+[vectordb](../README.md) / [Exports](../modules.md) / CompactionMetrics
+
+# Interface: CompactionMetrics
+
+## Table of contents
+
+### Properties
+
+- [filesAdded](CompactionMetrics.md#filesadded)
+- [filesRemoved](CompactionMetrics.md#filesremoved)
+- [fragmentsAdded](CompactionMetrics.md#fragmentsadded)
+- [fragmentsRemoved](CompactionMetrics.md#fragmentsremoved)
+
+## Properties
+
+### filesAdded
+
+• **filesAdded**: `number`
+
+The number of files added. This is typically equal to the number of
+fragments added.
+
+#### Defined in
+
+[index.ts:692](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L692)
+
+___
+
+### filesRemoved
+
+• **filesRemoved**: `number`
+
+The number of files that were removed. Each fragment may have more than one
+file.
+
+#### Defined in
+
+[index.ts:687](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L687)
+
+___
+
+### fragmentsAdded
+
+• **fragmentsAdded**: `number`
+
+The number of new fragments that were created.
+
+#### Defined in
+
+[index.ts:682](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L682)
+
+___
+
+### fragmentsRemoved
+
+• **fragmentsRemoved**: `number`
+
+The number of fragments that were removed.
+
+#### Defined in
+
+[index.ts:678](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L678)

docs/src/javascript/interfaces/CompactionOptions.md

@@ -0,0 +1,80 @@
+[vectordb](../README.md) / [Exports](../modules.md) / CompactionOptions
+
+# Interface: CompactionOptions
+
+## Table of contents
+
+### Properties
+
+- [materializeDeletions](CompactionOptions.md#materializedeletions)
+- [materializeDeletionsThreshold](CompactionOptions.md#materializedeletionsthreshold)
+- [maxRowsPerGroup](CompactionOptions.md#maxrowspergroup)
+- [numThreads](CompactionOptions.md#numthreads)
+- [targetRowsPerFragment](CompactionOptions.md#targetrowsperfragment)
+
+## Properties
+
+### materializeDeletions
+
+• `Optional` **materializeDeletions**: `boolean`
+
+If true, fragments that have rows that are deleted may be compacted to
+remove the deleted rows. This can improve the performance of queries.
+Default is true.
+
+#### Defined in
+
+[index.ts:660](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L660)
+
+___
+
+### materializeDeletionsThreshold
+
+• `Optional` **materializeDeletionsThreshold**: `number`
+
+A number between 0 and 1, representing the proportion of rows that must be
+marked deleted before a fragment is a candidate for compaction to remove
+the deleted rows. Default is 10%.
+
+#### Defined in
+
+[index.ts:666](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L666)
+
+___
+
+### maxRowsPerGroup
+
+• `Optional` **maxRowsPerGroup**: `number`
+
+The maximum number of rows per group. Defaults to 1024.
+
+#### Defined in
+
+[index.ts:654](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L654)
+
+___
+
+### numThreads
+
+• `Optional` **numThreads**: `number`
+
+The number of threads to use for compaction. If not provided, defaults to
+the number of cores on the machine.
+
+#### Defined in
+
+[index.ts:671](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L671)
+
+___
+
+### targetRowsPerFragment
+
+• `Optional` **targetRowsPerFragment**: `number`
+
+The number of rows per fragment to target. Fragments that have fewer rows
+will be compacted into adjacent fragments to produce larger fragments.
+Defaults to 1024 * 1024.
+
+#### Defined in
+
+[index.ts:650](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L650)

docs/src/javascript/interfaces/Connection.md

@@ -19,7 +19,6 @@ Connection could be local against filesystem or remote against a server.
 ### Methods
 
 - [createTable](Connection.md#createtable)
-- [createTableArrow](Connection.md#createtablearrow)
 - [dropTable](Connection.md#droptable)
 - [openTable](Connection.md#opentable)
 - [tableNames](Connection.md#tablenames)
@@ -32,13 +31,76 @@ Connection could be local against filesystem or remote against a server.
 
 #### Defined in
 
-[index.ts:70](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L70)
+[index.ts:125](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L125)
 
 ## Methods
 
 ### createTable
 
-▸ **createTable**<`T`\>(`name`, `data`, `mode?`, `embeddings?`): `Promise`<[`Table`](Table.md)<`T`\>\>
+▸ **createTable**\<`T`\>(`«destructured»`): `Promise`\<[`Table`](Table.md)\<`T`\>\>
+
+Creates a new Table, optionally initializing it with new data.
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `«destructured»` | [`CreateTableOptions`](CreateTableOptions.md)\<`T`\> |
+
+#### Returns
+
+`Promise`\<[`Table`](Table.md)\<`T`\>\>
+
+#### Defined in
+
+[index.ts:146](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L146)
+
+▸ **createTable**(`name`, `data`): `Promise`\<[`Table`](Table.md)\<`number`[]\>\>
+
+Creates a new Table and initialize it with new data.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `data` | `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
+
+#### Returns
+
+`Promise`\<[`Table`](Table.md)\<`number`[]\>\>
+
+#### Defined in
+
+[index.ts:154](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L154)
+
+▸ **createTable**(`name`, `data`, `options`): `Promise`\<[`Table`](Table.md)\<`number`[]\>\>
+
+Creates a new Table and initialize it with new data.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `data` | `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
+| `options` | [`WriteOptions`](WriteOptions.md) | The write options to use when creating the table. |
+
+#### Returns
+
+`Promise`\<[`Table`](Table.md)\<`number`[]\>\>
+
+#### Defined in
+
+[index.ts:163](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L163)
+
+▸ **createTable**\<`T`\>(`name`, `data`, `embeddings`): `Promise`\<[`Table`](Table.md)\<`T`\>\>
 
 Creates a new Table and initialize it with new data.
 
@@ -53,44 +115,49 @@ Creates a new Table and initialize it with new data.
 | Name | Type | Description |
 | :------ | :------ | :------ |
 | `name` | `string` | The name of the table. |
-| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
-| `mode?` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
-| `embeddings?` | [`EmbeddingFunction`](EmbeddingFunction.md)<`T`\> | An embedding function to use on this table |
+| `data` | `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
+| `embeddings` | [`EmbeddingFunction`](EmbeddingFunction.md)\<`T`\> | An embedding function to use on this table |
 
 #### Returns
 
-`Promise`<[`Table`](Table.md)<`T`\>\>
+`Promise`\<[`Table`](Table.md)\<`T`\>\>
 
 #### Defined in
 
-[index.ts:90](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L90)
+[index.ts:172](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L172)
 
-___
+▸ **createTable**\<`T`\>(`name`, `data`, `embeddings`, `options`): `Promise`\<[`Table`](Table.md)\<`T`\>\>
 
-### createTableArrow
+Creates a new Table and initialize it with new data.
 
-▸ **createTableArrow**(`name`, `table`): `Promise`<[`Table`](Table.md)<`number`[]\>\>
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
 
 #### Parameters
 
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` |
-| `table` | `Table`<`any`\> |
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `data` | `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
+| `embeddings` | [`EmbeddingFunction`](EmbeddingFunction.md)\<`T`\> | An embedding function to use on this table |
+| `options` | [`WriteOptions`](WriteOptions.md) | The write options to use when creating the table. |
 
 #### Returns
 
-`Promise`<[`Table`](Table.md)<`number`[]\>\>
+`Promise`\<[`Table`](Table.md)\<`T`\>\>
 
 #### Defined in
 
-[index.ts:92](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L92)
+[index.ts:181](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L181)
 
 ___
 
 ### dropTable
 
-▸ **dropTable**(`name`): `Promise`<`void`\>
+▸ **dropTable**(`name`): `Promise`\<`void`\>
 
 Drop an existing table.
 
@@ -102,17 +169,17 @@ Drop an existing table.
 
 #### Returns
 
-`Promise`<`void`\>
+`Promise`\<`void`\>
 
 #### Defined in
 
-[index.ts:98](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L98)
+[index.ts:187](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L187)
 
 ___
 
 ### openTable
 
-▸ **openTable**<`T`\>(`name`, `embeddings?`): `Promise`<[`Table`](Table.md)<`T`\>\>
+▸ **openTable**\<`T`\>(`name`, `embeddings?`): `Promise`\<[`Table`](Table.md)\<`T`\>\>
 
 Open a table in the database.
 
@@ -127,26 +194,26 @@ Open a table in the database.
 | Name | Type | Description |
 | :------ | :------ | :------ |
 | `name` | `string` | The name of the table. |
-| `embeddings?` | [`EmbeddingFunction`](EmbeddingFunction.md)<`T`\> | An embedding function to use on this table |
+| `embeddings?` | [`EmbeddingFunction`](EmbeddingFunction.md)\<`T`\> | An embedding function to use on this table |
 
 #### Returns
 
-`Promise`<[`Table`](Table.md)<`T`\>\>
+`Promise`\<[`Table`](Table.md)\<`T`\>\>
 
 #### Defined in
 
-[index.ts:80](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L80)
+[index.ts:135](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L135)
 
 ___
 
 ### tableNames
 
-▸ **tableNames**(): `Promise`<`string`[]\>
+▸ **tableNames**(): `Promise`\<`string`[]\>
 
 #### Returns
 
-`Promise`<`string`[]\>
+`Promise`\<`string`[]\>
 
 #### Defined in
 
-[index.ts:72](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L72)
+[index.ts:127](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L127)

docs/src/javascript/interfaces/ConnectionOptions.md

@@ -6,18 +6,62 @@
 
 ### Properties
 
+- [apiKey](ConnectionOptions.md#apikey)
 - [awsCredentials](ConnectionOptions.md#awscredentials)
+- [awsRegion](ConnectionOptions.md#awsregion)
+- [hostOverride](ConnectionOptions.md#hostoverride)
+- [region](ConnectionOptions.md#region)
 - [uri](ConnectionOptions.md#uri)
 
 ## Properties
 
+### apiKey
+
+• `Optional` **apiKey**: `string`
+
+#### Defined in
+
+[index.ts:49](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L49)
+
+___
+
 ### awsCredentials
 
 • `Optional` **awsCredentials**: [`AwsCredentials`](AwsCredentials.md)
 
 #### Defined in
 
-[index.ts:40](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L40)
+[index.ts:44](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L44)
+
+___
+
+### awsRegion
+
+• `Optional` **awsRegion**: `string`
+
+#### Defined in
+
+[index.ts:46](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L46)
+
+___
+
+### hostOverride
+
+• `Optional` **hostOverride**: `string`
+
+#### Defined in
+
+[index.ts:54](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L54)
+
+___
+
+### region
+
+• `Optional` **region**: `string`
+
+#### Defined in
+
+[index.ts:51](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L51)
 
 ___
 
@@ -27,4 +71,4 @@ ___
 
 #### Defined in
 
-[index.ts:39](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L39)
+[index.ts:42](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L42)

docs/src/javascript/interfaces/CreateTableOptions.md

@@ -0,0 +1,69 @@
+[vectordb](../README.md) / [Exports](../modules.md) / CreateTableOptions
+
+# Interface: CreateTableOptions\<T\>
+
+## Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+## Table of contents
+
+### Properties
+
+- [data](CreateTableOptions.md#data)
+- [embeddingFunction](CreateTableOptions.md#embeddingfunction)
+- [name](CreateTableOptions.md#name)
+- [schema](CreateTableOptions.md#schema)
+- [writeOptions](CreateTableOptions.md#writeoptions)
+
+## Properties
+
+### data
+
+• `Optional` **data**: `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[]
+
+#### Defined in
+
+[index.ts:79](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L79)
+
+___
+
+### embeddingFunction
+
+• `Optional` **embeddingFunction**: [`EmbeddingFunction`](EmbeddingFunction.md)\<`T`\>
+
+#### Defined in
+
+[index.ts:85](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L85)
+
+___
+
+### name
+
+• **name**: `string`
+
+#### Defined in
+
+[index.ts:76](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L76)
+
+___
+
+### schema
+
+• `Optional` **schema**: `Schema`\<`any`\>
+
+#### Defined in
+
+[index.ts:82](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L82)
+
+___
+
+### writeOptions
+
+• `Optional` **writeOptions**: [`WriteOptions`](WriteOptions.md)
+
+#### Defined in
+
+[index.ts:88](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L88)

docs/src/javascript/interfaces/EmbeddingFunction.md

@@ -1,6 +1,6 @@
 [vectordb](../README.md) / [Exports](../modules.md) / EmbeddingFunction
 
-# Interface: EmbeddingFunction<T\>
+# Interface: EmbeddingFunction\<T\>
 
 An embedding function that automatically creates vector representation for a given column.
 
@@ -25,11 +25,11 @@ An embedding function that automatically creates vector representation for a giv
 
 ### embed
 
-• **embed**: (`data`: `T`[]) => `Promise`<`number`[][]\>
+• **embed**: (`data`: `T`[]) => `Promise`\<`number`[][]\>
 
 #### Type declaration
 
-▸ (`data`): `Promise`<`number`[][]\>
+▸ (`data`): `Promise`\<`number`[][]\>
 
 Creates a vector representation for the given values.
 
@@ -41,11 +41,11 @@ Creates a vector representation for the given values.
 
 ##### Returns
 
-`Promise`<`number`[][]\>
+`Promise`\<`number`[][]\>
 
 #### Defined in
 
-[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/embedding_function.ts#L27)
+[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/embedding_function.ts#L27)
 
 ___
 
@@ -57,4 +57,4 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/embedding_function.ts#L22)
+[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/embedding_function.ts#L22)

docs/src/javascript/interfaces/IndexStats.md

@@ -0,0 +1,30 @@
+[vectordb](../README.md) / [Exports](../modules.md) / IndexStats
+
+# Interface: IndexStats
+
+## Table of contents
+
+### Properties
+
+- [numIndexedRows](IndexStats.md#numindexedrows)
+- [numUnindexedRows](IndexStats.md#numunindexedrows)
+
+## Properties
+
+### numIndexedRows
+
+• **numIndexedRows**: ``null`` \| `number`
+
+#### Defined in
+
+[index.ts:344](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L344)
+
+___
+
+### numUnindexedRows
+
+• **numUnindexedRows**: ``null`` \| `number`
+
+#### Defined in
+
+[index.ts:345](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L345)

docs/src/javascript/interfaces/IvfPQIndexConfig.md

@@ -7,6 +7,7 @@
 ### Properties
 
 - [column](IvfPQIndexConfig.md#column)
+- [index\_cache\_size](IvfPQIndexConfig.md#index_cache_size)
 - [index\_name](IvfPQIndexConfig.md#index_name)
 - [max\_iters](IvfPQIndexConfig.md#max_iters)
 - [max\_opq\_iters](IvfPQIndexConfig.md#max_opq_iters)
@@ -28,7 +29,19 @@ The column to be indexed
 
 #### Defined in
 
-[index.ts:382](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L382)
+[index.ts:701](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L701)
+
+___
+
+### index\_cache\_size
+
+• `Optional` **index\_cache\_size**: `number`
+
+Cache size of the index
+
+#### Defined in
+
+[index.ts:750](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L750)
 
 ___
 
@@ -40,7 +53,7 @@ A unique name for the index
 
 #### Defined in
 
-[index.ts:387](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L387)
+[index.ts:706](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L706)
 
 ___
 
@@ -52,7 +65,7 @@ The max number of iterations for kmeans training.
 
 #### Defined in
 
-[index.ts:402](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L402)
+[index.ts:721](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L721)
 
 ___
 
@@ -64,7 +77,7 @@ Max number of iterations to train OPQ, if `use_opq` is true.
 
 #### Defined in
 
-[index.ts:421](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L421)
+[index.ts:740](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L740)
 
 ___
 
@@ -76,7 +89,7 @@ Metric type, L2 or Cosine
 
 #### Defined in
 
-[index.ts:392](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L392)
+[index.ts:711](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L711)
 
 ___
 
@@ -88,7 +101,7 @@ The number of bits to present one PQ centroid.
 
 #### Defined in
 
-[index.ts:416](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L416)
+[index.ts:735](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L735)
 
 ___
 
@@ -100,7 +113,7 @@ The number of partitions this index
 
 #### Defined in
 
-[index.ts:397](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L397)
+[index.ts:716](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L716)
 
 ___
 
@@ -112,7 +125,7 @@ Number of subvectors to build PQ code
 
 #### Defined in
 
-[index.ts:412](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L412)
+[index.ts:731](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L731)
 
 ___
 
@@ -124,7 +137,7 @@ Replace an existing index with the same name if it exists.
 
 #### Defined in
 
-[index.ts:426](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L426)
+[index.ts:745](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L745)
 
 ___
 
@@ -134,7 +147,7 @@ ___
 
 #### Defined in
 
-[index.ts:428](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L428)
+[index.ts:752](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L752)
 
 ___
 
@@ -146,4 +159,4 @@ Train as optimized product quantization.
 
 #### Defined in
 
-[index.ts:407](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L407)
+[index.ts:726](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L726)

docs/src/javascript/interfaces/Table.md

@@ -1,6 +1,6 @@
 [vectordb](../README.md) / [Exports](../modules.md) / Table
 
-# Interface: Table<T\>
+# Interface: Table\<T\>
 
 A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
 
@@ -22,19 +22,22 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 - [countRows](Table.md#countrows)
 - [createIndex](Table.md#createindex)
 - [delete](Table.md#delete)
+- [indexStats](Table.md#indexstats)
+- [listIndices](Table.md#listindices)
 - [name](Table.md#name)
 - [overwrite](Table.md#overwrite)
 - [search](Table.md#search)
+- [update](Table.md#update)
 
 ## Properties
 
 ### add
 
-• **add**: (`data`: `Record`<`string`, `unknown`\>[]) => `Promise`<`number`\>
+• **add**: (`data`: `Record`\<`string`, `unknown`\>[]) => `Promise`\<`number`\>
 
 #### Type declaration
 
-▸ (`data`): `Promise`<`number`\>
+▸ (`data`): `Promise`\<`number`\>
 
 Insert records into this Table.
 
@@ -42,54 +45,50 @@ Insert records into this Table.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
 
 ##### Returns
 
-`Promise`<`number`\>
+`Promise`\<`number`\>
 
 The number of rows added to the table
 
 #### Defined in
 
-[index.ts:120](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L120)
+[index.ts:209](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L209)
 
 ___
 
 ### countRows
 
-• **countRows**: () => `Promise`<`number`\>
+• **countRows**: () => `Promise`\<`number`\>
 
 #### Type declaration
 
-▸ (): `Promise`<`number`\>
+▸ (): `Promise`\<`number`\>
 
 Returns the number of rows in this table.
 
 ##### Returns
 
-`Promise`<`number`\>
+`Promise`\<`number`\>
 
 #### Defined in
 
-[index.ts:140](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L140)
+[index.ts:229](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L229)
 
 ___
 
 ### createIndex
 
-• **createIndex**: (`indexParams`: [`IvfPQIndexConfig`](IvfPQIndexConfig.md)) => `Promise`<`any`\>
+• **createIndex**: (`indexParams`: [`IvfPQIndexConfig`](IvfPQIndexConfig.md)) => `Promise`\<`any`\>
 
 #### Type declaration
 
-▸ (`indexParams`): `Promise`<`any`\>
+▸ (`indexParams`): `Promise`\<`any`\>
 
 Create an ANN index on this Table vector index.
 
-**`See`**
-
-VectorIndexParams.
-
 ##### Parameters
 
 | Name | Type | Description |
@@ -98,27 +97,41 @@ VectorIndexParams.
 
 ##### Returns
 
-`Promise`<`any`\>
+`Promise`\<`any`\>
+
+**`See`**
+
+VectorIndexParams.
 
 #### Defined in
 
-[index.ts:135](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L135)
+[index.ts:224](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L224)
 
 ___
 
 ### delete
 
-• **delete**: (`filter`: `string`) => `Promise`<`void`\>
+• **delete**: (`filter`: `string`) => `Promise`\<`void`\>
 
 #### Type declaration
 
-▸ (`filter`): `Promise`<`void`\>
+▸ (`filter`): `Promise`\<`void`\>
 
 Delete rows from this table.
 
 This can be used to delete a single row, many rows, all rows, or
 sometimes no rows (if your predicate matches nothing).
 
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `filter` | `string` | A filter in the same format used by a sql WHERE clause. The filter must not be empty. |
+
+##### Returns
+
+`Promise`\<`void`\>
+
 **`Examples`**
 
 ```ts
@@ -142,19 +155,55 @@ await tbl.delete(`id IN (${to_remove.join(",")})`)
 await tbl.countRows() // Returns 1
 ```
 
+#### Defined in
+
+[index.ts:263](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L263)
+
+___
+
+### indexStats
+
+• **indexStats**: (`indexUuid`: `string`) => `Promise`\<[`IndexStats`](IndexStats.md)\>
+
+#### Type declaration
+
+▸ (`indexUuid`): `Promise`\<[`IndexStats`](IndexStats.md)\>
+
+Get statistics about an index.
+
 ##### Parameters
 
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `filter` | `string` | A filter in the same format used by a sql WHERE clause. The filter must not be empty. |
+| Name | Type |
+| :------ | :------ |
+| `indexUuid` | `string` |
 
 ##### Returns
 
-`Promise`<`void`\>
+`Promise`\<[`IndexStats`](IndexStats.md)\>
 
 #### Defined in
 
-[index.ts:174](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L174)
+[index.ts:306](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L306)
+
+___
+
+### listIndices
+
+• **listIndices**: () => `Promise`\<[`VectorIndex`](VectorIndex.md)[]\>
+
+#### Type declaration
+
+▸ (): `Promise`\<[`VectorIndex`](VectorIndex.md)[]\>
+
+List the indicies on this table.
+
+##### Returns
+
+`Promise`\<[`VectorIndex`](VectorIndex.md)[]\>
+
+#### Defined in
+
+[index.ts:301](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L301)
 
 ___
 
@@ -164,17 +213,17 @@ ___
 
 #### Defined in
 
-[index.ts:106](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L106)
+[index.ts:195](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L195)
 
 ___
 
 ### overwrite
 
-• **overwrite**: (`data`: `Record`<`string`, `unknown`\>[]) => `Promise`<`number`\>
+• **overwrite**: (`data`: `Record`\<`string`, `unknown`\>[]) => `Promise`\<`number`\>
 
 #### Type declaration
 
-▸ (`data`): `Promise`<`number`\>
+▸ (`data`): `Promise`\<`number`\>
 
 Insert records into this Table, replacing its contents.
 
@@ -182,27 +231,27 @@ Insert records into this Table, replacing its contents.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
 
 ##### Returns
 
-`Promise`<`number`\>
+`Promise`\<`number`\>
 
 The number of rows added to the table
 
 #### Defined in
 
-[index.ts:128](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L128)
+[index.ts:217](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L217)
 
 ___
 
 ### search
 
-• **search**: (`query`: `T`) => [`Query`](../classes/Query.md)<`T`\>
+• **search**: (`query`: `T`) => [`Query`](../classes/Query.md)\<`T`\>
 
 #### Type declaration
 
-▸ (`query`): [`Query`](../classes/Query.md)<`T`\>
+▸ (`query`): [`Query`](../classes/Query.md)\<`T`\>
 
 Creates a search query to find the nearest neighbors of the given search term
 
@@ -214,8 +263,59 @@ Creates a search query to find the nearest neighbors of the given search term
 
 ##### Returns
 
-[`Query`](../classes/Query.md)<`T`\>
+[`Query`](../classes/Query.md)\<`T`\>
 
 #### Defined in
 
-[index.ts:112](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L112)
+[index.ts:201](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L201)
+
+___
+
+### update
+
+• **update**: (`args`: [`UpdateArgs`](UpdateArgs.md) \| [`UpdateSqlArgs`](UpdateSqlArgs.md)) => `Promise`\<`void`\>
+
+#### Type declaration
+
+▸ (`args`): `Promise`\<`void`\>
+
+Update rows in this table.
+
+This can be used to update a single row, many rows, all rows, or
+sometimes no rows (if your predicate matches nothing).
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `args` | [`UpdateArgs`](UpdateArgs.md) \| [`UpdateSqlArgs`](UpdateSqlArgs.md) | see [UpdateArgs](UpdateArgs.md) and [UpdateSqlArgs](UpdateSqlArgs.md) for more details |
+
+##### Returns
+
+`Promise`\<`void`\>
+
+**`Examples`**
+
+```ts
+const con = await lancedb.connect("./.lancedb")
+const data = [
+   {id: 1, vector: [3, 3], name: 'Ye'},
+   {id: 2, vector: [4, 4], name: 'Mike'},
+];
+const tbl = await con.createTable("my_table", data)
+
+await tbl.update({
+  filter: "id = 2",
+  updates: { vector: [2, 2], name: "Michael" },
+})
+
+let results = await tbl.search([1, 1]).execute();
+// Returns [
+//   {id: 2, vector: [2, 2], name: 'Michael'}
+//   {id: 1, vector: [3, 3], name: 'Ye'}
+// ]
+```
+
+#### Defined in
+
+[index.ts:296](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L296)

docs/src/javascript/interfaces/UpdateArgs.md

@@ -0,0 +1,36 @@
+[vectordb](../README.md) / [Exports](../modules.md) / UpdateArgs
+
+# Interface: UpdateArgs
+
+## Table of contents
+
+### Properties
+
+- [values](UpdateArgs.md#values)
+- [where](UpdateArgs.md#where)
+
+## Properties
+
+### values
+
+• **values**: `Record`\<`string`, `Literal`\>
+
+A key-value map of updates. The keys are the column names, and the values are the
+new values to set
+
+#### Defined in
+
+[index.ts:320](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L320)
+
+___
+
+### where
+
+• `Optional` **where**: `string`
+
+A filter in the same format used by a sql WHERE clause. The filter may be empty,
+in which case all rows will be updated.
+
+#### Defined in
+
+[index.ts:314](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L314)

docs/src/javascript/interfaces/UpdateSqlArgs.md

@@ -0,0 +1,36 @@
+[vectordb](../README.md) / [Exports](../modules.md) / UpdateSqlArgs
+
+# Interface: UpdateSqlArgs
+
+## Table of contents
+
+### Properties
+
+- [valuesSql](UpdateSqlArgs.md#valuessql)
+- [where](UpdateSqlArgs.md#where)
+
+## Properties
+
+### valuesSql
+
+• **valuesSql**: `Record`\<`string`, `string`\>
+
+A key-value map of updates. The keys are the column names, and the values are the
+new values to set as SQL expressions.
+
+#### Defined in
+
+[index.ts:334](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L334)
+
+___
+
+### where
+
+• `Optional` **where**: `string`
+
+A filter in the same format used by a sql WHERE clause. The filter may be empty,
+in which case all rows will be updated.
+
+#### Defined in
+
+[index.ts:328](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L328)

docs/src/javascript/interfaces/VectorIndex.md

@@ -0,0 +1,41 @@
+[vectordb](../README.md) / [Exports](../modules.md) / VectorIndex
+
+# Interface: VectorIndex
+
+## Table of contents
+
+### Properties
+
+- [columns](VectorIndex.md#columns)
+- [name](VectorIndex.md#name)
+- [uuid](VectorIndex.md#uuid)
+
+## Properties
+
+### columns
+
+• **columns**: `string`[]
+
+#### Defined in
+
+[index.ts:338](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L338)
+
+___
+
+### name
+
+• **name**: `string`
+
+#### Defined in
+
+[index.ts:339](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L339)
+
+___
+
+### uuid
+
+• **uuid**: `string`
+
+#### Defined in
+
+[index.ts:340](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L340)

docs/src/javascript/interfaces/WriteOptions.md

@@ -0,0 +1,27 @@
+[vectordb](../README.md) / [Exports](../modules.md) / WriteOptions
+
+# Interface: WriteOptions
+
+Write options when creating a Table.
+
+## Implemented by
+
+- [`DefaultWriteOptions`](../classes/DefaultWriteOptions.md)
+
+## Table of contents
+
+### Properties
+
+- [writeMode](WriteOptions.md#writemode)
+
+## Properties
+
+### writeMode
+
+• `Optional` **writeMode**: [`WriteMode`](../enums/WriteMode.md)
+
+A [WriteMode](../enums/WriteMode.md) to use on this operation
+
+#### Defined in
+
+[index.ts:774](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L774)

docs/src/javascript/modules.md

@@ -11,6 +11,7 @@
 
 ### Classes
 
+- [DefaultWriteOptions](classes/DefaultWriteOptions.md)
 - [LocalConnection](classes/LocalConnection.md)
 - [LocalTable](classes/LocalTable.md)
 - [OpenAIEmbeddingFunction](classes/OpenAIEmbeddingFunction.md)
@@ -19,11 +20,20 @@
 ### Interfaces
 
 - [AwsCredentials](interfaces/AwsCredentials.md)
+- [CleanupStats](interfaces/CleanupStats.md)
+- [CompactionMetrics](interfaces/CompactionMetrics.md)
+- [CompactionOptions](interfaces/CompactionOptions.md)
 - [Connection](interfaces/Connection.md)
 - [ConnectionOptions](interfaces/ConnectionOptions.md)
+- [CreateTableOptions](interfaces/CreateTableOptions.md)
 - [EmbeddingFunction](interfaces/EmbeddingFunction.md)
+- [IndexStats](interfaces/IndexStats.md)
 - [IvfPQIndexConfig](interfaces/IvfPQIndexConfig.md)
 - [Table](interfaces/Table.md)
+- [UpdateArgs](interfaces/UpdateArgs.md)
+- [UpdateSqlArgs](interfaces/UpdateSqlArgs.md)
+- [VectorIndex](interfaces/VectorIndex.md)
+- [WriteOptions](interfaces/WriteOptions.md)
 
 ### Type Aliases
 
@@ -32,6 +42,7 @@
 ### Functions
 
 - [connect](modules.md#connect)
+- [isWriteOptions](modules.md#iswriteoptions)
 
 ## Type Aliases
 
@@ -41,13 +52,13 @@
 
 #### Defined in
 
-[index.ts:431](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L431)
+[index.ts:755](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L755)
 
 ## Functions
 
 ### connect
 
-▸ **connect**(`uri`): `Promise`<[`Connection`](interfaces/Connection.md)\>
+▸ **connect**(`uri`): `Promise`\<[`Connection`](interfaces/Connection.md)\>
 
 Connect to a LanceDB instance at the given URI
 
@@ -59,24 +70,44 @@ Connect to a LanceDB instance at the given URI
 
 #### Returns
 
-`Promise`<[`Connection`](interfaces/Connection.md)\>
+`Promise`\<[`Connection`](interfaces/Connection.md)\>
 
 #### Defined in
 
-[index.ts:47](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L47)
+[index.ts:95](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L95)
 
-▸ **connect**(`opts`): `Promise`<[`Connection`](interfaces/Connection.md)\>
+▸ **connect**(`opts`): `Promise`\<[`Connection`](interfaces/Connection.md)\>
 
 #### Parameters
 
 | Name | Type |
 | :------ | :------ |
-| `opts` | `Partial`<[`ConnectionOptions`](interfaces/ConnectionOptions.md)\> |
+| `opts` | `Partial`\<[`ConnectionOptions`](interfaces/ConnectionOptions.md)\> |
 
 #### Returns
 
-`Promise`<[`Connection`](interfaces/Connection.md)\>
+`Promise`\<[`Connection`](interfaces/Connection.md)\>
 
 #### Defined in
 
-[index.ts:48](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L48)
+[index.ts:96](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L96)
+
+___
+
+### isWriteOptions
+
+▸ **isWriteOptions**(`value`): value is WriteOptions
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `value` | `any` |
+
+#### Returns
+
+value is WriteOptions
+
+#### Defined in
+
+[index.ts:781](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L781)

docs/src/javascript/saas-modules.md

@@ -0,0 +1,92 @@
+# Table of contents
+
+## Installation
+
+```bash
+npm install vectordb
+```
+
+This will download the appropriate native library for your platform. We currently
+support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
+yet support Windows or musl-based Linux (such as Alpine Linux).
+
+
+## Classes 
+- [RemoteConnection](classes/RemoteConnection.md)
+- [RemoteTable](classes/RemoteTable.md)
+- [RemoteQuery](classes/RemoteQuery.md)
+
+
+## Methods
+
+- [add](classes/RemoteTable.md#add)
+- [countRows](classes/RemoteTable.md#countrows)
+- [createIndex](classes/RemoteTable.md#createindex)
+- [createTable](classes/RemoteConnection.md#createtable)
+- [delete](classes/RemoteTable.md#delete)
+- [dropTable](classes/RemoteConnection.md#droptable)
+- [listIndices](classes/RemoteTable.md#listindices)
+- [indexStats](classes/RemoteTable.md#liststats)
+- [openTable](classes/RemoteConnection.md#opentable)
+- [overwrite](classes/RemoteTable.md#overwrite)
+- [schema](classes/RemoteTable.md#schema)
+- [search](classes/RemoteTable.md#search)
+- [tableNames](classes/RemoteConnection.md#tablenames)
+- [update](classes/RemoteTable.md#update)
+
+
+## Example code 
+```javascript
+
+const lancedb = require('vectordb');
+const { Schema, Field, Int32, Float32, Utf8, FixedSizeList } = require ("apache-arrow/Arrow.node")
+
+// connect to a remote DB
+const devApiKey = process.env.LANCEDB_DEV_API_KEY
+const dbURI = process.env.LANCEDB_URI
+const db = await lancedb.connect({
+  uri: dbURI, // replace dbURI with your project, e.g. "db://your-project-name"
+  apiKey: devApiKey,  // replace dbURI with your api key
+  region: "us-east-1-dev"
+});
+// create a new table
+const tableName = "my_table_000"
+const data = [
+    { id: 1, vector: [0.1, 1.0], item: "foo", price: 10.0 },
+    { id: 2, vector: [3.9, 0.5], item: "bar", price: 20.0 }
+]
+const schema = new Schema(
+    [
+        new Field('id', new Int32()), 
+        new Field('vector', new FixedSizeList(2, new Field('float32', new Float32()))),
+        new Field('item', new Utf8()),
+        new Field('price', new Float32())
+    ]
+)
+const table = await db.createTable({
+    name: tableName,
+    schema,
+}, data)
+
+// list the table
+const tableNames_1 = await db.tableNames('')
+// add some data and search should be okay
+const newData = [
+      { id: 3, vector: [10.3, 1.9], item: "test1", price: 30.0 }, 
+      { id: 4, vector: [6.2, 9.2], item: "test2", price: 40.0 }
+]
+await table.add(newData)
+// create the index for the table
+await table.createIndex({
+      metric_type: "L2", 
+      column: "vector"
+})
+let result = await table.search([2.8, 4.3]).select(["vector", "price"]).limit(1).execute()
+// update the data
+await table.update({ 
+      where: "id == 1", 
+      values: { item: "foo1" } 
+})
+//drop the table
+await db.dropTable(tableName)
+```

docs/src/notebooks/code_qa_bot.ipynb

@@ -44,15 +44,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import openai\n",
+    "from openai import OpenAI\n",
     "import os\n",
     "\n",
     "# Configuring the environment variable OPENAI_API_KEY\n",
     "if \"OPENAI_API_KEY\" not in os.environ:\n",
-    "    # OR set the key here as a variable\n",
-    "    openai.api_key = \"sk-...\"\n",
-    "    \n",
-    "assert len(openai.Model.list()[\"data\"]) > 0"
+    "    os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"\n",
+    "client = OpenAI()\n",
+    "assert len(client.models.list().data) > 0"
    ]
   },
   {
@@ -144,7 +143,7 @@
    "source": [
     "# Pre-processing and loading the documentation\n",
     "\n",
-    "Next, let's pre-process and load the documentation. To make sure we don't need to do this repeatedly if we were updating code, we're caching it using pickle so we can retrieve it again (this could take a few minutes to run the first time yyou do it). We'll also add some more metadata to the docs here such as the title and version of the code:"
+    "Next, let's pre-process and load the documentation. To make sure we don't need to do this repeatedly if we were updating code, we're caching it using pickle so we can retrieve it again (this could take a few minutes to run the first time you do it). We'll also add some more metadata to the docs here such as the title and version of the code:"
    ]
   },
   {
@@ -255,7 +254,7 @@
    "id": "28d93b85",
    "metadata": {},
    "source": [
-    "And thats it! We're all setup. The next step is to run some queries, let's try a few:"
+    "And that's it! We're all set up. The next step is to run some queries, let's try a few:"
    ]
   },
   {

docs/src/notebooks/multimodal_search.ipynb

@@ -19,11 +19,11 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
+      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip available: \u001B[0m\u001B[31;49m22.3.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m23.1.2\u001B[0m\n",
+      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpip install --upgrade pip\u001B[0m\n",
       "\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
+      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip available: \u001B[0m\u001B[31;49m22.3.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m23.1.2\u001B[0m\n",
+      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpip install --upgrade pip\u001B[0m\n"
      ]
     }
    ],
@@ -39,6 +39,7 @@
    "outputs": [],
    "source": [
     "import io\n",
+    "\n",
     "import PIL\n",
     "import duckdb\n",
     "import lancedb"
@@ -158,18 +159,18 @@
     "        \"db = lancedb.connect('~/datasets/demo')\\n\"\n",
     "        \"tbl = db.open_table('diffusiondb')\\n\\n\"\n",
     "        f\"embedding = embed_func('{query}')\\n\"\n",
-    "        \"tbl.search(embedding).limit(9).to_df()\"\n",
+    "        \"tbl.search(embedding).limit(9).to_pandas()\"\n",
     "    )\n",
-    "    return (_extract(tbl.search(emb).limit(9).to_df()), code)\n",
+    "    return (_extract(tbl.search(emb).limit(9).to_pandas()), code)\n",
     "\n",
     "def find_image_keywords(query):\n",
     "    code = (\n",
     "        \"import lancedb\\n\"\n",
     "        \"db = lancedb.connect('~/datasets/demo')\\n\"\n",
     "        \"tbl = db.open_table('diffusiondb')\\n\\n\"\n",
-    "        f\"tbl.search('{query}').limit(9).to_df()\"\n",
+    "        f\"tbl.search('{query}').limit(9).to_pandas()\"\n",
     "    )\n",
-    "    return (_extract(tbl.search(query).limit(9).to_df()), code)\n",
+    "    return (_extract(tbl.search(query).limit(9).to_pandas()), code)\n",
     "\n",
     "def find_image_sql(query):\n",
     "    code = (\n",

docs/src/notebooks/tables_guide.ipynb

@@ -114,13 +114,10 @@
     }
    ],
    "source": [
-    "import pandas as pd\n",
-    "\n",
-    "data = pd.DataFrame({\n",
-    "    \"vector\": [[1.1, 1.2], [0.2, 1.8]],\n",
-    "    \"lat\": [45.5, 40.1],\n",
-    "    \"long\": [-122.7, -74.1]\n",
-    "})\n",
+    "data = [\n",
+    "    {\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7},\n",
+    "    {\"vector\": [0.2, 1.8], \"lat\": 40.1, \"long\": -74.1},\n",
+    "]\n",
     "\n",
     "db.create_table(\"table2\", data)\n",
     "\n",
@@ -249,11 +246,11 @@
     }
    ],
    "source": [
-    "from lancedb.pydantic import vector, LanceModel\n",
+    "from lancedb.pydantic import Vector, LanceModel\n",
     "\n",
     "class Content(LanceModel):\n",
     "    movie_id: int\n",
-    "    vector: vector(128)\n",
+    "    vector: Vector(128)\n",
     "    genres: str\n",
     "    title: str\n",
     "    imdb_id: int\n",
@@ -359,18 +356,18 @@
     "import pandas as pd\n",
     "\n",
     "class PydanticSchema(LanceModel):\n",
-    "    vector: vector(2)\n",
+    "    vector: Vector(2)\n",
     "    item: str\n",
     "    price: float\n",
     "\n",
     "def make_batches():\n",
     "    for i in range(5):\n",
     "        yield pd.DataFrame(\n",
-    "                    {\n",
-    "                        \"vector\": [[3.1, 4.1], [1, 1]],\n",
-    "                        \"item\": [\"foo\", \"bar\"],\n",
-    "                        \"price\": [10.0, 20.0],\n",
-    "                    })\n",
+    "                {\n",
+    "                    \"vector\": [[3.1, 4.1], [1, 1]],\n",
+    "                    \"item\": [\"foo\", \"bar\"],\n",
+    "                    \"price\": [10.0, 20.0],\n",
+    "                })\n",
     "\n",
     "tbl = db.create_table(\"table5\", make_batches(), schema=PydanticSchema)\n",
     "tbl.schema"
@@ -394,10 +391,10 @@
    "outputs": [],
    "source": [
     "import lancedb\n",
-    "from lancedb.pydantic import LanceModel, vector\n",
+    "from lancedb.pydantic import LanceModel, Vector\n",
     "\n",
     "class Model(LanceModel):\n",
-    "      vector: vector(2)\n",
+    "      vector: Vector(2)\n",
     "\n",
     "tbl = db.create_table(\"table6\", schema=Model.to_arrow_schema())"
    ]
@@ -572,9 +569,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "df = pd.DataFrame([{\"vector\": [1.3, 1.4], \"item\": \"fizz\", \"price\": 100.0},\n",
-    "              {\"vector\": [9.5, 56.2], \"item\": \"buzz\", \"price\": 200.0}])\n",
-    "tbl.add(df)"
+    "data = [\n",
+    "        {\"vector\": [1.3, 1.4], \"item\": \"fizz\", \"price\": 100.0},\n",
+    "        {\"vector\": [9.5, 56.2], \"item\": \"buzz\", \"price\": 200.0}\n",
+    "]\n",
+    "tbl.add(data)"
    ]
   },
   {
@@ -596,17 +595,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "\n",
-    "import pandas as pd\n",
-    "\n",
     "def make_batches():\n",
     "    for i in range(5):\n",
-    "        yield pd.DataFrame(\n",
-    "                {\n",
-    "                    \"vector\": [[3.1, 4.1], [1, 1]],\n",
-    "                    \"item\": [\"foo\", \"bar\"],\n",
-    "                    \"price\": [10.0, 20.0],\n",
-    "                })\n",
+    "        yield [\n",
+    "                  {\"vector\": [3.1, 4.1], \"item\": \"foo\", \"price\": 10.0},\n",
+    "                  {\"vector\": [1, 1], \"item\": \"bar\", \"price\": 20.0},\n",
+    "              ]\n",
     "tbl.add(make_batches())"
    ]
   },

docs/src/notebooks/youtube_transcript_search.ipynb

@@ -184,7 +184,7 @@
     "df = (contextualize(data.to_pandas())\n",
     "      .groupby(\"title\").text_col(\"text\")\n",
     "      .window(20).stride(4)\n",
-    "      .to_df())\n",
+    "      .to_pandas())\n",
     "df.head(1)"
    ]
   },
@@ -206,15 +206,16 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import openai\n",
+    "from openai import OpenAI\n",
     "import os\n",
     "\n",
     "# Configuring the environment variable OPENAI_API_KEY\n",
     "if \"OPENAI_API_KEY\" not in os.environ:\n",
     "    # OR set the key here as a variable\n",
-    "    openai.api_key = \"sk-...\"\n",
+    "    os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"\n",
     "    \n",
-    "assert len(openai.Model.list()[\"data\"]) > 0"
+    "client = OpenAI()\n",
+    "assert len(client.models.list().data) > 0"
    ]
   },
   {
@@ -234,8 +235,8 @@
    "outputs": [],
    "source": [
     "def embed_func(c):    \n",
-    "    rs = openai.Embedding.create(input=c, engine=\"text-embedding-ada-002\")\n",
-    "    return [record[\"embedding\"] for record in rs[\"data\"]]"
+    "    rs = client.embeddings.create(input=c, model=\"text-embedding-ada-002\")\n",
+    "    return [rs.data[0].embedding]"
    ]
   },
   {
@@ -536,9 +537,8 @@
    ],
    "source": [
     "def complete(prompt):\n",
-    "    # query text-davinci-003\n",
-    "    res = openai.Completion.create(\n",
-    "        engine='text-davinci-003',\n",
+    "    res = client.completions.create(\n",
+    "        model='text-davinci-003',\n",
     "        prompt=prompt,\n",
     "        temperature=0,\n",
     "        max_tokens=400,\n",
@@ -547,7 +547,7 @@
     "        presence_penalty=0,\n",
     "        stop=None\n",
     "    )\n",
-    "    return res['choices'][0]['text'].strip()\n",
+    "    return res.choices[0].text\n",
     "\n",
     "# check that it works\n",
     "query = \"who was the 12th person on the moon and when did they land?\"\n",
@@ -603,7 +603,7 @@
    "outputs": [],
    "source": [
     "# Use LanceDB to get top 3 most relevant context\n",
-    "context = tbl.search(emb).limit(3).to_df()"
+    "context = tbl.search(emb).limit(3).to_pandas()"
    ]
   },
   {

docs/src/python/arrow.md

@@ -39,7 +39,6 @@ to lazily generate data:
 
 from typing import Iterable
 import pyarrow as pa
-import lancedb
 
 def make_batches() -> Iterable[pa.RecordBatch]:
     for i in range(5):
@@ -74,7 +73,7 @@ table = db.open_table("pd_table")
 
 query_vector = [100, 100]
 # Pandas DataFrame
-df = table.search(query_vector).limit(1).to_df()
+df = table.search(query_vector).limit(1).to_pandas()
 print(df)
 ```
 
@@ -89,12 +88,12 @@ If you have more complex criteria, you can always apply the filter to the result
 ```python
 
 # Apply the filter via LanceDB
-results = table.search([100, 100]).where("price < 15").to_df()
+results = table.search([100, 100]).where("price < 15").to_pandas()
 assert len(results) == 1
 assert results["item"].iloc[0] == "foo"
 
 # Apply the filter via Pandas
-df = results = table.search([100, 100]).to_df()
+df = results = table.search([100, 100]).to_pandas()
 results = df[df.price < 15]
 assert len(results) == 1
 assert results["item"].iloc[0] == "foo"

docs/src/python/duckdb.md

@@ -11,15 +11,13 @@ pip install duckdb lancedb
 We will re-use [the dataset created previously](./arrow.md):
 
 ```python
-import pandas as pd
 import lancedb
 
 db = lancedb.connect("data/sample-lancedb")
-data = pd.DataFrame({
-    "vector": [[3.1, 4.1], [5.9, 26.5]],
-    "item": ["foo", "bar"],
-    "price": [10.0, 20.0]
-})
+data = [
+    {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+    {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}
+]
 table = db.create_table("pd_table", data=data)
 arrow_table = table.to_arrow()
 ```

docs/src/python/pydantic.md

@@ -7,16 +7,16 @@ LanceDB integrates with Pydantic for schema inference, data ingestion, and query
 
 LanceDB supports to create Apache Arrow Schema from a
 [Pydantic BaseModel](https://docs.pydantic.dev/latest/api/main/#pydantic.main.BaseModel)
-via [pydantic_to_schema()](python.md##lancedb.pydantic.pydantic_to_schema) method.
+via [pydantic_to_schema()](python.md#lancedb.pydantic.pydantic_to_schema) method.
 
 ::: lancedb.pydantic.pydantic_to_schema
 
 ## Vector Field
 
-LanceDB provides a [`vector(dim)`](python.md#lancedb.pydantic.vector) method to define a
+LanceDB provides a [`Vector(dim)`](python.md#lancedb.pydantic.Vector) method to define a
 vector Field in a Pydantic Model.
 
-::: lancedb.pydantic.vector
+::: lancedb.pydantic.Vector
 
 ## Type Conversion
 
@@ -33,4 +33,4 @@ Current supported type conversions:
 | `str`               | `pyarrow.utf8()`    |
 | `list`              | `pyarrow.List`    |
 | `BaseModel`         | `pyarrow.Struct`    |
-| `vector(n)`         | `pyarrow.FixedSizeList(float32, n)` |
+| `Vector(n)`         | `pyarrow.FixedSizeList(float32, n)` |

docs/src/python/python.md

@@ -22,13 +22,21 @@ pip install lancedb
 
 ::: lancedb.query.LanceQueryBuilder
 
-::: lancedb.query.LanceFtsQueryBuilder
-
 ## Embeddings
 
-::: lancedb.embeddings.with_embeddings
+::: lancedb.embeddings.registry.EmbeddingFunctionRegistry
 
-::: lancedb.embeddings.EmbeddingFunction
+::: lancedb.embeddings.base.EmbeddingFunction
+
+::: lancedb.embeddings.base.TextEmbeddingFunction
+
+::: lancedb.embeddings.sentence_transformers.SentenceTransformerEmbeddings
+
+::: lancedb.embeddings.openai.OpenAIEmbeddings
+
+::: lancedb.embeddings.open_clip.OpenClipEmbeddings
+
+::: lancedb.embeddings.with_embeddings
 
 ## Context
 
@@ -46,7 +54,7 @@ pip install lancedb
 
 ## Utilities
 
-::: lancedb.vector
+::: lancedb.schema.vector
 
 ## Integrations
 

docs/src/python/saas-python.md

@@ -0,0 +1,18 @@
+# LanceDB Python API Reference
+
+## Installation
+
+```shell
+pip install lancedb
+```
+
+## Connection
+
+::: lancedb.connect
+
+::: lancedb.remote.db.RemoteDBConnection
+
+## Table
+
+::: lancedb.remote.table.RemoteTable
+

docs/src/robots.txt

@@ -0,0 +1 @@
+User-agent: *

docs/src/scripts/posthog.js

@@ -0,0 +1,4 @@
+window.addEventListener("DOMContentLoaded", (event) => {
+    !function(t,e){var o,n,p,r;e.__SV||(window.posthog=e,e._i=[],e.init=function(i,s,a){function g(t,e){var o=e.split(".");2==o.length&&(t=t[o[0]],e=o[1]),t[e]=function(){t.push([e].concat(Array.prototype.slice.call(arguments,0)))}}(p=t.createElement("script")).type="text/javascript",p.async=!0,p.src=s.api_host+"/static/array.js",(r=t.getElementsByTagName("script")[0]).parentNode.insertBefore(p,r);var u=e;for(void 0!==a?u=e[a]=[]:a="posthog",u.people=u.people||[],u.toString=function(t){var e="posthog";return"posthog"!==a&&(e+="."+a),t||(e+=" (stub)"),e},u.people.toString=function(){return u.toString(1)+".people (stub)"},o="capture identify alias people.set people.set_once set_config register register_once unregister opt_out_capturing has_opted_out_capturing opt_in_capturing reset isFeatureEnabled onFeatureFlags getFeatureFlag getFeatureFlagPayload reloadFeatureFlags group updateEarlyAccessFeatureEnrollment getEarlyAccessFeatures getActiveMatchingSurveys getSurveys".split(" "),n=0;n<o.length;n++)g(u,o[n]);e._i.push([i,s,a])},e.__SV=1)}(document,window.posthog||[]);
+    posthog.init('phc_oENDjGgHtmIDrV6puUiFem2RB4JA8gGWulfdulmMdZP',{api_host:'https://app.posthog.com'})
+});

docs/src/search.md

@@ -4,7 +4,7 @@
 In a recommendation system or search engine, you can find similar products from
 the one you searched.
 In LLM and other AI applications,
-each data point can be [presented by the embeddings generated from some models](embedding.md),
+each data point can be [presented by the embeddings generated from some models](embeddings/index.md),
 it returns the most relevant features.
 
 A search in high-dimensional vector space, is to find `K-Nearest-Neighbors (KNN)` of the query vector.
@@ -25,8 +25,8 @@ Currently, we support the following metrics:
 
 ### Flat Search
 
-If LanceDB does not create a vector index, LanceDB would need to scan (`Flat Search`) the entire vector column
-and compute the distance for each vector in order to find the closest matches.
+If you do not create a vector index, LanceDB would need to exhaustively scan the entire vector column (via `Flat Search`)
+and compute the distance for *every* vector in order to find the closest matches. This is effectively a KNN search.
 
 
 <!-- Setup Code
@@ -67,7 +67,7 @@ await db_setup.createTable('my_vectors', data)
 
     df = tbl.search(np.random.random((1536))) \
         .limit(10) \
-        .to_df()
+        .to_list()
     ```
 
 === "JavaScript"
@@ -92,7 +92,7 @@ as well.
     df = tbl.search(np.random.random((1536))) \
         .metric("cosine") \
         .limit(10) \
-        .to_df()
+        .to_list()
     ```
 
 
@@ -110,7 +110,7 @@ as well.
 
 To accelerate vector retrievals, it is common to build vector indices.
 A vector index is a data structure specifically designed to efficiently organize and
-search vector data based on their similarity or distance metrics.
+search vector data based on their similarity via the chosen distance metric.
 By constructing a vector index, you can reduce the search space and avoid the need
 for brute-force scanning of the entire vector column.
 
@@ -119,3 +119,100 @@ This is why it is often called **Approximate Nearest Neighbors (ANN)** search, w
 always returns 100% recall.
 
 See [ANN Index](ann_indexes.md) for more details.
+
+
+### Output formats
+
+LanceDB returns results in many different formats commonly used in python.
+Let's create a LanceDB table with a nested schema:
+
+```python
+from datetime import datetime
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+import numpy as np
+from pydantic import BaseModel
+uri = "data/sample-lancedb-nested"
+
+class Metadata(BaseModel):
+    source: str
+    timestamp: datetime
+
+class Document(BaseModel):
+    content: str
+    meta: Metadata
+
+class LanceSchema(LanceModel):
+    id: str
+    vector: Vector(1536)
+    payload: Document
+
+# Let's add 100 sample rows to our dataset
+data = [LanceSchema(
+    id=f"id{i}",
+    vector=np.random.randn(1536),
+    payload=Document(
+        content=f"document{i}", meta=Metadata(source=f"source{i%10}", timestamp=datetime.now())
+    ),
+) for i in range(100)]
+
+tbl = db.create_table("documents", data=data)
+```
+
+#### As a pyarrow table
+
+Using `to_arrow()` we can get the results back as a pyarrow Table.
+This result table has the same columns as the LanceDB table, with 
+the addition of an `_distance` column for vector search or a `score`
+column for full text search.
+
+```python
+tbl.search(np.random.randn(1536)).to_arrow()
+```
+
+#### As a pandas dataframe
+
+You can also get the results as a pandas dataframe.
+
+```python
+tbl.search(np.random.randn(1536)).to_pandas()
+```
+
+While other formats like Arrow/Pydantic/Python dicts have a natural 
+way to handle nested schemas, pandas can only store nested data as a 
+python dict column, which makes it difficult to support nested references.
+So for convenience, you can also tell LanceDB to flatten a nested schema 
+when creating the pandas dataframe. 
+
+```python
+tbl.search(np.random.randn(1536)).to_pandas(flatten=True)
+```
+
+If your table has a deeply nested struct, you can control how many levels
+of nesting to flatten by passing in a positive integer.
+
+```python
+tbl.search(np.random.randn(1536)).to_pandas(flatten=1)
+```
+
+
+#### As a list of python dicts
+
+You can of course return results as a list of python dicts.
+
+```python
+tbl.search(np.random.randn(1536)).to_list()
+```
+
+#### As a list of pydantic models
+
+We can add data using pydantic models, and we can certainly
+retrieve results as pydantic models
+
+```python
+tbl.search(np.random.randn(1536)).to_pydantic(LanceSchema)
+```
+
+Note that in this case the extra `_distance` field is discarded since
+it's not part of the LanceSchema.
+

docs/src/sql.md

@@ -1,7 +1,7 @@
 # SQL filters
 
 LanceDB embraces the utilization of standard SQL expressions as predicates for hybrid
-filters. It can be used during hybrid vector search and deletion operations.
+filters. It can be used during hybrid vector search, update, and deletion operations.
 
 Currently, Lance supports a growing list of expressions.
 
@@ -22,7 +22,7 @@ import numpy as np
 uri = "data/sample-lancedb"
 db = lancedb.connect(uri)
 
-data = [{"vector": row, "item": f"item {i}"}
+data = [{"vector": row, "item": f"item {i}", "id": i}
      for i, row in enumerate(np.random.random((10_000, 2)).astype('int'))]
 
 tbl = db.create_table("my_vectors", data=data)
@@ -35,33 +35,25 @@ const db = await vectordb.connect('data/sample-lancedb')
 
 let data = []
 for (let i = 0; i < 10_000; i++) {
-     data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
+     data.push({vector: Array(1536).fill(i), id: i, item: `item ${i}`, strId: `${i}`})
 }
-const tbl = await db.createTable('my_vectors', data)
+const tbl = await db.createTable('myVectors', data)
 ```
 -->
 === "Python"
 
     ```python
     tbl.search([100, 102]) \
-       .where("""(
-        (label IN [10, 20])
-        AND
-        (note.email IS NOT NULL)
-        ) OR NOT note.created
-        """)
-
+       .where("(item IN ('item 0', 'item 2')) AND (id > 10)") \
+       .to_arrow()
     ```
+
 === "Javascript"
 
     ```javascript
-    tbl.search([100, 102])
-       .where(`(
-            (label IN [10, 20])
-            AND
-            (note.email IS NOT NULL)
-        ) OR NOT note.created
-       `)
+    await tbl.search(Array(1536).fill(0))
+       .where("(item IN ('item 0', 'item 2')) AND (id > 10)")
+       .execute()
     ```
 
 
@@ -118,3 +110,22 @@ The mapping from SQL types to Arrow types is:
 
 [^1]: See precision mapping in previous table.
 
+
+## Filtering without Vector Search
+
+You can also filter your data without search.
+
+=== "Python"
+      ```python
+      tbl.search().where("id=10").limit(10).to_arrow()
+      ```
+
+=== "JavaScript"
+      ```javascript
+      await tbl.where('id=10').limit(10).execute()
+      ```
+
+!!! warning
+    If your table is large, this could potentially return a very large
+    amount of data. Please be sure to use a `limit` clause unless
+    you're sure you want to return the whole result set.

docs/test/md_testing.js

@@ -8,6 +8,7 @@ const excludedGlobs = [
   "../src/embedding.md",
   "../src/examples/*.md",
   "../src/guides/tables.md",
+  "../src/embeddings/*.md",
 ];
 
 const nodePrefix = "javascript";

docs/test/md_testing.py

@@ -8,7 +8,9 @@ excluded_globs = [
     "../src/embedding.md",
     "../src/examples/*.md",
     "../src/integrations/voxel51.md",
-    "../src/guides/tables.md"
+    "../src/guides/tables.md",
+    "../src/python/duckdb.md",
+    "../src/embeddings/*.md",
 ]
 
 python_prefix = "py"
@@ -16,28 +18,44 @@ python_file = ".py"
 python_folder = "python"
 
 files = glob.glob(glob_string, recursive=True)
-excluded_files = [f for excluded_glob in excluded_globs for f in glob.glob(excluded_glob, recursive=True)]
+excluded_files = [
+    f
+    for excluded_glob in excluded_globs
+    for f in glob.glob(excluded_glob, recursive=True)
+]
+
 
 def yield_lines(lines: Iterator[str], prefix: str, suffix: str):
     in_code_block = False
     # Python code has strict indentation
     strip_length = 0
+    skip_test = False
     for line in lines:
+        if "skip-test" in line:
+            skip_test = True
         if line.strip().startswith(prefix + python_prefix):
             in_code_block = True
             strip_length = len(line) - len(line.lstrip())
         elif in_code_block and line.strip().startswith(suffix):
             in_code_block = False
-            yield "\n"
+            if not skip_test:
+                yield "\n"
+            skip_test = False
         elif in_code_block:
-            yield line[strip_length:]
+            if not skip_test:
+                yield line[strip_length:]
 
 for file in filter(lambda file: file not in excluded_files, files):
     with open(file, "r") as f:
         lines = list(yield_lines(iter(f), "```", "```"))
 
     if len(lines) > 0:
-        out_path = Path(python_folder) / Path(file).name.strip(".md") / (Path(file).name.strip(".md") + python_file)
+        print(lines)
+        out_path = (
+            Path(python_folder)
+            / Path(file).name.strip(".md")
+            / (Path(file).name.strip(".md") + python_file)
+        )
         print(out_path)
         out_path.parent.mkdir(exist_ok=True, parents=True)
         with open(out_path, "w") as out:

docs/test/requirements.txt

@@ -1,5 +1,8 @@
-lancedb @ git+https://github.com/lancedb/lancedb.git#egg=subdir&subdirectory=python
+-e ../../python
 numpy
 pandas
 pylance
 duckdb
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch
+

node/README.md

@@ -9,8 +9,13 @@ npm install vectordb
 ```
 
 This will download the appropriate native library for your platform. We currently
-support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
-yet support Windows or musl-based Linux (such as Alpine Linux).
+support:
+
+* Linux (x86_64 and aarch64)
+* MacOS (Intel and ARM/M1/M2)
+* Windows (x86_64 only)
+
+We do not yet support musl-based Linux (such as Alpine Linux) or aarch64 Windows.
 
 ## Usage
 

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.2.4",
+  "version": "0.4.2",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -9,6 +9,7 @@
     "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json",
     "build-release": "npm run build -- --release",
     "test": "npm run tsc && mocha -recursive dist/test",
+    "integration-test": "npm run tsc && mocha -recursive dist/integration_test",
     "lint": "eslint native.js src --ext .js,.ts",
     "clean": "rm -rf node_modules *.node dist/",
     "pack-build": "neon pack-build",
@@ -34,6 +35,7 @@
     "@types/node": "^18.16.2",
     "@types/sinon": "^10.0.15",
     "@types/temp": "^0.9.1",
+    "@types/uuid": "^9.0.3",
     "@typescript-eslint/eslint-plugin": "^5.59.1",
     "cargo-cp-artifact": "^0.1",
     "chai": "^4.3.7",
@@ -51,12 +53,13 @@
     "ts-node-dev": "^2.0.0",
     "typedoc": "^0.24.7",
     "typedoc-plugin-markdown": "^3.15.3",
-    "typescript": "*"
+    "typescript": "*",
+    "uuid": "^9.0.0"
   },
   "dependencies": {
-    "@apache-arrow/ts": "^12.0.0",
+    "@apache-arrow/ts": "^14.0.2",
     "@neon-rs/load": "^0.0.74",
-    "apache-arrow": "^12.0.0",
+    "apache-arrow": "^14.0.2",
     "axios": "^1.4.0"
   },
   "os": [
@@ -78,10 +81,10 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.2.4",
-    "@lancedb/vectordb-darwin-x64": "0.2.4",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.2.4",
-    "@lancedb/vectordb-linux-x64-gnu": "0.2.4",
-    "@lancedb/vectordb-win32-x64-msvc": "0.2.4"
+    "@lancedb/vectordb-darwin-arm64": "0.4.2",
+    "@lancedb/vectordb-darwin-x64": "0.4.2",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.4.2",
+    "@lancedb/vectordb-linux-x64-gnu": "0.4.2",
+    "@lancedb/vectordb-win32-x64-msvc": "0.4.2"
   }
 }

node/src/arrow.ts

@@ -17,10 +17,9 @@ import {
   Float32,
   makeBuilder,
   RecordBatchFileWriter,
-  Utf8,
-  type Vector,
+  Utf8, type Vector,
   FixedSizeList,
-  vectorFromArray, type Schema, Table as ArrowTable
+  vectorFromArray, type Schema, Table as ArrowTable, RecordBatchStreamWriter, List, Float64, RecordBatch, makeData, Struct
 } from 'apache-arrow'
 import { type EmbeddingFunction } from './index'
 
@@ -59,7 +58,26 @@ export async function convertToTable<T> (data: Array<Record<string, unknown>>, e
       if (typeof values[0] === 'string') {
         // `vectorFromArray` converts strings into dictionary vectors, forcing it back to a string column
         records[columnsKey] = vectorFromArray(values, new Utf8())
+      } else if (Array.isArray(values[0])) {
+        const elementType = getElementType(values[0])
+        let innerType
+        if (elementType === 'string') {
+          innerType = new Utf8()
+        } else if (elementType === 'number') {
+          innerType = new Float64()
+        } else {
+          // TODO: pass in schema if it exists, else keep going to the next element
+          throw new Error(`Unsupported array element type ${elementType}`)
+        }
+        const listBuilder = makeBuilder({
+          type: new List(new Field('item', innerType, true))
+        })
+        for (const value of values) {
+          listBuilder.append(value)
+        }
+        records[columnsKey] = listBuilder.finish().toVector()
       } else {
+        // TODO if this is a struct field then recursively align the subfields
         records[columnsKey] = vectorFromArray(values)
       }
     }
@@ -68,6 +86,14 @@ export async function convertToTable<T> (data: Array<Record<string, unknown>>, e
   return new ArrowTable(records)
 }
 
+function getElementType (arr: any[]): string {
+  if (arr.length === 0) {
+    return 'undefined'
+  }
+
+  return typeof arr[0]
+}
+
 // Creates a new Arrow ListBuilder that stores a Vector column
 function newVectorBuilder (dim: number): FixedSizeListBuilder<Float32> {
   return makeBuilder({
@@ -77,19 +103,34 @@ function newVectorBuilder (dim: number): FixedSizeListBuilder<Float32> {
 
 // Creates the Arrow Type for a Vector column with dimension `dim`
 function newVectorType (dim: number): FixedSizeList<Float32> {
-  const children = new Field<Float32>('item', new Float32())
+  // Somewhere we always default to have the elements nullable, so we need to set it to true
+  // otherwise we often get schema mismatches because the stored data always has schema with nullable elements
+  const children = new Field<Float32>('item', new Float32(), true)
   return new FixedSizeList(dim, children)
 }
 
 // Converts an Array of records into Arrow IPC format
-export async function fromRecordsToBuffer<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Buffer> {
-  const table = await convertToTable(data, embeddings)
+export async function fromRecordsToBuffer<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<Buffer> {
+  let table = await convertToTable(data, embeddings)
+  if (schema !== undefined) {
+    table = alignTable(table, schema)
+  }
   const writer = RecordBatchFileWriter.writeAll(table)
   return Buffer.from(await writer.toUint8Array())
 }
 
+// Converts an Array of records into Arrow IPC stream format
+export async function fromRecordsToStreamBuffer<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<Buffer> {
+  let table = await convertToTable(data, embeddings)
+  if (schema !== undefined) {
+    table = alignTable(table, schema)
+  }
+  const writer = RecordBatchStreamWriter.writeAll(table)
+  return Buffer.from(await writer.toUint8Array())
+}
+
 // Converts an Arrow Table into Arrow IPC format
-export async function fromTableToBuffer<T> (table: ArrowTable, embeddings?: EmbeddingFunction<T>): Promise<Buffer> {
+export async function fromTableToBuffer<T> (table: ArrowTable, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<Buffer> {
   if (embeddings !== undefined) {
     const source = table.getChild(embeddings.sourceColumn)
 
@@ -101,10 +142,56 @@ export async function fromTableToBuffer<T> (table: ArrowTable, embeddings?: Embe
     const column = vectorFromArray(vectors, newVectorType(vectors[0].length))
     table = table.assign(new ArrowTable({ vector: column }))
   }
+  if (schema !== undefined) {
+    table = alignTable(table, schema)
+  }
   const writer = RecordBatchFileWriter.writeAll(table)
   return Buffer.from(await writer.toUint8Array())
 }
 
+// Converts an Arrow Table into Arrow IPC stream format
+export async function fromTableToStreamBuffer<T> (table: ArrowTable, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<Buffer> {
+  if (embeddings !== undefined) {
+    const source = table.getChild(embeddings.sourceColumn)
+
+    if (source === null) {
+      throw new Error(`The embedding source column ${embeddings.sourceColumn} was not found in the Arrow Table`)
+    }
+
+    const vectors = await embeddings.embed(source.toArray() as T[])
+    const column = vectorFromArray(vectors, newVectorType(vectors[0].length))
+    table = table.assign(new ArrowTable({ vector: column }))
+  }
+  if (schema !== undefined) {
+    table = alignTable(table, schema)
+  }
+  const writer = RecordBatchStreamWriter.writeAll(table)
+  return Buffer.from(await writer.toUint8Array())
+}
+
+function alignBatch (batch: RecordBatch, schema: Schema): RecordBatch {
+  const alignedChildren = []
+  for (const field of schema.fields) {
+    const indexInBatch = batch.schema.fields?.findIndex((f) => f.name === field.name)
+    if (indexInBatch < 0) {
+      throw new Error(`The column ${field.name} was not found in the Arrow Table`)
+    }
+    alignedChildren.push(batch.data.children[indexInBatch])
+  }
+  const newData = makeData({
+    type: new Struct(schema.fields),
+    length: batch.numRows,
+    nullCount: batch.nullCount,
+    children: alignedChildren
+  })
+  return new RecordBatch(schema, newData)
+}
+
+function alignTable (table: ArrowTable, schema: Schema): ArrowTable {
+  const alignedBatches = table.batches.map(batch => alignBatch(batch, schema))
+  return new ArrowTable(schema, alignedBatches)
+}
+
 // Creates an empty Arrow Table
 export function createEmptyTable (schema: Schema): ArrowTable {
   return new ArrowTable(schema)

node/src/index.ts

@@ -14,16 +14,18 @@
 
 import {
   type Schema,
-  Table as ArrowTable
+  Table as ArrowTable,
+  tableFromIPC
 } from 'apache-arrow'
 import { createEmptyTable, fromRecordsToBuffer, fromTableToBuffer } from './arrow'
 import type { EmbeddingFunction } from './embedding/embedding_function'
 import { RemoteConnection } from './remote'
 import { Query } from './query'
 import { isEmbeddingFunction } from './embedding/embedding_function'
+import { type Literal, toSQL } from './util'
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete } = require('../native.js')
+const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateScalarIndex, tableCreateVectorIndex, tableCountRows, tableDelete, tableUpdate, tableCleanupOldVersions, tableCompactFiles, tableListIndices, tableIndexStats, tableSchema } = require('../native.js')
 
 export { Query }
 export type { EmbeddingFunction }
@@ -222,6 +224,56 @@ export interface Table<T = number[]> {
    */
   createIndex: (indexParams: VectorIndexParams) => Promise<any>
 
+  /**
+   * Create a scalar index on this Table for the given column
+   *
+   * @param column The column to index
+   * @param replace If false, fail if an index already exists on the column
+   *
+   * Scalar indices, like vector indices, can be used to speed up scans.  A scalar
+   * index can speed up scans that contain filter expressions on the indexed column.
+   * For example, the following scan will be faster if the column `my_col` has
+   * a scalar index:
+   *
+   * ```ts
+   * const con = await lancedb.connect('./.lancedb');
+   * const table = await con.openTable('images');
+   * const results = await table.where('my_col = 7').execute();
+   * ```
+   *
+   * Scalar indices can also speed up scans containing a vector search and a
+   * prefilter:
+   *
+   * ```ts
+   * const con = await lancedb.connect('././lancedb');
+   * const table = await con.openTable('images');
+   * const results = await table.search([1.0, 2.0]).where('my_col != 7').prefilter(true);
+   * ```
+   *
+   * Scalar indices can only speed up scans for basic filters using
+   * equality, comparison, range (e.g. `my_col BETWEEN 0 AND 100`), and set
+   * membership (e.g. `my_col IN (0, 1, 2)`)
+   *
+   * Scalar indices can be used if the filter contains multiple indexed columns and
+   * the filter criteria are AND'd or OR'd together
+   * (e.g. `my_col < 0 AND other_col> 100`)
+   *
+   * Scalar indices may be used if the filter contains non-indexed columns but,
+   * depending on the structure of the filter, they may not be usable.  For example,
+   * if the column `not_indexed` does not have a scalar index then the filter
+   * `my_col = 0 OR not_indexed = 1` will not be able to use any scalar index on
+   * `my_col`.
+   *
+   * @examples
+   *
+   * ```ts
+   * const con = await lancedb.connect('././lancedb')
+   * const table = await con.openTable('images')
+   * await table.createScalarIndex('my_col')
+   * ```
+   */
+  createScalarIndex: (column: string, replace: boolean) => Promise<void>
+
   /**
    * Returns the number of rows in this table.
    */
@@ -260,6 +312,90 @@ export interface Table<T = number[]> {
    * ```
    */
   delete: (filter: string) => Promise<void>
+
+  /**
+   * Update rows in this table.
+   *
+   * This can be used to update a single row, many rows, all rows, or
+   * sometimes no rows (if your predicate matches nothing).
+   *
+   * @param args see {@link UpdateArgs} and {@link UpdateSqlArgs} for more details
+   *
+   * @examples
+   *
+   * ```ts
+   * const con = await lancedb.connect("./.lancedb")
+   * const data = [
+   *    {id: 1, vector: [3, 3], name: 'Ye'},
+   *    {id: 2, vector: [4, 4], name: 'Mike'},
+   * ];
+   * const tbl = await con.createTable("my_table", data)
+   *
+   * await tbl.update({
+   *   where: "id = 2",
+   *   values: { vector: [2, 2], name: "Michael" },
+   * })
+   *
+   * let results = await tbl.search([1, 1]).execute();
+   * // Returns [
+   * //   {id: 2, vector: [2, 2], name: 'Michael'}
+   * //   {id: 1, vector: [3, 3], name: 'Ye'}
+   * // ]
+   * ```
+   *
+   */
+  update: (args: UpdateArgs | UpdateSqlArgs) => Promise<void>
+
+  /**
+   * List the indicies on this table.
+   */
+  listIndices: () => Promise<VectorIndex[]>
+
+  /**
+   * Get statistics about an index.
+   */
+  indexStats: (indexUuid: string) => Promise<IndexStats>
+
+  schema: Promise<Schema>
+}
+
+export interface UpdateArgs {
+  /**
+   * A filter in the same format used by a sql WHERE clause. The filter may be empty,
+   * in which case all rows will be updated.
+   */
+  where?: string
+
+  /**
+   * A key-value map of updates. The keys are the column names, and the values are the
+   * new values to set
+   */
+  values: Record<string, Literal>
+}
+
+export interface UpdateSqlArgs {
+  /**
+   * A filter in the same format used by a sql WHERE clause. The filter may be empty,
+   * in which case all rows will be updated.
+   */
+  where?: string
+
+  /**
+   * A key-value map of updates. The keys are the column names, and the values are the
+   * new values to set as SQL expressions.
+   */
+  valuesSql: Record<string, string>
+}
+
+export interface VectorIndex {
+  columns: string[]
+  name: string
+  uuid: string
+}
+
+export interface IndexStats {
+  numIndexedRows: number | null
+  numUnindexedRows: number | null
 }
 
 /**
@@ -349,10 +485,10 @@ export class LocalConnection implements Connection {
       }
       buffer = await fromTableToBuffer(createEmptyTable(schema))
     } else if (data instanceof ArrowTable) {
-      buffer = await fromTableToBuffer(data, embeddingFunction)
+      buffer = await fromTableToBuffer(data, embeddingFunction, schema)
     } else {
       // data is Array<Record<...>>
-      buffer = await fromRecordsToBuffer(data, embeddingFunction)
+      buffer = await fromRecordsToBuffer(data, embeddingFunction, schema)
     }
 
     const tbl = await tableCreate.call(this._db, name, buffer, writeOptions?.writeMode?.toString(), ...getAwsArgs(this._options()))
@@ -375,6 +511,7 @@ export class LocalConnection implements Connection {
 export class LocalTable<T = number[]> implements Table<T> {
   private _tbl: any
   private readonly _name: string
+  private readonly _isElectron: boolean
   private readonly _embeddings?: EmbeddingFunction<T>
   private readonly _options: () => ConnectionOptions
 
@@ -391,6 +528,7 @@ export class LocalTable<T = number[]> implements Table<T> {
     this._name = name
     this._embeddings = embeddings
     this._options = () => options
+    this._isElectron = this.checkElectron()
   }
 
   get name (): string {
@@ -405,6 +543,16 @@ export class LocalTable<T = number[]> implements Table<T> {
     return new Query(query, this._tbl, this._embeddings)
   }
 
+  /**
+   * Creates a filter query to find all rows matching the specified criteria
+   * @param value The filter criteria (like SQL where clause syntax)
+   */
+  filter (value: string): Query<T> {
+    return new Query(undefined, this._tbl, this._embeddings).filter(value)
+  }
+
+  where = this.filter
+
   /**
    * Insert records into this Table.
    *
@@ -412,9 +560,10 @@ export class LocalTable<T = number[]> implements Table<T> {
    * @return The number of rows added to the table
    */
   async add (data: Array<Record<string, unknown>>): Promise<number> {
+    const schema = await this.schema
     return tableAdd.call(
       this._tbl,
-      await fromRecordsToBuffer(data, this._embeddings),
+      await fromRecordsToBuffer(data, this._embeddings, schema),
       WriteMode.Append.toString(),
       ...getAwsArgs(this._options())
     ).then((newTable: any) => { this._tbl = newTable })
@@ -444,6 +593,10 @@ export class LocalTable<T = number[]> implements Table<T> {
     return tableCreateVectorIndex.call(this._tbl, indexParams).then((newTable: any) => { this._tbl = newTable })
   }
 
+  async createScalarIndex (column: string, replace: boolean): Promise<void> {
+    return tableCreateScalarIndex.call(this._tbl, column, replace)
+  }
+
   /**
    * Returns the number of rows in this table.
    */
@@ -459,6 +612,165 @@ export class LocalTable<T = number[]> implements Table<T> {
   async delete (filter: string): Promise<void> {
     return tableDelete.call(this._tbl, filter).then((newTable: any) => { this._tbl = newTable })
   }
+
+  /**
+   * Update rows in this table.
+   *
+   * @param args see {@link UpdateArgs} and {@link UpdateSqlArgs} for more details
+   *
+   * @returns
+   */
+  async update (args: UpdateArgs | UpdateSqlArgs): Promise<void> {
+    let filter: string | null
+    let updates: Record<string, string>
+
+    if ('valuesSql' in args) {
+      filter = args.where ?? null
+      updates = args.valuesSql
+    } else {
+      filter = args.where ?? null
+      updates = {}
+      for (const [key, value] of Object.entries(args.values)) {
+        updates[key] = toSQL(value)
+      }
+    }
+
+    return tableUpdate.call(this._tbl, filter, updates).then((newTable: any) => { this._tbl = newTable })
+  }
+
+  /**
+   * Clean up old versions of the table, freeing disk space.
+   *
+   * @param olderThan The minimum age in minutes of the versions to delete. If not
+   *                  provided, defaults to two weeks.
+   * @param deleteUnverified Because they may be part of an in-progress
+   *                  transaction, uncommitted files newer than 7 days old are
+   *                  not deleted by default. This means that failed transactions
+   *                  can leave around data that takes up disk space for up to
+   *                  7 days. You can override this safety mechanism by setting
+   *                 this option to `true`, only if you promise there are no
+   *                 in progress writes while you run this operation. Failure to
+   *                 uphold this promise can lead to corrupted tables.
+   * @returns
+   */
+  async cleanupOldVersions (olderThan?: number, deleteUnverified?: boolean): Promise<CleanupStats> {
+    return tableCleanupOldVersions.call(this._tbl, olderThan, deleteUnverified)
+      .then((res: { newTable: any, metrics: CleanupStats }) => {
+        this._tbl = res.newTable
+        return res.metrics
+      })
+  }
+
+  /**
+   * Run the compaction process on the table.
+   *
+   * This can be run after making several small appends to optimize the table
+   * for faster reads.
+   *
+   * @param options Advanced options configuring compaction. In most cases, you
+   *               can omit this arguments, as the default options are sensible
+   *               for most tables.
+   * @returns Metrics about the compaction operation.
+   */
+  async compactFiles (options?: CompactionOptions): Promise<CompactionMetrics> {
+    const optionsArg = options ?? {}
+    return tableCompactFiles.call(this._tbl, optionsArg)
+      .then((res: { newTable: any, metrics: CompactionMetrics }) => {
+        this._tbl = res.newTable
+        return res.metrics
+      })
+  }
+
+  async listIndices (): Promise<VectorIndex[]> {
+    return tableListIndices.call(this._tbl)
+  }
+
+  async indexStats (indexUuid: string): Promise<IndexStats> {
+    return tableIndexStats.call(this._tbl, indexUuid)
+  }
+
+  get schema (): Promise<Schema> {
+    // empty table
+    return this.getSchema()
+  }
+
+  private async getSchema (): Promise<Schema> {
+    const buffer = await tableSchema.call(this._tbl, this._isElectron)
+    const table = tableFromIPC(buffer)
+    return table.schema
+  }
+
+  // See https://github.com/electron/electron/issues/2288
+  private checkElectron (): boolean {
+    try {
+      // eslint-disable-next-line no-prototype-builtins
+      return (process?.versions?.hasOwnProperty('electron') || navigator?.userAgent?.toLowerCase()?.includes(' electron'))
+    } catch (e) {
+      return false
+    }
+  }
+}
+
+export interface CleanupStats {
+  /**
+   * The number of bytes removed from disk.
+   */
+  bytesRemoved: number
+  /**
+   * The number of old table versions removed.
+   */
+  oldVersions: number
+}
+
+export interface CompactionOptions {
+  /**
+   * The number of rows per fragment to target. Fragments that have fewer rows
+   * will be compacted into adjacent fragments to produce larger fragments.
+   * Defaults to 1024 * 1024.
+   */
+  targetRowsPerFragment?: number
+  /**
+   * The maximum number of rows per group. Defaults to 1024.
+   */
+  maxRowsPerGroup?: number
+  /**
+   * If true, fragments that have rows that are deleted may be compacted to
+   * remove the deleted rows. This can improve the performance of queries.
+   * Default is true.
+   */
+  materializeDeletions?: boolean
+  /**
+   * A number between 0 and 1, representing the proportion of rows that must be
+   * marked deleted before a fragment is a candidate for compaction to remove
+   * the deleted rows. Default is 10%.
+   */
+  materializeDeletionsThreshold?: number
+  /**
+   * The number of threads to use for compaction. If not provided, defaults to
+   * the number of cores on the machine.
+   */
+  numThreads?: number
+}
+
+export interface CompactionMetrics {
+  /**
+   * The number of fragments that were removed.
+   */
+  fragmentsRemoved: number
+  /**
+   * The number of new fragments that were created.
+   */
+  fragmentsAdded: number
+  /**
+   * The number of files that were removed. Each fragment may have more than one
+   * file.
+   */
+  filesRemoved: number
+  /**
+   * The number of files added. This is typically equal to the number of
+   * fragments added.
+   */
+  filesAdded: number
 }
 
 /// Config to build IVF_PQ index.
@@ -513,6 +825,11 @@ export interface IvfPQIndexConfig {
    */
   replace?: boolean
 
+  /**
+   * Cache size of the index
+   */
+  index_cache_size?: number
+
   type: 'ivf_pq'
 }
 

node/src/integration_test/test.ts

@@ -0,0 +1,180 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { describe } from 'mocha'
+import * as chai from 'chai'
+import * as chaiAsPromised from 'chai-as-promised'
+import { v4 as uuidv4 } from 'uuid'
+
+import * as lancedb from '../index'
+import { tmpdir } from 'os'
+import * as fs from 'fs'
+import * as path from 'path'
+
+const assert = chai.assert
+chai.use(chaiAsPromised)
+
+describe('LanceDB AWS Integration test', function () {
+  it('s3+ddb schema is processed correctly', async function () {
+    this.timeout(15000)
+
+    // WARNING: specifying engine is NOT a publicly supported feature in lancedb yet
+    // THE API WILL CHANGE
+    const conn = await lancedb.connect('s3://lancedb-integtest?engine=ddb&ddbTableName=lancedb-integtest')
+    const data = [{ vector: Array(128).fill(1.0) }]
+
+    const tableName = uuidv4()
+    let table = await conn.createTable(tableName, data, { writeMode: lancedb.WriteMode.Overwrite })
+
+    const futs = [table.add(data), table.add(data), table.add(data), table.add(data), table.add(data)]
+    await Promise.allSettled(futs)
+
+    table = await conn.openTable(tableName)
+    assert.equal(await table.countRows(), 6)
+  })
+})
+
+describe('LanceDB Mirrored Store Integration test', function () {
+  it('s3://...?mirroredStore=... param is processed correctly', async function () {
+    this.timeout(600000)
+
+    const dir = tmpdir()
+    console.log(dir)
+    const conn = await lancedb.connect(`s3://lancedb-integtest?mirroredStore=${dir}`)
+    const data = Array(200).fill({ vector: Array(128).fill(1.0), id: 0 })
+    data.push(...Array(200).fill({ vector: Array(128).fill(1.0), id: 1 }))
+    data.push(...Array(200).fill({ vector: Array(128).fill(1.0), id: 2 }))
+    data.push(...Array(200).fill({ vector: Array(128).fill(1.0), id: 3 }))
+
+    const tableName = uuidv4()
+
+    // try create table and check if it's mirrored
+    const t = await conn.createTable(tableName, data, { writeMode: lancedb.WriteMode.Overwrite })
+
+    const mirroredPath = path.join(dir, `${tableName}.lance`)
+    fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
+      if (err != null) throw err
+      // there should be three dirs
+      assert.equal(files.length, 3)
+      assert.isTrue(files[0].isDirectory())
+      assert.isTrue(files[1].isDirectory())
+
+      fs.readdir(path.join(mirroredPath, '_transactions'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].name.endsWith('.txn'))
+      })
+
+      fs.readdir(path.join(mirroredPath, '_versions'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].name.endsWith('.manifest'))
+      })
+
+      fs.readdir(path.join(mirroredPath, 'data'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].name.endsWith('.lance'))
+      })
+    })
+
+    // try create index and check if it's mirrored
+    await t.createIndex({ column: 'vector', type: 'ivf_pq' })
+
+    fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
+      if (err != null) throw err
+      // there should be four dirs
+      assert.equal(files.length, 4)
+      assert.isTrue(files[0].isDirectory())
+      assert.isTrue(files[1].isDirectory())
+      assert.isTrue(files[2].isDirectory())
+
+      // Two TXs now
+      fs.readdir(path.join(mirroredPath, '_transactions'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 2)
+        assert.isTrue(files[0].name.endsWith('.txn'))
+        assert.isTrue(files[1].name.endsWith('.txn'))
+      })
+
+      fs.readdir(path.join(mirroredPath, 'data'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].name.endsWith('.lance'))
+      })
+
+      fs.readdir(path.join(mirroredPath, '_indices'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].isDirectory())
+
+        fs.readdir(path.join(mirroredPath, '_indices', files[0].name), { withFileTypes: true }, (err, files) => {
+          if (err != null) throw err
+
+          assert.equal(files.length, 1)
+          assert.isTrue(files[0].isFile())
+          assert.isTrue(files[0].name.endsWith('.idx'))
+        })
+      })
+    })
+
+    // try delete and check if it's mirrored
+    await t.delete('id = 0')
+
+    fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
+      if (err != null) throw err
+      // there should be five dirs
+      assert.equal(files.length, 5)
+      assert.isTrue(files[0].isDirectory())
+      assert.isTrue(files[1].isDirectory())
+      assert.isTrue(files[2].isDirectory())
+      assert.isTrue(files[3].isDirectory())
+      assert.isTrue(files[4].isDirectory())
+
+      // Three TXs now
+      fs.readdir(path.join(mirroredPath, '_transactions'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 3)
+        assert.isTrue(files[0].name.endsWith('.txn'))
+        assert.isTrue(files[1].name.endsWith('.txn'))
+      })
+
+      fs.readdir(path.join(mirroredPath, 'data'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].name.endsWith('.lance'))
+      })
+
+      fs.readdir(path.join(mirroredPath, '_indices'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].isDirectory())
+
+        fs.readdir(path.join(mirroredPath, '_indices', files[0].name), { withFileTypes: true }, (err, files) => {
+          if (err != null) throw err
+
+          assert.equal(files.length, 1)
+          assert.isTrue(files[0].isFile())
+          assert.isTrue(files[0].name.endsWith('.idx'))
+        })
+      })
+
+      fs.readdir(path.join(mirroredPath, '_deletions'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].name.endsWith('.arrow'))
+      })
+    })
+  })
+})

node/src/query.ts

@@ -23,27 +23,29 @@ const { tableSearch } = require('../native.js')
  * A builder for nearest neighbor queries for LanceDB.
  */
 export class Query<T = number[]> {
-  private readonly _query: T
+  private readonly _query?: T
   private readonly _tbl?: any
   private _queryVector?: number[]
-  private _limit: number
+  private _limit?: number
   private _refineFactor?: number
   private _nprobes: number
   private _select?: string[]
   private _filter?: string
   private _metricType?: MetricType
+  private _prefilter: boolean
   protected readonly _embeddings?: EmbeddingFunction<T>
 
-  constructor (query: T, tbl?: any, embeddings?: EmbeddingFunction<T>) {
+  constructor (query?: T, tbl?: any, embeddings?: EmbeddingFunction<T>) {
     this._tbl = tbl
     this._query = query
-    this._limit = 10
+    this._limit = undefined
     this._nprobes = 20
     this._refineFactor = undefined
     this._select = undefined
     this._filter = undefined
     this._metricType = undefined
     this._embeddings = embeddings
+    this._prefilter = false
   }
 
   /***
@@ -102,14 +104,21 @@ export class Query<T = number[]> {
     return this
   }
 
+  prefilter (value: boolean): Query<T> {
+    this._prefilter = value
+    return this
+  }
+
   /**
      * Execute the query and return the results as an Array of Objects
      */
   async execute<T = Record<string, unknown>> (): Promise<T[]> {
-    if (this._embeddings !== undefined) {
-      this._queryVector = (await this._embeddings.embed([this._query]))[0]
-    } else {
-      this._queryVector = this._query as number[]
+    if (this._query !== undefined) {
+      if (this._embeddings !== undefined) {
+        this._queryVector = (await this._embeddings.embed([this._query]))[0]
+      } else {
+        this._queryVector = this._query as number[]
+      }
     }
 
     const isElectron = this.isElectron()

node/src/remote/client.ts

@@ -38,6 +38,7 @@ export class HttpLancedbClient {
     vector: number[],
     k: number,
     nprobes: number,
+    prefilter: boolean,
     refineFactor?: number,
     columns?: string[],
     filter?: string
@@ -50,7 +51,8 @@ export class HttpLancedbClient {
                 nprobes,
                 refineFactor,
                 columns,
-                filter
+                filter,
+                prefilter
               },
               {
                 headers: {
@@ -63,6 +65,9 @@ export class HttpLancedbClient {
               }
     ).catch((err) => {
       console.error('error: ', err)
+      if (err.response === undefined) {
+        throw new Error(`Network Error: ${err.message as string}`)
+      }
       return err.response
     })
     if (response.status !== 200) {
@@ -86,13 +91,17 @@ export class HttpLancedbClient {
       {
         headers: {
           'Content-Type': 'application/json',
-          'x-api-key': this._apiKey()
+          'x-api-key': this._apiKey(),
+          ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
         },
         params,
         timeout: 10000
       }
     ).catch((err) => {
       console.error('error: ', err)
+      if (err.response === undefined) {
+        throw new Error(`Network Error: ${err.message as string}`)
+      }
       return err.response
     })
     if (response.status !== 200) {
@@ -108,13 +117,18 @@ export class HttpLancedbClient {
   /**
    * Sent POST request.
    */
-  public async post (path: string, data?: any, params?: Record<string, string | number>): Promise<AxiosResponse> {
+  public async post (
+    path: string,
+    data?: any,
+    params?: Record<string, string | number>,
+    content?: string | undefined
+  ): Promise<AxiosResponse> {
     const response = await axios.post(
         `${this._url}${path}`,
         data,
         {
           headers: {
-            'Content-Type': 'application/json',
+            'Content-Type': content ?? 'application/json',
             'x-api-key': this._apiKey(),
             ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
           },
@@ -123,6 +137,9 @@ export class HttpLancedbClient {
         }
     ).catch((err) => {
       console.error('error: ', err)
+      if (err.response === undefined) {
+        throw new Error(`Network Error: ${err.message as string}`)
+      }
       return err.response
     })
     if (response.status !== 200) {

node/src/remote/index.ts

@@ -14,12 +14,18 @@
 
 import {
   type EmbeddingFunction, type Table, type VectorIndexParams, type Connection,
-  type ConnectionOptions, type CreateTableOptions, type WriteOptions
+  type ConnectionOptions, type CreateTableOptions, type VectorIndex,
+  type WriteOptions,
+  type IndexStats,
+  type UpdateArgs, type UpdateSqlArgs
 } from '../index'
 import { Query } from '../query'
 
-import { Vector } from 'apache-arrow'
+import { Vector, Table as ArrowTable } from 'apache-arrow'
 import { HttpLancedbClient } from './client'
+import { isEmbeddingFunction } from '../embedding/embedding_function'
+import { createEmptyTable, fromRecordsToStreamBuffer, fromTableToStreamBuffer } from '../arrow'
+import { toSQL } from '../util'
 
 /**
  * Remote connection.
@@ -51,8 +57,8 @@ export class RemoteConnection implements Connection {
     return 'db://' + this._client.uri
   }
 
-  async tableNames (): Promise<string[]> {
-    const response = await this._client.get('/v1/table/')
+  async tableNames (pageToken: string = '', limit: number = 10): Promise<string[]> {
+    const response = await this._client.get('/v1/table/', { limit, page_token: pageToken })
     return response.data.tables
   }
 
@@ -66,8 +72,60 @@ export class RemoteConnection implements Connection {
     }
   }
 
-  async createTable<T> (name: string | CreateTableOptions<T>, data?: Array<Record<string, unknown>>, optsOrEmbedding?: WriteOptions | EmbeddingFunction<T>, opt?: WriteOptions): Promise<Table<T>> {
-    throw new Error('Not implemented')
+  async createTable<T> (nameOrOpts: string | CreateTableOptions<T>, data?: Array<Record<string, unknown>>, optsOrEmbedding?: WriteOptions | EmbeddingFunction<T>, opt?: WriteOptions): Promise<Table<T>> {
+    // Logic copied from LocatlConnection, refactor these to a base class + connectionImpl pattern
+    let schema
+    let embeddings: undefined | EmbeddingFunction<T>
+    let tableName: string
+    if (typeof nameOrOpts === 'string') {
+      if (optsOrEmbedding !== undefined && isEmbeddingFunction(optsOrEmbedding)) {
+        embeddings = optsOrEmbedding
+      }
+      tableName = nameOrOpts
+    } else {
+      schema = nameOrOpts.schema
+      embeddings = nameOrOpts.embeddingFunction
+      tableName = nameOrOpts.name
+    }
+
+    let buffer: Buffer
+
+    function isEmpty (data: Array<Record<string, unknown>> | ArrowTable<any>): boolean {
+      if (data instanceof ArrowTable) {
+        return data.data.length === 0
+      }
+      return data.length === 0
+    }
+
+    if ((data === undefined) || isEmpty(data)) {
+      if (schema === undefined) {
+        throw new Error('Either data or schema needs to defined')
+      }
+      buffer = await fromTableToStreamBuffer(createEmptyTable(schema))
+    } else if (data instanceof ArrowTable) {
+      buffer = await fromTableToStreamBuffer(data, embeddings)
+    } else {
+      // data is Array<Record<...>>
+      buffer = await fromRecordsToStreamBuffer(data, embeddings)
+    }
+
+    const res = await this._client.post(
+      `/v1/table/${tableName}/create/`,
+      buffer,
+      undefined,
+      'application/vnd.apache.arrow.stream'
+    )
+    if (res.status !== 200) {
+      throw new Error(`Server Error, status: ${res.status}, ` +
+      // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+        `message: ${res.statusText}: ${res.data}`)
+    }
+
+    if (embeddings === undefined) {
+      return new RemoteTable(this._client, tableName)
+    } else {
+      return new RemoteTable(this._client, tableName, embeddings)
+    }
   }
 
   async dropTable (name: string): Promise<void> {
@@ -98,6 +156,7 @@ export class RemoteQuery<T = number[]> extends Query<T> {
       queryVector,
       (this as any)._limit,
       (this as any)._nprobes,
+      (this as any)._prefilter,
       (this as any)._refineFactor,
       (this as any)._select,
       (this as any)._filter
@@ -136,27 +195,141 @@ export class RemoteTable<T = number[]> implements Table<T> {
     return this._name
   }
 
+  get schema (): Promise<any> {
+    return this._client.post(`/v1/table/${this._name}/describe/`).then(res => {
+      if (res.status !== 200) {
+        throw new Error(`Server Error, status: ${res.status}, ` +
+        // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+          `message: ${res.statusText}: ${res.data}`)
+      }
+      return res.data?.schema
+    })
+  }
+
   search (query: T): Query<T> {
     return new RemoteQuery(query, this._client, this._name)//, this._embeddings_new)
   }
 
   async add (data: Array<Record<string, unknown>>): Promise<number> {
-    throw new Error('Not implemented')
+    const buffer = await fromRecordsToStreamBuffer(data, this._embeddings)
+    const res = await this._client.post(
+      `/v1/table/${this._name}/insert/`,
+      buffer,
+      {
+        mode: 'append'
+      },
+      'application/vnd.apache.arrow.stream'
+    )
+    if (res.status !== 200) {
+      throw new Error(`Server Error, status: ${res.status}, ` +
+      // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+        `message: ${res.statusText}: ${res.data}`)
+    }
+    return data.length
   }
 
   async overwrite (data: Array<Record<string, unknown>>): Promise<number> {
-    throw new Error('Not implemented')
+    const buffer = await fromRecordsToStreamBuffer(data, this._embeddings)
+    const res = await this._client.post(
+      `/v1/table/${this._name}/insert/`,
+      buffer,
+      {
+        mode: 'overwrite'
+      },
+      'application/vnd.apache.arrow.stream'
+    )
+    if (res.status !== 200) {
+      throw new Error(`Server Error, status: ${res.status}, ` +
+      // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+        `message: ${res.statusText}: ${res.data}`)
+    }
+    return data.length
   }
 
-  async createIndex (indexParams: VectorIndexParams): Promise<any> {
+  async createIndex (indexParams: VectorIndexParams): Promise<void> {
+    const unsupportedParams = [
+      'index_name',
+      'num_partitions',
+      'max_iters',
+      'use_opq',
+      'num_sub_vectors',
+      'num_bits',
+      'max_opq_iters',
+      'replace'
+    ]
+    for (const param of unsupportedParams) {
+      // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
+      if (indexParams[param as keyof VectorIndexParams]) {
+        throw new Error(`${param} is not supported for remote connections`)
+      }
+    }
+
+    const column = indexParams.column ?? 'vector'
+    const indexType = 'vector' // only vector index is supported for remote connections
+    const metricType = indexParams.metric_type ?? 'L2'
+    const indexCacheSize = indexParams.index_cache_size ?? null
+
+    const data = {
+      column,
+      index_type: indexType,
+      metric_type: metricType,
+      index_cache_size: indexCacheSize
+    }
+    const res = await this._client.post(`/v1/table/${this._name}/create_index/`, data)
+    if (res.status !== 200) {
+      throw new Error(`Server Error, status: ${res.status}, ` +
+      // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+        `message: ${res.statusText}: ${res.data}`)
+    }
+  }
+
+  async createScalarIndex (column: string, replace: boolean): Promise<void> {
     throw new Error('Not implemented')
   }
 
   async countRows (): Promise<number> {
-    throw new Error('Not implemented')
+    const result = await this._client.post(`/v1/table/${this._name}/describe/`)
+    return result.data?.stats?.num_rows
   }
 
   async delete (filter: string): Promise<void> {
-    throw new Error('Not implemented')
+    await this._client.post(`/v1/table/${this._name}/delete/`, { predicate: filter })
+  }
+
+  async update (args: UpdateArgs | UpdateSqlArgs): Promise<void> {
+    let filter: string | null
+    let updates: Record<string, string>
+
+    if ('valuesSql' in args) {
+      filter = args.where ?? null
+      updates = args.valuesSql
+    } else {
+      filter = args.where ?? null
+      updates = {}
+      for (const [key, value] of Object.entries(args.values)) {
+        updates[key] = toSQL(value)
+      }
+    }
+    await this._client.post(`/v1/table/${this._name}/update/`, {
+      predicate: filter,
+      updates: Object.entries(updates).map(([key, value]) => [key, value])
+    })
+  }
+
+  async listIndices (): Promise<VectorIndex[]> {
+    const results = await this._client.post(`/v1/table/${this._name}/index/list/`)
+    return results.data.indexes?.map((index: any) => ({
+      columns: index.columns,
+      name: index.index_name,
+      uuid: index.index_uuid
+    }))
+  }
+
+  async indexStats (indexUuid: string): Promise<IndexStats> {
+    const results = await this._client.post(`/v1/table/${this._name}/index/${indexUuid}/stats/`)
+    return {
+      numIndexedRows: results.data.num_indexed_rows,
+      numUnindexedRows: results.data.num_unindexed_rows
+    }
   }
 }

node/src/test/test.ts

@@ -18,8 +18,8 @@ import * as chai from 'chai'
 import * as chaiAsPromised from 'chai-as-promised'
 
 import * as lancedb from '../index'
-import { type AwsCredentials, type EmbeddingFunction, MetricType, Query, WriteMode, DefaultWriteOptions, isWriteOptions } from '../index'
-import { Field, Int32, makeVector, Schema, Utf8, Table as ArrowTable, vectorFromArray } from 'apache-arrow'
+import { type AwsCredentials, type EmbeddingFunction, MetricType, Query, WriteMode, DefaultWriteOptions, isWriteOptions, type LocalTable } from '../index'
+import { FixedSizeList, Field, Int32, makeVector, Schema, Utf8, Table as ArrowTable, vectorFromArray, Float32 } from 'apache-arrow'
 
 const expect = chai.expect
 const assert = chai.assert
@@ -78,12 +78,31 @@ describe('LanceDB client', function () {
     })
 
     it('limits # of results', async function () {
-      const uri = await createTestDB()
+      const uri = await createTestDB(2, 100)
       const con = await lancedb.connect(uri)
       const table = await con.openTable('vectors')
-      const results = await table.search([0.1, 0.3]).limit(1).execute()
+      let results = await table.search([0.1, 0.3]).limit(1).execute()
       assert.equal(results.length, 1)
       assert.equal(results[0].id, 1)
+
+      // there is a default limit if unspecified
+      results = await table.search([0.1, 0.3]).execute()
+      assert.equal(results.length, 10)
+    })
+
+    it('uses a filter / where clause without vector search', async function () {
+      // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
+      const assertResults = (results: Array<Record<string, unknown>>) => {
+        assert.equal(results.length, 50)
+      }
+
+      const uri = await createTestDB(2, 100)
+      const con = await lancedb.connect(uri)
+      const table = (await con.openTable('vectors')) as LocalTable
+      let results = await table.filter('id % 2 = 0').execute()
+      assertResults(results)
+      results = await table.where('id % 2 = 0').execute()
+      assertResults(results)
     })
 
     it('uses a filter / where clause', async function () {
@@ -102,6 +121,31 @@ describe('LanceDB client', function () {
       assertResults(results)
     })
 
+    it('should correctly process prefilter/postfilter', async function () {
+      const uri = await createTestDB(16, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+      // post filter should return less than the limit
+      let results = await table.search(new Array(16).fill(0.1)).limit(10).filter('id >= 10').prefilter(false).execute()
+      assert.isTrue(results.length < 10)
+
+      // pre filter should return exactly the limit
+      results = await table.search(new Array(16).fill(0.1)).limit(10).filter('id >= 10').prefilter(true).execute()
+      assert.isTrue(results.length === 10)
+    })
+
+    it('should allow creation and use of scalar indices', async function () {
+      const uri = await createTestDB(16, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      await table.createScalarIndex('id', true)
+
+      // Prefiltering should still work the same
+      const results = await table.search(new Array(16).fill(0.1)).limit(10).filter('id >= 10').prefilter(true).execute()
+      assert.isTrue(results.length === 10)
+    })
+
     it('select only a subset of columns', async function () {
       const uri = await createTestDB()
       const con = await lancedb.connect(uri)
@@ -132,6 +176,26 @@ describe('LanceDB client', function () {
       assert.deepEqual(await con.tableNames(), ['vectors'])
     })
 
+    it('create a table with a schema and records', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const schema = new Schema(
+        [new Field('id', new Int32()),
+          new Field('name', new Utf8()),
+          new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true)), false)
+        ]
+      )
+      const data = [
+        { vector: [0.5, 0.2], name: 'foo', id: 0 },
+        { vector: [0.3, 0.1], name: 'bar', id: 1 }
+      ]
+      // even thought the keys in data is out of order it should still work
+      const table = await con.createTable({ name: 'vectors', data, schema })
+      assert.equal(table.name, 'vectors')
+      assert.deepEqual(await con.tableNames(), ['vectors'])
+    })
+
     it('create a table with a empty data array', async function () {
       const dir = await track().mkdir('lancejs')
       const con = await lancedb.connect(dir)
@@ -174,6 +238,25 @@ describe('LanceDB client', function () {
       assert.equal(await table.countRows(), 2)
     })
 
+    it('creates a new table from javascript objects with variable sized list', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const data = [
+        { id: 1, vector: [0.1, 0.2], list_of_str: ['a', 'b', 'c'], list_of_num: [1, 2, 3] },
+        { id: 2, vector: [1.1, 1.2], list_of_str: ['x', 'y'], list_of_num: [4, 5, 6] }
+      ]
+
+      const tableName = 'with_variable_sized_list'
+      const table = await con.createTable(tableName, data) as LocalTable
+      assert.equal(table.name, tableName)
+      assert.equal(await table.countRows(), 2)
+      const rs = await table.filter('id>1').execute()
+      assert.equal(rs.length, 1)
+      assert.deepEqual(rs[0].list_of_str, ['x', 'y'])
+      assert.isTrue(rs[0].list_of_num instanceof Float64Array)
+    })
+
     it('fails to create a new table when the vector column is missing', async function () {
       const dir = await track().mkdir('lancejs')
       const con = await lancedb.connect(dir)
@@ -231,6 +314,25 @@ describe('LanceDB client', function () {
       assert.equal(await table.countRows(), 4)
     })
 
+    it('appends records with fields in a different order', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const data = [
+        { id: 1, vector: [0.1, 0.2], price: 10, name: 'a' },
+        { id: 2, vector: [1.1, 1.2], price: 50, name: 'b' }
+      ]
+
+      const table = await con.createTable('vectors', data)
+
+      const dataAdd = [
+        { id: 3, vector: [2.1, 2.2], name: 'c', price: 10 },
+        { id: 4, vector: [3.1, 3.2], name: 'd', price: 50 }
+      ]
+      await table.add(dataAdd)
+      assert.equal(await table.countRows(), 4)
+    })
+
     it('overwrite all records in a table', async function () {
       const uri = await createTestDB()
       const con = await lancedb.connect(uri)
@@ -246,6 +348,46 @@ describe('LanceDB client', function () {
       assert.equal(await table.countRows(), 2)
     })
 
+    it('can update records in the table', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+
+      const table = await con.openTable('vectors')
+      assert.equal(await table.countRows(), 2)
+
+      await table.update({ where: 'price = 10', valuesSql: { price: '100' } })
+      const results = await table.search([0.1, 0.2]).execute()
+      assert.equal(results[0].price, 100)
+      assert.equal(results[1].price, 11)
+    })
+
+    it('can update the records using a literal value', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+
+      const table = await con.openTable('vectors')
+      assert.equal(await table.countRows(), 2)
+
+      await table.update({ where: 'price = 10', values: { price: 100 } })
+      const results = await table.search([0.1, 0.2]).execute()
+      assert.equal(results[0].price, 100)
+      assert.equal(results[1].price, 11)
+    })
+
+    it('can update every record in the table', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+
+      const table = await con.openTable('vectors')
+      assert.equal(await table.countRows(), 2)
+
+      await table.update({ valuesSql: { price: '100' } })
+      const results = await table.search([0.1, 0.2]).execute()
+
+      assert.equal(results[0].price, 100)
+      assert.equal(results[1].price, 100)
+    })
+
     it('can delete records from a table', async function () {
       const uri = await createTestDB()
       const con = await lancedb.connect(uri)
@@ -258,6 +400,37 @@ describe('LanceDB client', function () {
     })
   })
 
+  describe('when searching an empty dataset', function () {
+    it('should not fail', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const schema = new Schema(
+        [new Field('vector', new FixedSizeList(128, new Field('float32', new Float32())))]
+      )
+      const table = await con.createTable({ name: 'vectors', schema })
+      const result = await table.search(Array(128).fill(0.1)).execute()
+      assert.isEmpty(result)
+    })
+  })
+
+  describe('when searching an empty-after-delete dataset', function () {
+    it('should not fail', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const schema = new Schema(
+        [new Field('vector', new FixedSizeList(128, new Field('float32', new Float32())))]
+      )
+      const table = await con.createTable({ name: 'vectors', schema })
+      await table.add([{ vector: Array(128).fill(0.1) }])
+      // https://github.com/lancedb/lance/issues/1635
+      await table.delete('true')
+      const result = await table.search(Array(128).fill(0.1)).execute()
+      assert.isEmpty(result)
+    })
+  })
+
   describe('when creating a vector index', function () {
     it('overwrite all records in a table', async function () {
       const uri = await createTestDB(32, 300)
@@ -298,6 +471,24 @@ describe('LanceDB client', function () {
       const createIndex = table.createIndex({ type: 'ivf_pq', column: 'name', num_partitions: -1, max_iters: 2, num_sub_vectors: 2 })
       await expect(createIndex).to.be.rejectedWith('num_partitions: must be > 0')
     })
+
+    it('should be able to list index and stats', async function () {
+      const uri = await createTestDB(32, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+
+      const indices = await table.listIndices()
+      expect(indices).to.have.lengthOf(1)
+      expect(indices[0].name).to.equal('vector_idx')
+      expect(indices[0].uuid).to.not.be.equal(undefined)
+      expect(indices[0].columns).to.have.lengthOf(1)
+      expect(indices[0].columns[0]).to.equal('vector')
+
+      const stats = await table.indexStats(indices[0].uuid)
+      expect(stats.numIndexedRows).to.equal(300)
+      expect(stats.numUnindexedRows).to.equal(0)
+    }).timeout(50_000)
   })
 
   describe('when using a custom embedding function', function () {
@@ -346,6 +537,61 @@ describe('LanceDB client', function () {
       assert.equal(results.length, 2)
     })
   })
+
+  describe('when inspecting the schema', function () {
+    it('should return the schema', async function () {
+      const uri = await createTestDB()
+      const db = await lancedb.connect(uri)
+      // the fsl inner field must be named 'item' and be nullable
+      const expectedSchema = new Schema(
+        [
+          new Field('id', new Int32()),
+          new Field('vector', new FixedSizeList(128, new Field('item', new Float32(), true))),
+          new Field('s', new Utf8())
+        ]
+      )
+      const table = await db.createTable({
+        name: 'some_table',
+        schema: expectedSchema
+      })
+      const schema = await table.schema
+      assert.deepEqual(expectedSchema, schema)
+    })
+  })
+})
+
+describe('Remote LanceDB client', function () {
+  describe('when the server is not reachable', function () {
+    it('produces a network error', async function () {
+      const con = await lancedb.connect({
+        uri: 'db://test-1234',
+        region: 'asdfasfasfdf',
+        apiKey: 'some-api-key'
+      })
+
+      // GET
+      try {
+        await con.tableNames()
+      } catch (err) {
+        expect(err).to.have.property('message', 'Network Error: getaddrinfo ENOTFOUND test-1234.asdfasfasfdf.api.lancedb.com')
+      }
+
+      // POST
+      try {
+        await con.createTable({ name: 'vectors', schema: new Schema([]) })
+      } catch (err) {
+        expect(err).to.have.property('message', 'Network Error: getaddrinfo ENOTFOUND test-1234.asdfasfasfdf.api.lancedb.com')
+      }
+
+      // Search
+      const table = await con.openTable('vectors')
+      try {
+        await table.search([0.1, 0.3]).execute()
+      } catch (err) {
+        expect(err).to.have.property('message', 'Network Error: getaddrinfo ENOTFOUND test-1234.asdfasfasfdf.api.lancedb.com')
+      }
+    })
+  })
 })
 
 describe('Query object', function () {
@@ -416,3 +662,45 @@ describe('WriteOptions', function () {
     })
   })
 })
+
+describe('Compact and cleanup', function () {
+  it('can cleanup after compaction', async function () {
+    const dir = await track().mkdir('lancejs')
+    const con = await lancedb.connect(dir)
+
+    const data = [
+      { price: 10, name: 'foo', vector: [1, 2, 3] },
+      { price: 50, name: 'bar', vector: [4, 5, 6] }
+    ]
+    const table = await con.createTable('t1', data) as LocalTable
+
+    const newData = [
+      { price: 30, name: 'baz', vector: [7, 8, 9] }
+    ]
+    await table.add(newData)
+
+    const compactionMetrics = await table.compactFiles({
+      numThreads: 2
+    })
+    assert.equal(compactionMetrics.fragmentsRemoved, 2)
+    assert.equal(compactionMetrics.fragmentsAdded, 1)
+    assert.equal(await table.countRows(), 3)
+
+    await table.cleanupOldVersions()
+    assert.equal(await table.countRows(), 3)
+
+    // should have no effect, but this validates the arguments are parsed.
+    await table.compactFiles({
+      targetRowsPerFragment: 102410,
+      maxRowsPerGroup: 1024,
+      materializeDeletions: true,
+      materializeDeletionsThreshold: 0.5,
+      numThreads: 2
+    })
+
+    const cleanupMetrics = await table.cleanupOldVersions(0, true)
+    assert.isAtLeast(cleanupMetrics.bytesRemoved, 1)
+    assert.isAtLeast(cleanupMetrics.oldVersions, 1)
+    assert.equal(await table.countRows(), 3)
+  })
+})

node/src/test/util.ts

@@ -0,0 +1,45 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { toSQL } from '../util'
+import * as chai from 'chai'
+
+const expect = chai.expect
+
+describe('toSQL', function () {
+  it('should turn string to SQL expression', function () {
+    expect(toSQL('foo')).to.equal("'foo'")
+  })
+
+  it('should turn number to SQL expression', function () {
+    expect(toSQL(123)).to.equal('123')
+  })
+
+  it('should turn boolean to SQL expression', function () {
+    expect(toSQL(true)).to.equal('TRUE')
+  })
+
+  it('should turn null to SQL expression', function () {
+    expect(toSQL(null)).to.equal('NULL')
+  })
+
+  it('should turn Date to SQL expression', function () {
+    const date = new Date('05 October 2011 14:48 UTC')
+    expect(toSQL(date)).to.equal("'2011-10-05T14:48:00.000Z'")
+  })
+
+  it('should turn array to SQL expression', function () {
+    expect(toSQL(['foo', 'bar', true, 1])).to.equal("['foo', 'bar', TRUE, 1]")
+  })
+})

node/src/util.ts

@@ -0,0 +1,44 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+export type Literal = string | number | boolean | null | Date | Literal[]
+
+export function toSQL (value: Literal): string {
+  if (typeof value === 'string') {
+    return `'${value}'`
+  }
+
+  if (typeof value === 'number') {
+    return value.toString()
+  }
+
+  if (typeof value === 'boolean') {
+    return value ? 'TRUE' : 'FALSE'
+  }
+
+  if (value === null) {
+    return 'NULL'
+  }
+
+  if (value instanceof Date) {
+    return `'${value.toISOString()}'`
+  }
+
+  if (Array.isArray(value)) {
+    return `[${value.map(toSQL).join(', ')}]`
+  }
+
+  // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+  throw new Error(`Unsupported value type: ${typeof value} value: (${value})`)
+}

python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.2.2
+current_version = 0.4.4
 commit = True
 message = [python] Bump version: {current_version} → {new_version}
 tag = True

python/LICENSE

@@ -0,0 +1 @@
+../LICENSE

python/README.md

@@ -16,7 +16,7 @@ pip install lancedb
 import lancedb
 db = lancedb.connect('<PATH_TO_LANCEDB_DATASET>')
 table = db.open_table('my_table')
-results = table.search([0.1, 0.3]).limit(20).to_df()
+results = table.search([0.1, 0.3]).limit(20).to_list()
 print(results)
 ```
 
@@ -45,8 +45,8 @@ pytest
 To run linter and automatically fix all errors:
 
 ```bash
-black .
-isort .
+ruff format python
+ruff --fix python
 ```
 
 If any packages are missing, install them with:

python/lancedb/__init__.py

@@ -11,18 +11,23 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import importlib.metadata
 from typing import Optional
 
-from .db import URI, DBConnection, LanceDBConnection
+__version__ = importlib.metadata.version("lancedb")
+
+from .common import URI
+from .db import DBConnection, LanceDBConnection
 from .remote.db import RemoteDBConnection
-from .schema import vector
+from .schema import vector  # noqa: F401
+from .utils import sentry_log  # noqa: F401
 
 
 def connect(
     uri: URI,
     *,
     api_key: Optional[str] = None,
-    region: str = "us-west-2",
+    region: str = "us-east-1",
     host_override: Optional[str] = None,
 ) -> DBConnection:
     """Connect to a LanceDB database.
@@ -31,9 +36,13 @@ def connect(
     ----------
     uri: str or Path
         The uri of the database.
-    api_token: str, optional
+    api_key: str, optional
         If presented, connect to LanceDB cloud.
         Otherwise, connect to a database on file system or cloud storage.
+    region: str, default "us-east-1"
+        The region to use for LanceDB Cloud.
+    host_override: str, optional
+        The override url for LanceDB Cloud.
 
     Examples
     --------

python/lancedb/cli/__init__.py

@@ -0,0 +1,12 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.

python/lancedb/cli/cli.py

@@ -0,0 +1,46 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import click
+
+from lancedb.utils import CONFIG
+
+
+@click.group()
+@click.version_option(help="LanceDB command line interface entry point")
+def cli():
+    "LanceDB command line interface"
+
+
+diagnostics_help = """
+Enable or disable LanceDB diagnostics. When enabled, LanceDB will send anonymous events to help us improve LanceDB.
+These diagnostics are used only for error reporting and no data is collected. You can find more about diagnosis on
+our docs: https://lancedb.github.io/lancedb/cli_config/
+"""
+
+
+@cli.command(help=diagnostics_help)
+@click.option("--enabled/--disabled", default=True)
+def diagnostics(enabled):
+    CONFIG.update({"diagnostics": True if enabled else False})
+    click.echo("LanceDB diagnostics is %s" % ("enabled" if enabled else "disabled"))
+
+
+@cli.command(help="Show current LanceDB configuration")
+def config():
+    # TODO: pretty print as table with colors and formatting
+    click.echo("Current LanceDB configuration:")
+    cfg = CONFIG.copy()
+    cfg.pop("uuid")  # Don't show uuid as it is not configurable
+    for item, amount in cfg.items():
+        click.echo("{} ({})".format(item, amount))

python/lancedb/conftest.py

@@ -1,7 +1,12 @@
 import os
+import time
+from typing import Any
 
+import numpy as np
 import pytest
 
+from .embeddings import EmbeddingFunctionRegistry, TextEmbeddingFunction
+
 # import lancedb so we don't have to in every example
 
 
@@ -14,3 +19,47 @@ def doctest_setup(monkeypatch, tmpdir):
     monkeypatch.setitem(os.environ, "COLUMNS", "80")
     # Work in a temporary directory
     monkeypatch.chdir(tmpdir)
+
+
+registry = EmbeddingFunctionRegistry.get_instance()
+
+
+@registry.register("test")
+class MockTextEmbeddingFunction(TextEmbeddingFunction):
+    """
+    Return the hash of the first 10 characters
+    """
+
+    def generate_embeddings(self, texts):
+        return [self._compute_one_embedding(row) for row in texts]
+
+    def _compute_one_embedding(self, row):
+        emb = np.array([float(hash(c)) for c in row[:10]])
+        emb /= np.linalg.norm(emb)
+        return emb
+
+    def ndims(self):
+        return 10
+
+
+class RateLimitedAPI:
+    rate_limit = 0.1  # 1 request per 0.1 second
+    last_request_time = 0
+
+    @staticmethod
+    def make_request():
+        current_time = time.time()
+
+        if current_time - RateLimitedAPI.last_request_time < RateLimitedAPI.rate_limit:
+            raise Exception("Rate limit exceeded. Please try again later.")
+
+        # Simulate a successful request
+        RateLimitedAPI.last_request_time = current_time
+        return "Request successful"
+
+
+@registry.register("test-rate-limited")
+class MockRateLimitedEmbeddingFunction(MockTextEmbeddingFunction):
+    def generate_embeddings(self, texts):
+        RateLimitedAPI.make_request()
+        return [self._compute_one_embedding(row) for row in texts]

python/lancedb/context.py

@@ -12,6 +12,9 @@
 #  limitations under the License.
 from __future__ import annotations
 
+import deprecation
+
+from . import __version__
 from .exceptions import MissingColumnError, MissingValueError
 from .util import safe_import_pandas
 
@@ -43,7 +46,7 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
     this how many tokens, but depending on the input data, it could be sentences,
     paragraphs, messages, etc.
 
-    >>> contextualize(data).window(3).stride(1).text_col('token').to_df()
+    >>> contextualize(data).window(3).stride(1).text_col('token').to_pandas()
                     token  document_id
     0     The quick brown            1
     1     quick brown fox            1
@@ -56,7 +59,7 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
     8          dog I love            1
     9   I love sandwiches            2
     10    love sandwiches            2
-    >>> contextualize(data).window(7).stride(1).min_window_size(7).text_col('token').to_df()
+    >>> contextualize(data).window(7).stride(1).min_window_size(7).text_col('token').to_pandas()
                                       token  document_id
     0   The quick brown fox jumped over the            1
     1  quick brown fox jumped over the lazy            1
@@ -68,7 +71,7 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
     ``stride`` determines how many rows to skip between each window start. This can
     be used to reduce the total number of windows generated.
 
-    >>> contextualize(data).window(4).stride(2).text_col('token').to_df()
+    >>> contextualize(data).window(4).stride(2).text_col('token').to_pandas()
                         token  document_id
     0     The quick brown fox            1
     2   brown fox jumped over            1
@@ -81,7 +84,9 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
     context windows that don't cross document boundaries. In this case, we can
     pass ``document_id`` as the group by.
 
-    >>> contextualize(data).window(4).stride(2).text_col('token').groupby('document_id').to_df()
+    >>> (contextualize(data)
+    ...     .window(4).stride(2).text_col('token').groupby('document_id')
+    ...     .to_pandas())
                        token  document_id
     0    The quick brown fox            1
     2  brown fox jumped over            1
@@ -89,18 +94,24 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
     6           the lazy dog            1
     9      I love sandwiches            2
 
-    ``min_window_size`` determines the minimum size of the  context windows that are generated
-    This can be used to trim the last few context windows which have size less than
-    ``min_window_size``. By default context windows of size 1 are skipped.
+    ``min_window_size`` determines the minimum size of the context windows
+    that are generated.This can be used to trim the last few context windows
+    which have size less than ``min_window_size``.
+    By default context windows of size 1 are skipped.
 
-    >>> contextualize(data).window(6).stride(3).text_col('token').groupby('document_id').to_df()
+    >>> (contextualize(data)
+    ...     .window(6).stride(3).text_col('token').groupby('document_id')
+    ...     .to_pandas())
                                  token  document_id
     0  The quick brown fox jumped over            1
     3     fox jumped over the lazy dog            1
     6                     the lazy dog            1
     9                I love sandwiches            2
 
-    >>> contextualize(data).window(6).stride(3).min_window_size(4).text_col('token').groupby('document_id').to_df()
+    >>> (contextualize(data)
+    ...     .window(6).stride(3).min_window_size(4).text_col('token')
+    ...     .groupby('document_id')
+    ...     .to_pandas())
                                  token  document_id
     0  The quick brown fox jumped over            1
     3     fox jumped over the lazy dog            1
@@ -110,7 +121,9 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
 
 
 class Contextualizer:
-    """Create context windows from a DataFrame. See [lancedb.context.contextualize][]."""
+    """Create context windows from a DataFrame.
+    See [lancedb.context.contextualize][].
+    """
 
     def __init__(self, raw_df):
         self._text_col = None
@@ -176,7 +189,16 @@ class Contextualizer:
         self._min_window_size = min_window_size
         return self
 
+    @deprecation.deprecated(
+        deprecated_in="0.3.1",
+        removed_in="0.4.0",
+        current_version=__version__,
+        details="Use to_pandas() instead",
+    )
     def to_df(self) -> "pd.DataFrame":
+        return self.to_pandas()
+
+    def to_pandas(self) -> "pd.DataFrame":
         """Create the context windows and return a DataFrame."""
         if pd is None:
             raise ImportError(