[python] Bump version: 0.4.0 → 0.4.1

Modify some grammar, punctuation, and spelling errors.

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.2
+current_version = 0.4.1
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True

.github/ISSUE_TEMPLATE/bug-node.yml

@@ -0,0 +1,33 @@
+name: Bug Report - Node / Typescript
+description: File a bug report
+title: "bug(node): "
+labels: [bug, typescript]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: input
+    id: version
+    attributes:
+      label: LanceDB version
+      description: What version of LanceDB are you using? `npm list | grep vectordb`.
+      placeholder: v0.3.2
+    validations:
+      required: false
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: Also tell us, what did you expect to happen?
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Are there known steps to reproduce?
+      description: |
+        Let us know how to reproduce the bug and we may be able to fix it more
+        quickly. This is not required, but it is helpful.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/bug-python.yml

@@ -0,0 +1,33 @@
+name: Bug Report - Python
+description: File a bug report
+title: "bug(python): "
+labels: [bug, python]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: input
+    id: version
+    attributes:
+      label: LanceDB version
+      description: What version of LanceDB are you using? `python -c "import lancedb; print(lancedb.__version__)"`.
+      placeholder: v0.3.2
+    validations:
+      required: false
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: Also tell us, what did you expect to happen?
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Are there known steps to reproduce?
+      description: |
+        Let us know how to reproduce the bug and we may be able to fix it more
+        quickly. This is not required, but it is helpful.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Discord Community Support
+    url: https://discord.com/invite/zMM32dvNtd
+    about: Please ask and answer questions here.

.github/ISSUE_TEMPLATE/documentation.yml

@@ -0,0 +1,23 @@
+name: 'Documentation improvement'
+description: Report an issue with the documentation.
+labels: [documentation]
+
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: >
+        Describe the issue with the documentation and how it can be fixed or improved.
+    validations:
+      required: true
+
+  - type: input
+    id: link
+    attributes:
+      label: Link
+      description: >
+        Provide a link to the existing documentation, if applicable.
+      placeholder: ex. https://lancedb.github.io/lancedb/guides/tables/...
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/feature.yml

@@ -0,0 +1,31 @@
+name: Feature suggestion
+description: Suggestion a new feature for LanceDB
+title: "Feature: "
+labels: [enhancement]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Share a new idea for a feature or improvement. Be sure to search existing
+        issues first to avoid duplicates.
+  - type: dropdown
+    id: sdk
+    attributes:
+      label: SDK
+      description: Which SDK are you using? This helps us prioritize.
+      options:
+        - Python
+        - Node
+        - Rust
+      default: 0
+    validations:
+      required: false
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: |
+        Describe the feature and why it would be useful. If applicable, consider
+        providing a code example of what it might be like to use the feature.
+    validations:
+      required: true

.github/workflows/node.yml

@@ -11,6 +11,10 @@ on:
       - .github/workflows/node.yml
       - docker-compose.yml
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   # Disable full debug symbol generation to speed up CI build and keep memory down
   # "1" means line tables only, which is useful for panic tracebacks.

.github/workflows/npm-publish.yml

@@ -38,13 +38,17 @@ jobs:
             node/vectordb-*.tgz
 
   node-macos:
-    runs-on: macos-12
+    strategy:
+      matrix:
+        config:
+          - arch: x86_64-apple-darwin
+            runner: macos-13
+          - arch: aarch64-apple-darwin
+            # xlarge is implicitly arm64.
+            runner: macos-13-xlarge
+    runs-on: ${{ matrix.config.runner }}
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64-apple-darwin, aarch64-apple-darwin]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -54,17 +58,15 @@ jobs:
         run: |
           cd node
           npm ci
-      - name: Install rustup target
-        if: ${{ matrix.target == 'aarch64-apple-darwin' }}
-        run: rustup target add aarch64-apple-darwin
       - name: Build MacOS native node modules
-        run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
+        run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }}
       - name: Upload Darwin Artifacts
         uses: actions/upload-artifact@v3
         with:
           name: native-darwin
           path: |
             node/dist/lancedb-vectordb-darwin*.tgz
+         
 
   node-linux:
     name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu

.github/workflows/python.yml

@@ -8,6 +8,11 @@ on:
     paths:
       - python/**
       - .github/workflows/python.yml
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   linux:
     timeout-minutes: 30
@@ -32,18 +37,26 @@ jobs:
       run: |
         pip install -e .[tests]
         pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock black isort
-    - name: Black
-      run: black --check --diff --no-color --quiet .
-    - name: isort
-      run: isort --check --diff --quiet .
+        pip install pytest pytest-mock ruff
+    - name: Lint
+      run: ruff format --check .
     - name: Run tests
       run: pytest -m "not slow" -x -v --durations=30 tests
     - name: doctest
       run: pytest --doctest-modules lancedb
-  mac:
+  platform:
+    name: "Platform: ${{ matrix.config.name }}"
     timeout-minutes: 30
-    runs-on: "macos-12"
+    strategy:
+      matrix:
+        config: 
+          - name: x86 Mac
+            runner: macos-13
+          - name: Arm Mac
+            runner: macos-13-xlarge
+          - name: x86 Windows
+            runner: windows-latest
+    runs-on: "${{ matrix.config.runner }}"
     defaults:
       run:
         shell: bash
@@ -62,8 +75,6 @@ jobs:
         pip install -e .[tests]
         pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
         pip install pytest pytest-mock black
-    - name: Black
-      run: black --check --diff --no-color --quiet .
     - name: Run tests
       run: pytest -m "not slow" -x -v --durations=30 tests
   pydantic1x:
@@ -87,12 +98,8 @@ jobs:
         pip install "pydantic<2"
         pip install -e .[tests]
         pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock black isort
-    - name: Black
-      run: black --check --diff --no-color --quiet .
-    - name: isort
-      run: isort --check --diff --quiet .
+        pip install pytest pytest-mock
     - name: Run tests
       run: pytest -m "not slow" -x -v --durations=30 tests
     - name: doctest
-      run: pytest --doctest-modules lancedb
+      run: pytest --doctest-modules lancedb

.github/workflows/rust.yml

@@ -10,6 +10,10 @@ on:
       - rust/**
       - .github/workflows/rust.yml
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   # This env var is used by Swatinem/rust-cache@v2 for the cache
   # key, so we set it to make sure it is always consistent.
@@ -20,6 +24,29 @@ env:
   RUST_BACKTRACE: "1"
 
 jobs:
+  lint:
+    timeout-minutes: 30
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+        working-directory: rust
+    steps:
+    - uses: actions/checkout@v3
+      with:
+          fetch-depth: 0
+          lfs: true
+    - uses: Swatinem/rust-cache@v2
+      with:
+        workspaces: rust
+    - name: Install dependencies
+      run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+    - name: Run format
+      run: cargo fmt --all -- --check
+    - name: Run clippy
+      run: cargo clippy --all --all-features -- -D warnings
   linux:
     timeout-minutes: 30
     runs-on: ubuntu-22.04
@@ -44,8 +71,11 @@ jobs:
     - name: Run tests
       run: cargo test --all-features
   macos:
-    runs-on: macos-12
     timeout-minutes: 30
+    strategy:
+      matrix:
+        mac-runner: [ "macos-13", "macos-13-xlarge" ]
+    runs-on: "${{ matrix.mac-runner }}"
     defaults:
       run:
         shell: bash

Cargo.toml

@@ -5,23 +5,24 @@ exclude = ["python"]
 resolver = "2"
 
 [workspace.dependencies]
-lance = { "version" = "=0.8.5", "features" = ["dynamodb"] }
-lance-linalg = { "version" = "=0.8.5" }
-lance-testing = { "version" = "=0.8.5" }
+lance = { "version" = "=0.9.1", "features" = ["dynamodb"] }
+lance-index = { "version" = "=0.9.1" }
+lance-linalg = { "version" = "=0.9.1" }
+lance-testing = { "version" = "=0.9.1" }
 # Note that this one does not include pyarrow
-arrow = { version = "47.0.0", optional = false }
-arrow-array = "47.0"
-arrow-data = "47.0"
-arrow-ipc = "47.0"
-arrow-ord = "47.0"
-arrow-schema = "47.0"
-arrow-arith = "47.0"
-arrow-cast = "47.0"
+arrow = { version = "49.0.0", optional = false }
+arrow-array = "49.0"
+arrow-data = "49.0"
+arrow-ipc = "49.0"
+arrow-ord = "49.0"
+arrow-schema = "49.0"
+arrow-arith = "49.0"
+arrow-cast = "49.0"
 chrono = "0.4.23"
-half = { "version" = "=2.2.1", default-features = false, features = [
-    "num-traits"
+half = { "version" = "=2.3.1", default-features = false, features = [
+    "num-traits",
 ] }
 log = "0.4"
-object_store = "0.7.1"
+object_store = "0.8.0"
 snafu = "0.7.4"
 url = "2"

README.md

@@ -5,10 +5,11 @@
 
 **Developer-friendly, serverless vector database for AI applications**
 
-<a href="https://lancedb.github.io/lancedb/">Documentation</a> •
-<a href="https://blog.lancedb.com/">Blog</a> •
-<a href="https://discord.gg/zMM32dvNtd">Discord</a> •
-<a href="https://twitter.com/lancedb">Twitter</a>
+<a href='https://github.com/lancedb/vectordb-recipes/tree/main' target="_blank"><img alt='LanceDB' src='https://img.shields.io/badge/VectorDB_Recipes-100000?style=for-the-badge&logo=LanceDB&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
+<a href='https://lancedb.github.io/lancedb/' target="_blank"><img alt='lancdb' src='https://img.shields.io/badge/DOCS-100000?style=for-the-badge&logo=lancdb&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
+[![Medium](https://img.shields.io/badge/Medium-12100E?style=for-the-badge&logo=medium&logoColor=white)](https://blog.lancedb.com/) 
+[![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/zMM32dvNtd) 
+[![Twitter](https://img.shields.io/badge/Twitter-%231DA1F2.svg?style=for-the-badge&logo=Twitter&logoColor=white)](https://twitter.com/lancedb)
 
 </p>
 

ci/build_macos_artifacts.sh

@@ -1,6 +1,7 @@
 # Builds the macOS artifacts (node binaries).
 # Usage: ./ci/build_macos_artifacts.sh [target]
 # Targets supported: x86_64-apple-darwin aarch64-apple-darwin
+set -e
 
 prebuild_rust() {
     # Building here for the sake of easier debugging.

docs/README.md

@@ -0,0 +1,26 @@
+# LanceDB Documentation
+
+LanceDB docs are deployed to https://lancedb.github.io/lancedb/.
+
+Docs is built and deployed automatically by [Github Actions](.github/workflows/docs.yml)
+whenever a commit is pushed to the `main` branch. So it is possible for the docs to show
+unreleased features.
+
+## Building the docs
+
+### Setup
+1. Install LanceDB. From LanceDB repo root: `pip install -e python`
+2. Install dependencies. From LanceDB repo root: `pip install -r docs/requirements.txt`
+3. Make sure you have node and npm setup
+4. Make sure protobuf and libssl are installed
+
+### Building node module and create markdown files
+
+See [Javascript docs README](docs/src/javascript/README.md)
+
+### Build docs
+From LanceDB repo root:
+
+Run: `PYTHONPATH=. mkdocs build -f docs/mkdocs.yml`
+
+If successful, you should see a `docs/site` directory that you can verify locally.

docs/mkdocs.yml

@@ -1,4 +1,5 @@
 site_name: LanceDB Docs
+site_url: https://lancedb.github.io/lancedb/
 repo_url: https://github.com/lancedb/lancedb
 edit_uri: https://github.com/lancedb/lancedb/tree/main/docs/src
 repo_name: lancedb/lancedb
@@ -73,12 +74,14 @@ nav:
     - Vector Search: search.md
     - SQL filters: sql.md
     - Indexing: ann_indexes.md
+    - Versioning & Reproducibility: notebooks/reproducibility.ipynb
   - 🧬 Embeddings:
     - embeddings/index.md
     - Ingest Embedding Functions: embeddings/embedding_functions.md
     - Available Functions: embeddings/default_embedding_functions.md
     - Create Custom Embedding Functions: embeddings/api.md
-    - Example- MultiModal CLIP Embeddings: notebooks/DisappearingEmbeddingFunction.ipynb
+    - Example - Multi-lingual semantic search: notebooks/multi_lingual_example.ipynb
+    - Example - MultiModal CLIP Embeddings: notebooks/DisappearingEmbeddingFunction.ipynb
   - 🔍 Python full-text search: fts.md
   - 🔌 Integrations:
     - integrations/index.md
@@ -95,6 +98,7 @@ nav:
     - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
     - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
     - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
+    - Example - Calculate CLIP Embeddings with Roboflow Inference: examples/image_embeddings_roboflow.md
     - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
     - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
   - 🌐 Javascript examples:
@@ -110,12 +114,14 @@ nav:
   - Vector Search: search.md
   - SQL filters: sql.md
   - Indexing: ann_indexes.md
+  - Versioning & Reproducibility: notebooks/reproducibility.ipynb
 - Embeddings:
   - embeddings/index.md
   - Ingest Embedding Functions: embeddings/embedding_functions.md
   - Available Functions: embeddings/default_embedding_functions.md
   - Create Custom Embedding Functions: embeddings/api.md
-  - Example- MultiModal CLIP Embeddings: notebooks/DisappearingEmbeddingFunction.ipynb
+  - Example - Multi-lingual semantic search: notebooks/multi_lingual_example.ipynb
+  - Example - MultiModal CLIP Embeddings: notebooks/DisappearingEmbeddingFunction.ipynb
 - Python full-text search: fts.md
 - Integrations:
   - integrations/index.md
@@ -140,7 +146,8 @@ nav:
   - Serverless Chatbot from any website: examples/serverless_website_chatbot.md
   - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
 - API references:
-  - Python API: python/python.md
+  - OSS Python API: python/python.md
+  - SaaS Python API: python/saas-python.md
   - Javascript API: javascript/modules.md
 - LanceDB Cloud↗: https://noteforms.com/forms/lancedb-mailing-list-cloud-kty1o5?notionforms=1&utm_source=notionforms
 

docs/requirements.txt

@@ -2,3 +2,4 @@ mkdocs==1.4.2
 mkdocs-jupyter==0.24.1
 mkdocs-material==9.1.3
 mkdocstrings[python]==0.20.0
+pydantic

docs/src/ann_indexes.md

@@ -71,9 +71,41 @@ a single PQ code.
 ### Use GPU to build vector index
 
 Lance Python SDK has experimental GPU support for creating IVF index.
+Using GPU for index creation requires [PyTorch>2.0](https://pytorch.org/) being installed.
+
 You can specify the GPU device to train IVF partitions via
 
-- **accelerator**: Specify to `"cuda"`` to enable GPU training.
+- **accelerator**: Specify to ``cuda`` or ``mps`` (on Apple Silicon) to enable GPU training.
+
+=== "Linux"
+
+     <!-- skip-test -->
+     ``` { .python .copy }
+     # Create index using CUDA on Nvidia GPUs.
+     tbl.create_index(
+          num_partitions=256,
+          num_sub_vectors=96,
+          accelerator="cuda"
+     )
+     ```
+
+=== "Macos"
+
+     <!-- skip-test -->
+     ```python
+     # Create index using MPS on Apple Silicon.
+     tbl.create_index(
+          num_partitions=256,
+          num_sub_vectors=96,
+          accelerator="mps"
+     )
+     ```
+
+Trouble shootings:
+
+If you see ``AssertionError: Torch not compiled with CUDA enabled``, you need to [install
+PyTorch with CUDA support](https://pytorch.org/get-started/locally/).
+
 
 ## Querying an ANN Index
 

docs/src/basic.md

@@ -64,18 +64,26 @@ We'll cover the basics of using LanceDB on your local machine in this section.
       tbl = db.create_table("table_from_df", data=df)
       ```
 
+      !!! warning
+
+            If the table already exists, LanceDB will raise an error by default.
+            If you want to overwrite the table, you can pass in `mode="overwrite"`
+            to the `createTable` function.
+
 === "Javascript"
       ```javascript
-      const tb = await db.createTable("my_table",
-                        data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                              {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+      const tb = await db.createTable(
+        "myTable",
+        [{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+         {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
       ```
-      
-!!! warning
 
-      If the table already exists, LanceDB will raise an error by default.
-      If you want to overwrite the table, you can pass in `mode="overwrite"`
-      to the `createTable` function.
+      !!! warning
+
+            If the table already exists, LanceDB will raise an error by default.
+            If you want to overwrite the table, you can pass in `"overwrite"`
+            to the `createTable` function like this: `await con.createTable(tableName, data, { writeMode: WriteMode.Overwrite })`
+      
 
 ??? info "Under the hood, LanceDB is converting the input data into an Apache Arrow table and persisting it to disk in [Lance format](https://www.github.com/lancedb/lance)."
 
@@ -108,7 +116,7 @@ Once created, you can open a table using the following code:
 
 === "Javascript"
       ```javascript
-      const tbl = await db.openTable("my_table");
+      const tbl = await db.openTable("myTable");
       ```
 
       If you forget the name of your table, you can always get a listing of all table names:
@@ -194,10 +202,17 @@ Use the `drop_table()` method on the database to remove a table.
       db.drop_table("my_table")
       ```
 
-This permanently removes the table and is not recoverable, unlike deleting rows.
-By default, if the table does not exist an exception is raised. To suppress this,
-you can pass in `ignore_missing=True`.
+      This permanently removes the table and is not recoverable, unlike deleting rows.
+      By default, if the table does not exist an exception is raised. To suppress this,
+      you can pass in `ignore_missing=True`.
 
+=== "JavaScript"
+      ```javascript
+      await db.dropTable('myTable')
+      ```
+
+      This permanently removes the table and is not recoverable, unlike deleting rows.
+      If the table does not exist an exception is raised. 
 
 ## What's next
 

docs/src/embeddings/default_embedding_functions.md

@@ -1,7 +1,9 @@
-There are various Embedding functions available out of the box with lancedb. We're working on supporting other popular embedding APIs.
+There are various Embedding functions available out of the box with LanceDB. We're working on supporting other popular embedding APIs.
 
 ## Text Embedding Functions
-Here are the text embedding functions registered by default
+Here are the text embedding functions registered by default.
+Embedding functions have an inbuilt rate limit handler wrapper for source and query embedding function calls that retry with exponential standoff. 
+Each `EmbeddingFunction` implementation automatically takes `max_retries` as an argument which has the default value of 7.
 
 ### Sentence Transformers
 Here are the parameters that you can set when registering a `sentence-transformers` object, and their default values:
@@ -66,11 +68,61 @@ actual = table.search(query).limit(1).to_pydantic(Words)[0]
 print(actual.text)
 ```
 
+### Instructor Embeddings
+Instructor is an instruction-finetuned text embedding model that can generate text embeddings tailored to any task (e.g. classification, retrieval, clustering, text evaluation, etc.) and domains (e.g. science, finance, etc.) by simply providing the task instruction, without any finetuning.
+
+If you want to calculate customized embeddings for specific sentences, you may follow the unified template to write instructions:
+
+                          Represent the `domain` `text_type` for `task_objective`:
+
+* `domain` is optional, and it specifies the domain of the text, e.g. science, finance, medicine, etc.
+* `text_type` is required, and it specifies the encoding unit, e.g. sentence, document, paragraph, etc.
+* `task_objective` is optional, and it specifies the objective of embedding, e.g. retrieve a document, classify the sentence, etc.
+
+More information about the model can be found here - https://github.com/xlang-ai/instructor-embedding
+
+| Argument | Type | Default | Description |
+|---|---|---|---|
+| `name` | `str` | "hkunlp/instructor-base" | The name of the model to use |
+| `batch_size` | `int` | `32` | The batch size to use when generating embeddings |
+| `device` | `str` | `"cpu"` | The device to use when generating embeddings |
+| `show_progress_bar` | `bool` | `True` | Whether to show a progress bar when generating embeddings |
+| `normalize_embeddings` | `bool` | `True` | Whether to normalize the embeddings |
+| `quantize` | `bool` | `False` | Whether to quantize the model |
+| `source_instruction` | `str` | `"represent the docuement for retreival"` | The instruction for the source column |
+| `query_instruction` | `str` | `"represent the document for retreiving the most similar documents"` | The instruction for the query |
+
+
+
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry, InstuctorEmbeddingFunction
+
+instructor = get_registry().get("instructor").create(
+                            source_instruction="represent the docuement for retreival",
+                            query_instruction="represent the document for retreiving the most similar documents"
+                            )
+
+class Schema(LanceModel):
+    vector: Vector(instructor.ndims()) = instructor.VectorField()
+    text: str = instructor.SourceField()
+
+db = lancedb.connect("~/.lancedb")
+tbl = db.create_table("test", schema=Schema, mode="overwrite")
+
+texts = [{"text": "Capitalism has been dominant in the Western world since the end of feudalism, but most feel[who?] that..."},
+        {"text": "The disparate impact theory is especially controversial under the Fair Housing Act because the Act..."},
+        {"text": "Disparate impact in United States labor law refers to practices in employment, housing, and other areas that.."}]
+
+tbl.add(texts)
+```
+
 ## Multi-modal embedding functions
-Multi-modal embedding functions allow you query your table using both images and text.
+Multi-modal embedding functions allow you to query your table using both images and text.
 
 ### OpenClipEmbeddings
-We support CLIP model embeddings using the open souce alternbative, open-clip which support various customizations. It is registered as `open-clip` and supports following customizations.
+We support CLIP model embeddings using the open source alternative, open-clip which supports various customizations. It is registered as `open-clip` and supports the following customizations:
 
 
 | Parameter | Type | Default Value | Description |
@@ -153,4 +205,4 @@ print(actual.label)
 
 ```
 
-If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue.
+If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue.

docs/src/embeddings/embedding_functions.md

@@ -57,6 +57,19 @@ query_image = Image.open(p)
 table.search(query_image)
 
 ```
+### Rate limit Handling
+`EmbeddingFunction` class wraps the calls for source and query embedding generation inside a rate limit handler that retries the requests with exponential backoff after successive failures. By default the maximum retires is set to 7. You can tune it by setting it to a different number or disable it by setting it to 0.
+Example
+----
+
+```python
+clip = registry.get("open-clip").create() # Defaults to 7 max retries
+clip = registry.get("open-clip").create(max_retries=10) # Increase max retries to 10
+clip = registry.get("open-clip").create(max_retries=0) # Retries disabled
+````
+
+NOTE:
+Embedding functions can also fail due to other errors that have nothing to do with rate limits. This is why the error is also logged.
 
 ### A little fun with PyDantic
 LanceDB is integrated with PyDantic. Infact we've used the integration in the above example to define the schema. It is also being used behing the scene by the embdding function API to ingest useful information as table metadata.

docs/src/examples/image_embeddings_roboflow.md

@@ -0,0 +1,165 @@
+# How to Load Image Embeddings into LanceDB
+
+With the rise of Large Multimodal Models (LMMs) such as [GPT-4 Vision](https://blog.roboflow.com/gpt-4-vision/), the need for storing image embeddings is growing. The most effective way to store text and image embeddings is in a vector database such as LanceDB. Vector databases are a special kind of data store that enables efficient search over stored embeddings. 
+
+[CLIP](https://blog.roboflow.com/openai-clip/), a multimodal model developed by OpenAI, is commonly used to calculate image embeddings. These embeddings can then be used with a vector database to build a semantic search engine that you can query using images or text. For example, you could use LanceDB and CLIP embeddings to build a search engine for a database of folders.
+
+In this guide, we are going to show you how to use Roboflow Inference to load image embeddings into LanceDB. Without further ado, let’s get started!
+
+## Step #1: Install Roboflow Inference
+
+[Roboflow Inference](https://inference.roboflow.com) enables you to run state-of-the-art computer vision models with minimal configuration. Inference supports a range of models, from fine-tuned object detection, classification, and segmentation models to foundation models like CLIP. We will use Inference to calculate CLIP image embeddings.
+
+Inference provides a HTTP API through which you can run vision models.
+
+Inference powers the Roboflow hosted API, and is available as an open source utility. In this guide, we are going to run Inference locally, which enables you to calculate CLIP embeddings on your own hardware. We will also show you how to use the hosted Roboflow CLIP API, which is ideal if you need to scale and do not want to manage a system for calculating embeddings.
+
+To get started, first install the Inference CLI:
+
+```
+pip install inference-cli
+```
+
+Next, install Docker. Refer to the official Docker installation instructions for your operating system to get Docker set up. Once Docker is ready, you can start Inference using the following command:
+
+```
+inference server start
+```
+
+An Inference server will start running at ‘http://localhost:9001’.
+
+## Step #2: Set Up a LanceDB Vector Database
+
+Now that we have Inference running, we can set up a LanceDB vector database. You can run LanceDB in JavaScript and Python. For this guide, we will use the Python API. But, you can take the HTTP requests we make below and change them to JavaScript if required.
+
+For this guide, we are going to search the [COCO 128 dataset](https://universe.roboflow.com/team-roboflow/coco-128), which contains a wide range of objects. The variability in objects present in this dataset makes it a good dataset to demonstrate the capabilities of vector search. If you want to use this dataset, you can download [COCO 128 from Roboflow Universe](https://universe.roboflow.com/team-roboflow/coco-128). With that said, you can search whatever folder of images you want.
+
+Once you have a dataset ready, install LanceDB with the following command:
+
+```
+pip install lancedb
+```
+
+We also need to install a specific commit of `tantivy`, a dependency of the LanceDB full text search engine we will use later in this guide:
+
+```
+pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+```
+
+Create a new Python file and add the following code:
+
+```python
+import cv2
+import supervision as sv
+import requests
+
+import lancedb
+
+db = lancedb.connect("./embeddings")
+
+IMAGE_DIR = "images/"
+API_KEY = os.environ.get("ROBOFLOW_API_KEY")
+SERVER_URL = "http://localhost:9001"
+
+results = []
+
+for i, image in enumerate(os.listdir(IMAGE_DIR)):
+    infer_clip_payload = {
+        #Images can be provided as urls or as base64 encoded strings
+        "image": {
+            "type": "base64",
+            "value": base64.b64encode(open(IMAGE_DIR + image, "rb").read()).decode("utf-8"),
+        },
+    }
+
+    res = requests.post(
+        f"{SERVER_URL}/clip/embed_image?api_key={API_KEY}",
+        json=infer_clip_payload,
+    )
+
+    embeddings = res.json()['embeddings']
+
+    print("Calculated embedding for image: ", image)
+
+    image = {"vector": embeddings[0], "name": os.path.join(IMAGE_DIR, image)}
+
+    results.append(image)
+
+tbl = db.create_table("images", data=results)
+
+tbl.create_fts_index("name")
+```
+
+To use the code above, you will need a Roboflow API key. [Learn how to retrieve a Roboflow API key](https://docs.roboflow.com/api-reference/authentication#retrieve-an-api-key). Run the following command to set up your API key in your environment:
+
+```
+export ROBOFLOW_API_KEY=""
+```
+
+Replace the `IMAGE_DIR` value with the folder in which you are storing the images for which you want to calculate embeddings. If you want to use the Roboflow CLIP API to calculate embeddings, replace the `SERVER_URL` value with `https://infer.roboflow.com`.
+
+Run the script above to create a new LanceDB database. This database will be stored on your local machine. The database will be called `embeddings` and the table will be called `images`.
+
+The script above calculates all embeddings for a folder then creates a new table. To add additional images, use the following code:
+
+```python
+def make_batches():
+    for i in range(5):
+        yield [
+                {"vector": [3.1, 4.1], "name": "image1.png"},
+                {"vector": [5.9, 26.5], "name": "image2.png"}
+            ]
+
+tbl = db.open_table("images")
+tbl.add(make_batches())
+```
+
+Replacing the `make_batches()` function with code to load embeddings for images.
+
+## Step #3: Run a Search Query
+
+We are now ready to run a search query. To run a search query, we need a text embedding that represents a text query. We can use this embedding to search our LanceDB database for an entry.
+
+Let’s calculate a text embedding for the query “cat”, then run a search query:
+
+```python
+infer_clip_payload = {
+    "text": "cat",
+}
+
+res = requests.post(
+    f"{SERVER_URL}/clip/embed_text?api_key={API_KEY}",
+    json=infer_clip_payload,
+)
+
+embeddings = res.json()['embeddings']
+
+df = tbl.search(embeddings[0]).limit(3).to_list()
+
+print("Results:")
+
+for i in df:
+    print(i["name"])
+```
+
+This code will search for the three images most closely related to the prompt “cat”. The names of the most similar three images will be printed to the console. Here are the three top results:
+
+```
+dataset/images/train/000000000650_jpg.rf.1b74ba165c5a3513a3211d4a80b69e1c.jpg
+dataset/images/train/000000000138_jpg.rf.af439ef1c55dd8a4e4b142d186b9c957.jpg
+dataset/images/train/000000000165_jpg.rf.eae14d5509bf0c9ceccddbb53a5f0c66.jpg
+```
+
+Let’s open the top image:
+
+![Cat](https://media.roboflow.com/cat_lancedb.jpg)
+
+The top image was a cat. Our search was successful.
+
+## Conclusion
+
+LanceDB is a vector database that you can use to store and efficiently search your image embeddings. You can use Roboflow Inference, a scalable computer vision inference server, to calculate CLIP embeddings that you can store in LanceDB.
+
+You can use Inference and LanceDB together to build a range of applications with image embeddings, from a media search engine to a retrieval-augmented generation pipeline for use with LMMs.
+
+To learn more about Inference and its capabilities, refer to the Inference documentation.

docs/src/guides/tables.md

@@ -1,5 +1,7 @@
  <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/tables_guide.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
-A Table is a collection of Records in a LanceDB Database. You can follow along on colab!
+A Table is a collection of Records in a LanceDB Database. Tables in Lance have a schema that defines the columns and their types. These schemas can include nested columns and can evolve over time.
+
+This guide will show how to create tables, insert data into them, and update the data. You can follow along on colab!
 
 ## Creating a LanceDB Table
 
@@ -201,8 +203,8 @@ A Table is a collection of Records in a LanceDB Database. You can follow along o
     ```javascript
     data
     const tb = await db.createTable("my_table",
-                  data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+      [{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+       {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
     ```
 
     !!! info "Note"
@@ -361,19 +363,28 @@ Use the `delete()` method on tables to delete rows from a table. To choose which
     await tbl.countRows() // Returns 1
     ```
 
-### Updating a Table [Experimental]
-EXPERIMENTAL: Update rows in the table (not threadsafe).
+## Updating a Table
 
-This can be used to update zero to all rows depending on how many rows match the where clause.
+This can be used to update zero to all rows depending on how many rows match the where clause. The update queries follow the form of a SQL UPDATE statement. The `where` parameter is a SQL filter that matches on the metadata columns. The `values` or `values_sql` parameters are used to provide the new values for the columns.
 
-| Parameter | Type | Description |
+| Parameter   | Type | Description |
 |---|---|---|
 | `where` | `str` | The SQL where clause to use when updating rows. For example, `'x = 2'` or `'x IN (1, 2, 3)'`. The filter must not be empty, or it will error. |
 | `values` | `dict` | The values to update. The keys are the column names and the values are the values to set. |
+| `values_sql` | `dict` | The values to update. The keys are the column names and the values are the SQL expressions to set. For example, `{'x': 'x + 1'}` will increment the value of the `x` column by 1. |
 
+!!! info "SQL syntax"
+
+    See [SQL filters](sql.md) for more information on the supported SQL syntax.
+
+!!! warning "Warning"
+
+    Updating nested columns is not yet supported.
 
 === "Python"
 
+    API Reference: [lancedb.table.Table.update][]
+
     ```python
     import lancedb
     import pandas as pd
@@ -403,6 +414,55 @@ This can be used to update zero to all rows depending on how many rows match the
     2  2  [10.0, 10.0]
     ```
 
+=== "Javascript/Typescript"
+
+    API Reference: [vectordb.Table.update](../../javascript/interfaces/Table/#update)
+
+    ```javascript
+    const lancedb = require("vectordb");
+
+    const db = await lancedb.connect("./.lancedb");
+
+    const data = [
+      {x: 1, vector: [1, 2]},
+      {x: 2, vector: [3, 4]},
+      {x: 3, vector: [5, 6]},
+    ];
+    const tbl = await db.createTable("my_table", data)
+
+    await tbl.update({ where: "x = 2", values: {vector: [10, 10]} })
+    ```
+
+The `values` parameter is used to provide the new values for the columns as literal values. You can also use the `values_sql` / `valuesSql` parameter to provide SQL expressions for the new values. For example, you can use `values_sql="x + 1"` to increment the value of the `x` column by 1.
+
+=== "Python"
+
+    ```python
+    # Update the table where x = 2
+    table.update(valuesSql={"x": "x + 1"})
+
+    print(table.to_pandas())
+    ```
+
+    Output
+    ```shell
+        x  vector
+    0  2  [1.0, 2.0]
+    1  4  [5.0, 6.0]
+    2  3  [10.0, 10.0]
+    ```
+
+=== "Javascript/Typescript"
+
+    ```javascript
+    await tbl.update({ valuesSql: { x: "x + 1" } })
+    ```
+
+!!! info "Note"
+
+    When rows are updated, they are moved out of the index. The row will still show up in ANN queries, but the query will not be as fast as it would be if the row was in the index. If you update a large proportion of rows, consider rebuilding the index afterwards.
+
+
 ## What's Next?
 
 Learn how to Query your tables and create indices

docs/src/index.md

@@ -67,7 +67,7 @@ LanceDB's core is written in Rust 🦀 and is built using <a href="https://githu
 
 ## Documentation Quick Links
 * [`Basic Operations`](basic.md) - basic functionality of LanceDB.
-* [`Embedding Functions`](embedding.md) - functions for working with embeddings.
+* [`Embedding Functions`](embeddings/index.md) - functions for working with embeddings.
 * [`Indexing`](ann_indexes.md) - create vector indexes to speed up queries.
 * [`Full text search`](fts.md) - [EXPERIMENTAL] full-text search API
 * [`Ecosystem Integrations`](python/integration.md) - integrating LanceDB with python data tooling ecosystem.

docs/src/javascript/README.md

@@ -11,8 +11,13 @@ npm install vectordb
 ```
 
 This will download the appropriate native library for your platform. We currently
-support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
-yet support Windows or musl-based Linux (such as Alpine Linux).
+support:
+
+* Linux (x86_64 and aarch64)
+* MacOS (Intel and ARM/M1/M2)
+* Windows (x86_64 only)
+
+We do not yet support musl-based Linux (such as Alpine Linux) or aarch64 Windows.
 
 ## Usage
 

docs/src/javascript/classes/DefaultWriteOptions.md

@@ -0,0 +1,41 @@
+[vectordb](../README.md) / [Exports](../modules.md) / DefaultWriteOptions
+
+# Class: DefaultWriteOptions
+
+Write options when creating a Table.
+
+## Implements
+
+- [`WriteOptions`](../interfaces/WriteOptions.md)
+
+## Table of contents
+
+### Constructors
+
+- [constructor](DefaultWriteOptions.md#constructor)
+
+### Properties
+
+- [writeMode](DefaultWriteOptions.md#writemode)
+
+## Constructors
+
+### constructor
+
+• **new DefaultWriteOptions**()
+
+## Properties
+
+### writeMode
+
+• **writeMode**: [`WriteMode`](../enums/WriteMode.md) = `WriteMode.Create`
+
+A [WriteMode](../enums/WriteMode.md) to use on this operation
+
+#### Implementation of
+
+[WriteOptions](../interfaces/WriteOptions.md).[writeMode](../interfaces/WriteOptions.md#writemode)
+
+#### Defined in
+
+[index.ts:778](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L778)

docs/src/javascript/classes/LocalConnection.md

@@ -26,7 +26,7 @@ A connection to a LanceDB database.
 ### Methods
 
 - [createTable](LocalConnection.md#createtable)
-- [createTableArrow](LocalConnection.md#createtablearrow)
+- [createTableImpl](LocalConnection.md#createtableimpl)
 - [dropTable](LocalConnection.md#droptable)
 - [openTable](LocalConnection.md#opentable)
 - [tableNames](LocalConnection.md#tablenames)
@@ -46,7 +46,7 @@ A connection to a LanceDB database.
 
 #### Defined in
 
-[index.ts:184](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L184)
+[index.ts:355](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L355)
 
 ## Properties
 
@@ -56,17 +56,25 @@ A connection to a LanceDB database.
 
 #### Defined in
 
-[index.ts:182](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L182)
+[index.ts:353](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L353)
 
 ___
 
 ### \_options
 
-• `Private` `Readonly` **\_options**: [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
+• `Private` `Readonly` **\_options**: () => [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
+
+#### Type declaration
+
+▸ (): [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
+
+##### Returns
+
+[`ConnectionOptions`](../interfaces/ConnectionOptions.md)
 
 #### Defined in
 
-[index.ts:181](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L181)
+[index.ts:352](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L352)
 
 ## Accessors
 
@@ -84,27 +92,34 @@ ___
 
 #### Defined in
 
-[index.ts:189](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L189)
+[index.ts:360](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L360)
 
 ## Methods
 
 ### createTable
 
-▸ **createTable**(`name`, `data`, `mode?`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+▸ **createTable**\<`T`\>(`name`, `data?`, `optsOrEmbedding?`, `opt?`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
-Creates a new Table and initialize it with new data.
+Creates a new Table, optionally initializing it with new data.
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
 
 #### Parameters
 
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
-| `mode?` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` \| [`CreateTableOptions`](../interfaces/CreateTableOptions.md)\<`T`\> |
+| `data?` | `Record`\<`string`, `unknown`\>[] |
+| `optsOrEmbedding?` | [`WriteOptions`](../interfaces/WriteOptions.md) \| [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> |
+| `opt?` | [`WriteOptions`](../interfaces/WriteOptions.md) |
 
 #### Returns
 
-`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+`Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
 #### Implementation of
 
@@ -112,120 +127,44 @@ Creates a new Table and initialize it with new data.
 
 #### Defined in
 
-[index.ts:230](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L230)
-
-▸ **createTable**(`name`, `data`, `mode`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` |
-| `data` | `Record`<`string`, `unknown`\>[] |
-| `mode` | [`WriteMode`](../enums/WriteMode.md) |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-#### Implementation of
-
-Connection.createTable
-
-#### Defined in
-
-[index.ts:231](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L231)
-
-▸ **createTable**<`T`\>(`name`, `data`, `mode`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-Creates a new Table and initialize it with new data.
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
-| `mode` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-#### Implementation of
-
-Connection.createTable
-
-#### Defined in
-
-[index.ts:241](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L241)
-
-▸ **createTable**<`T`\>(`name`, `data`, `mode`, `embeddings?`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` |
-| `data` | `Record`<`string`, `unknown`\>[] |
-| `mode` | [`WriteMode`](../enums/WriteMode.md) |
-| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
-
-#### Returns
-
-`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
-
-#### Implementation of
-
-Connection.createTable
-
-#### Defined in
-
-[index.ts:242](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L242)
+[index.ts:395](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L395)
 
 ___
 
-### createTableArrow
+### createTableImpl
 
-▸ **createTableArrow**(`name`, `table`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+▸ `Private` **createTableImpl**\<`T`\>(`«destructured»`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
 
 #### Parameters
 
 | Name | Type |
 | :------ | :------ |
-| `name` | `string` |
-| `table` | `Table`<`any`\> |
+| `«destructured»` | `Object` |
+| › `data?` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] |
+| › `embeddingFunction?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> |
+| › `name` | `string` |
+| › `schema?` | `Schema`\<`any`\> |
+| › `writeOptions?` | [`WriteOptions`](../interfaces/WriteOptions.md) |
 
 #### Returns
 
-`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
-
-#### Implementation of
-
-[Connection](../interfaces/Connection.md).[createTableArrow](../interfaces/Connection.md#createtablearrow)
+`Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
 #### Defined in
 
-[index.ts:266](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L266)
+[index.ts:413](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L413)
 
 ___
 
 ### dropTable
 
-▸ **dropTable**(`name`): `Promise`<`void`\>
+▸ **dropTable**(`name`): `Promise`\<`void`\>
 
 Drop an existing table.
 
@@ -237,7 +176,7 @@ Drop an existing table.
 
 #### Returns
 
-`Promise`<`void`\>
+`Promise`\<`void`\>
 
 #### Implementation of
 
@@ -245,13 +184,13 @@ Drop an existing table.
 
 #### Defined in
 
-[index.ts:276](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L276)
+[index.ts:453](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L453)
 
 ___
 
 ### openTable
 
-▸ **openTable**(`name`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+▸ **openTable**(`name`): `Promise`\<[`Table`](../interfaces/Table.md)\<`number`[]\>\>
 
 Open a table in the database.
 
@@ -263,7 +202,7 @@ Open a table in the database.
 
 #### Returns
 
-`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+`Promise`\<[`Table`](../interfaces/Table.md)\<`number`[]\>\>
 
 #### Implementation of
 
@@ -271,9 +210,9 @@ Open a table in the database.
 
 #### Defined in
 
-[index.ts:205](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L205)
+[index.ts:376](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L376)
 
-▸ **openTable**<`T`\>(`name`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+▸ **openTable**\<`T`\>(`name`, `embeddings`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
 Open a table in the database.
 
@@ -288,11 +227,11 @@ Open a table in the database.
 | Name | Type | Description |
 | :------ | :------ | :------ |
 | `name` | `string` | The name of the table. |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> | An embedding function to use on this Table |
 
 #### Returns
 
-`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+`Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
 #### Implementation of
 
@@ -300,9 +239,9 @@ Connection.openTable
 
 #### Defined in
 
-[index.ts:212](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L212)
+[index.ts:384](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L384)
 
-▸ **openTable**<`T`\>(`name`, `embeddings?`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+▸ **openTable**\<`T`\>(`name`, `embeddings?`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
 #### Type parameters
 
@@ -315,11 +254,11 @@ Connection.openTable
 | Name | Type |
 | :------ | :------ |
 | `name` | `string` |
-| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
+| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> |
 
 #### Returns
 
-`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+`Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
 #### Implementation of
 
@@ -327,19 +266,19 @@ Connection.openTable
 
 #### Defined in
 
-[index.ts:213](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L213)
+[index.ts:385](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L385)
 
 ___
 
 ### tableNames
 
-▸ **tableNames**(): `Promise`<`string`[]\>
+▸ **tableNames**(): `Promise`\<`string`[]\>
 
 Get the names of all tables in the database.
 
 #### Returns
 
-`Promise`<`string`[]\>
+`Promise`\<`string`[]\>
 
 #### Implementation of
 
@@ -347,4 +286,4 @@ Get the names of all tables in the database.
 
 #### Defined in
 
-[index.ts:196](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L196)
+[index.ts:367](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L367)

docs/src/javascript/classes/LocalTable.md

@@ -1,6 +1,6 @@
 [vectordb](../README.md) / [Exports](../modules.md) / LocalTable
 
-# Class: LocalTable<T\>
+# Class: LocalTable\<T\>
 
 A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
 
@@ -12,7 +12,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 ## Implements
 
-- [`Table`](../interfaces/Table.md)<`T`\>
+- [`Table`](../interfaces/Table.md)\<`T`\>
 
 ## Table of contents
 
@@ -26,6 +26,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 - [\_name](LocalTable.md#_name)
 - [\_options](LocalTable.md#_options)
 - [\_tbl](LocalTable.md#_tbl)
+- [where](LocalTable.md#where)
 
 ### Accessors
 
@@ -34,17 +35,23 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 ### Methods
 
 - [add](LocalTable.md#add)
+- [cleanupOldVersions](LocalTable.md#cleanupoldversions)
+- [compactFiles](LocalTable.md#compactfiles)
 - [countRows](LocalTable.md#countrows)
 - [createIndex](LocalTable.md#createindex)
 - [delete](LocalTable.md#delete)
+- [filter](LocalTable.md#filter)
+- [indexStats](LocalTable.md#indexstats)
+- [listIndices](LocalTable.md#listindices)
 - [overwrite](LocalTable.md#overwrite)
 - [search](LocalTable.md#search)
+- [update](LocalTable.md#update)
 
 ## Constructors
 
 ### constructor
 
-• **new LocalTable**<`T`\>(`tbl`, `name`, `options`)
+• **new LocalTable**\<`T`\>(`tbl`, `name`, `options`)
 
 #### Type parameters
 
@@ -62,9 +69,9 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 #### Defined in
 
-[index.ts:287](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L287)
+[index.ts:464](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L464)
 
-• **new LocalTable**<`T`\>(`tbl`, `name`, `options`, `embeddings`)
+• **new LocalTable**\<`T`\>(`tbl`, `name`, `options`, `embeddings`)
 
 #### Type parameters
 
@@ -79,21 +86,21 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 | `tbl` | `any` |  |
 | `name` | `string` |  |
 | `options` | [`ConnectionOptions`](../interfaces/ConnectionOptions.md) |  |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use when interacting with this table |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> | An embedding function to use when interacting with this table |
 
 #### Defined in
 
-[index.ts:294](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L294)
+[index.ts:471](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L471)
 
 ## Properties
 
 ### \_embeddings
 
-• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
+• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\>
 
 #### Defined in
 
-[index.ts:284](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L284)
+[index.ts:461](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L461)
 
 ___
 
@@ -103,27 +110,61 @@ ___
 
 #### Defined in
 
-[index.ts:283](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L283)
+[index.ts:460](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L460)
 
 ___
 
 ### \_options
 
-• `Private` `Readonly` **\_options**: [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
+• `Private` `Readonly` **\_options**: () => [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
+
+#### Type declaration
+
+▸ (): [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
+
+##### Returns
+
+[`ConnectionOptions`](../interfaces/ConnectionOptions.md)
 
 #### Defined in
 
-[index.ts:285](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L285)
+[index.ts:462](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L462)
 
 ___
 
 ### \_tbl
 
-• `Private` `Readonly` **\_tbl**: `any`
+• `Private` **\_tbl**: `any`
 
 #### Defined in
 
-[index.ts:282](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L282)
+[index.ts:459](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L459)
+
+___
+
+### where
+
+• **where**: (`value`: `string`) => [`Query`](Query.md)\<`T`\>
+
+#### Type declaration
+
+▸ (`value`): [`Query`](Query.md)\<`T`\>
+
+Creates a filter query to find all rows matching the specified criteria
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `value` | `string` | The filter criteria (like SQL where clause syntax) |
+
+##### Returns
+
+[`Query`](Query.md)\<`T`\>
+
+#### Defined in
+
+[index.ts:499](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L499)
 
 ## Accessors
 
@@ -141,13 +182,13 @@ ___
 
 #### Defined in
 
-[index.ts:302](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L302)
+[index.ts:479](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L479)
 
 ## Methods
 
 ### add
 
-▸ **add**(`data`): `Promise`<`number`\>
+▸ **add**(`data`): `Promise`\<`number`\>
 
 Insert records into this Table.
 
@@ -155,11 +196,11 @@ Insert records into this Table.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
 
 #### Returns
 
-`Promise`<`number`\>
+`Promise`\<`number`\>
 
 The number of rows added to the table
 
@@ -169,19 +210,69 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:320](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L320)
+[index.ts:507](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L507)
+
+___
+
+### cleanupOldVersions
+
+▸ **cleanupOldVersions**(`olderThan?`, `deleteUnverified?`): `Promise`\<[`CleanupStats`](../interfaces/CleanupStats.md)\>
+
+Clean up old versions of the table, freeing disk space.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `olderThan?` | `number` | The minimum age in minutes of the versions to delete. If not provided, defaults to two weeks. |
+| `deleteUnverified?` | `boolean` | Because they may be part of an in-progress transaction, uncommitted files newer than 7 days old are not deleted by default. This means that failed transactions can leave around data that takes up disk space for up to 7 days. You can override this safety mechanism by setting this option to `true`, only if you promise there are no in progress writes while you run this operation. Failure to uphold this promise can lead to corrupted tables. |
+
+#### Returns
+
+`Promise`\<[`CleanupStats`](../interfaces/CleanupStats.md)\>
+
+#### Defined in
+
+[index.ts:596](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L596)
+
+___
+
+### compactFiles
+
+▸ **compactFiles**(`options?`): `Promise`\<[`CompactionMetrics`](../interfaces/CompactionMetrics.md)\>
+
+Run the compaction process on the table.
+
+This can be run after making several small appends to optimize the table
+for faster reads.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `options?` | [`CompactionOptions`](../interfaces/CompactionOptions.md) | Advanced options configuring compaction. In most cases, you can omit this arguments, as the default options are sensible for most tables. |
+
+#### Returns
+
+`Promise`\<[`CompactionMetrics`](../interfaces/CompactionMetrics.md)\>
+
+Metrics about the compaction operation.
+
+#### Defined in
+
+[index.ts:615](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L615)
 
 ___
 
 ### countRows
 
-▸ **countRows**(): `Promise`<`number`\>
+▸ **countRows**(): `Promise`\<`number`\>
 
 Returns the number of rows in this table.
 
 #### Returns
 
-`Promise`<`number`\>
+`Promise`\<`number`\>
 
 #### Implementation of
 
@@ -189,20 +280,16 @@ Returns the number of rows in this table.
 
 #### Defined in
 
-[index.ts:362](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L362)
+[index.ts:543](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L543)
 
 ___
 
 ### createIndex
 
-▸ **createIndex**(`indexParams`): `Promise`<`any`\>
+▸ **createIndex**(`indexParams`): `Promise`\<`any`\>
 
 Create an ANN index on this Table vector index.
 
-**`See`**
-
-VectorIndexParams.
-
 #### Parameters
 
 | Name | Type | Description |
@@ -211,7 +298,11 @@ VectorIndexParams.
 
 #### Returns
 
-`Promise`<`any`\>
+`Promise`\<`any`\>
+
+**`See`**
+
+VectorIndexParams.
 
 #### Implementation of
 
@@ -219,13 +310,13 @@ VectorIndexParams.
 
 #### Defined in
 
-[index.ts:355](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L355)
+[index.ts:536](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L536)
 
 ___
 
 ### delete
 
-▸ **delete**(`filter`): `Promise`<`void`\>
+▸ **delete**(`filter`): `Promise`\<`void`\>
 
 Delete rows from this table.
 
@@ -237,7 +328,7 @@ Delete rows from this table.
 
 #### Returns
 
-`Promise`<`void`\>
+`Promise`\<`void`\>
 
 #### Implementation of
 
@@ -245,13 +336,81 @@ Delete rows from this table.
 
 #### Defined in
 
-[index.ts:371](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L371)
+[index.ts:552](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L552)
+
+___
+
+### filter
+
+▸ **filter**(`value`): [`Query`](Query.md)\<`T`\>
+
+Creates a filter query to find all rows matching the specified criteria
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `value` | `string` | The filter criteria (like SQL where clause syntax) |
+
+#### Returns
+
+[`Query`](Query.md)\<`T`\>
+
+#### Defined in
+
+[index.ts:495](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L495)
+
+___
+
+### indexStats
+
+▸ **indexStats**(`indexUuid`): `Promise`\<[`IndexStats`](../interfaces/IndexStats.md)\>
+
+Get statistics about an index.
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `indexUuid` | `string` |
+
+#### Returns
+
+`Promise`\<[`IndexStats`](../interfaces/IndexStats.md)\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[indexStats](../interfaces/Table.md#indexstats)
+
+#### Defined in
+
+[index.ts:628](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L628)
+
+___
+
+### listIndices
+
+▸ **listIndices**(): `Promise`\<[`VectorIndex`](../interfaces/VectorIndex.md)[]\>
+
+List the indicies on this table.
+
+#### Returns
+
+`Promise`\<[`VectorIndex`](../interfaces/VectorIndex.md)[]\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[listIndices](../interfaces/Table.md#listindices)
+
+#### Defined in
+
+[index.ts:624](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L624)
 
 ___
 
 ### overwrite
 
-▸ **overwrite**(`data`): `Promise`<`number`\>
+▸ **overwrite**(`data`): `Promise`\<`number`\>
 
 Insert records into this Table, replacing its contents.
 
@@ -259,11 +418,11 @@ Insert records into this Table, replacing its contents.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
 
 #### Returns
 
-`Promise`<`number`\>
+`Promise`\<`number`\>
 
 The number of rows added to the table
 
@@ -273,13 +432,13 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:338](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L338)
+[index.ts:522](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L522)
 
 ___
 
 ### search
 
-▸ **search**(`query`): [`Query`](Query.md)<`T`\>
+▸ **search**(`query`): [`Query`](Query.md)\<`T`\>
 
 Creates a search query to find the nearest neighbors of the given search term
 
@@ -291,7 +450,7 @@ Creates a search query to find the nearest neighbors of the given search term
 
 #### Returns
 
-[`Query`](Query.md)<`T`\>
+[`Query`](Query.md)\<`T`\>
 
 #### Implementation of
 
@@ -299,4 +458,30 @@ Creates a search query to find the nearest neighbors of the given search term
 
 #### Defined in
 
-[index.ts:310](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L310)
+[index.ts:487](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L487)
+
+___
+
+### update
+
+▸ **update**(`args`): `Promise`\<`void`\>
+
+Update rows in this table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `args` | [`UpdateArgs`](../interfaces/UpdateArgs.md) \| [`UpdateSqlArgs`](../interfaces/UpdateSqlArgs.md) | see [UpdateArgs](../interfaces/UpdateArgs.md) and [UpdateSqlArgs](../interfaces/UpdateSqlArgs.md) for more details |
+
+#### Returns
+
+`Promise`\<`void`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[update](../interfaces/Table.md#update)
+
+#### Defined in
+
+[index.ts:563](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L563)

docs/src/javascript/classes/OpenAIEmbeddingFunction.md

@@ -6,7 +6,7 @@ An embedding function that automatically creates vector representation for a giv
 
 ## Implements
 
-- [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`string`\>
+- [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`string`\>
 
 ## Table of contents
 
@@ -40,7 +40,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L21)
+[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L21)
 
 ## Properties
 
@@ -50,7 +50,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L19)
+[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L19)
 
 ___
 
@@ -60,7 +60,7 @@ ___
 
 #### Defined in
 
-[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L18)
+[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L18)
 
 ___
 
@@ -76,13 +76,13 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L50)
+[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L50)
 
 ## Methods
 
 ### embed
 
-▸ **embed**(`data`): `Promise`<`number`[][]\>
+▸ **embed**(`data`): `Promise`\<`number`[][]\>
 
 Creates a vector representation for the given values.
 
@@ -94,7 +94,7 @@ Creates a vector representation for the given values.
 
 #### Returns
 
-`Promise`<`number`[][]\>
+`Promise`\<`number`[][]\>
 
 #### Implementation of
 
@@ -102,4 +102,4 @@ Creates a vector representation for the given values.
 
 #### Defined in
 
-[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L38)
+[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L38)

docs/src/javascript/classes/Query.md

@@ -1,6 +1,6 @@
 [vectordb](../README.md) / [Exports](../modules.md) / Query
 
-# Class: Query<T\>
+# Class: Query\<T\>
 
 A builder for nearest neighbor queries for LanceDB.
 
@@ -23,6 +23,7 @@ A builder for nearest neighbor queries for LanceDB.
 - [\_limit](Query.md#_limit)
 - [\_metricType](Query.md#_metrictype)
 - [\_nprobes](Query.md#_nprobes)
+- [\_prefilter](Query.md#_prefilter)
 - [\_query](Query.md#_query)
 - [\_queryVector](Query.md#_queryvector)
 - [\_refineFactor](Query.md#_refinefactor)
@@ -34,9 +35,11 @@ A builder for nearest neighbor queries for LanceDB.
 
 - [execute](Query.md#execute)
 - [filter](Query.md#filter)
+- [isElectron](Query.md#iselectron)
 - [limit](Query.md#limit)
 - [metricType](Query.md#metrictype)
 - [nprobes](Query.md#nprobes)
+- [prefilter](Query.md#prefilter)
 - [refineFactor](Query.md#refinefactor)
 - [select](Query.md#select)
 
@@ -44,7 +47,7 @@ A builder for nearest neighbor queries for LanceDB.
 
 ### constructor
 
-• **new Query**<`T`\>(`tbl`, `query`, `embeddings?`)
+• **new Query**\<`T`\>(`query?`, `tbl?`, `embeddings?`)
 
 #### Type parameters
 
@@ -56,23 +59,23 @@ A builder for nearest neighbor queries for LanceDB.
 
 | Name | Type |
 | :------ | :------ |
-| `tbl` | `any` |
-| `query` | `T` |
-| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
+| `query?` | `T` |
+| `tbl?` | `any` |
+| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> |
 
 #### Defined in
 
-[index.ts:448](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L448)
+[query.ts:38](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L38)
 
 ## Properties
 
 ### \_embeddings
 
-• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
+• `Protected` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\>
 
 #### Defined in
 
-[index.ts:446](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L446)
+[query.ts:36](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L36)
 
 ___
 
@@ -82,17 +85,17 @@ ___
 
 #### Defined in
 
-[index.ts:444](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L444)
+[query.ts:33](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L33)
 
 ___
 
 ### \_limit
 
-• `Private` **\_limit**: `number`
+• `Private` `Optional` **\_limit**: `number`
 
 #### Defined in
 
-[index.ts:440](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L440)
+[query.ts:29](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L29)
 
 ___
 
@@ -102,7 +105,7 @@ ___
 
 #### Defined in
 
-[index.ts:445](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L445)
+[query.ts:34](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L34)
 
 ___
 
@@ -112,17 +115,27 @@ ___
 
 #### Defined in
 
-[index.ts:442](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L442)
+[query.ts:31](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L31)
+
+___
+
+### \_prefilter
+
+• `Private` **\_prefilter**: `boolean`
+
+#### Defined in
+
+[query.ts:35](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L35)
 
 ___
 
 ### \_query
 
-• `Private` `Readonly` **\_query**: `T`
+• `Private` `Optional` `Readonly` **\_query**: `T`
 
 #### Defined in
 
-[index.ts:438](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L438)
+[query.ts:26](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L26)
 
 ___
 
@@ -132,7 +145,7 @@ ___
 
 #### Defined in
 
-[index.ts:439](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L439)
+[query.ts:28](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L28)
 
 ___
 
@@ -142,7 +155,7 @@ ___
 
 #### Defined in
 
-[index.ts:441](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L441)
+[query.ts:30](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L30)
 
 ___
 
@@ -152,27 +165,27 @@ ___
 
 #### Defined in
 
-[index.ts:443](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L443)
+[query.ts:32](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L32)
 
 ___
 
 ### \_tbl
 
-• `Private` `Readonly` **\_tbl**: `any`
+• `Private` `Optional` `Readonly` **\_tbl**: `any`
 
 #### Defined in
 
-[index.ts:437](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L437)
+[query.ts:27](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L27)
 
 ___
 
 ### where
 
-• **where**: (`value`: `string`) => [`Query`](Query.md)<`T`\>
+• **where**: (`value`: `string`) => [`Query`](Query.md)\<`T`\>
 
 #### Type declaration
 
-▸ (`value`): [`Query`](Query.md)<`T`\>
+▸ (`value`): [`Query`](Query.md)\<`T`\>
 
 A filter statement to be applied to this query.
 
@@ -184,17 +197,17 @@ A filter statement to be applied to this query.
 
 ##### Returns
 
-[`Query`](Query.md)<`T`\>
+[`Query`](Query.md)\<`T`\>
 
 #### Defined in
 
-[index.ts:496](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L496)
+[query.ts:87](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L87)
 
 ## Methods
 
 ### execute
 
-▸ **execute**<`T`\>(): `Promise`<`T`[]\>
+▸ **execute**\<`T`\>(): `Promise`\<`T`[]\>
 
 Execute the query and return the results as an Array of Objects
 
@@ -202,21 +215,21 @@ Execute the query and return the results as an Array of Objects
 
 | Name | Type |
 | :------ | :------ |
-| `T` | `Record`<`string`, `unknown`\> |
+| `T` | `Record`\<`string`, `unknown`\> |
 
 #### Returns
 
-`Promise`<`T`[]\>
+`Promise`\<`T`[]\>
 
 #### Defined in
 
-[index.ts:519](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L519)
+[query.ts:115](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L115)
 
 ___
 
 ### filter
 
-▸ **filter**(`value`): [`Query`](Query.md)<`T`\>
+▸ **filter**(`value`): [`Query`](Query.md)\<`T`\>
 
 A filter statement to be applied to this query.
 
@@ -228,17 +241,31 @@ A filter statement to be applied to this query.
 
 #### Returns
 
-[`Query`](Query.md)<`T`\>
+[`Query`](Query.md)\<`T`\>
 
 #### Defined in
 
-[index.ts:491](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L491)
+[query.ts:82](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L82)
+
+___
+
+### isElectron
+
+▸ `Private` **isElectron**(): `boolean`
+
+#### Returns
+
+`boolean`
+
+#### Defined in
+
+[query.ts:142](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L142)
 
 ___
 
 ### limit
 
-▸ **limit**(`value`): [`Query`](Query.md)<`T`\>
+▸ **limit**(`value`): [`Query`](Query.md)\<`T`\>
 
 Sets the number of results that will be returned
 
@@ -250,24 +277,20 @@ Sets the number of results that will be returned
 
 #### Returns
 
-[`Query`](Query.md)<`T`\>
+[`Query`](Query.md)\<`T`\>
 
 #### Defined in
 
-[index.ts:464](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L464)
+[query.ts:55](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L55)
 
 ___
 
 ### metricType
 
-▸ **metricType**(`value`): [`Query`](Query.md)<`T`\>
+▸ **metricType**(`value`): [`Query`](Query.md)\<`T`\>
 
 The MetricType used for this Query.
 
-**`See`**
-
-MetricType for the different options
-
 #### Parameters
 
 | Name | Type | Description |
@@ -276,17 +299,21 @@ MetricType for the different options
 
 #### Returns
 
-[`Query`](Query.md)<`T`\>
+[`Query`](Query.md)\<`T`\>
+
+**`See`**
+
+MetricType for the different options
 
 #### Defined in
 
-[index.ts:511](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L511)
+[query.ts:102](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L102)
 
 ___
 
 ### nprobes
 
-▸ **nprobes**(`value`): [`Query`](Query.md)<`T`\>
+▸ **nprobes**(`value`): [`Query`](Query.md)\<`T`\>
 
 The number of probes used. A higher number makes search more accurate but also slower.
 
@@ -298,17 +325,37 @@ The number of probes used. A higher number makes search more accurate but also s
 
 #### Returns
 
-[`Query`](Query.md)<`T`\>
+[`Query`](Query.md)\<`T`\>
 
 #### Defined in
 
-[index.ts:482](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L482)
+[query.ts:73](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L73)
+
+___
+
+### prefilter
+
+▸ **prefilter**(`value`): [`Query`](Query.md)\<`T`\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `value` | `boolean` |
+
+#### Returns
+
+[`Query`](Query.md)\<`T`\>
+
+#### Defined in
+
+[query.ts:107](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L107)
 
 ___
 
 ### refineFactor
 
-▸ **refineFactor**(`value`): [`Query`](Query.md)<`T`\>
+▸ **refineFactor**(`value`): [`Query`](Query.md)\<`T`\>
 
 Refine the results by reading extra elements and re-ranking them in memory.
 
@@ -320,17 +367,17 @@ Refine the results by reading extra elements and re-ranking them in memory.
 
 #### Returns
 
-[`Query`](Query.md)<`T`\>
+[`Query`](Query.md)\<`T`\>
 
 #### Defined in
 
-[index.ts:473](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L473)
+[query.ts:64](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L64)
 
 ___
 
 ### select
 
-▸ **select**(`value`): [`Query`](Query.md)<`T`\>
+▸ **select**(`value`): [`Query`](Query.md)\<`T`\>
 
 Return only the specified columns.
 
@@ -342,8 +389,8 @@ Return only the specified columns.
 
 #### Returns
 
-[`Query`](Query.md)<`T`\>
+[`Query`](Query.md)\<`T`\>
 
 #### Defined in
 
-[index.ts:502](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L502)
+[query.ts:93](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L93)

docs/src/javascript/enums/MetricType.md

@@ -22,7 +22,7 @@ Cosine distance
 
 #### Defined in
 
-[index.ts:567](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L567)
+[index.ts:798](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L798)
 
 ___
 
@@ -34,7 +34,7 @@ Dot product
 
 #### Defined in
 
-[index.ts:572](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L572)
+[index.ts:803](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L803)
 
 ___
 
@@ -46,4 +46,4 @@ Euclidean distance
 
 #### Defined in
 
-[index.ts:562](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L562)
+[index.ts:793](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L793)

docs/src/javascript/enums/WriteMode.md

@@ -22,7 +22,7 @@ Append new data to the table.
 
 #### Defined in
 
-[index.ts:552](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L552)
+[index.ts:766](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L766)
 
 ___
 
@@ -34,7 +34,7 @@ Create a new [Table](../interfaces/Table.md).
 
 #### Defined in
 
-[index.ts:548](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L548)
+[index.ts:762](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L762)
 
 ___
 
@@ -46,4 +46,4 @@ Overwrite the existing [Table](../interfaces/Table.md) if presented.
 
 #### Defined in
 
-[index.ts:550](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L550)
+[index.ts:764](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L764)

docs/src/javascript/interfaces/AwsCredentials.md

@@ -18,7 +18,7 @@
 
 #### Defined in
 
-[index.ts:31](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L31)
+[index.ts:34](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L34)
 
 ___
 
@@ -28,7 +28,7 @@ ___
 
 #### Defined in
 
-[index.ts:33](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L33)
+[index.ts:36](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L36)
 
 ___
 
@@ -38,4 +38,4 @@ ___
 
 #### Defined in
 
-[index.ts:35](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L35)
+[index.ts:38](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L38)

docs/src/javascript/interfaces/CleanupStats.md

@@ -0,0 +1,34 @@
+[vectordb](../README.md) / [Exports](../modules.md) / CleanupStats
+
+# Interface: CleanupStats
+
+## Table of contents
+
+### Properties
+
+- [bytesRemoved](CleanupStats.md#bytesremoved)
+- [oldVersions](CleanupStats.md#oldversions)
+
+## Properties
+
+### bytesRemoved
+
+• **bytesRemoved**: `number`
+
+The number of bytes removed from disk.
+
+#### Defined in
+
+[index.ts:637](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L637)
+
+___
+
+### oldVersions
+
+• **oldVersions**: `number`
+
+The number of old table versions removed.
+
+#### Defined in
+
+[index.ts:641](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L641)

docs/src/javascript/interfaces/CompactionMetrics.md

@@ -0,0 +1,62 @@
+[vectordb](../README.md) / [Exports](../modules.md) / CompactionMetrics
+
+# Interface: CompactionMetrics
+
+## Table of contents
+
+### Properties
+
+- [filesAdded](CompactionMetrics.md#filesadded)
+- [filesRemoved](CompactionMetrics.md#filesremoved)
+- [fragmentsAdded](CompactionMetrics.md#fragmentsadded)
+- [fragmentsRemoved](CompactionMetrics.md#fragmentsremoved)
+
+## Properties
+
+### filesAdded
+
+• **filesAdded**: `number`
+
+The number of files added. This is typically equal to the number of
+fragments added.
+
+#### Defined in
+
+[index.ts:692](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L692)
+
+___
+
+### filesRemoved
+
+• **filesRemoved**: `number`
+
+The number of files that were removed. Each fragment may have more than one
+file.
+
+#### Defined in
+
+[index.ts:687](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L687)
+
+___
+
+### fragmentsAdded
+
+• **fragmentsAdded**: `number`
+
+The number of new fragments that were created.
+
+#### Defined in
+
+[index.ts:682](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L682)
+
+___
+
+### fragmentsRemoved
+
+• **fragmentsRemoved**: `number`
+
+The number of fragments that were removed.
+
+#### Defined in
+
+[index.ts:678](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L678)

docs/src/javascript/interfaces/CompactionOptions.md

@@ -0,0 +1,80 @@
+[vectordb](../README.md) / [Exports](../modules.md) / CompactionOptions
+
+# Interface: CompactionOptions
+
+## Table of contents
+
+### Properties
+
+- [materializeDeletions](CompactionOptions.md#materializedeletions)
+- [materializeDeletionsThreshold](CompactionOptions.md#materializedeletionsthreshold)
+- [maxRowsPerGroup](CompactionOptions.md#maxrowspergroup)
+- [numThreads](CompactionOptions.md#numthreads)
+- [targetRowsPerFragment](CompactionOptions.md#targetrowsperfragment)
+
+## Properties
+
+### materializeDeletions
+
+• `Optional` **materializeDeletions**: `boolean`
+
+If true, fragments that have rows that are deleted may be compacted to
+remove the deleted rows. This can improve the performance of queries.
+Default is true.
+
+#### Defined in
+
+[index.ts:660](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L660)
+
+___
+
+### materializeDeletionsThreshold
+
+• `Optional` **materializeDeletionsThreshold**: `number`
+
+A number between 0 and 1, representing the proportion of rows that must be
+marked deleted before a fragment is a candidate for compaction to remove
+the deleted rows. Default is 10%.
+
+#### Defined in
+
+[index.ts:666](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L666)
+
+___
+
+### maxRowsPerGroup
+
+• `Optional` **maxRowsPerGroup**: `number`
+
+The maximum number of rows per group. Defaults to 1024.
+
+#### Defined in
+
+[index.ts:654](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L654)
+
+___
+
+### numThreads
+
+• `Optional` **numThreads**: `number`
+
+The number of threads to use for compaction. If not provided, defaults to
+the number of cores on the machine.
+
+#### Defined in
+
+[index.ts:671](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L671)
+
+___
+
+### targetRowsPerFragment
+
+• `Optional` **targetRowsPerFragment**: `number`
+
+The number of rows per fragment to target. Fragments that have fewer rows
+will be compacted into adjacent fragments to produce larger fragments.
+Defaults to 1024 * 1024.
+
+#### Defined in
+
+[index.ts:650](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L650)

docs/src/javascript/interfaces/Connection.md

@@ -19,7 +19,6 @@ Connection could be local against filesystem or remote against a server.
 ### Methods
 
 - [createTable](Connection.md#createtable)
-- [createTableArrow](Connection.md#createtablearrow)
 - [dropTable](Connection.md#droptable)
 - [openTable](Connection.md#opentable)
 - [tableNames](Connection.md#tablenames)
@@ -32,13 +31,76 @@ Connection could be local against filesystem or remote against a server.
 
 #### Defined in
 
-[index.ts:70](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L70)
+[index.ts:125](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L125)
 
 ## Methods
 
 ### createTable
 
-▸ **createTable**<`T`\>(`name`, `data`, `mode?`, `embeddings?`): `Promise`<[`Table`](Table.md)<`T`\>\>
+▸ **createTable**\<`T`\>(`«destructured»`): `Promise`\<[`Table`](Table.md)\<`T`\>\>
+
+Creates a new Table, optionally initializing it with new data.
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `«destructured»` | [`CreateTableOptions`](CreateTableOptions.md)\<`T`\> |
+
+#### Returns
+
+`Promise`\<[`Table`](Table.md)\<`T`\>\>
+
+#### Defined in
+
+[index.ts:146](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L146)
+
+▸ **createTable**(`name`, `data`): `Promise`\<[`Table`](Table.md)\<`number`[]\>\>
+
+Creates a new Table and initialize it with new data.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `data` | `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
+
+#### Returns
+
+`Promise`\<[`Table`](Table.md)\<`number`[]\>\>
+
+#### Defined in
+
+[index.ts:154](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L154)
+
+▸ **createTable**(`name`, `data`, `options`): `Promise`\<[`Table`](Table.md)\<`number`[]\>\>
+
+Creates a new Table and initialize it with new data.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `data` | `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
+| `options` | [`WriteOptions`](WriteOptions.md) | The write options to use when creating the table. |
+
+#### Returns
+
+`Promise`\<[`Table`](Table.md)\<`number`[]\>\>
+
+#### Defined in
+
+[index.ts:163](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L163)
+
+▸ **createTable**\<`T`\>(`name`, `data`, `embeddings`): `Promise`\<[`Table`](Table.md)\<`T`\>\>
 
 Creates a new Table and initialize it with new data.
 
@@ -53,44 +115,49 @@ Creates a new Table and initialize it with new data.
 | Name | Type | Description |
 | :------ | :------ | :------ |
 | `name` | `string` | The name of the table. |
-| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
-| `mode?` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
-| `embeddings?` | [`EmbeddingFunction`](EmbeddingFunction.md)<`T`\> | An embedding function to use on this table |
+| `data` | `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
+| `embeddings` | [`EmbeddingFunction`](EmbeddingFunction.md)\<`T`\> | An embedding function to use on this table |
 
 #### Returns
 
-`Promise`<[`Table`](Table.md)<`T`\>\>
+`Promise`\<[`Table`](Table.md)\<`T`\>\>
 
 #### Defined in
 
-[index.ts:90](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L90)
+[index.ts:172](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L172)
 
-___
+▸ **createTable**\<`T`\>(`name`, `data`, `embeddings`, `options`): `Promise`\<[`Table`](Table.md)\<`T`\>\>
 
-### createTableArrow
+Creates a new Table and initialize it with new data.
 
-▸ **createTableArrow**(`name`, `table`): `Promise`<[`Table`](Table.md)<`number`[]\>\>
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
 
 #### Parameters
 
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` |
-| `table` | `Table`<`any`\> |
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `data` | `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
+| `embeddings` | [`EmbeddingFunction`](EmbeddingFunction.md)\<`T`\> | An embedding function to use on this table |
+| `options` | [`WriteOptions`](WriteOptions.md) | The write options to use when creating the table. |
 
 #### Returns
 
-`Promise`<[`Table`](Table.md)<`number`[]\>\>
+`Promise`\<[`Table`](Table.md)\<`T`\>\>
 
 #### Defined in
 
-[index.ts:92](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L92)
+[index.ts:181](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L181)
 
 ___
 
 ### dropTable
 
-▸ **dropTable**(`name`): `Promise`<`void`\>
+▸ **dropTable**(`name`): `Promise`\<`void`\>
 
 Drop an existing table.
 
@@ -102,17 +169,17 @@ Drop an existing table.
 
 #### Returns
 
-`Promise`<`void`\>
+`Promise`\<`void`\>
 
 #### Defined in
 
-[index.ts:98](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L98)
+[index.ts:187](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L187)
 
 ___
 
 ### openTable
 
-▸ **openTable**<`T`\>(`name`, `embeddings?`): `Promise`<[`Table`](Table.md)<`T`\>\>
+▸ **openTable**\<`T`\>(`name`, `embeddings?`): `Promise`\<[`Table`](Table.md)\<`T`\>\>
 
 Open a table in the database.
 
@@ -127,26 +194,26 @@ Open a table in the database.
 | Name | Type | Description |
 | :------ | :------ | :------ |
 | `name` | `string` | The name of the table. |
-| `embeddings?` | [`EmbeddingFunction`](EmbeddingFunction.md)<`T`\> | An embedding function to use on this table |
+| `embeddings?` | [`EmbeddingFunction`](EmbeddingFunction.md)\<`T`\> | An embedding function to use on this table |
 
 #### Returns
 
-`Promise`<[`Table`](Table.md)<`T`\>\>
+`Promise`\<[`Table`](Table.md)\<`T`\>\>
 
 #### Defined in
 
-[index.ts:80](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L80)
+[index.ts:135](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L135)
 
 ___
 
 ### tableNames
 
-▸ **tableNames**(): `Promise`<`string`[]\>
+▸ **tableNames**(): `Promise`\<`string`[]\>
 
 #### Returns
 
-`Promise`<`string`[]\>
+`Promise`\<`string`[]\>
 
 #### Defined in
 
-[index.ts:72](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L72)
+[index.ts:127](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L127)

docs/src/javascript/interfaces/ConnectionOptions.md

@@ -6,18 +6,62 @@
 
 ### Properties
 
+- [apiKey](ConnectionOptions.md#apikey)
 - [awsCredentials](ConnectionOptions.md#awscredentials)
+- [awsRegion](ConnectionOptions.md#awsregion)
+- [hostOverride](ConnectionOptions.md#hostoverride)
+- [region](ConnectionOptions.md#region)
 - [uri](ConnectionOptions.md#uri)
 
 ## Properties
 
+### apiKey
+
+• `Optional` **apiKey**: `string`
+
+#### Defined in
+
+[index.ts:49](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L49)
+
+___
+
 ### awsCredentials
 
 • `Optional` **awsCredentials**: [`AwsCredentials`](AwsCredentials.md)
 
 #### Defined in
 
-[index.ts:40](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L40)
+[index.ts:44](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L44)
+
+___
+
+### awsRegion
+
+• `Optional` **awsRegion**: `string`
+
+#### Defined in
+
+[index.ts:46](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L46)
+
+___
+
+### hostOverride
+
+• `Optional` **hostOverride**: `string`
+
+#### Defined in
+
+[index.ts:54](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L54)
+
+___
+
+### region
+
+• `Optional` **region**: `string`
+
+#### Defined in
+
+[index.ts:51](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L51)
 
 ___
 
@@ -27,4 +71,4 @@ ___
 
 #### Defined in
 
-[index.ts:39](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L39)
+[index.ts:42](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L42)

docs/src/javascript/interfaces/CreateTableOptions.md

@@ -0,0 +1,69 @@
+[vectordb](../README.md) / [Exports](../modules.md) / CreateTableOptions
+
+# Interface: CreateTableOptions\<T\>
+
+## Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+## Table of contents
+
+### Properties
+
+- [data](CreateTableOptions.md#data)
+- [embeddingFunction](CreateTableOptions.md#embeddingfunction)
+- [name](CreateTableOptions.md#name)
+- [schema](CreateTableOptions.md#schema)
+- [writeOptions](CreateTableOptions.md#writeoptions)
+
+## Properties
+
+### data
+
+• `Optional` **data**: `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[]
+
+#### Defined in
+
+[index.ts:79](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L79)
+
+___
+
+### embeddingFunction
+
+• `Optional` **embeddingFunction**: [`EmbeddingFunction`](EmbeddingFunction.md)\<`T`\>
+
+#### Defined in
+
+[index.ts:85](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L85)
+
+___
+
+### name
+
+• **name**: `string`
+
+#### Defined in
+
+[index.ts:76](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L76)
+
+___
+
+### schema
+
+• `Optional` **schema**: `Schema`\<`any`\>
+
+#### Defined in
+
+[index.ts:82](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L82)
+
+___
+
+### writeOptions
+
+• `Optional` **writeOptions**: [`WriteOptions`](WriteOptions.md)
+
+#### Defined in
+
+[index.ts:88](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L88)

docs/src/javascript/interfaces/EmbeddingFunction.md

@@ -1,6 +1,6 @@
 [vectordb](../README.md) / [Exports](../modules.md) / EmbeddingFunction
 
-# Interface: EmbeddingFunction<T\>
+# Interface: EmbeddingFunction\<T\>
 
 An embedding function that automatically creates vector representation for a given column.
 
@@ -25,11 +25,11 @@ An embedding function that automatically creates vector representation for a giv
 
 ### embed
 
-• **embed**: (`data`: `T`[]) => `Promise`<`number`[][]\>
+• **embed**: (`data`: `T`[]) => `Promise`\<`number`[][]\>
 
 #### Type declaration
 
-▸ (`data`): `Promise`<`number`[][]\>
+▸ (`data`): `Promise`\<`number`[][]\>
 
 Creates a vector representation for the given values.
 
@@ -41,11 +41,11 @@ Creates a vector representation for the given values.
 
 ##### Returns
 
-`Promise`<`number`[][]\>
+`Promise`\<`number`[][]\>
 
 #### Defined in
 
-[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/embedding_function.ts#L27)
+[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/embedding_function.ts#L27)
 
 ___
 
@@ -57,4 +57,4 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/embedding_function.ts#L22)
+[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/embedding_function.ts#L22)

docs/src/javascript/interfaces/IndexStats.md

@@ -0,0 +1,30 @@
+[vectordb](../README.md) / [Exports](../modules.md) / IndexStats
+
+# Interface: IndexStats
+
+## Table of contents
+
+### Properties
+
+- [numIndexedRows](IndexStats.md#numindexedrows)
+- [numUnindexedRows](IndexStats.md#numunindexedrows)
+
+## Properties
+
+### numIndexedRows
+
+• **numIndexedRows**: ``null`` \| `number`
+
+#### Defined in
+
+[index.ts:344](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L344)
+
+___
+
+### numUnindexedRows
+
+• **numUnindexedRows**: ``null`` \| `number`
+
+#### Defined in
+
+[index.ts:345](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L345)

docs/src/javascript/interfaces/IvfPQIndexConfig.md

@@ -7,6 +7,7 @@
 ### Properties
 
 - [column](IvfPQIndexConfig.md#column)
+- [index\_cache\_size](IvfPQIndexConfig.md#index_cache_size)
 - [index\_name](IvfPQIndexConfig.md#index_name)
 - [max\_iters](IvfPQIndexConfig.md#max_iters)
 - [max\_opq\_iters](IvfPQIndexConfig.md#max_opq_iters)
@@ -28,7 +29,19 @@ The column to be indexed
 
 #### Defined in
 
-[index.ts:382](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L382)
+[index.ts:701](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L701)
+
+___
+
+### index\_cache\_size
+
+• `Optional` **index\_cache\_size**: `number`
+
+Cache size of the index
+
+#### Defined in
+
+[index.ts:750](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L750)
 
 ___
 
@@ -40,7 +53,7 @@ A unique name for the index
 
 #### Defined in
 
-[index.ts:387](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L387)
+[index.ts:706](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L706)
 
 ___
 
@@ -52,7 +65,7 @@ The max number of iterations for kmeans training.
 
 #### Defined in
 
-[index.ts:402](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L402)
+[index.ts:721](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L721)
 
 ___
 
@@ -64,7 +77,7 @@ Max number of iterations to train OPQ, if `use_opq` is true.
 
 #### Defined in
 
-[index.ts:421](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L421)
+[index.ts:740](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L740)
 
 ___
 
@@ -76,7 +89,7 @@ Metric type, L2 or Cosine
 
 #### Defined in
 
-[index.ts:392](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L392)
+[index.ts:711](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L711)
 
 ___
 
@@ -88,7 +101,7 @@ The number of bits to present one PQ centroid.
 
 #### Defined in
 
-[index.ts:416](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L416)
+[index.ts:735](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L735)
 
 ___
 
@@ -100,7 +113,7 @@ The number of partitions this index
 
 #### Defined in
 
-[index.ts:397](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L397)
+[index.ts:716](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L716)
 
 ___
 
@@ -112,7 +125,7 @@ Number of subvectors to build PQ code
 
 #### Defined in
 
-[index.ts:412](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L412)
+[index.ts:731](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L731)
 
 ___
 
@@ -124,7 +137,7 @@ Replace an existing index with the same name if it exists.
 
 #### Defined in
 
-[index.ts:426](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L426)
+[index.ts:745](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L745)
 
 ___
 
@@ -134,7 +147,7 @@ ___
 
 #### Defined in
 
-[index.ts:428](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L428)
+[index.ts:752](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L752)
 
 ___
 
@@ -146,4 +159,4 @@ Train as optimized product quantization.
 
 #### Defined in
 
-[index.ts:407](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L407)
+[index.ts:726](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L726)

docs/src/javascript/interfaces/Table.md

@@ -1,6 +1,6 @@
 [vectordb](../README.md) / [Exports](../modules.md) / Table
 
-# Interface: Table<T\>
+# Interface: Table\<T\>
 
 A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
 
@@ -22,19 +22,22 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 - [countRows](Table.md#countrows)
 - [createIndex](Table.md#createindex)
 - [delete](Table.md#delete)
+- [indexStats](Table.md#indexstats)
+- [listIndices](Table.md#listindices)
 - [name](Table.md#name)
 - [overwrite](Table.md#overwrite)
 - [search](Table.md#search)
+- [update](Table.md#update)
 
 ## Properties
 
 ### add
 
-• **add**: (`data`: `Record`<`string`, `unknown`\>[]) => `Promise`<`number`\>
+• **add**: (`data`: `Record`\<`string`, `unknown`\>[]) => `Promise`\<`number`\>
 
 #### Type declaration
 
-▸ (`data`): `Promise`<`number`\>
+▸ (`data`): `Promise`\<`number`\>
 
 Insert records into this Table.
 
@@ -42,54 +45,50 @@ Insert records into this Table.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
 
 ##### Returns
 
-`Promise`<`number`\>
+`Promise`\<`number`\>
 
 The number of rows added to the table
 
 #### Defined in
 
-[index.ts:120](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L120)
+[index.ts:209](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L209)
 
 ___
 
 ### countRows
 
-• **countRows**: () => `Promise`<`number`\>
+• **countRows**: () => `Promise`\<`number`\>
 
 #### Type declaration
 
-▸ (): `Promise`<`number`\>
+▸ (): `Promise`\<`number`\>
 
 Returns the number of rows in this table.
 
 ##### Returns
 
-`Promise`<`number`\>
+`Promise`\<`number`\>
 
 #### Defined in
 
-[index.ts:140](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L140)
+[index.ts:229](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L229)
 
 ___
 
 ### createIndex
 
-• **createIndex**: (`indexParams`: [`IvfPQIndexConfig`](IvfPQIndexConfig.md)) => `Promise`<`any`\>
+• **createIndex**: (`indexParams`: [`IvfPQIndexConfig`](IvfPQIndexConfig.md)) => `Promise`\<`any`\>
 
 #### Type declaration
 
-▸ (`indexParams`): `Promise`<`any`\>
+▸ (`indexParams`): `Promise`\<`any`\>
 
 Create an ANN index on this Table vector index.
 
-**`See`**
-
-VectorIndexParams.
-
 ##### Parameters
 
 | Name | Type | Description |
@@ -98,27 +97,41 @@ VectorIndexParams.
 
 ##### Returns
 
-`Promise`<`any`\>
+`Promise`\<`any`\>
+
+**`See`**
+
+VectorIndexParams.
 
 #### Defined in
 
-[index.ts:135](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L135)
+[index.ts:224](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L224)
 
 ___
 
 ### delete
 
-• **delete**: (`filter`: `string`) => `Promise`<`void`\>
+• **delete**: (`filter`: `string`) => `Promise`\<`void`\>
 
 #### Type declaration
 
-▸ (`filter`): `Promise`<`void`\>
+▸ (`filter`): `Promise`\<`void`\>
 
 Delete rows from this table.
 
 This can be used to delete a single row, many rows, all rows, or
 sometimes no rows (if your predicate matches nothing).
 
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `filter` | `string` | A filter in the same format used by a sql WHERE clause. The filter must not be empty. |
+
+##### Returns
+
+`Promise`\<`void`\>
+
 **`Examples`**
 
 ```ts
@@ -142,19 +155,55 @@ await tbl.delete(`id IN (${to_remove.join(",")})`)
 await tbl.countRows() // Returns 1
 ```
 
+#### Defined in
+
+[index.ts:263](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L263)
+
+___
+
+### indexStats
+
+• **indexStats**: (`indexUuid`: `string`) => `Promise`\<[`IndexStats`](IndexStats.md)\>
+
+#### Type declaration
+
+▸ (`indexUuid`): `Promise`\<[`IndexStats`](IndexStats.md)\>
+
+Get statistics about an index.
+
 ##### Parameters
 
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `filter` | `string` | A filter in the same format used by a sql WHERE clause. The filter must not be empty. |
+| Name | Type |
+| :------ | :------ |
+| `indexUuid` | `string` |
 
 ##### Returns
 
-`Promise`<`void`\>
+`Promise`\<[`IndexStats`](IndexStats.md)\>
 
 #### Defined in
 
-[index.ts:174](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L174)
+[index.ts:306](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L306)
+
+___
+
+### listIndices
+
+• **listIndices**: () => `Promise`\<[`VectorIndex`](VectorIndex.md)[]\>
+
+#### Type declaration
+
+▸ (): `Promise`\<[`VectorIndex`](VectorIndex.md)[]\>
+
+List the indicies on this table.
+
+##### Returns
+
+`Promise`\<[`VectorIndex`](VectorIndex.md)[]\>
+
+#### Defined in
+
+[index.ts:301](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L301)
 
 ___
 
@@ -164,17 +213,17 @@ ___
 
 #### Defined in
 
-[index.ts:106](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L106)
+[index.ts:195](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L195)
 
 ___
 
 ### overwrite
 
-• **overwrite**: (`data`: `Record`<`string`, `unknown`\>[]) => `Promise`<`number`\>
+• **overwrite**: (`data`: `Record`\<`string`, `unknown`\>[]) => `Promise`\<`number`\>
 
 #### Type declaration
 
-▸ (`data`): `Promise`<`number`\>
+▸ (`data`): `Promise`\<`number`\>
 
 Insert records into this Table, replacing its contents.
 
@@ -182,27 +231,27 @@ Insert records into this Table, replacing its contents.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
 
 ##### Returns
 
-`Promise`<`number`\>
+`Promise`\<`number`\>
 
 The number of rows added to the table
 
 #### Defined in
 
-[index.ts:128](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L128)
+[index.ts:217](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L217)
 
 ___
 
 ### search
 
-• **search**: (`query`: `T`) => [`Query`](../classes/Query.md)<`T`\>
+• **search**: (`query`: `T`) => [`Query`](../classes/Query.md)\<`T`\>
 
 #### Type declaration
 
-▸ (`query`): [`Query`](../classes/Query.md)<`T`\>
+▸ (`query`): [`Query`](../classes/Query.md)\<`T`\>
 
 Creates a search query to find the nearest neighbors of the given search term
 
@@ -214,8 +263,59 @@ Creates a search query to find the nearest neighbors of the given search term
 
 ##### Returns
 
-[`Query`](../classes/Query.md)<`T`\>
+[`Query`](../classes/Query.md)\<`T`\>
 
 #### Defined in
 
-[index.ts:112](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L112)
+[index.ts:201](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L201)
+
+___
+
+### update
+
+• **update**: (`args`: [`UpdateArgs`](UpdateArgs.md) \| [`UpdateSqlArgs`](UpdateSqlArgs.md)) => `Promise`\<`void`\>
+
+#### Type declaration
+
+▸ (`args`): `Promise`\<`void`\>
+
+Update rows in this table.
+
+This can be used to update a single row, many rows, all rows, or
+sometimes no rows (if your predicate matches nothing).
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `args` | [`UpdateArgs`](UpdateArgs.md) \| [`UpdateSqlArgs`](UpdateSqlArgs.md) | see [UpdateArgs](UpdateArgs.md) and [UpdateSqlArgs](UpdateSqlArgs.md) for more details |
+
+##### Returns
+
+`Promise`\<`void`\>
+
+**`Examples`**
+
+```ts
+const con = await lancedb.connect("./.lancedb")
+const data = [
+   {id: 1, vector: [3, 3], name: 'Ye'},
+   {id: 2, vector: [4, 4], name: 'Mike'},
+];
+const tbl = await con.createTable("my_table", data)
+
+await tbl.update({
+  filter: "id = 2",
+  updates: { vector: [2, 2], name: "Michael" },
+})
+
+let results = await tbl.search([1, 1]).execute();
+// Returns [
+//   {id: 2, vector: [2, 2], name: 'Michael'}
+//   {id: 1, vector: [3, 3], name: 'Ye'}
+// ]
+```
+
+#### Defined in
+
+[index.ts:296](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L296)

docs/src/javascript/interfaces/UpdateArgs.md

@@ -0,0 +1,36 @@
+[vectordb](../README.md) / [Exports](../modules.md) / UpdateArgs
+
+# Interface: UpdateArgs
+
+## Table of contents
+
+### Properties
+
+- [values](UpdateArgs.md#values)
+- [where](UpdateArgs.md#where)
+
+## Properties
+
+### values
+
+• **values**: `Record`\<`string`, `Literal`\>
+
+A key-value map of updates. The keys are the column names, and the values are the
+new values to set
+
+#### Defined in
+
+[index.ts:320](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L320)
+
+___
+
+### where
+
+• `Optional` **where**: `string`
+
+A filter in the same format used by a sql WHERE clause. The filter may be empty,
+in which case all rows will be updated.
+
+#### Defined in
+
+[index.ts:314](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L314)

docs/src/javascript/interfaces/UpdateSqlArgs.md

@@ -0,0 +1,36 @@
+[vectordb](../README.md) / [Exports](../modules.md) / UpdateSqlArgs
+
+# Interface: UpdateSqlArgs
+
+## Table of contents
+
+### Properties
+
+- [valuesSql](UpdateSqlArgs.md#valuessql)
+- [where](UpdateSqlArgs.md#where)
+
+## Properties
+
+### valuesSql
+
+• **valuesSql**: `Record`\<`string`, `string`\>
+
+A key-value map of updates. The keys are the column names, and the values are the
+new values to set as SQL expressions.
+
+#### Defined in
+
+[index.ts:334](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L334)
+
+___
+
+### where
+
+• `Optional` **where**: `string`
+
+A filter in the same format used by a sql WHERE clause. The filter may be empty,
+in which case all rows will be updated.
+
+#### Defined in
+
+[index.ts:328](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L328)

docs/src/javascript/interfaces/VectorIndex.md

@@ -0,0 +1,41 @@
+[vectordb](../README.md) / [Exports](../modules.md) / VectorIndex
+
+# Interface: VectorIndex
+
+## Table of contents
+
+### Properties
+
+- [columns](VectorIndex.md#columns)
+- [name](VectorIndex.md#name)
+- [uuid](VectorIndex.md#uuid)
+
+## Properties
+
+### columns
+
+• **columns**: `string`[]
+
+#### Defined in
+
+[index.ts:338](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L338)
+
+___
+
+### name
+
+• **name**: `string`
+
+#### Defined in
+
+[index.ts:339](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L339)
+
+___
+
+### uuid
+
+• **uuid**: `string`
+
+#### Defined in
+
+[index.ts:340](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L340)

docs/src/javascript/interfaces/WriteOptions.md

@@ -0,0 +1,27 @@
+[vectordb](../README.md) / [Exports](../modules.md) / WriteOptions
+
+# Interface: WriteOptions
+
+Write options when creating a Table.
+
+## Implemented by
+
+- [`DefaultWriteOptions`](../classes/DefaultWriteOptions.md)
+
+## Table of contents
+
+### Properties
+
+- [writeMode](WriteOptions.md#writemode)
+
+## Properties
+
+### writeMode
+
+• `Optional` **writeMode**: [`WriteMode`](../enums/WriteMode.md)
+
+A [WriteMode](../enums/WriteMode.md) to use on this operation
+
+#### Defined in
+
+[index.ts:774](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L774)

docs/src/javascript/modules.md

@@ -11,6 +11,7 @@
 
 ### Classes
 
+- [DefaultWriteOptions](classes/DefaultWriteOptions.md)
 - [LocalConnection](classes/LocalConnection.md)
 - [LocalTable](classes/LocalTable.md)
 - [OpenAIEmbeddingFunction](classes/OpenAIEmbeddingFunction.md)
@@ -19,11 +20,20 @@
 ### Interfaces
 
 - [AwsCredentials](interfaces/AwsCredentials.md)
+- [CleanupStats](interfaces/CleanupStats.md)
+- [CompactionMetrics](interfaces/CompactionMetrics.md)
+- [CompactionOptions](interfaces/CompactionOptions.md)
 - [Connection](interfaces/Connection.md)
 - [ConnectionOptions](interfaces/ConnectionOptions.md)
+- [CreateTableOptions](interfaces/CreateTableOptions.md)
 - [EmbeddingFunction](interfaces/EmbeddingFunction.md)
+- [IndexStats](interfaces/IndexStats.md)
 - [IvfPQIndexConfig](interfaces/IvfPQIndexConfig.md)
 - [Table](interfaces/Table.md)
+- [UpdateArgs](interfaces/UpdateArgs.md)
+- [UpdateSqlArgs](interfaces/UpdateSqlArgs.md)
+- [VectorIndex](interfaces/VectorIndex.md)
+- [WriteOptions](interfaces/WriteOptions.md)
 
 ### Type Aliases
 
@@ -32,6 +42,7 @@
 ### Functions
 
 - [connect](modules.md#connect)
+- [isWriteOptions](modules.md#iswriteoptions)
 
 ## Type Aliases
 
@@ -41,13 +52,13 @@
 
 #### Defined in
 
-[index.ts:431](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L431)
+[index.ts:755](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L755)
 
 ## Functions
 
 ### connect
 
-▸ **connect**(`uri`): `Promise`<[`Connection`](interfaces/Connection.md)\>
+▸ **connect**(`uri`): `Promise`\<[`Connection`](interfaces/Connection.md)\>
 
 Connect to a LanceDB instance at the given URI
 
@@ -59,24 +70,44 @@ Connect to a LanceDB instance at the given URI
 
 #### Returns
 
-`Promise`<[`Connection`](interfaces/Connection.md)\>
+`Promise`\<[`Connection`](interfaces/Connection.md)\>
 
 #### Defined in
 
-[index.ts:47](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L47)
+[index.ts:95](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L95)
 
-▸ **connect**(`opts`): `Promise`<[`Connection`](interfaces/Connection.md)\>
+▸ **connect**(`opts`): `Promise`\<[`Connection`](interfaces/Connection.md)\>
 
 #### Parameters
 
 | Name | Type |
 | :------ | :------ |
-| `opts` | `Partial`<[`ConnectionOptions`](interfaces/ConnectionOptions.md)\> |
+| `opts` | `Partial`\<[`ConnectionOptions`](interfaces/ConnectionOptions.md)\> |
 
 #### Returns
 
-`Promise`<[`Connection`](interfaces/Connection.md)\>
+`Promise`\<[`Connection`](interfaces/Connection.md)\>
 
 #### Defined in
 
-[index.ts:48](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L48)
+[index.ts:96](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L96)
+
+___
+
+### isWriteOptions
+
+▸ **isWriteOptions**(`value`): value is WriteOptions
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `value` | `any` |
+
+#### Returns
+
+value is WriteOptions
+
+#### Defined in
+
+[index.ts:781](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L781)

docs/src/notebooks/DisappearingEmbeddingFunction.ipynb

@@ -1,5 +1,13 @@
 {
   "cells": [
+    {
+      "cell_type": "markdown",
+      "id": "88c1af18",
+      "metadata": {},
+      "source": [
+        "# Example - MultiModal CLIP Embeddings"
+      ]
+    },
     {
       "cell_type": "markdown",
       "id": "c6b5d346-2c2a-4341-a132-00e53543f8d1",

docs/src/python/python.md

@@ -22,21 +22,19 @@ pip install lancedb
 
 ::: lancedb.query.LanceQueryBuilder
 
-::: lancedb.query.LanceFtsQueryBuilder
-
 ## Embeddings
 
-::: lancedb.embeddings.functions.EmbeddingFunctionRegistry
+::: lancedb.embeddings.registry.EmbeddingFunctionRegistry
 
-::: lancedb.embeddings.functions.EmbeddingFunction
+::: lancedb.embeddings.base.EmbeddingFunction
 
-::: lancedb.embeddings.functions.TextEmbeddingFunction
+::: lancedb.embeddings.base.TextEmbeddingFunction
 
-::: lancedb.embeddings.functions.SentenceTransformerEmbeddings
+::: lancedb.embeddings.sentence_transformers.SentenceTransformerEmbeddings
 
-::: lancedb.embeddings.functions.OpenAIEmbeddings
+::: lancedb.embeddings.openai.OpenAIEmbeddings
 
-::: lancedb.embeddings.functions.OpenClipEmbeddings
+::: lancedb.embeddings.open_clip.OpenClipEmbeddings
 
 ::: lancedb.embeddings.with_embeddings
 
@@ -56,7 +54,7 @@ pip install lancedb
 
 ## Utilities
 
-::: lancedb.vector
+::: lancedb.schema.vector
 
 ## Integrations
 

docs/src/python/saas-python.md

@@ -0,0 +1,18 @@
+# LanceDB Python API Reference
+
+## Installation
+
+```shell
+pip install lancedb
+```
+
+## Connection
+
+::: lancedb.connect
+
+::: lancedb.remote.db.RemoteDBConnection
+
+## Table
+
+::: lancedb.remote.table.RemoteTable
+

docs/src/robots.txt

@@ -0,0 +1 @@
+User-agent: *

docs/src/scripts/posthog.js

@@ -0,0 +1,4 @@
+window.addEventListener("DOMContentLoaded", (event) => {
+    !function(t,e){var o,n,p,r;e.__SV||(window.posthog=e,e._i=[],e.init=function(i,s,a){function g(t,e){var o=e.split(".");2==o.length&&(t=t[o[0]],e=o[1]),t[e]=function(){t.push([e].concat(Array.prototype.slice.call(arguments,0)))}}(p=t.createElement("script")).type="text/javascript",p.async=!0,p.src=s.api_host+"/static/array.js",(r=t.getElementsByTagName("script")[0]).parentNode.insertBefore(p,r);var u=e;for(void 0!==a?u=e[a]=[]:a="posthog",u.people=u.people||[],u.toString=function(t){var e="posthog";return"posthog"!==a&&(e+="."+a),t||(e+=" (stub)"),e},u.people.toString=function(){return u.toString(1)+".people (stub)"},o="capture identify alias people.set people.set_once set_config register register_once unregister opt_out_capturing has_opted_out_capturing opt_in_capturing reset isFeatureEnabled onFeatureFlags getFeatureFlag getFeatureFlagPayload reloadFeatureFlags group updateEarlyAccessFeatureEnrollment getEarlyAccessFeatures getActiveMatchingSurveys getSurveys".split(" "),n=0;n<o.length;n++)g(u,o[n]);e._i.push([i,s,a])},e.__SV=1)}(document,window.posthog||[]);
+    posthog.init('phc_oENDjGgHtmIDrV6puUiFem2RB4JA8gGWulfdulmMdZP',{api_host:'https://app.posthog.com'})
+});

docs/src/search.md

@@ -4,7 +4,7 @@
 In a recommendation system or search engine, you can find similar products from
 the one you searched.
 In LLM and other AI applications,
-each data point can be [presented by the embeddings generated from some models](embedding.md),
+each data point can be [presented by the embeddings generated from some models](embeddings/index.md),
 it returns the most relevant features.
 
 A search in high-dimensional vector space, is to find `K-Nearest-Neighbors (KNN)` of the query vector.
@@ -118,4 +118,101 @@ However, fast vector search using indices often entails making a trade-off with
 This is why it is often called **Approximate Nearest Neighbors (ANN)** search, while the Flat Search (KNN)
 always returns 100% recall.
 
-See [ANN Index](ann_indexes.md) for more details.
+See [ANN Index](ann_indexes.md) for more details.
+
+
+### Output formats
+
+LanceDB returns results in many different formats commonly used in python.
+Let's create a LanceDB table with a nested schema:
+
+```python
+from datetime import datetime
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+import numpy as np
+from pydantic import BaseModel
+uri = "data/sample-lancedb-nested"
+
+class Metadata(BaseModel):
+    source: str
+    timestamp: datetime
+
+class Document(BaseModel):
+    content: str
+    meta: Metadata
+
+class LanceSchema(LanceModel):
+    id: str
+    vector: Vector(1536)
+    payload: Document
+
+# Let's add 100 sample rows to our dataset
+data = [LanceSchema(
+    id=f"id{i}",
+    vector=np.random.randn(1536),
+    payload=Document(
+        content=f"document{i}", meta=Metadata(source=f"source{i%10}", timestamp=datetime.now())
+    ),
+) for i in range(100)]
+
+tbl = db.create_table("documents", data=data)
+```
+
+#### As a pyarrow table
+
+Using `to_arrow()` we can get the results back as a pyarrow Table.
+This result table has the same columns as the LanceDB table, with 
+the addition of an `_distance` column for vector search or a `score`
+column for full text search.
+
+```python
+tbl.search(np.random.randn(1536)).to_arrow()
+```
+
+#### As a pandas dataframe
+
+You can also get the results as a pandas dataframe.
+
+```python
+tbl.search(np.random.randn(1536)).to_pandas()
+```
+
+While other formats like Arrow/Pydantic/Python dicts have a natural 
+way to handle nested schemas, pandas can only store nested data as a 
+python dict column, which makes it difficult to support nested references.
+So for convenience, you can also tell LanceDB to flatten a nested schema 
+when creating the pandas dataframe. 
+
+```python
+tbl.search(np.random.randn(1536)).to_pandas(flatten=True)
+```
+
+If your table has a deeply nested struct, you can control how many levels
+of nesting to flatten by passing in a positive integer.
+
+```python
+tbl.search(np.random.randn(1536)).to_pandas(flatten=1)
+```
+
+
+#### As a list of python dicts
+
+You can of course return results as a list of python dicts.
+
+```python
+tbl.search(np.random.randn(1536)).to_list()
+```
+
+#### As a list of pydantic models
+
+We can add data using pydantic models, and we can certainly
+retrieve results as pydantic models
+
+```python
+tbl.search(np.random.randn(1536)).to_pydantic(LanceSchema)
+```
+
+Note that in this case the extra `_distance` field is discarded since
+it's not part of the LanceSchema.
+

docs/src/sql.md

@@ -1,7 +1,7 @@
 # SQL filters
 
 LanceDB embraces the utilization of standard SQL expressions as predicates for hybrid
-filters. It can be used during hybrid vector search and deletion operations.
+filters. It can be used during hybrid vector search, update, and deletion operations.
 
 Currently, Lance supports a growing list of expressions.
 
@@ -22,7 +22,7 @@ import numpy as np
 uri = "data/sample-lancedb"
 db = lancedb.connect(uri)
 
-data = [{"vector": row, "item": f"item {i}"}
+data = [{"vector": row, "item": f"item {i}", "id": i}
      for i, row in enumerate(np.random.random((10_000, 2)).astype('int'))]
 
 tbl = db.create_table("my_vectors", data=data)
@@ -35,33 +35,25 @@ const db = await vectordb.connect('data/sample-lancedb')
 
 let data = []
 for (let i = 0; i < 10_000; i++) {
-     data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
+     data.push({vector: Array(1536).fill(i), id: i, item: `item ${i}`, strId: `${i}`})
 }
-const tbl = await db.createTable('my_vectors', data)
+const tbl = await db.createTable('myVectors', data)
 ```
 -->
 === "Python"
 
     ```python
     tbl.search([100, 102]) \
-       .where("""(
-        (label IN [10, 20])
-        AND
-        (note.email IS NOT NULL)
-        ) OR NOT note.created
-        """)
-
+       .where("(item IN ('item 0', 'item 2')) AND (id > 10)") \
+       .to_arrow()
     ```
+
 === "Javascript"
 
     ```javascript
-    tbl.search([100, 102])
-       .where(`(
-            (label IN [10, 20])
-            AND
-            (note.email IS NOT NULL)
-        ) OR NOT note.created
-       `)
+    await tbl.search(Array(1536).fill(0))
+       .where("(item IN ('item 0', 'item 2')) AND (id > 10)")
+       .execute()
     ```
 
 
@@ -118,3 +110,22 @@ The mapping from SQL types to Arrow types is:
 
 [^1]: See precision mapping in previous table.
 
+
+## Filtering without Vector Search
+
+You can also filter your data without search.
+
+=== "Python"
+      ```python
+      tbl.search().where("id=10").limit(10).to_arrow()
+      ```
+
+=== "JavaScript"
+      ```javascript
+      await tbl.where('id=10').limit(10).execute()
+      ```
+
+!!! warning
+    If your table is large, this could potentially return a very large
+    amount of data. Please be sure to use a `limit` clause unless
+    you're sure you want to return the whole result set.

docs/test/md_testing.py

@@ -18,29 +18,45 @@ python_file = ".py"
 python_folder = "python"
 
 files = glob.glob(glob_string, recursive=True)
-excluded_files = [f for excluded_glob in excluded_globs for f in glob.glob(excluded_glob, recursive=True)]
+excluded_files = [
+    f
+    for excluded_glob in excluded_globs
+    for f in glob.glob(excluded_glob, recursive=True)
+]
+
 
 def yield_lines(lines: Iterator[str], prefix: str, suffix: str):
     in_code_block = False
     # Python code has strict indentation
     strip_length = 0
+    skip_test = False
     for line in lines:
+        if "skip-test" in line:
+            skip_test = True
         if line.strip().startswith(prefix + python_prefix):
             in_code_block = True
             strip_length = len(line) - len(line.lstrip())
         elif in_code_block and line.strip().startswith(suffix):
             in_code_block = False
-            yield "\n"
+            if not skip_test:
+                yield "\n"
+            skip_test = False
         elif in_code_block:
-            yield line[strip_length:]
+            if not skip_test:
+                yield line[strip_length:]
 
 for file in filter(lambda file: file not in excluded_files, files):
     with open(file, "r") as f:
         lines = list(yield_lines(iter(f), "```", "```"))
 
     if len(lines) > 0:
-        out_path = Path(python_folder) / Path(file).name.strip(".md") / (Path(file).name.strip(".md") + python_file)
+        print(lines)
+        out_path = (
+            Path(python_folder)
+            / Path(file).name.strip(".md")
+            / (Path(file).name.strip(".md") + python_file)
+        )
         print(out_path)
         out_path.parent.mkdir(exist_ok=True, parents=True)
         with open(out_path, "w") as out:
-            out.writelines(lines)
+            out.writelines(lines)

node/README.md

@@ -9,8 +9,13 @@ npm install vectordb
 ```
 
 This will download the appropriate native library for your platform. We currently
-support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
-yet support Windows or musl-based Linux (such as Alpine Linux).
+support:
+
+* Linux (x86_64 and aarch64)
+* MacOS (Intel and ARM/M1/M2)
+* Windows (x86_64 only)
+
+We do not yet support musl-based Linux (such as Alpine Linux) or aarch64 Windows.
 
 ## Usage
 

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.3.1",
+      "version": "0.4.0",
       "cpu": [
         "x64",
         "arm64"
@@ -53,11 +53,11 @@
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.3.1",
-        "@lancedb/vectordb-darwin-x64": "0.3.1",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.3.1",
-        "@lancedb/vectordb-linux-x64-gnu": "0.3.1",
-        "@lancedb/vectordb-win32-x64-msvc": "0.3.1"
+        "@lancedb/vectordb-darwin-arm64": "0.4.0",
+        "@lancedb/vectordb-darwin-x64": "0.4.0",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.4.0",
+        "@lancedb/vectordb-linux-x64-gnu": "0.4.0",
+        "@lancedb/vectordb-win32-x64-msvc": "0.4.0"
       }
     },
     "node_modules/@apache-arrow/ts": {
@@ -317,9 +317,9 @@
       }
     },
     "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.3.1.tgz",
-      "integrity": "sha512-h3yUP249xaO3rrRuVC4oRxEm5/9T66CGKiI8OwYCJUOEFrfz/jj+6PK8geMn7IqbPnOY9YRPSEi/Cc3EdFd6Sg==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.0.tgz",
+      "integrity": "sha512-cP6zGtBWXEcJHCI4uLNIP5ILtRvexvwmL8Uri1dnHG8dT8g12Ykug3BHO6Wt6wp/xASd2jJRIF/VAJsN9IeP1A==",
       "cpu": [
         "arm64"
       ],
@@ -329,9 +329,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.3.1.tgz",
-      "integrity": "sha512-SQ32iMMVfvjXgvFGSGdsXcSnVDypR6eE06d7VIXsuKAg6P9e1XUhB4YcsHGeAEEv3gEoUSgsljo92ZvXJcWouQ==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.4.0.tgz",
+      "integrity": "sha512-ig0gV5ol1sFe2lb1HOatK0rizyj9I91WbnH79i7OdUl3nAQIcWm70CnxrPLtx0DS2NTGh2kFJbYCWcaUlu6YfA==",
       "cpu": [
         "x64"
       ],
@@ -341,9 +341,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.3.1.tgz",
-      "integrity": "sha512-+jk2nJnaIWTqcOAyix2y+ClLNM5ECIdwyHZp5KjDqOlP6Z7eb5V2Xsah0AFp8nX3BiRRvqj3zR3zi26D7OBnYw==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.4.0.tgz",
+      "integrity": "sha512-gMXIDT2kriAPDwWIRKXdaTCNdOeFGEok1S9Y30AOruHXddW1vCIo4JNJIYbBqHnwAeI4wI3ae6GRCFaf1UxO3g==",
       "cpu": [
         "arm64"
       ],
@@ -353,9 +353,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.3.1.tgz",
-      "integrity": "sha512-I42Zf2lH8SUZLLYDDG4kzZ8iPq2wf1cXMh9iKNiLwgl5BnRsZVQ5A5k0uCX7IV7FcnHL/febKOxixXQyoKNAzw==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.4.0.tgz",
+      "integrity": "sha512-ZQ3lDrDSz1IKdx/mS9Lz08agFO+OD5oSFrrcFNCoT1+H93eS1mCLdmCoEARu3jKbx0tMs38l5J9yXZ2QmJye3w==",
       "cpu": [
         "x64"
       ],
@@ -365,9 +365,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.3.1.tgz",
-      "integrity": "sha512-3OBS+fc4kcwhkqIy5b2Nump/iYoAgQd6gmYIJux3LJbMCc4yDcPJdFGVQkWu43JfBh7YOWPfOng2NSCUDBGmoA==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.4.0.tgz",
+      "integrity": "sha512-toNcNwBRE1sdsSf5hr7W8QiqZ33csc/knVEek4CyvYkZHJGh4Z6WI+DJUIASo5wzUez4TX7qUPpRPL9HuaPMCg==",
       "cpu": [
         "x64"
       ],
@@ -4869,33 +4869,33 @@
       }
     },
     "@lancedb/vectordb-darwin-arm64": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.3.1.tgz",
-      "integrity": "sha512-h3yUP249xaO3rrRuVC4oRxEm5/9T66CGKiI8OwYCJUOEFrfz/jj+6PK8geMn7IqbPnOY9YRPSEi/Cc3EdFd6Sg==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.0.tgz",
+      "integrity": "sha512-cP6zGtBWXEcJHCI4uLNIP5ILtRvexvwmL8Uri1dnHG8dT8g12Ykug3BHO6Wt6wp/xASd2jJRIF/VAJsN9IeP1A==",
       "optional": true
     },
     "@lancedb/vectordb-darwin-x64": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.3.1.tgz",
-      "integrity": "sha512-SQ32iMMVfvjXgvFGSGdsXcSnVDypR6eE06d7VIXsuKAg6P9e1XUhB4YcsHGeAEEv3gEoUSgsljo92ZvXJcWouQ==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.4.0.tgz",
+      "integrity": "sha512-ig0gV5ol1sFe2lb1HOatK0rizyj9I91WbnH79i7OdUl3nAQIcWm70CnxrPLtx0DS2NTGh2kFJbYCWcaUlu6YfA==",
       "optional": true
     },
     "@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.3.1.tgz",
-      "integrity": "sha512-+jk2nJnaIWTqcOAyix2y+ClLNM5ECIdwyHZp5KjDqOlP6Z7eb5V2Xsah0AFp8nX3BiRRvqj3zR3zi26D7OBnYw==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.4.0.tgz",
+      "integrity": "sha512-gMXIDT2kriAPDwWIRKXdaTCNdOeFGEok1S9Y30AOruHXddW1vCIo4JNJIYbBqHnwAeI4wI3ae6GRCFaf1UxO3g==",
       "optional": true
     },
     "@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.3.1.tgz",
-      "integrity": "sha512-I42Zf2lH8SUZLLYDDG4kzZ8iPq2wf1cXMh9iKNiLwgl5BnRsZVQ5A5k0uCX7IV7FcnHL/febKOxixXQyoKNAzw==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.4.0.tgz",
+      "integrity": "sha512-ZQ3lDrDSz1IKdx/mS9Lz08agFO+OD5oSFrrcFNCoT1+H93eS1mCLdmCoEARu3jKbx0tMs38l5J9yXZ2QmJye3w==",
       "optional": true
     },
     "@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.3.1.tgz",
-      "integrity": "sha512-3OBS+fc4kcwhkqIy5b2Nump/iYoAgQd6gmYIJux3LJbMCc4yDcPJdFGVQkWu43JfBh7YOWPfOng2NSCUDBGmoA==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.4.0.tgz",
+      "integrity": "sha512-toNcNwBRE1sdsSf5hr7W8QiqZ33csc/knVEek4CyvYkZHJGh4Z6WI+DJUIASo5wzUez4TX7qUPpRPL9HuaPMCg==",
       "optional": true
     },
     "@neon-rs/cli": {

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.3.2",
+  "version": "0.4.1",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -81,10 +81,10 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.3.2",
-    "@lancedb/vectordb-darwin-x64": "0.3.2",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.3.2",
-    "@lancedb/vectordb-linux-x64-gnu": "0.3.2",
-    "@lancedb/vectordb-win32-x64-msvc": "0.3.2"
+    "@lancedb/vectordb-darwin-arm64": "0.4.1",
+    "@lancedb/vectordb-darwin-x64": "0.4.1",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.4.1",
+    "@lancedb/vectordb-linux-x64-gnu": "0.4.1",
+    "@lancedb/vectordb-win32-x64-msvc": "0.4.1"
   }
 }

node/src/index.ts

@@ -21,9 +21,10 @@ import type { EmbeddingFunction } from './embedding/embedding_function'
 import { RemoteConnection } from './remote'
 import { Query } from './query'
 import { isEmbeddingFunction } from './embedding/embedding_function'
+import { type Literal, toSQL } from './util'
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete, tableCleanupOldVersions, tableCompactFiles } = require('../native.js')
+const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateScalarIndex, tableCreateVectorIndex, tableCountRows, tableDelete, tableUpdate, tableCleanupOldVersions, tableCompactFiles, tableListIndices, tableIndexStats } = require('../native.js')
 
 export { Query }
 export type { EmbeddingFunction }
@@ -222,6 +223,56 @@ export interface Table<T = number[]> {
    */
   createIndex: (indexParams: VectorIndexParams) => Promise<any>
 
+  /**
+   * Create a scalar index on this Table for the given column
+   *
+   * @param column The column to index
+   * @param replace If false, fail if an index already exists on the column
+   *
+   * Scalar indices, like vector indices, can be used to speed up scans.  A scalar
+   * index can speed up scans that contain filter expressions on the indexed column.
+   * For example, the following scan will be faster if the column `my_col` has
+   * a scalar index:
+   *
+   * ```ts
+   * const con = await lancedb.connect('./.lancedb');
+   * const table = await con.openTable('images');
+   * const results = await table.where('my_col = 7').execute();
+   * ```
+   *
+   * Scalar indices can also speed up scans containing a vector search and a
+   * prefilter:
+   *
+   * ```ts
+   * const con = await lancedb.connect('././lancedb');
+   * const table = await con.openTable('images');
+   * const results = await table.search([1.0, 2.0]).where('my_col != 7').prefilter(true);
+   * ```
+   *
+   * Scalar indices can only speed up scans for basic filters using
+   * equality, comparison, range (e.g. `my_col BETWEEN 0 AND 100`), and set
+   * membership (e.g. `my_col IN (0, 1, 2)`)
+   *
+   * Scalar indices can be used if the filter contains multiple indexed columns and
+   * the filter criteria are AND'd or OR'd together
+   * (e.g. `my_col < 0 AND other_col> 100`)
+   *
+   * Scalar indices may be used if the filter contains non-indexed columns but,
+   * depending on the structure of the filter, they may not be usable.  For example,
+   * if the column `not_indexed` does not have a scalar index then the filter
+   * `my_col = 0 OR not_indexed = 1` will not be able to use any scalar index on
+   * `my_col`.
+   *
+   * @examples
+   *
+   * ```ts
+   * const con = await lancedb.connect('././lancedb')
+   * const table = await con.openTable('images')
+   * await table.createScalarIndex('my_col')
+   * ```
+   */
+  createScalarIndex: (column: string, replace: boolean) => Promise<void>
+
   /**
    * Returns the number of rows in this table.
    */
@@ -260,6 +311,88 @@ export interface Table<T = number[]> {
    * ```
    */
   delete: (filter: string) => Promise<void>
+
+  /**
+   * Update rows in this table.
+   *
+   * This can be used to update a single row, many rows, all rows, or
+   * sometimes no rows (if your predicate matches nothing).
+   *
+   * @param args see {@link UpdateArgs} and {@link UpdateSqlArgs} for more details
+   *
+   * @examples
+   *
+   * ```ts
+   * const con = await lancedb.connect("./.lancedb")
+   * const data = [
+   *    {id: 1, vector: [3, 3], name: 'Ye'},
+   *    {id: 2, vector: [4, 4], name: 'Mike'},
+   * ];
+   * const tbl = await con.createTable("my_table", data)
+   *
+   * await tbl.update({
+   *   where: "id = 2",
+   *   values: { vector: [2, 2], name: "Michael" },
+   * })
+   *
+   * let results = await tbl.search([1, 1]).execute();
+   * // Returns [
+   * //   {id: 2, vector: [2, 2], name: 'Michael'}
+   * //   {id: 1, vector: [3, 3], name: 'Ye'}
+   * // ]
+   * ```
+   *
+   */
+  update: (args: UpdateArgs | UpdateSqlArgs) => Promise<void>
+
+  /**
+   * List the indicies on this table.
+   */
+  listIndices: () => Promise<VectorIndex[]>
+
+  /**
+   * Get statistics about an index.
+   */
+  indexStats: (indexUuid: string) => Promise<IndexStats>
+}
+
+export interface UpdateArgs {
+  /**
+   * A filter in the same format used by a sql WHERE clause. The filter may be empty,
+   * in which case all rows will be updated.
+   */
+  where?: string
+
+  /**
+   * A key-value map of updates. The keys are the column names, and the values are the
+   * new values to set
+   */
+  values: Record<string, Literal>
+}
+
+export interface UpdateSqlArgs {
+  /**
+   * A filter in the same format used by a sql WHERE clause. The filter may be empty,
+   * in which case all rows will be updated.
+   */
+  where?: string
+
+  /**
+   * A key-value map of updates. The keys are the column names, and the values are the
+   * new values to set as SQL expressions.
+   */
+  valuesSql: Record<string, string>
+}
+
+export interface VectorIndex {
+  columns: string[]
+  name: string
+  uuid: string
+}
+
+export interface IndexStats {
+  numIndexedRows: number | null
+  numUnindexedRows: number | null
 }
 
 /**
@@ -405,6 +538,16 @@ export class LocalTable<T = number[]> implements Table<T> {
     return new Query(query, this._tbl, this._embeddings)
   }
 
+  /**
+   * Creates a filter query to find all rows matching the specified criteria
+   * @param value The filter criteria (like SQL where clause syntax)
+   */
+  filter (value: string): Query<T> {
+    return new Query(undefined, this._tbl, this._embeddings).filter(value)
+  }
+
+  where = this.filter
+
   /**
    * Insert records into this Table.
    *
@@ -444,6 +587,10 @@ export class LocalTable<T = number[]> implements Table<T> {
     return tableCreateVectorIndex.call(this._tbl, indexParams).then((newTable: any) => { this._tbl = newTable })
   }
 
+  async createScalarIndex (column: string, replace: boolean): Promise<void> {
+    return tableCreateScalarIndex.call(this._tbl, column, replace)
+  }
+
   /**
    * Returns the number of rows in this table.
    */
@@ -460,6 +607,31 @@ export class LocalTable<T = number[]> implements Table<T> {
     return tableDelete.call(this._tbl, filter).then((newTable: any) => { this._tbl = newTable })
   }
 
+  /**
+   * Update rows in this table.
+   *
+   * @param args see {@link UpdateArgs} and {@link UpdateSqlArgs} for more details
+   *
+   * @returns
+   */
+  async update (args: UpdateArgs | UpdateSqlArgs): Promise<void> {
+    let filter: string | null
+    let updates: Record<string, string>
+
+    if ('valuesSql' in args) {
+      filter = args.where ?? null
+      updates = args.valuesSql
+    } else {
+      filter = args.where ?? null
+      updates = {}
+      for (const [key, value] of Object.entries(args.values)) {
+        updates[key] = toSQL(value)
+      }
+    }
+
+    return tableUpdate.call(this._tbl, filter, updates).then((newTable: any) => { this._tbl = newTable })
+  }
+
   /**
    * Clean up old versions of the table, freeing disk space.
    *
@@ -502,6 +674,14 @@ export class LocalTable<T = number[]> implements Table<T> {
         return res.metrics
       })
   }
+
+  async listIndices (): Promise<VectorIndex[]> {
+    return tableListIndices.call(this._tbl)
+  }
+
+  async indexStats (indexUuid: string): Promise<IndexStats> {
+    return tableIndexStats.call(this._tbl, indexUuid)
+  }
 }
 
 export interface CleanupStats {
@@ -618,6 +798,11 @@ export interface IvfPQIndexConfig {
    */
   replace?: boolean
 
+  /**
+   * Cache size of the index
+   */
+  index_cache_size?: number
+
   type: 'ivf_pq'
 }
 

node/src/integration_test/test.ts

@@ -65,8 +65,8 @@ describe('LanceDB Mirrored Store Integration test', function () {
     const mirroredPath = path.join(dir, `${tableName}.lance`)
     fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
       if (err != null) throw err
-      // there should be two dirs
-      assert.equal(files.length, 2)
+      // there should be three dirs
+      assert.equal(files.length, 3)
       assert.isTrue(files[0].isDirectory())
       assert.isTrue(files[1].isDirectory())
 
@@ -76,6 +76,12 @@ describe('LanceDB Mirrored Store Integration test', function () {
         assert.isTrue(files[0].name.endsWith('.txn'))
       })
 
+      fs.readdir(path.join(mirroredPath, '_versions'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].name.endsWith('.manifest'))
+      })
+
       fs.readdir(path.join(mirroredPath, 'data'), { withFileTypes: true }, (err, files) => {
         if (err != null) throw err
         assert.equal(files.length, 1)
@@ -88,8 +94,8 @@ describe('LanceDB Mirrored Store Integration test', function () {
 
     fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
       if (err != null) throw err
-      // there should be two dirs
-      assert.equal(files.length, 3)
+      // there should be four dirs
+      assert.equal(files.length, 4)
       assert.isTrue(files[0].isDirectory())
       assert.isTrue(files[1].isDirectory())
       assert.isTrue(files[2].isDirectory())
@@ -128,12 +134,13 @@ describe('LanceDB Mirrored Store Integration test', function () {
 
     fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
       if (err != null) throw err
-      // there should be two dirs
-      assert.equal(files.length, 4)
+      // there should be five dirs
+      assert.equal(files.length, 5)
       assert.isTrue(files[0].isDirectory())
       assert.isTrue(files[1].isDirectory())
       assert.isTrue(files[2].isDirectory())
       assert.isTrue(files[3].isDirectory())
+      assert.isTrue(files[4].isDirectory())
 
       // Three TXs now
       fs.readdir(path.join(mirroredPath, '_transactions'), { withFileTypes: true }, (err, files) => {

node/src/query.ts

@@ -23,27 +23,29 @@ const { tableSearch } = require('../native.js')
  * A builder for nearest neighbor queries for LanceDB.
  */
 export class Query<T = number[]> {
-  private readonly _query: T
+  private readonly _query?: T
   private readonly _tbl?: any
   private _queryVector?: number[]
-  private _limit: number
+  private _limit?: number
   private _refineFactor?: number
   private _nprobes: number
   private _select?: string[]
   private _filter?: string
   private _metricType?: MetricType
+  private _prefilter: boolean
   protected readonly _embeddings?: EmbeddingFunction<T>
 
-  constructor (query: T, tbl?: any, embeddings?: EmbeddingFunction<T>) {
+  constructor (query?: T, tbl?: any, embeddings?: EmbeddingFunction<T>) {
     this._tbl = tbl
     this._query = query
-    this._limit = 10
+    this._limit = undefined
     this._nprobes = 20
     this._refineFactor = undefined
     this._select = undefined
     this._filter = undefined
     this._metricType = undefined
     this._embeddings = embeddings
+    this._prefilter = false
   }
 
   /***
@@ -102,14 +104,21 @@ export class Query<T = number[]> {
     return this
   }
 
+  prefilter (value: boolean): Query<T> {
+    this._prefilter = value
+    return this
+  }
+
   /**
      * Execute the query and return the results as an Array of Objects
      */
   async execute<T = Record<string, unknown>> (): Promise<T[]> {
-    if (this._embeddings !== undefined) {
-      this._queryVector = (await this._embeddings.embed([this._query]))[0]
-    } else {
-      this._queryVector = this._query as number[]
+    if (this._query !== undefined) {
+      if (this._embeddings !== undefined) {
+        this._queryVector = (await this._embeddings.embed([this._query]))[0]
+      } else {
+        this._queryVector = this._query as number[]
+      }
     }
 
     const isElectron = this.isElectron()

node/src/remote/client.ts

@@ -38,6 +38,7 @@ export class HttpLancedbClient {
     vector: number[],
     k: number,
     nprobes: number,
+    prefilter: boolean,
     refineFactor?: number,
     columns?: string[],
     filter?: string
@@ -50,7 +51,8 @@ export class HttpLancedbClient {
                 nprobes,
                 refineFactor,
                 columns,
-                filter
+                filter,
+                prefilter
               },
               {
                 headers: {
@@ -63,6 +65,9 @@ export class HttpLancedbClient {
               }
     ).catch((err) => {
       console.error('error: ', err)
+      if (err.response === undefined) {
+        throw new Error(`Network Error: ${err.message as string}`)
+      }
       return err.response
     })
     if (response.status !== 200) {
@@ -86,13 +91,17 @@ export class HttpLancedbClient {
       {
         headers: {
           'Content-Type': 'application/json',
-          'x-api-key': this._apiKey()
+          'x-api-key': this._apiKey(),
+          ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
         },
         params,
         timeout: 10000
       }
     ).catch((err) => {
       console.error('error: ', err)
+      if (err.response === undefined) {
+        throw new Error(`Network Error: ${err.message as string}`)
+      }
       return err.response
     })
     if (response.status !== 200) {
@@ -128,6 +137,9 @@ export class HttpLancedbClient {
         }
     ).catch((err) => {
       console.error('error: ', err)
+      if (err.response === undefined) {
+        throw new Error(`Network Error: ${err.message as string}`)
+      }
       return err.response
     })
     if (response.status !== 200) {

node/src/remote/index.ts

@@ -14,7 +14,10 @@
 
 import {
   type EmbeddingFunction, type Table, type VectorIndexParams, type Connection,
-  type ConnectionOptions, type CreateTableOptions, type WriteOptions
+  type ConnectionOptions, type CreateTableOptions, type VectorIndex,
+  type WriteOptions,
+  type IndexStats,
+  type UpdateArgs, type UpdateSqlArgs
 } from '../index'
 import { Query } from '../query'
 
@@ -22,6 +25,7 @@ import { Vector, Table as ArrowTable } from 'apache-arrow'
 import { HttpLancedbClient } from './client'
 import { isEmbeddingFunction } from '../embedding/embedding_function'
 import { createEmptyTable, fromRecordsToStreamBuffer, fromTableToStreamBuffer } from '../arrow'
+import { toSQL } from '../util'
 
 /**
  * Remote connection.
@@ -53,8 +57,8 @@ export class RemoteConnection implements Connection {
     return 'db://' + this._client.uri
   }
 
-  async tableNames (): Promise<string[]> {
-    const response = await this._client.get('/v1/table/')
+  async tableNames (pageToken: string = '', limit: number = 10): Promise<string[]> {
+    const response = await this._client.get('/v1/table/', { limit, page_token: pageToken })
     return response.data.tables
   }
 
@@ -152,6 +156,7 @@ export class RemoteQuery<T = number[]> extends Query<T> {
       queryVector,
       (this as any)._limit,
       (this as any)._nprobes,
+      (this as any)._prefilter,
       (this as any)._refineFactor,
       (this as any)._select,
       (this as any)._filter
@@ -190,6 +195,17 @@ export class RemoteTable<T = number[]> implements Table<T> {
     return this._name
   }
 
+  get schema (): Promise<any> {
+    return this._client.post(`/v1/table/${this._name}/describe/`).then(res => {
+      if (res.status !== 200) {
+        throw new Error(`Server Error, status: ${res.status}, ` +
+        // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+          `message: ${res.statusText}: ${res.data}`)
+      }
+      return res.data?.schema
+    })
+  }
+
   search (query: T): Query<T> {
     return new RemoteQuery(query, this._client, this._name)//, this._embeddings_new)
   }
@@ -230,15 +246,90 @@ export class RemoteTable<T = number[]> implements Table<T> {
     return data.length
   }
 
-  async createIndex (indexParams: VectorIndexParams): Promise<any> {
+  async createIndex (indexParams: VectorIndexParams): Promise<void> {
+    const unsupportedParams = [
+      'index_name',
+      'num_partitions',
+      'max_iters',
+      'use_opq',
+      'num_sub_vectors',
+      'num_bits',
+      'max_opq_iters',
+      'replace'
+    ]
+    for (const param of unsupportedParams) {
+      // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
+      if (indexParams[param as keyof VectorIndexParams]) {
+        throw new Error(`${param} is not supported for remote connections`)
+      }
+    }
+
+    const column = indexParams.column ?? 'vector'
+    const indexType = 'vector' // only vector index is supported for remote connections
+    const metricType = indexParams.metric_type ?? 'L2'
+    const indexCacheSize = indexParams ?? null
+
+    const data = {
+      column,
+      index_type: indexType,
+      metric_type: metricType,
+      index_cache_size: indexCacheSize
+    }
+    const res = await this._client.post(`/v1/table/${this._name}/create_index/`, data)
+    if (res.status !== 200) {
+      throw new Error(`Server Error, status: ${res.status}, ` +
+      // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+        `message: ${res.statusText}: ${res.data}`)
+    }
+  }
+
+  async createScalarIndex (column: string, replace: boolean): Promise<void> {
     throw new Error('Not implemented')
   }
 
   async countRows (): Promise<number> {
-    throw new Error('Not implemented')
+    const result = await this._client.post(`/v1/table/${this._name}/describe/`)
+    return result.data?.stats?.num_rows
   }
 
   async delete (filter: string): Promise<void> {
     await this._client.post(`/v1/table/${this._name}/delete/`, { predicate: filter })
   }
+
+  async update (args: UpdateArgs | UpdateSqlArgs): Promise<void> {
+    let filter: string | null
+    let updates: Record<string, string>
+
+    if ('valuesSql' in args) {
+      filter = args.where ?? null
+      updates = args.valuesSql
+    } else {
+      filter = args.where ?? null
+      updates = {}
+      for (const [key, value] of Object.entries(args.values)) {
+        updates[key] = toSQL(value)
+      }
+    }
+    await this._client.post(`/v1/table/${this._name}/update/`, {
+      predicate: filter,
+      updates: Object.entries(updates).map(([key, value]) => [key, value])
+    })
+  }
+
+  async listIndices (): Promise<VectorIndex[]> {
+    const results = await this._client.post(`/v1/table/${this._name}/index/list/`)
+    return results.data.indexes?.map((index: any) => ({
+      columns: index.columns,
+      name: index.index_name,
+      uuid: index.index_uuid
+    }))
+  }
+
+  async indexStats (indexUuid: string): Promise<IndexStats> {
+    const results = await this._client.post(`/v1/table/${this._name}/index/${indexUuid}/stats/`)
+    return {
+      numIndexedRows: results.data.num_indexed_rows,
+      numUnindexedRows: results.data.num_unindexed_rows
+    }
+  }
 }

node/src/test/test.ts

@@ -78,12 +78,31 @@ describe('LanceDB client', function () {
     })
 
     it('limits # of results', async function () {
-      const uri = await createTestDB()
+      const uri = await createTestDB(2, 100)
       const con = await lancedb.connect(uri)
       const table = await con.openTable('vectors')
-      const results = await table.search([0.1, 0.3]).limit(1).execute()
+      let results = await table.search([0.1, 0.3]).limit(1).execute()
       assert.equal(results.length, 1)
       assert.equal(results[0].id, 1)
+
+      // there is a default limit if unspecified
+      results = await table.search([0.1, 0.3]).execute()
+      assert.equal(results.length, 10)
+    })
+
+    it('uses a filter / where clause without vector search', async function () {
+      // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
+      const assertResults = (results: Array<Record<string, unknown>>) => {
+        assert.equal(results.length, 50)
+      }
+
+      const uri = await createTestDB(2, 100)
+      const con = await lancedb.connect(uri)
+      const table = (await con.openTable('vectors')) as LocalTable
+      let results = await table.filter('id % 2 = 0').execute()
+      assertResults(results)
+      results = await table.where('id % 2 = 0').execute()
+      assertResults(results)
     })
 
     it('uses a filter / where clause', async function () {
@@ -102,6 +121,31 @@ describe('LanceDB client', function () {
       assertResults(results)
     })
 
+    it('should correctly process prefilter/postfilter', async function () {
+      const uri = await createTestDB(16, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+      // post filter should return less than the limit
+      let results = await table.search(new Array(16).fill(0.1)).limit(10).filter('id >= 10').prefilter(false).execute()
+      assert.isTrue(results.length < 10)
+
+      // pre filter should return exactly the limit
+      results = await table.search(new Array(16).fill(0.1)).limit(10).filter('id >= 10').prefilter(true).execute()
+      assert.isTrue(results.length === 10)
+    })
+
+    it('should allow creation and use of scalar indices', async function () {
+      const uri = await createTestDB(16, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      await table.createScalarIndex('id', true)
+
+      // Prefiltering should still work the same
+      const results = await table.search(new Array(16).fill(0.1)).limit(10).filter('id >= 10').prefilter(true).execute()
+      assert.isTrue(results.length === 10)
+    })
+
     it('select only a subset of columns', async function () {
       const uri = await createTestDB()
       const con = await lancedb.connect(uri)
@@ -246,6 +290,46 @@ describe('LanceDB client', function () {
       assert.equal(await table.countRows(), 2)
     })
 
+    it('can update records in the table', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+
+      const table = await con.openTable('vectors')
+      assert.equal(await table.countRows(), 2)
+
+      await table.update({ where: 'price = 10', valuesSql: { price: '100' } })
+      const results = await table.search([0.1, 0.2]).execute()
+      assert.equal(results[0].price, 100)
+      assert.equal(results[1].price, 11)
+    })
+
+    it('can update the records using a literal value', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+
+      const table = await con.openTable('vectors')
+      assert.equal(await table.countRows(), 2)
+
+      await table.update({ where: 'price = 10', values: { price: 100 } })
+      const results = await table.search([0.1, 0.2]).execute()
+      assert.equal(results[0].price, 100)
+      assert.equal(results[1].price, 11)
+    })
+
+    it('can update every record in the table', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+
+      const table = await con.openTable('vectors')
+      assert.equal(await table.countRows(), 2)
+
+      await table.update({ valuesSql: { price: '100' } })
+      const results = await table.search([0.1, 0.2]).execute()
+
+      assert.equal(results[0].price, 100)
+      assert.equal(results[1].price, 100)
+    })
+
     it('can delete records from a table', async function () {
       const uri = await createTestDB()
       const con = await lancedb.connect(uri)
@@ -282,7 +366,8 @@ describe('LanceDB client', function () {
       )
       const table = await con.createTable({ name: 'vectors', schema })
       await table.add([{ vector: Array(128).fill(0.1) }])
-      await table.delete('vector IS NOT NULL')
+      // https://github.com/lancedb/lance/issues/1635
+      await table.delete('true')
       const result = await table.search(Array(128).fill(0.1)).execute()
       assert.isEmpty(result)
     })
@@ -328,6 +413,24 @@ describe('LanceDB client', function () {
       const createIndex = table.createIndex({ type: 'ivf_pq', column: 'name', num_partitions: -1, max_iters: 2, num_sub_vectors: 2 })
       await expect(createIndex).to.be.rejectedWith('num_partitions: must be > 0')
     })
+
+    it('should be able to list index and stats', async function () {
+      const uri = await createTestDB(32, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+
+      const indices = await table.listIndices()
+      expect(indices).to.have.lengthOf(1)
+      expect(indices[0].name).to.equal('vector_idx')
+      expect(indices[0].uuid).to.not.be.equal(undefined)
+      expect(indices[0].columns).to.have.lengthOf(1)
+      expect(indices[0].columns[0]).to.equal('vector')
+
+      const stats = await table.indexStats(indices[0].uuid)
+      expect(stats.numIndexedRows).to.equal(300)
+      expect(stats.numUnindexedRows).to.equal(0)
+    }).timeout(50_000)
   })
 
   describe('when using a custom embedding function', function () {
@@ -378,6 +481,40 @@ describe('LanceDB client', function () {
   })
 })
 
+describe('Remote LanceDB client', function () {
+  describe('when the server is not reachable', function () {
+    it('produces a network error', async function () {
+      const con = await lancedb.connect({
+        uri: 'db://test-1234',
+        region: 'asdfasfasfdf',
+        apiKey: 'some-api-key'
+      })
+
+      // GET
+      try {
+        await con.tableNames()
+      } catch (err) {
+        expect(err).to.have.property('message', 'Network Error: getaddrinfo ENOTFOUND test-1234.asdfasfasfdf.api.lancedb.com')
+      }
+
+      // POST
+      try {
+        await con.createTable({ name: 'vectors', schema: new Schema([]) })
+      } catch (err) {
+        expect(err).to.have.property('message', 'Network Error: getaddrinfo ENOTFOUND test-1234.asdfasfasfdf.api.lancedb.com')
+      }
+
+      // Search
+      const table = await con.openTable('vectors')
+      try {
+        await table.search([0.1, 0.3]).execute()
+      } catch (err) {
+        expect(err).to.have.property('message', 'Network Error: getaddrinfo ENOTFOUND test-1234.asdfasfasfdf.api.lancedb.com')
+      }
+    })
+  })
+})
+
 describe('Query object', function () {
   it('sets custom parameters', async function () {
     const query = new Query([0.1, 0.3])
@@ -475,7 +612,7 @@ describe('Compact and cleanup', function () {
 
     // should have no effect, but this validates the arguments are parsed.
     await table.compactFiles({
-      targetRowsPerFragment: 1024 * 10,
+      targetRowsPerFragment: 102410,
       maxRowsPerGroup: 1024,
       materializeDeletions: true,
       materializeDeletionsThreshold: 0.5,

node/src/test/util.ts

@@ -0,0 +1,45 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { toSQL } from '../util'
+import * as chai from 'chai'
+
+const expect = chai.expect
+
+describe('toSQL', function () {
+  it('should turn string to SQL expression', function () {
+    expect(toSQL('foo')).to.equal("'foo'")
+  })
+
+  it('should turn number to SQL expression', function () {
+    expect(toSQL(123)).to.equal('123')
+  })
+
+  it('should turn boolean to SQL expression', function () {
+    expect(toSQL(true)).to.equal('TRUE')
+  })
+
+  it('should turn null to SQL expression', function () {
+    expect(toSQL(null)).to.equal('NULL')
+  })
+
+  it('should turn Date to SQL expression', function () {
+    const date = new Date('05 October 2011 14:48 UTC')
+    expect(toSQL(date)).to.equal("'2011-10-05T14:48:00.000Z'")
+  })
+
+  it('should turn array to SQL expression', function () {
+    expect(toSQL(['foo', 'bar', true, 1])).to.equal("['foo', 'bar', TRUE, 1]")
+  })
+})

node/src/util.ts

@@ -0,0 +1,44 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+export type Literal = string | number | boolean | null | Date | Literal[]
+
+export function toSQL (value: Literal): string {
+  if (typeof value === 'string') {
+    return `'${value}'`
+  }
+
+  if (typeof value === 'number') {
+    return value.toString()
+  }
+
+  if (typeof value === 'boolean') {
+    return value ? 'TRUE' : 'FALSE'
+  }
+
+  if (value === null) {
+    return 'NULL'
+  }
+
+  if (value instanceof Date) {
+    return `'${value.toISOString()}'`
+  }
+
+  if (Array.isArray(value)) {
+    return `[${value.map(toSQL).join(', ')}]`
+  }
+
+  // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+  throw new Error(`Unsupported value type: ${typeof value} value: (${value})`)
+}

python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.1
+current_version = 0.4.1
 commit = True
 message = [python] Bump version: {current_version} → {new_version}
 tag = True

python/lancedb/__init__.py

@@ -16,17 +16,18 @@ from typing import Optional
 
 __version__ = importlib.metadata.version("lancedb")
 
-from .db import URI, DBConnection, LanceDBConnection
+from .common import URI
+from .db import DBConnection, LanceDBConnection
 from .remote.db import RemoteDBConnection
-from .schema import vector
-from .utils import sentry_log
+from .schema import vector  # noqa: F401
+from .utils import sentry_log  # noqa: F401
 
 
 def connect(
     uri: URI,
     *,
     api_key: Optional[str] = None,
-    region: str = "us-west-2",
+    region: str = "us-east-1",
     host_override: Optional[str] = None,
 ) -> DBConnection:
     """Connect to a LanceDB database.
@@ -38,7 +39,7 @@ def connect(
     api_key: str, optional
         If presented, connect to LanceDB cloud.
         Otherwise, connect to a database on file system or cloud storage.
-    region: str, default "us-west-2"
+    region: str, default "us-east-1"
         The region to use for LanceDB Cloud.
     host_override: str, optional
         The override url for LanceDB Cloud.

python/lancedb/conftest.py

@@ -1,4 +1,6 @@
 import os
+import time
+from typing import Any
 
 import numpy as np
 import pytest
@@ -38,3 +40,26 @@ class MockTextEmbeddingFunction(TextEmbeddingFunction):
 
     def ndims(self):
         return 10
+
+
+class RateLimitedAPI:
+    rate_limit = 0.1  # 1 request per 0.1 second
+    last_request_time = 0
+
+    @staticmethod
+    def make_request():
+        current_time = time.time()
+
+        if current_time - RateLimitedAPI.last_request_time < RateLimitedAPI.rate_limit:
+            raise Exception("Rate limit exceeded. Please try again later.")
+
+        # Simulate a successful request
+        RateLimitedAPI.last_request_time = current_time
+        return "Request successful"
+
+
+@registry.register("test-rate-limited")
+class MockRateLimitedEmbeddingFunction(MockTextEmbeddingFunction):
+    def generate_embeddings(self, texts):
+        RateLimitedAPI.make_request()
+        return [self._compute_one_embedding(row) for row in texts]

python/lancedb/context.py

@@ -84,7 +84,9 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
     context windows that don't cross document boundaries. In this case, we can
     pass ``document_id`` as the group by.
 
-    >>> contextualize(data).window(4).stride(2).text_col('token').groupby('document_id').to_pandas()
+    >>> (contextualize(data)
+    ...     .window(4).stride(2).text_col('token').groupby('document_id')
+    ...     .to_pandas())
                        token  document_id
     0    The quick brown fox            1
     2  brown fox jumped over            1
@@ -92,18 +94,24 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
     6           the lazy dog            1
     9      I love sandwiches            2
 
-    ``min_window_size`` determines the minimum size of the  context windows that are generated
-    This can be used to trim the last few context windows which have size less than
-    ``min_window_size``. By default context windows of size 1 are skipped.
+    ``min_window_size`` determines the minimum size of the context windows
+    that are generated.This can be used to trim the last few context windows
+    which have size less than ``min_window_size``.
+    By default context windows of size 1 are skipped.
 
-    >>> contextualize(data).window(6).stride(3).text_col('token').groupby('document_id').to_pandas()
+    >>> (contextualize(data)
+    ...     .window(6).stride(3).text_col('token').groupby('document_id')
+    ...     .to_pandas())
                                  token  document_id
     0  The quick brown fox jumped over            1
     3     fox jumped over the lazy dog            1
     6                     the lazy dog            1
     9                I love sandwiches            2
 
-    >>> contextualize(data).window(6).stride(3).min_window_size(4).text_col('token').groupby('document_id').to_pandas()
+    >>> (contextualize(data)
+    ...     .window(6).stride(3).min_window_size(4).text_col('token')
+    ...     .groupby('document_id')
+    ...     .to_pandas())
                                  token  document_id
     0  The quick brown fox jumped over            1
     3     fox jumped over the lazy dog            1
@@ -113,7 +121,9 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
 
 
 class Contextualizer:
-    """Create context windows from a DataFrame. See [lancedb.context.contextualize][]."""
+    """Create context windows from a DataFrame.
+    See [lancedb.context.contextualize][].
+    """
 
     def __init__(self, raw_df):
         self._text_col = None
@@ -183,7 +193,7 @@ class Contextualizer:
         deprecated_in="0.3.1",
         removed_in="0.4.0",
         current_version=__version__,
-        details="Use the bar function instead",
+        details="Use to_pandas() instead",
     )
     def to_df(self) -> "pd.DataFrame":
         return self.to_pandas()

python/lancedb/db.py

@@ -14,26 +14,39 @@
 from __future__ import annotations
 
 import os
-from abc import ABC, abstractmethod
+from abc import abstractmethod
 from pathlib import Path
-from typing import List, Optional, Union
+from typing import TYPE_CHECKING, Iterable, List, Optional, Union
 
 import pyarrow as pa
+from overrides import EnforceOverrides, override
 from pyarrow import fs
 
-from .common import DATA, URI
-from .embeddings import EmbeddingFunctionConfig
-from .pydantic import LanceModel
 from .table import LanceTable, Table
-from .util import fs_from_uri, get_uri_location, get_uri_scheme
+from .util import fs_from_uri, get_uri_location, get_uri_scheme, join_uri
+
+if TYPE_CHECKING:
+    from .common import DATA, URI
+    from .embeddings import EmbeddingFunctionConfig
+    from .pydantic import LanceModel
 
 
-class DBConnection(ABC):
+class DBConnection(EnforceOverrides):
     """An active LanceDB connection interface."""
 
     @abstractmethod
-    def table_names(self) -> list[str]:
-        """List all table names in the database."""
+    def table_names(
+        self, page_token: Optional[str] = None, limit: int = 10
+    ) -> Iterable[str]:
+        """List all table in this database
+
+        Parameters
+        ----------
+        page_token: str, optional
+            The token to use for pagination. If not present, start from the beginning.
+        limit: int, default 10
+            The size of the page to return.
+        """
         pass
 
     @abstractmethod
@@ -45,6 +58,7 @@ class DBConnection(ABC):
         mode: str = "create",
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
+        embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
     ) -> Table:
         """Create a [Table][lancedb.table.Table] in the database.
 
@@ -52,12 +66,24 @@ class DBConnection(ABC):
         ----------
         name: str
             The name of the table.
-        data: list, tuple, dict, pd.DataFrame; optional
-            The data to initialize the table. User must provide at least one of `data` or `schema`.
-        schema: pyarrow.Schema or LanceModel; optional
-            The schema of the table.
+        data: The data to initialize the table, *optional*
+            User must provide at least one of `data` or `schema`.
+            Acceptable types are:
+
+            - dict or list-of-dict
+
+            - pandas.DataFrame
+
+            - pyarrow.Table or pyarrow.RecordBatch
+        schema: The schema of the table, *optional*
+            Acceptable types are:
+
+            - pyarrow.Schema
+
+            - [LanceModel][lancedb.pydantic.LanceModel]
         mode: str; default "create"
-            The mode to use when creating the table. Can be either "create" or "overwrite".
+            The mode to use when creating the table.
+            Can be either "create" or "overwrite".
             By default, if the table already exists, an exception is raised.
             If you want to overwrite the table, use mode="overwrite".
         on_bad_vectors: str, default "error"
@@ -150,7 +176,8 @@ class DBConnection(ABC):
         ...     for i in range(5):
         ...         yield pa.RecordBatch.from_arrays(
         ...             [
-        ...                 pa.array([[3.1, 4.1], [5.9, 26.5]], pa.list_(pa.float32(), 2)),
+        ...                 pa.array([[3.1, 4.1], [5.9, 26.5]],
+        ...                     pa.list_(pa.float32(), 2)),
         ...                 pa.array(["foo", "bar"]),
         ...                 pa.array([10.0, 20.0]),
         ...             ],
@@ -249,23 +276,25 @@ class LanceDBConnection(DBConnection):
     def uri(self) -> str:
         return self._uri
 
-    def table_names(self) -> list[str]:
-        """Get the names of all tables in the database.
+    @override
+    def table_names(
+        self, page_token: Optional[str] = None, limit: int = 10
+    ) -> Iterable[str]:
+        """Get the names of all tables in the database. The names are sorted.
 
         Returns
         -------
-        list of str
+        Iterator of str.
             A list of table names.
         """
         try:
-            filesystem, path = fs_from_uri(self.uri)
+            filesystem = fs_from_uri(self.uri)[0]
         except pa.ArrowInvalid:
             raise NotImplementedError("Unsupported scheme: " + self.uri)
 
         try:
-            paths = filesystem.get_file_info(
-                fs.FileSelector(get_uri_location(self.uri))
-            )
+            loc = get_uri_location(self.uri)
+            paths = filesystem.get_file_info(fs.FileSelector(loc))
         except FileNotFoundError:
             # It is ok if the file does not exist since it will be created
             paths = []
@@ -274,6 +303,7 @@ class LanceDBConnection(DBConnection):
             for file_info in paths
             if file_info.extension == "lance"
         ]
+        tables.sort()
         return tables
 
     def __len__(self) -> int:
@@ -282,6 +312,7 @@ class LanceDBConnection(DBConnection):
     def __contains__(self, name: str) -> bool:
         return name in self.table_names()
 
+    @override
     def create_table(
         self,
         name: str,
@@ -313,6 +344,7 @@ class LanceDBConnection(DBConnection):
         )
         return tbl
 
+    @override
     def open_table(self, name: str) -> LanceTable:
         """Open a table in the database.
 
@@ -327,6 +359,7 @@ class LanceDBConnection(DBConnection):
         """
         return LanceTable.open(self, name)
 
+    @override
     def drop_table(self, name: str, ignore_missing: bool = False):
         """Drop a table from the database.
 
@@ -339,12 +372,13 @@ class LanceDBConnection(DBConnection):
         """
         try:
             filesystem, path = fs_from_uri(self.uri)
-            table_path = os.path.join(path, name + ".lance")
+            table_path = join_uri(path, name + ".lance")
             filesystem.delete_dir(table_path)
         except FileNotFoundError:
             if not ignore_missing:
                 raise
 
+    @override
     def drop_database(self):
         filesystem, path = fs_from_uri(self.uri)
         filesystem.delete_dir(path)

python/lancedb/embeddings/__init__.py

@@ -11,16 +11,12 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-
+# ruff: noqa: F401
+from .base import EmbeddingFunction, EmbeddingFunctionConfig, TextEmbeddingFunction
 from .cohere import CohereEmbeddingFunction
-from .functions import (
-    EmbeddingFunction,
-    EmbeddingFunctionConfig,
-    EmbeddingFunctionRegistry,
-    OpenAIEmbeddings,
-    OpenClipEmbeddings,
-    SentenceTransformerEmbeddings,
-    TextEmbeddingFunction,
-    register,
-)
+from .instructor import InstructorEmbeddingFunction
+from .open_clip import OpenClipEmbeddings
+from .openai import OpenAIEmbeddings
+from .registry import EmbeddingFunctionRegistry, get_registry
+from .sentence_transformers import SentenceTransformerEmbeddings
 from .utils import with_embeddings

python/lancedb/embeddings/base.py

@@ -0,0 +1,181 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+import importlib
+from abc import ABC, abstractmethod
+from typing import List, Union
+
+import numpy as np
+import pyarrow as pa
+from pydantic import BaseModel, Field, PrivateAttr
+
+from .utils import TEXT, retry_with_exponential_backoff
+
+
+class EmbeddingFunction(BaseModel, ABC):
+    """
+    An ABC for embedding functions.
+
+    All concrete embedding functions must implement the following:
+    1. compute_query_embeddings() which takes a query and returns a list of embeddings
+    2. get_source_embeddings() which returns a list of embeddings for the source column
+    For text data, the two will be the same. For multi-modal data, the source column
+    might be images and the vector column might be text.
+    3. ndims method which returns the number of dimensions of the vector column
+    """
+
+    __slots__ = ("__weakref__",)  # pydantic 1.x compatibility
+    max_retries: int = (
+        7  # Setitng 0 disables retires. Maybe this should not be enabled by default,
+    )
+    _ndims: int = PrivateAttr()
+
+    @classmethod
+    def create(cls, **kwargs):
+        """
+        Create an instance of the embedding function
+        """
+        return cls(**kwargs)
+
+    @abstractmethod
+    def compute_query_embeddings(self, *args, **kwargs) -> List[np.array]:
+        """
+        Compute the embeddings for a given user query
+        """
+        pass
+
+    @abstractmethod
+    def compute_source_embeddings(self, *args, **kwargs) -> List[np.array]:
+        """
+        Compute the embeddings for the source column in the database
+        """
+        pass
+
+    def compute_query_embeddings_with_retry(self, *args, **kwargs) -> List[np.array]:
+        """
+        Compute the embeddings for a given user query with retries
+        """
+        return retry_with_exponential_backoff(
+            self.compute_query_embeddings, max_retries=self.max_retries
+        )(
+            *args,
+            **kwargs,
+        )
+
+    def compute_source_embeddings_with_retry(self, *args, **kwargs) -> List[np.array]:
+        """
+        Compute the embeddings for the source column in the database with retries
+        """
+        return retry_with_exponential_backoff(
+            self.compute_source_embeddings, max_retries=self.max_retries
+        )(*args, **kwargs)
+
+    def sanitize_input(self, texts: TEXT) -> Union[List[str], np.ndarray]:
+        """
+        Sanitize the input to the embedding function.
+        """
+        if isinstance(texts, str):
+            texts = [texts]
+        elif isinstance(texts, pa.Array):
+            texts = texts.to_pylist()
+        elif isinstance(texts, pa.ChunkedArray):
+            texts = texts.combine_chunks().to_pylist()
+        return texts
+
+    @classmethod
+    def safe_import(cls, module: str, mitigation=None):
+        """
+        Import the specified module. If the module is not installed,
+        raise an ImportError with a helpful message.
+
+        Parameters
+        ----------
+        module : str
+            The name of the module to import
+        mitigation : Optional[str]
+            The package(s) to install to mitigate the error.
+            If not provided then the module name will be used.
+        """
+        try:
+            return importlib.import_module(module)
+        except ImportError:
+            raise ImportError(f"Please install {mitigation or module}")
+
+    def safe_model_dump(self):
+        from ..pydantic import PYDANTIC_VERSION
+
+        if PYDANTIC_VERSION.major < 2:
+            return dict(self)
+        return self.model_dump()
+
+    @abstractmethod
+    def ndims(self):
+        """
+        Return the dimensions of the vector column
+        """
+        pass
+
+    def SourceField(self, **kwargs):
+        """
+        Creates a pydantic Field that can automatically annotate
+        the source column for this embedding function
+        """
+        return Field(json_schema_extra={"source_column_for": self}, **kwargs)
+
+    def VectorField(self, **kwargs):
+        """
+        Creates a pydantic Field that can automatically annotate
+        the target vector column for this embedding function
+        """
+        return Field(json_schema_extra={"vector_column_for": self}, **kwargs)
+
+    def __eq__(self, __value: object) -> bool:
+        if not hasattr(__value, "__dict__"):
+            return False
+        return vars(self) == vars(__value)
+
+    def __hash__(self) -> int:
+        return hash(frozenset(vars(self).items()))
+
+
+class EmbeddingFunctionConfig(BaseModel):
+    """
+    This model encapsulates the configuration for a embedding function
+    in a lancedb table. It holds the embedding function, the source column,
+    and the vector column
+    """
+
+    vector_column: str
+    source_column: str
+    function: EmbeddingFunction
+
+
+class TextEmbeddingFunction(EmbeddingFunction):
+    """
+    A callable ABC for embedding functions that take text as input
+    """
+
+    def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]:
+        return self.compute_source_embeddings(query, *args, **kwargs)
+
+    def compute_source_embeddings(self, texts: TEXT, *args, **kwargs) -> List[np.array]:
+        texts = self.sanitize_input(texts)
+        return self.generate_embeddings(texts)
+
+    @abstractmethod
+    def generate_embeddings(
+        self, texts: Union[List[str], np.ndarray]
+    ) -> List[np.array]:
+        """
+        Generate the embeddings for the given texts
+        """
+        pass

python/lancedb/embeddings/cohere.py

@@ -16,7 +16,8 @@ from typing import ClassVar, List, Union
 
 import numpy as np
 
-from .functions import TextEmbeddingFunction, register
+from .base import TextEmbeddingFunction
+from .registry import register
 from .utils import api_key_not_found_help
 
 
@@ -30,7 +31,8 @@ class CohereEmbeddingFunction(TextEmbeddingFunction):
     Parameters
     ----------
     name: str, default "embed-multilingual-v2.0"
-        The name of the model to use. See the Cohere documentation for a list of available models.
+        The name of the model to use. See the Cohere documentation for
+        a list of available models.
 
     Examples
     --------
@@ -38,7 +40,10 @@ class CohereEmbeddingFunction(TextEmbeddingFunction):
     from lancedb.pydantic import LanceModel, Vector
     from lancedb.embeddings import EmbeddingFunctionRegistry
 
-    cohere = EmbeddingFunctionRegistry.get_instance().get("cohere").create(name="embed-multilingual-v2.0")
+    cohere = EmbeddingFunctionRegistry
+        .get_instance()
+        .get("cohere")
+        .create(name="embed-multilingual-v2.0")
 
     class TextModel(LanceModel):
         text: str = cohere.SourceField()

python/lancedb/embeddings/functions.py

@@ -1,578 +0,0 @@
-#  Copyright (c) 2023. LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-import concurrent.futures
-import importlib
-import io
-import json
-import os
-import socket
-import urllib.error
-import urllib.parse as urlparse
-import urllib.request
-from abc import ABC, abstractmethod
-from typing import Dict, List, Optional, Union
-
-import numpy as np
-import pyarrow as pa
-from cachetools import cached
-from pydantic import BaseModel, Field, PrivateAttr
-from tqdm import tqdm
-
-
-class EmbeddingFunctionRegistry:
-    """
-    This is a singleton class used to register embedding functions
-    and fetch them by name. It also handles serializing and deserializing.
-    You can implement your own embedding function by subclassing EmbeddingFunction
-    or TextEmbeddingFunction and registering it with the registry.
-
-    Examples
-    --------
-    >>> registry = EmbeddingFunctionRegistry.get_instance()
-    >>> @registry.register("my-embedding-function")
-    ... class MyEmbeddingFunction(EmbeddingFunction):
-    ...     def ndims(self) -> int:
-    ...         return 128
-    ...
-    ...     def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]:
-    ...         return self.compute_source_embeddings(query, *args, **kwargs)
-    ...
-    ...     def compute_source_embeddings(self, texts: TEXT, *args, **kwargs) -> List[np.array]:
-    ...         return [np.random.rand(self.ndims()) for _ in range(len(texts))]
-    ...
-    >>> registry.get("my-embedding-function")
-    <class 'lancedb.embeddings.functions.MyEmbeddingFunction'>
-    """
-
-    @classmethod
-    def get_instance(cls):
-        return __REGISTRY__
-
-    def __init__(self):
-        self._functions = {}
-
-    def register(self, alias: str = None):
-        """
-        This creates a decorator that can be used to register
-        an EmbeddingFunction.
-
-        Parameters
-        ----------
-        alias : Optional[str]
-            a human friendly name for the embedding function. If not
-            provided, the class name will be used.
-        """
-
-        # This is a decorator for a class that inherits from BaseModel
-        # It adds the class to the registry
-        def decorator(cls):
-            if not issubclass(cls, EmbeddingFunction):
-                raise TypeError("Must be a subclass of EmbeddingFunction")
-            if cls.__name__ in self._functions:
-                raise KeyError(f"{cls.__name__} was already registered")
-            key = alias or cls.__name__
-            self._functions[key] = cls
-            cls.__embedding_function_registry_alias__ = alias
-            return cls
-
-        return decorator
-
-    def reset(self):
-        """
-        Reset the registry to its initial state
-        """
-        self._functions = {}
-
-    def get(self, name: str):
-        """
-        Fetch an embedding function class by name
-
-        Parameters
-        ----------
-        name : str
-            The name of the embedding function to fetch
-            Either the alias or the class name if no alias was provided
-            during registration
-        """
-        return self._functions[name]
-
-    def parse_functions(
-        self, metadata: Optional[Dict[bytes, bytes]]
-    ) -> Dict[str, "EmbeddingFunctionConfig"]:
-        """
-        Parse the metadata from an arrow table and
-        return a mapping of the vector column to the
-        embedding function and source column
-
-        Parameters
-        ----------
-        metadata : Optional[Dict[bytes, bytes]]
-            The metadata from an arrow table. Note that
-            the keys and values are bytes (pyarrow api)
-
-        Returns
-        -------
-        functions : dict
-            A mapping of vector column name to embedding function.
-            An empty dict is returned if input is None or does not
-            contain b"embedding_functions".
-        """
-        if metadata is None or b"embedding_functions" not in metadata:
-            return {}
-        serialized = metadata[b"embedding_functions"]
-        raw_list = json.loads(serialized.decode("utf-8"))
-        return {
-            obj["vector_column"]: EmbeddingFunctionConfig(
-                vector_column=obj["vector_column"],
-                source_column=obj["source_column"],
-                function=self.get(obj["name"])(**obj["model"]),
-            )
-            for obj in raw_list
-        }
-
-    def function_to_metadata(self, conf: "EmbeddingFunctionConfig"):
-        """
-        Convert the given embedding function and source / vector column configs
-        into a config dictionary that can be serialized into arrow metadata
-        """
-        func = conf.function
-        name = getattr(
-            func, "__embedding_function_registry_alias__", func.__class__.__name__
-        )
-        json_data = func.safe_model_dump()
-        return {
-            "name": name,
-            "model": json_data,
-            "source_column": conf.source_column,
-            "vector_column": conf.vector_column,
-        }
-
-    def get_table_metadata(self, func_list):
-        """
-        Convert a list of embedding functions and source / vector configs
-        into a config dictionary that can be serialized into arrow metadata
-        """
-        if func_list is None or len(func_list) == 0:
-            return None
-        json_data = [self.function_to_metadata(func) for func in func_list]
-        # Note that metadata dictionary values must be bytes
-        # so we need to json dump then utf8 encode
-        metadata = json.dumps(json_data, indent=2).encode("utf-8")
-        return {"embedding_functions": metadata}
-
-
-# Global instance
-__REGISTRY__ = EmbeddingFunctionRegistry()
-
-
-TEXT = Union[str, List[str], pa.Array, pa.ChunkedArray, np.ndarray]
-IMAGES = Union[
-    str, bytes, List[str], List[bytes], pa.Array, pa.ChunkedArray, np.ndarray
-]
-
-
-class EmbeddingFunction(BaseModel, ABC):
-    """
-    An ABC for embedding functions.
-
-    All concrete embedding functions must implement the following:
-    1. compute_query_embeddings() which takes a query and returns a list of embeddings
-    2. get_source_embeddings() which returns a list of embeddings for the source column
-    For text data, the two will be the same. For multi-modal data, the source column
-    might be images and the vector column might be text.
-    3. ndims method which returns the number of dimensions of the vector column
-    """
-
-    _ndims: int = PrivateAttr()
-
-    @classmethod
-    def create(cls, **kwargs):
-        """
-        Create an instance of the embedding function
-        """
-        return cls(**kwargs)
-
-    @abstractmethod
-    def compute_query_embeddings(self, *args, **kwargs) -> List[np.array]:
-        """
-        Compute the embeddings for a given user query
-        """
-        pass
-
-    @abstractmethod
-    def compute_source_embeddings(self, *args, **kwargs) -> List[np.array]:
-        """
-        Compute the embeddings for the source column in the database
-        """
-        pass
-
-    def sanitize_input(self, texts: TEXT) -> Union[List[str], np.ndarray]:
-        """
-        Sanitize the input to the embedding function.
-        """
-        if isinstance(texts, str):
-            texts = [texts]
-        elif isinstance(texts, pa.Array):
-            texts = texts.to_pylist()
-        elif isinstance(texts, pa.ChunkedArray):
-            texts = texts.combine_chunks().to_pylist()
-        return texts
-
-    @classmethod
-    def safe_import(cls, module: str, mitigation=None):
-        """
-        Import the specified module. If the module is not installed,
-        raise an ImportError with a helpful message.
-
-        Parameters
-        ----------
-        module : str
-            The name of the module to import
-        mitigation : Optional[str]
-            The package(s) to install to mitigate the error.
-            If not provided then the module name will be used.
-        """
-        try:
-            return importlib.import_module(module)
-        except ImportError:
-            raise ImportError(f"Please install {mitigation or module}")
-
-    def safe_model_dump(self):
-        from ..pydantic import PYDANTIC_VERSION
-
-        if PYDANTIC_VERSION.major < 2:
-            return dict(self)
-        return self.model_dump()
-
-    @abstractmethod
-    def ndims(self):
-        """
-        Return the dimensions of the vector column
-        """
-        pass
-
-    def SourceField(self, **kwargs):
-        """
-        Creates a pydantic Field that can automatically annotate
-        the source column for this embedding function
-        """
-        return Field(json_schema_extra={"source_column_for": self}, **kwargs)
-
-    def VectorField(self, **kwargs):
-        """
-        Creates a pydantic Field that can automatically annotate
-        the target vector column for this embedding function
-        """
-        return Field(json_schema_extra={"vector_column_for": self}, **kwargs)
-
-
-class EmbeddingFunctionConfig(BaseModel):
-    """
-    This model encapsulates the configuration for a embedding function
-    in a lancedb table. It holds the embedding function, the source column,
-    and the vector column
-    """
-
-    vector_column: str
-    source_column: str
-    function: EmbeddingFunction
-
-
-class TextEmbeddingFunction(EmbeddingFunction):
-    """
-    A callable ABC for embedding functions that take text as input
-    """
-
-    def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]:
-        return self.compute_source_embeddings(query, *args, **kwargs)
-
-    def compute_source_embeddings(self, texts: TEXT, *args, **kwargs) -> List[np.array]:
-        texts = self.sanitize_input(texts)
-        return self.generate_embeddings(texts)
-
-    @abstractmethod
-    def generate_embeddings(
-        self, texts: Union[List[str], np.ndarray]
-    ) -> List[np.array]:
-        """
-        Generate the embeddings for the given texts
-        """
-        pass
-
-
-# @EmbeddingFunctionRegistry.get_instance().register(name) doesn't work in 3.8
-register = lambda name: EmbeddingFunctionRegistry.get_instance().register(name)
-
-
-@register("sentence-transformers")
-class SentenceTransformerEmbeddings(TextEmbeddingFunction):
-    """
-    An embedding function that uses the sentence-transformers library
-
-    https://huggingface.co/sentence-transformers
-    """
-
-    name: str = "all-MiniLM-L6-v2"
-    device: str = "cpu"
-    normalize: bool = True
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self._ndims = None
-
-    @property
-    def embedding_model(self):
-        """
-        Get the sentence-transformers embedding model specified by the
-        name and device. This is cached so that the model is only loaded
-        once per process.
-        """
-        return self.__class__.get_embedding_model(self.name, self.device)
-
-    def ndims(self):
-        if self._ndims is None:
-            self._ndims = len(self.generate_embeddings("foo")[0])
-        return self._ndims
-
-    def generate_embeddings(
-        self, texts: Union[List[str], np.ndarray]
-    ) -> List[np.array]:
-        """
-        Get the embeddings for the given texts
-
-        Parameters
-        ----------
-        texts: list[str] or np.ndarray (of str)
-            The texts to embed
-        """
-        return self.embedding_model.encode(
-            list(texts),
-            convert_to_numpy=True,
-            normalize_embeddings=self.normalize,
-        ).tolist()
-
-    @classmethod
-    @cached(cache={})
-    def get_embedding_model(cls, name, device):
-        """
-        Get the sentence-transformers embedding model specified by the
-        name and device. This is cached so that the model is only loaded
-        once per process.
-
-        Parameters
-        ----------
-        name : str
-            The name of the model to load
-        device : str
-            The device to load the model on
-
-        TODO: use lru_cache instead with a reasonable/configurable maxsize
-        """
-        sentence_transformers = cls.safe_import(
-            "sentence_transformers", "sentence-transformers"
-        )
-        return sentence_transformers.SentenceTransformer(name, device=device)
-
-
-@register("openai")
-class OpenAIEmbeddings(TextEmbeddingFunction):
-    """
-    An embedding function that uses the OpenAI API
-
-    https://platform.openai.com/docs/guides/embeddings
-    """
-
-    name: str = "text-embedding-ada-002"
-
-    def ndims(self):
-        # TODO don't hardcode this
-        return 1536
-
-    def generate_embeddings(
-        self, texts: Union[List[str], np.ndarray]
-    ) -> List[np.array]:
-        """
-        Get the embeddings for the given texts
-
-        Parameters
-        ----------
-        texts: list[str] or np.ndarray (of str)
-            The texts to embed
-        """
-        # TODO retry, rate limit, token limit
-        openai = self.safe_import("openai")
-        rs = openai.Embedding.create(input=texts, model=self.name)["data"]
-        return [v["embedding"] for v in rs]
-
-
-@register("open-clip")
-class OpenClipEmbeddings(EmbeddingFunction):
-    """
-    An embedding function that uses the OpenClip API
-    For multi-modal text-to-image search
-
-    https://github.com/mlfoundations/open_clip
-    """
-
-    name: str = "ViT-B-32"
-    pretrained: str = "laion2b_s34b_b79k"
-    device: str = "cpu"
-    batch_size: int = 64
-    normalize: bool = True
-    _model = PrivateAttr()
-    _preprocess = PrivateAttr()
-    _tokenizer = PrivateAttr()
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        open_clip = self.safe_import("open_clip", "open-clip")
-        model, _, preprocess = open_clip.create_model_and_transforms(
-            self.name, pretrained=self.pretrained
-        )
-        model.to(self.device)
-        self._model, self._preprocess = model, preprocess
-        self._tokenizer = open_clip.get_tokenizer(self.name)
-        self._ndims = None
-
-    def ndims(self):
-        if self._ndims is None:
-            self._ndims = self.generate_text_embeddings("foo").shape[0]
-        return self._ndims
-
-    def compute_query_embeddings(
-        self, query: Union[str, "PIL.Image.Image"], *args, **kwargs
-    ) -> List[np.ndarray]:
-        """
-        Compute the embeddings for a given user query
-
-        Parameters
-        ----------
-        query : Union[str, PIL.Image.Image]
-            The query to embed. A query can be either text or an image.
-        """
-        if isinstance(query, str):
-            return [self.generate_text_embeddings(query)]
-        else:
-            PIL = self.safe_import("PIL", "pillow")
-            if isinstance(query, PIL.Image.Image):
-                return [self.generate_image_embedding(query)]
-            else:
-                raise TypeError("OpenClip supports str or PIL Image as query")
-
-    def generate_text_embeddings(self, text: str) -> np.ndarray:
-        torch = self.safe_import("torch")
-        text = self.sanitize_input(text)
-        text = self._tokenizer(text)
-        text.to(self.device)
-        with torch.no_grad():
-            text_features = self._model.encode_text(text.to(self.device))
-            if self.normalize:
-                text_features /= text_features.norm(dim=-1, keepdim=True)
-            return text_features.cpu().numpy().squeeze()
-
-    def sanitize_input(self, images: IMAGES) -> Union[List[bytes], np.ndarray]:
-        """
-        Sanitize the input to the embedding function.
-        """
-        if isinstance(images, (str, bytes)):
-            images = [images]
-        elif isinstance(images, pa.Array):
-            images = images.to_pylist()
-        elif isinstance(images, pa.ChunkedArray):
-            images = images.combine_chunks().to_pylist()
-        return images
-
-    def compute_source_embeddings(
-        self, images: IMAGES, *args, **kwargs
-    ) -> List[np.array]:
-        """
-        Get the embeddings for the given images
-        """
-        images = self.sanitize_input(images)
-        embeddings = []
-        for i in range(0, len(images), self.batch_size):
-            j = min(i + self.batch_size, len(images))
-            batch = images[i:j]
-            embeddings.extend(self._parallel_get(batch))
-        return embeddings
-
-    def _parallel_get(self, images: Union[List[str], List[bytes]]) -> List[np.ndarray]:
-        """
-        Issue concurrent requests to retrieve the image data
-        """
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [
-                executor.submit(self.generate_image_embedding, image)
-                for image in images
-            ]
-            return [future.result() for future in tqdm(futures)]
-
-    def generate_image_embedding(
-        self, image: Union[str, bytes, "PIL.Image.Image"]
-    ) -> np.ndarray:
-        """
-        Generate the embedding for a single image
-
-        Parameters
-        ----------
-        image : Union[str, bytes, PIL.Image.Image]
-            The image to embed. If the image is a str, it is treated as a uri.
-            If the image is bytes, it is treated as the raw image bytes.
-        """
-        torch = self.safe_import("torch")
-        # TODO handle retry and errors for https
-        image = self._to_pil(image)
-        image = self._preprocess(image).unsqueeze(0)
-        with torch.no_grad():
-            return self._encode_and_normalize_image(image)
-
-    def _to_pil(self, image: Union[str, bytes]):
-        PIL = self.safe_import("PIL", "pillow")
-        if isinstance(image, bytes):
-            return PIL.Image.open(io.BytesIO(image))
-        if isinstance(image, PIL.Image.Image):
-            return image
-        elif isinstance(image, str):
-            parsed = urlparse.urlparse(image)
-            # TODO handle drive letter on windows.
-            if parsed.scheme == "file":
-                return PIL.Image.open(parsed.path)
-            elif parsed.scheme == "":
-                return PIL.Image.open(image if os.name == "nt" else parsed.path)
-            elif parsed.scheme.startswith("http"):
-                return PIL.Image.open(io.BytesIO(url_retrieve(image)))
-            else:
-                raise NotImplementedError("Only local and http(s) urls are supported")
-
-    def _encode_and_normalize_image(self, image_tensor: "torch.Tensor"):
-        """
-        encode a single image tensor and optionally normalize the output
-        """
-        image_features = self._model.encode_image(image_tensor.to(self.device))
-        if self.normalize:
-            image_features /= image_features.norm(dim=-1, keepdim=True)
-        return image_features.cpu().numpy().squeeze()
-
-
-def url_retrieve(url: str):
-    """
-    Parameters
-    ----------
-    url: str
-        URL to download from
-    """
-    try:
-        with urllib.request.urlopen(url) as conn:
-            return conn.read()
-    except (socket.gaierror, urllib.error.URLError) as err:
-        raise ConnectionError("could not download {} due to {}".format(url, err))

python/lancedb/embeddings/instructor.py

@@ -0,0 +1,137 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from typing import List
+
+import numpy as np
+
+from .base import TextEmbeddingFunction
+from .registry import register
+from .utils import TEXT, weak_lru
+
+
+@register("instructor")
+class InstructorEmbeddingFunction(TextEmbeddingFunction):
+    """
+    An embedding function that uses the InstructorEmbedding library. Instructor models support multi-task learning, and can be used for a
+    variety of tasks, including text classification, sentence similarity, and document retrieval.
+    If you want to calculate customized embeddings for specific sentences, you may follow the unified template to write instructions:
+        "Represent the `domain` `text_type` for `task_objective`":
+
+        * domain is optional, and it specifies the domain of the text, e.g., science, finance, medicine, etc.
+        * text_type is required, and it specifies the encoding unit, e.g., sentence, document, paragraph, etc.
+        * task_objective is optional, and it specifies the objective of embedding, e.g., retrieve a document, classify the sentence, etc.
+
+    For example, if you want to calculate embeddings for a document, you may write the instruction as follows:
+        "Represent the document for retreival"
+
+    Parameters
+    ----------
+    name: str
+        The name of the model to use. Available models are listed at https://github.com/xlang-ai/instructor-embedding#model-list;
+        The default model is hkunlp/instructor-base
+    batch_size: int, default 32
+        The batch size to use when generating embeddings
+    device: str, default "cpu"
+        The device to use when generating embeddings
+    show_progress_bar: bool, default True
+        Whether to show a progress bar when generating embeddings
+    normalize_embeddings: bool, default True
+        Whether to normalize the embeddings
+    quantize: bool, default False
+        Whether to quantize the model
+    source_instruction: str, default "represent the docuement for retreival"
+        The instruction for the source column
+    query_instruction: str, default "represent the document for retreiving the most similar documents"
+        The instruction for the query
+
+    Examples
+    --------
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import get_registry, InstuctorEmbeddingFunction
+
+    instructor = get_registry().get("instructor").create(
+                                source_instruction="represent the docuement for retreival",
+                                query_instruction="represent the document for retreiving the most similar documents"
+                                )
+
+    class Schema(LanceModel):
+        vector: Vector(instructor.ndims()) = instructor.VectorField()
+        text: str = instructor.SourceField()
+
+    db = lancedb.connect("~/.lancedb")
+    tbl = db.create_table("test", schema=Schema, mode="overwrite")
+
+    texts = [{"text": "Capitalism has been dominant in the Western world since the end of feudalism, but most feel[who?] that..."},
+            {"text": "The disparate impact theory is especially controversial under the Fair Housing Act because the Act..."},
+            {"text": "Disparate impact in United States labor law refers to practices in employment, housing, and other areas that.."}]
+
+    tbl.add(texts)
+
+    """
+
+    name: str = "hkunlp/instructor-base"
+    batch_size: int = 32
+    device: str = "cpu"
+    show_progress_bar: bool = True
+    normalize_embeddings: bool = True
+    quantize: bool = False
+    # convert_to_numpy: bool = True # Hardcoding this as numpy can be ingested directly
+
+    source_instruction: str = "represent the document for retrieval"
+    query_instruction: str = (
+        "represent the document for retrieving the most similar documents"
+    )
+
+    @weak_lru(maxsize=1)
+    def ndims(self):
+        model = self.get_model()
+        return model.encode("foo").shape[0]
+
+    def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]:
+        return self.generate_embeddings([[self.query_instruction, query]])
+
+    def compute_source_embeddings(self, texts: TEXT, *args, **kwargs) -> List[np.array]:
+        texts = self.sanitize_input(texts)
+        texts_formatted = []
+        for text in texts:
+            texts_formatted.append([self.source_instruction, text])
+        return self.generate_embeddings(texts_formatted)
+
+    def generate_embeddings(self, texts: List) -> List:
+        model = self.get_model()
+        res = model.encode(
+            texts,
+            batch_size=self.batch_size,
+            show_progress_bar=self.show_progress_bar,
+            normalize_embeddings=self.normalize_embeddings,
+        ).tolist()
+        return res
+
+    @weak_lru(maxsize=1)
+    def get_model(self):
+        instructor_embedding = self.safe_import(
+            "InstructorEmbedding", "InstructorEmbedding"
+        )
+        torch = self.safe_import("torch", "torch")
+
+        model = instructor_embedding.INSTRUCTOR(self.name)
+        if self.quantize:
+            if (
+                "qnnpack" in torch.backends.quantized.supported_engines
+            ):  # fix for https://github.com/pytorch/pytorch/issues/29327
+                torch.backends.quantized.engine = "qnnpack"
+            model = torch.quantization.quantize_dynamic(
+                model, {torch.nn.Linear}, dtype=torch.qint8
+            )
+        return model

python/lancedb/embeddings/open_clip.py

@@ -0,0 +1,175 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+import concurrent.futures
+import io
+import os
+import urllib.parse as urlparse
+from typing import List, Union
+
+import numpy as np
+import pyarrow as pa
+from pydantic import PrivateAttr
+from tqdm import tqdm
+
+from .base import EmbeddingFunction
+from .registry import register
+from .utils import IMAGES, url_retrieve
+
+
+@register("open-clip")
+class OpenClipEmbeddings(EmbeddingFunction):
+    """
+    An embedding function that uses the OpenClip API
+    For multi-modal text-to-image search
+
+    https://github.com/mlfoundations/open_clip
+    """
+
+    name: str = "ViT-B-32"
+    pretrained: str = "laion2b_s34b_b79k"
+    device: str = "cpu"
+    batch_size: int = 64
+    normalize: bool = True
+    _model = PrivateAttr()
+    _preprocess = PrivateAttr()
+    _tokenizer = PrivateAttr()
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        open_clip = self.safe_import("open_clip", "open-clip")
+        model, _, preprocess = open_clip.create_model_and_transforms(
+            self.name, pretrained=self.pretrained
+        )
+        model.to(self.device)
+        self._model, self._preprocess = model, preprocess
+        self._tokenizer = open_clip.get_tokenizer(self.name)
+        self._ndims = None
+
+    def ndims(self):
+        if self._ndims is None:
+            self._ndims = self.generate_text_embeddings("foo").shape[0]
+        return self._ndims
+
+    def compute_query_embeddings(
+        self, query: Union[str, "PIL.Image.Image"], *args, **kwargs
+    ) -> List[np.ndarray]:
+        """
+        Compute the embeddings for a given user query
+
+        Parameters
+        ----------
+        query : Union[str, PIL.Image.Image]
+            The query to embed. A query can be either text or an image.
+        """
+        if isinstance(query, str):
+            return [self.generate_text_embeddings(query)]
+        else:
+            PIL = self.safe_import("PIL", "pillow")
+            if isinstance(query, PIL.Image.Image):
+                return [self.generate_image_embedding(query)]
+            else:
+                raise TypeError("OpenClip supports str or PIL Image as query")
+
+    def generate_text_embeddings(self, text: str) -> np.ndarray:
+        torch = self.safe_import("torch")
+        text = self.sanitize_input(text)
+        text = self._tokenizer(text)
+        text.to(self.device)
+        with torch.no_grad():
+            text_features = self._model.encode_text(text.to(self.device))
+            if self.normalize:
+                text_features /= text_features.norm(dim=-1, keepdim=True)
+            return text_features.cpu().numpy().squeeze()
+
+    def sanitize_input(self, images: IMAGES) -> Union[List[bytes], np.ndarray]:
+        """
+        Sanitize the input to the embedding function.
+        """
+        if isinstance(images, (str, bytes)):
+            images = [images]
+        elif isinstance(images, pa.Array):
+            images = images.to_pylist()
+        elif isinstance(images, pa.ChunkedArray):
+            images = images.combine_chunks().to_pylist()
+        return images
+
+    def compute_source_embeddings(
+        self, images: IMAGES, *args, **kwargs
+    ) -> List[np.array]:
+        """
+        Get the embeddings for the given images
+        """
+        images = self.sanitize_input(images)
+        embeddings = []
+        for i in range(0, len(images), self.batch_size):
+            j = min(i + self.batch_size, len(images))
+            batch = images[i:j]
+            embeddings.extend(self._parallel_get(batch))
+        return embeddings
+
+    def _parallel_get(self, images: Union[List[str], List[bytes]]) -> List[np.ndarray]:
+        """
+        Issue concurrent requests to retrieve the image data
+        """
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [
+                executor.submit(self.generate_image_embedding, image)
+                for image in images
+            ]
+            return [future.result() for future in tqdm(futures)]
+
+    def generate_image_embedding(
+        self, image: Union[str, bytes, "PIL.Image.Image"]
+    ) -> np.ndarray:
+        """
+        Generate the embedding for a single image
+
+        Parameters
+        ----------
+        image : Union[str, bytes, PIL.Image.Image]
+            The image to embed. If the image is a str, it is treated as a uri.
+            If the image is bytes, it is treated as the raw image bytes.
+        """
+        torch = self.safe_import("torch")
+        # TODO handle retry and errors for https
+        image = self._to_pil(image)
+        image = self._preprocess(image).unsqueeze(0)
+        with torch.no_grad():
+            return self._encode_and_normalize_image(image)
+
+    def _to_pil(self, image: Union[str, bytes]):
+        PIL = self.safe_import("PIL", "pillow")
+        if isinstance(image, bytes):
+            return PIL.Image.open(io.BytesIO(image))
+        if isinstance(image, PIL.Image.Image):
+            return image
+        elif isinstance(image, str):
+            parsed = urlparse.urlparse(image)
+            # TODO handle drive letter on windows.
+            if parsed.scheme == "file":
+                return PIL.Image.open(parsed.path)
+            elif parsed.scheme == "":
+                return PIL.Image.open(image if os.name == "nt" else parsed.path)
+            elif parsed.scheme.startswith("http"):
+                return PIL.Image.open(io.BytesIO(url_retrieve(image)))
+            else:
+                raise NotImplementedError("Only local and http(s) urls are supported")
+
+    def _encode_and_normalize_image(self, image_tensor: "torch.Tensor"):
+        """
+        encode a single image tensor and optionally normalize the output
+        """
+        image_features = self._model.encode_image(image_tensor.to(self.device))
+        if self.normalize:
+            image_features /= image_features.norm(dim=-1, keepdim=True)
+        return image_features.cpu().numpy().squeeze()

python/lancedb/embeddings/openai.py

@@ -0,0 +1,49 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from typing import List, Union
+
+import numpy as np
+
+from .base import TextEmbeddingFunction
+from .registry import register
+
+
+@register("openai")
+class OpenAIEmbeddings(TextEmbeddingFunction):
+    """
+    An embedding function that uses the OpenAI API
+
+    https://platform.openai.com/docs/guides/embeddings
+    """
+
+    name: str = "text-embedding-ada-002"
+
+    def ndims(self):
+        # TODO don't hardcode this
+        return 1536
+
+    def generate_embeddings(
+        self, texts: Union[List[str], np.ndarray]
+    ) -> List[np.array]:
+        """
+        Get the embeddings for the given texts
+
+        Parameters
+        ----------
+        texts: list[str] or np.ndarray (of str)
+            The texts to embed
+        """
+        # TODO retry, rate limit, token limit
+        openai = self.safe_import("openai")
+        rs = openai.Embedding.create(input=texts, model=self.name)["data"]
+        return [v["embedding"] for v in rs]

python/lancedb/embeddings/registry.py

@@ -0,0 +1,186 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+import json
+from typing import Dict, Optional
+
+from .base import EmbeddingFunction, EmbeddingFunctionConfig
+
+
+class EmbeddingFunctionRegistry:
+    """
+    This is a singleton class used to register embedding functions
+    and fetch them by name. It also handles serializing and deserializing.
+    You can implement your own embedding function by subclassing EmbeddingFunction
+    or TextEmbeddingFunction and registering it with the registry.
+
+    NOTE: Here TEXT is a type alias for Union[str, List[str], pa.Array, pa.ChunkedArray, np.ndarray]
+    Examples
+    --------
+    >>> registry = EmbeddingFunctionRegistry.get_instance()
+    >>> @registry.register("my-embedding-function")
+    ... class MyEmbeddingFunction(EmbeddingFunction):
+    ...     def ndims(self) -> int:
+    ...         return 128
+    ...
+    ...     def compute_query_embeddings(self, query: str, *args, **kwargs):
+    ...         return self.compute_source_embeddings(query, *args, **kwargs)
+    ...
+    ...     def compute_source_embeddings(self, texts, *args, **kwargs):
+    ...         return [np.random.rand(self.ndims()) for _ in range(len(texts))]
+    ...
+    >>> registry.get("my-embedding-function")
+    <class 'lancedb.embeddings.registry.MyEmbeddingFunction'>
+    """
+
+    @classmethod
+    def get_instance(cls):
+        return __REGISTRY__
+
+    def __init__(self):
+        self._functions = {}
+
+    def register(self, alias: str = None):
+        """
+        This creates a decorator that can be used to register
+        an EmbeddingFunction.
+
+        Parameters
+        ----------
+        alias : Optional[str]
+            a human friendly name for the embedding function. If not
+            provided, the class name will be used.
+        """
+
+        # This is a decorator for a class that inherits from BaseModel
+        # It adds the class to the registry
+        def decorator(cls):
+            if not issubclass(cls, EmbeddingFunction):
+                raise TypeError("Must be a subclass of EmbeddingFunction")
+            if cls.__name__ in self._functions:
+                raise KeyError(f"{cls.__name__} was already registered")
+            key = alias or cls.__name__
+            self._functions[key] = cls
+            cls.__embedding_function_registry_alias__ = alias
+            return cls
+
+        return decorator
+
+    def reset(self):
+        """
+        Reset the registry to its initial state
+        """
+        self._functions = {}
+
+    def get(self, name: str):
+        """
+        Fetch an embedding function class by name
+
+        Parameters
+        ----------
+        name : str
+            The name of the embedding function to fetch
+            Either the alias or the class name if no alias was provided
+            during registration
+        """
+        return self._functions[name]
+
+    def parse_functions(
+        self, metadata: Optional[Dict[bytes, bytes]]
+    ) -> Dict[str, "EmbeddingFunctionConfig"]:
+        """
+        Parse the metadata from an arrow table and
+        return a mapping of the vector column to the
+        embedding function and source column
+
+        Parameters
+        ----------
+        metadata : Optional[Dict[bytes, bytes]]
+            The metadata from an arrow table. Note that
+            the keys and values are bytes (pyarrow api)
+
+        Returns
+        -------
+        functions : dict
+            A mapping of vector column name to embedding function.
+            An empty dict is returned if input is None or does not
+            contain b"embedding_functions".
+        """
+        if metadata is None or b"embedding_functions" not in metadata:
+            return {}
+        serialized = metadata[b"embedding_functions"]
+        raw_list = json.loads(serialized.decode("utf-8"))
+        return {
+            obj["vector_column"]: EmbeddingFunctionConfig(
+                vector_column=obj["vector_column"],
+                source_column=obj["source_column"],
+                function=self.get(obj["name"])(**obj["model"]),
+            )
+            for obj in raw_list
+        }
+
+    def function_to_metadata(self, conf: "EmbeddingFunctionConfig"):
+        """
+        Convert the given embedding function and source / vector column configs
+        into a config dictionary that can be serialized into arrow metadata
+        """
+        func = conf.function
+        name = getattr(
+            func, "__embedding_function_registry_alias__", func.__class__.__name__
+        )
+        json_data = func.safe_model_dump()
+        return {
+            "name": name,
+            "model": json_data,
+            "source_column": conf.source_column,
+            "vector_column": conf.vector_column,
+        }
+
+    def get_table_metadata(self, func_list):
+        """
+        Convert a list of embedding functions and source / vector configs
+        into a config dictionary that can be serialized into arrow metadata
+        """
+        if func_list is None or len(func_list) == 0:
+            return None
+        json_data = [self.function_to_metadata(func) for func in func_list]
+        # Note that metadata dictionary values must be bytes
+        # so we need to json dump then utf8 encode
+        metadata = json.dumps(json_data, indent=2).encode("utf-8")
+        return {"embedding_functions": metadata}
+
+
+# Global instance
+__REGISTRY__ = EmbeddingFunctionRegistry()
+
+
+# @EmbeddingFunctionRegistry.get_instance().register(name) doesn't work in 3.8
+register = lambda name: EmbeddingFunctionRegistry.get_instance().register(name)
+
+
+def get_registry():
+    """
+    Utility function to get the global instance of the registry
+
+    Returns
+    -------
+    EmbeddingFunctionRegistry
+        The global registry instance
+
+    Examples
+    --------
+    from lancedb.embeddings import get_registry
+
+    registry = get_registry()
+    openai = registry.get("openai").create()
+    """
+    return __REGISTRY__.get_instance()

python/lancedb/embeddings/sentence_transformers.py

@@ -0,0 +1,89 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from typing import List, Union
+
+import numpy as np
+from cachetools import cached
+
+from .base import TextEmbeddingFunction
+from .registry import register
+from .utils import weak_lru
+
+
+@register("sentence-transformers")
+class SentenceTransformerEmbeddings(TextEmbeddingFunction):
+    """
+    An embedding function that uses the sentence-transformers library
+
+    https://huggingface.co/sentence-transformers
+    """
+
+    name: str = "all-MiniLM-L6-v2"
+    device: str = "cpu"
+    normalize: bool = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._ndims = None
+
+    @property
+    def embedding_model(self):
+        """
+        Get the sentence-transformers embedding model specified by the
+        name and device. This is cached so that the model is only loaded
+        once per process.
+        """
+        return self.get_embedding_model()
+
+    def ndims(self):
+        if self._ndims is None:
+            self._ndims = len(self.generate_embeddings("foo")[0])
+        return self._ndims
+
+    def generate_embeddings(
+        self, texts: Union[List[str], np.ndarray]
+    ) -> List[np.array]:
+        """
+        Get the embeddings for the given texts
+
+        Parameters
+        ----------
+        texts: list[str] or np.ndarray (of str)
+            The texts to embed
+        """
+        return self.embedding_model.encode(
+            list(texts),
+            convert_to_numpy=True,
+            normalize_embeddings=self.normalize,
+        ).tolist()
+
+    @weak_lru(maxsize=1)
+    def get_embedding_model(self):
+        """
+        Get the sentence-transformers embedding model specified by the
+        name and device. This is cached so that the model is only loaded
+        once per process.
+
+        Parameters
+        ----------
+        name : str
+            The name of the model to load
+        device : str
+            The device to load the model on
+
+        TODO: use lru_cache instead with a reasonable/configurable maxsize
+        """
+        sentence_transformers = self.safe_import(
+            "sentence_transformers", "sentence-transformers"
+        )
+        return sentence_transformers.SentenceTransformer(self.name, device=self.device)

python/lancedb/embeddings/utils.py

@@ -11,9 +11,15 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import functools
 import math
+import random
+import socket
 import sys
-from typing import Callable, Union
+import time
+import urllib.error
+import weakref
+from typing import Callable, List, Union
 
 import numpy as np
 import pyarrow as pa
@@ -24,7 +30,12 @@ from ..util import safe_import_pandas
 from ..utils.general import LOGGER
 
 pd = safe_import_pandas()
+
 DATA = Union[pa.Table, "pd.DataFrame"]
+TEXT = Union[str, List[str], pa.Array, pa.ChunkedArray, np.ndarray]
+IMAGES = Union[
+    str, bytes, List[str], List[bytes], pa.Array, pa.ChunkedArray, np.ndarray
+]
 
 
 def with_embeddings(
@@ -155,6 +166,113 @@ class FunctionWrapper:
             yield from _chunker(arr)
 
 
+def weak_lru(maxsize=128):
+    """
+    LRU cache that keeps weak references to the objects it caches. Only caches the latest instance of the objects to make sure memory usage
+    is bounded.
+
+    Parameters
+    ----------
+    maxsize : int, default 128
+        The maximum number of objects to cache.
+
+    Returns
+    -------
+    Callable
+        A decorator that can be applied to a method.
+
+    Examples
+    --------
+    >>> class Foo:
+    ...     @weak_lru()
+    ...     def bar(self, x):
+    ...         return x
+    >>> foo = Foo()
+    >>> foo.bar(1)
+    1
+    >>> foo.bar(2)
+    2
+    >>> foo.bar(1)
+    1
+    """
+
+    def wrapper(func):
+        @functools.lru_cache(maxsize)
+        def _func(_self, *args, **kwargs):
+            return func(_self(), *args, **kwargs)
+
+        @functools.wraps(func)
+        def inner(self, *args, **kwargs):
+            return _func(weakref.ref(self), *args, **kwargs)
+
+        return inner
+
+    return wrapper
+
+
+def retry_with_exponential_backoff(
+    func,
+    initial_delay: float = 1,
+    exponential_base: float = 2,
+    jitter: bool = True,
+    max_retries: int = 7,
+    # errors: tuple = (),
+):
+    """Retry a function with exponential backoff.
+
+    Args:
+        func (function): The function to be retried.
+        initial_delay (float): Initial delay in seconds (default is 1).
+        exponential_base (float): The base for exponential backoff (default is 2).
+        jitter (bool): Whether to add jitter to the delay (default is True).
+        max_retries (int): Maximum number of retries (default is 10).
+        errors (tuple): Tuple of specific exceptions to retry on (default is (openai.error.RateLimitError,)).
+
+    Returns:
+        function: The decorated function.
+    """
+
+    def wrapper(*args, **kwargs):
+        num_retries = 0
+        delay = initial_delay
+
+        # Loop until a successful response or max_retries is hit or an exception is raised
+        while True:
+            try:
+                return func(*args, **kwargs)
+
+            # Currently retrying on all exceptions as there is no way to know the format of the error msgs used by different APIs
+            # We'll log the error and say that it is assumed that if this portion errors out, it's due to rate limit but the user
+            # should check the error message to be sure
+            except Exception as e:
+                num_retries += 1
+
+                if num_retries > max_retries:
+                    raise Exception(
+                        f"Maximum number of retries ({max_retries}) exceeded."
+                    )
+
+                delay *= exponential_base * (1 + jitter * random.random())
+                LOGGER.info(f"Retrying in {delay:.2f} seconds due to {e}")
+                time.sleep(delay)
+
+    return wrapper
+
+
+def url_retrieve(url: str):
+    """
+    Parameters
+    ----------
+    url: str
+        URL to download from
+    """
+    try:
+        with urllib.request.urlopen(url) as conn:
+            return conn.read()
+    except (socket.gaierror, urllib.error.URLError) as err:
+        raise ConnectionError("could not download {} due to {}".format(url, err))
+
+
 def api_key_not_found_help(provider):
     LOGGER.error(f"Could not find API key for {provider}.")
     raise ValueError(f"Please set the {provider.upper()}_API_KEY environment variable.")

python/lancedb/fts.py

@@ -75,8 +75,14 @@ def populate_index(index: tantivy.Index, table: LanceTable, fields: List[str]) -
         The number of rows indexed
     """
     # first check the fields exist and are string or large string type
+    nested = []
     for name in fields:
-        f = table.schema.field(name)  # raises KeyError if not found
+        try:
+            f = table.schema.field(name)  # raises KeyError if not found
+        except KeyError:
+            f = resolve_path(table.schema, name)
+            nested.append(name)
+
         if not pa.types.is_string(f.type) and not pa.types.is_large_string(f.type):
             raise TypeError(f"Field {name} is not a string type")
 
@@ -85,7 +91,16 @@ def populate_index(index: tantivy.Index, table: LanceTable, fields: List[str]) -
     # write data into index
     dataset = table.to_lance()
     row_id = 0
+
+    max_nested_level = 0
+    if len(nested) > 0:
+        max_nested_level = max([len(name.split(".")) for name in nested])
+
     for b in dataset.to_batches(columns=fields):
+        if max_nested_level > 0:
+            b = pa.Table.from_batches([b])
+            for _ in range(max_nested_level - 1):
+                b = b.flatten()
         for i in range(b.num_rows):
             doc = tantivy.Document()
             doc.add_integer("doc_id", row_id)
@@ -98,6 +113,30 @@ def populate_index(index: tantivy.Index, table: LanceTable, fields: List[str]) -
     return row_id
 
 
+def resolve_path(schema, field_name: str) -> pa.Field:
+    """
+    Resolve a nested field path to a list of field names
+
+    Parameters
+    ----------
+    field_name : str
+        The field name to resolve
+
+    Returns
+    -------
+    List[str]
+        The resolved path
+    """
+    path = field_name.split(".")
+    field = schema.field(path.pop(0))
+    for segment in path:
+        if pa.types.is_struct(field.type):
+            field = field.type.field(segment)
+        else:
+            raise KeyError(f"field {field_name} not found in schema {schema}")
+    return field
+
+
 def search_index(
     index: tantivy.Index, query: str, limit: int = 10
 ) -> Tuple[Tuple[int], Tuple[float]]:

python/lancedb/pydantic.py

@@ -19,6 +19,7 @@ import inspect
 import sys
 import types
 from abc import ABC, abstractmethod
+from datetime import date, datetime
 from typing import Any, Callable, Dict, Generator, List, Type, Union, _GenericAlias
 
 import numpy as np
@@ -159,6 +160,10 @@ def _py_type_to_arrow_type(py_type: Type[Any]) -> pa.DataType:
         return pa.bool_()
     elif py_type == bytes:
         return pa.binary()
+    elif py_type == date:
+        return pa.date32()
+    elif py_type == datetime:
+        return pa.timestamp("us")
     raise TypeError(
         f"Converting Pydantic type to Arrow Type: unsupported type {py_type}"
     )
@@ -322,7 +327,12 @@ class LanceModel(pydantic.BaseModel):
         for vec, func in vec_and_function:
             for source, field_info in cls.safe_get_fields().items():
                 src_func = get_extras(field_info, "source_column_for")
-                if src_func == func:
+                if src_func is func:
+                    # note we can't use == here since the function is a pydantic
+                    # model so two instances of the same function are ==, so if you
+                    # have multiple vector columns from multiple sources, both will
+                    # be mapped to the same source column
+                    # GH594
                     configs.append(
                         EmbeddingFunctionConfig(
                             source_column=source, vector_column=vec, function=func
@@ -338,3 +348,20 @@ def get_extras(field_info: pydantic.fields.FieldInfo, key: str) -> Any:
     if PYDANTIC_VERSION.major >= 2:
         return (field_info.json_schema_extra or {}).get(key)
     return (field_info.field_info.extra or {}).get("json_schema_extra", {}).get(key)
+
+
+if PYDANTIC_VERSION.major < 2:
+
+    def model_to_dict(model: pydantic.BaseModel) -> Dict[str, Any]:
+        """
+        Convert a Pydantic model to a dictionary.
+        """
+        return model.dict()
+
+else:
+
+    def model_to_dict(model: pydantic.BaseModel) -> Dict[str, Any]:
+        """
+        Convert a Pydantic model to a dictionary.
+        """
+        return model.model_dump()

python/lancedb/query.py

@@ -14,7 +14,7 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
-from typing import List, Literal, Optional, Type, Union
+from typing import TYPE_CHECKING, List, Literal, Optional, Type, Union
 
 import deprecation
 import numpy as np
@@ -23,14 +23,49 @@ import pydantic
 
 from . import __version__
 from .common import VECTOR_COLUMN_NAME
-from .pydantic import LanceModel
 from .util import safe_import_pandas
 
+if TYPE_CHECKING:
+    from .pydantic import LanceModel
+
 pd = safe_import_pandas()
 
 
 class Query(pydantic.BaseModel):
-    """A Query"""
+    """The LanceDB Query
+
+    Attributes
+    ----------
+    vector : List[float]
+        the vector to search for
+    filter : Optional[str]
+        sql filter to refine the query with, optional
+    prefilter : bool
+        if True then apply the filter before vector search
+    k : int
+        top k results to return
+    metric : str
+        the distance metric between a pair of vectors,
+
+        can support L2 (default), Cosine and Dot.
+        [metric definitions][search]
+    columns : Optional[List[str]]
+        which columns to return in the results
+    nprobes : int
+        The number of probes used - optional
+
+        - A higher number makes search more accurate but also slower.
+
+        - See discussion in [Querying an ANN Index][querying-an-ann-index] for
+          tuning advice.
+    refine_factor : Optional[int]
+        Refine the results by reading extra elements and re-ranking them in memory - optional
+
+        - A higher number makes search more accurate but also slower.
+
+        - See discussion in [Querying an ANN Index][querying-an-ann-index] for
+          tuning advice.
+    """
 
     vector_column: str = VECTOR_COLUMN_NAME
 
@@ -61,6 +96,10 @@ class Query(pydantic.BaseModel):
 
 
 class LanceQueryBuilder(ABC):
+    """Build LanceDB query based on specific query type:
+    vector or full text search.
+    """
+
     @classmethod
     def create(
         cls,
@@ -103,7 +142,7 @@ class LanceQueryBuilder(ABC):
             if not isinstance(query, (list, np.ndarray)):
                 conf = table.embedding_functions.get(vector_column_name)
                 if conf is not None:
-                    query = conf.function.compute_query_embeddings(query)[0]
+                    query = conf.function.compute_query_embeddings_with_retry(query)[0]
                 else:
                     msg = f"No embedding function for {vector_column_name}"
                     raise ValueError(msg)
@@ -114,7 +153,7 @@ class LanceQueryBuilder(ABC):
             else:
                 conf = table.embedding_functions.get(vector_column_name)
                 if conf is not None:
-                    query = conf.function.compute_query_embeddings(query)[0]
+                    query = conf.function.compute_query_embeddings_with_retry(query)[0]
                     return query, "vector"
                 else:
                     return query, "fts"
@@ -133,11 +172,11 @@ class LanceQueryBuilder(ABC):
         deprecated_in="0.3.1",
         removed_in="0.4.0",
         current_version=__version__,
-        details="Use the bar function instead",
+        details="Use to_pandas() instead",
     )
     def to_df(self) -> "pd.DataFrame":
         """
-        Deprecated alias for `to_pandas()`. Please use `to_pandas()` instead.
+        *Deprecated alias for `to_pandas()`. Please use `to_pandas()` instead.*
 
         Execute the query and return the results as a pandas DataFrame.
         In addition to the selected columns, LanceDB also returns a vector
@@ -146,14 +185,40 @@ class LanceQueryBuilder(ABC):
         """
         return self.to_pandas()
 
-    def to_pandas(self) -> "pd.DataFrame":
+    def to_pandas(self, flatten: Optional[Union[int, bool]] = None) -> "pd.DataFrame":
         """
         Execute the query and return the results as a pandas DataFrame.
         In addition to the selected columns, LanceDB also returns a vector
         and also the "_distance" column which is the distance between the query
         vector and the returned vector.
+
+        Parameters
+        ----------
+        flatten: Optional[Union[int, bool]]
+            If flatten is True, flatten all nested columns.
+            If flatten is an integer, flatten the nested columns up to the
+            specified depth.
+            If unspecified, do not flatten the nested columns.
         """
-        return self.to_arrow().to_pandas()
+        tbl = self.to_arrow()
+        if flatten is True:
+            while True:
+                tbl = tbl.flatten()
+                has_struct = False
+                # loop through all columns to check if there is any struct column
+                if any(pa.types.is_struct(col.type) for col in tbl.schema):
+                    continue
+                else:
+                    break
+        elif isinstance(flatten, int):
+            if flatten <= 0:
+                raise ValueError(
+                    "Please specify a positive integer for flatten or the boolean value `True`"
+                )
+            while flatten > 0:
+                tbl = tbl.flatten()
+                flatten -= 1
+        return tbl.to_pandas()
 
     @abstractmethod
     def to_arrow(self) -> pa.Table:
@@ -226,13 +291,20 @@ class LanceQueryBuilder(ABC):
         self._columns = columns
         return self
 
-    def where(self, where) -> LanceQueryBuilder:
+    def where(self, where: str, prefilter: bool = False) -> LanceQueryBuilder:
         """Set the where clause.
 
         Parameters
         ----------
         where: str
-            The where clause.
+            The where clause which is a valid SQL where clause. See
+            `Lance filter pushdown <https://lancedb.github.io/lance/read_and_write.html#filter-push-down>`_
+            for valid SQL expressions.
+        prefilter: bool, default False
+            If True, apply the filter before vector search, otherwise the
+            filter is applied on the result of vector search.
+            This feature is **EXPERIMENTAL** and may be removed and modified
+            without warning in the future.
 
         Returns
         -------
@@ -240,13 +312,12 @@ class LanceQueryBuilder(ABC):
             The LanceQueryBuilder object.
         """
         self._where = where
+        self._prefilter = prefilter
         return self
 
 
 class LanceVectorQueryBuilder(LanceQueryBuilder):
     """
-    A builder for nearest neighbor queries for LanceDB.
-
     Examples
     --------
     >>> import lancedb
@@ -302,7 +373,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
         Higher values will yield better recall (more likely to find vectors if
         they exist) at the expense of latency.
 
-        See discussion in [Querying an ANN Index][../querying-an-ann-index] for
+        See discussion in [Querying an ANN Index][querying-an-ann-index] for
         tuning advice.
 
         Parameters
@@ -369,14 +440,14 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
         Parameters
         ----------
         where: str
-            The where clause.
+            The where clause which is a valid SQL where clause. See
+            `Lance filter pushdown <https://lancedb.github.io/lance/read_and_write.html#filter-push-down>`_
+            for valid SQL expressions.
         prefilter: bool, default False
             If True, apply the filter before vector search, otherwise the
             filter is applied on the result of vector search.
             This feature is **EXPERIMENTAL** and may be removed and modified
-            without warning in the future. Currently this is only supported
-            in OSS and can only be used with a table that does not have an ANN
-            index.
+            without warning in the future.
 
         Returns
         -------
@@ -389,6 +460,8 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
 
 
 class LanceFtsQueryBuilder(LanceQueryBuilder):
+    """A builder for full text search for LanceDB."""
+
     def __init__(self, table: "lancedb.table.Table", query: str):
         super().__init__(table)
         self._query = query

python/lancedb/remote/__init__.py

@@ -18,6 +18,8 @@ import attrs
 import pyarrow as pa
 from pydantic import BaseModel
 
+from lancedb.common import VECTOR_COLUMN_NAME
+
 __all__ = ["LanceDBClient", "VectorQuery", "VectorQueryResult"]
 
 
@@ -43,6 +45,8 @@ class VectorQuery(BaseModel):
 
     refine_factor: Optional[int] = None
 
+    vector_column: str = VECTOR_COLUMN_NAME
+
 
 @attrs.define
 class VectorQueryResult:

python/lancedb/remote/client.py

@@ -13,7 +13,7 @@
 
 
 import functools
-from typing import Any, Callable, Dict, Optional, Union
+from typing import Any, Callable, Dict, Iterable, Optional, Union
 
 import aiohttp
 import attrs
@@ -151,9 +151,13 @@ class RestfulLanceDBClient:
             return await deserialize(resp)
 
     @_check_not_closed
-    async def list_tables(self):
+    async def list_tables(
+        self, limit: int, page_token: Optional[str] = None
+    ) -> Iterable[str]:
         """List all tables in the database."""
-        json = await self.get("/v1/table/", {})
+        if page_token is None:
+            page_token = ""
+        json = await self.get("/v1/table/", {"limit": limit, "page_token": page_token})
         return json["tables"]
 
     @_check_not_closed

python/lancedb/remote/db.py

@@ -12,17 +12,23 @@
 #  limitations under the License.
 
 import asyncio
+import inspect
+import logging
 import uuid
-from typing import List, Optional
+from typing import Iterable, List, Optional, Union
 from urllib.parse import urlparse
 
 import pyarrow as pa
+from overrides import override
 
 from ..common import DATA
 from ..db import DBConnection
+from ..embeddings import EmbeddingFunctionConfig
+from ..pydantic import LanceModel
 from ..table import Table, _sanitize_data
 from .arrow import to_ipc_binary
 from .client import ARROW_STREAM_CONTENT_TYPE, RestfulLanceDBClient
+from .errors import LanceDBClientError
 
 
 class RemoteDBConnection(DBConnection):
@@ -50,13 +56,37 @@ class RemoteDBConnection(DBConnection):
             self._loop = asyncio.get_event_loop()
 
     def __repr__(self) -> str:
-        return f"RemoveConnect(name={self.db_name})"
+        return f"RemoteConnect(name={self.db_name})"
 
-    def table_names(self) -> List[str]:
-        """List the names of all tables in the database."""
-        result = self._loop.run_until_complete(self._client.list_tables())
-        return result
+    @override
+    def table_names(
+        self, page_token: Optional[str] = None, limit: int = 10
+    ) -> Iterable[str]:
+        """List the names of all tables in the database.
 
+        Parameters
+        ----------
+        page_token: str
+            The last token to start the new page.
+        limit: int, default 10
+            The maximum number of tables to return for each page.
+
+        Returns
+        -------
+        An iterator of table names.
+        """
+        while True:
+            result = self._loop.run_until_complete(
+                self._client.list_tables(limit, page_token)
+            )
+            if len(result) > 0:
+                page_token = result[len(result) - 1]
+            else:
+                break
+            for item in result:
+                yield item
+
+    @override
     def open_table(self, name: str) -> Table:
         """Open a Lance Table in the database.
 
@@ -71,23 +101,142 @@ class RemoteDBConnection(DBConnection):
         """
         from .table import RemoteTable
 
-        # TODO: check if table exists
-
+        # check if table exists
+        try:
+            self._loop.run_until_complete(
+                self._client.post(f"/v1/table/{name}/describe/")
+            )
+        except LanceDBClientError as err:
+            if str(err).startswith("Not found"):
+                logging.error(
+                    f"Table {name} does not exist. "
+                    f"Please first call db.create_table({name}, data)"
+                )
         return RemoteTable(self, name)
 
+    @override
     def create_table(
         self,
         name: str,
         data: DATA = None,
-        schema: pa.Schema = None,
+        schema: Optional[Union[pa.Schema, LanceModel]] = None,
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
+        embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
     ) -> Table:
+        """Create a [Table][lancedb.table.Table] in the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+        data: The data to initialize the table, *optional*
+            User must provide at least one of `data` or `schema`.
+            Acceptable types are:
+
+            - dict or list-of-dict
+
+            - pandas.DataFrame
+
+            - pyarrow.Table or pyarrow.RecordBatch
+        schema: The schema of the table, *optional*
+            Acceptable types are:
+
+            - pyarrow.Schema
+
+            - [LanceModel][lancedb.pydantic.LanceModel]
+        on_bad_vectors: str, default "error"
+            What to do if any of the vectors are not the same size or contains NaNs.
+            One of "error", "drop", "fill".
+        fill_value: float
+            The value to use when filling vectors. Only used if on_bad_vectors="fill".
+
+        Returns
+        -------
+        LanceTable
+            A reference to the newly created table.
+
+        !!! note
+
+            The vector index won't be created by default.
+            To create the index, call the `create_index` method on the table.
+
+        Examples
+        --------
+
+        Can create with list of tuples or dictionaries:
+
+        >>> import lancedb
+        >>> db = lancedb.connect("db://...", api_key="...", region="...") # doctest: +SKIP
+        >>> data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
+        ...         {"vector": [0.2, 1.8], "lat": 40.1, "long":  -74.1}]
+        >>> db.create_table("my_table", data) # doctest: +SKIP
+        LanceTable(my_table)
+
+        You can also pass a pandas DataFrame:
+
+        >>> import pandas as pd
+        >>> data = pd.DataFrame({
+        ...    "vector": [[1.1, 1.2], [0.2, 1.8]],
+        ...    "lat": [45.5, 40.1],
+        ...    "long": [-122.7, -74.1]
+        ... })
+        >>> db.create_table("table2", data) # doctest: +SKIP
+        LanceTable(table2)
+
+        >>> custom_schema = pa.schema([
+        ...   pa.field("vector", pa.list_(pa.float32(), 2)),
+        ...   pa.field("lat", pa.float32()),
+        ...   pa.field("long", pa.float32())
+        ... ])
+        >>> db.create_table("table3", data, schema = custom_schema) # doctest: +SKIP
+        LanceTable(table3)
+
+        It is also possible to create an table from `[Iterable[pa.RecordBatch]]`:
+
+        >>> import pyarrow as pa
+        >>> def make_batches():
+        ...     for i in range(5):
+        ...         yield pa.RecordBatch.from_arrays(
+        ...             [
+        ...                 pa.array([[3.1, 4.1], [5.9, 26.5]],
+        ...                     pa.list_(pa.float32(), 2)),
+        ...                 pa.array(["foo", "bar"]),
+        ...                 pa.array([10.0, 20.0]),
+        ...             ],
+        ...             ["vector", "item", "price"],
+        ...         )
+        >>> schema=pa.schema([
+        ...     pa.field("vector", pa.list_(pa.float32(), 2)),
+        ...     pa.field("item", pa.utf8()),
+        ...     pa.field("price", pa.float32()),
+        ... ])
+        >>> db.create_table("table4", make_batches(), schema=schema) # doctest: +SKIP
+        LanceTable(table4)
+
+        """
         if data is None and schema is None:
             raise ValueError("Either data or schema must be provided.")
+        if embedding_functions is not None:
+            raise NotImplementedError(
+                "embedding_functions is not supported for remote databases."
+                "Please vote https://github.com/lancedb/lancedb/issues/626 "
+                "for this feature."
+            )
+
+        if inspect.isclass(schema) and issubclass(schema, LanceModel):
+            # convert LanceModel to pyarrow schema
+            # note that it's possible this contains
+            # embedding function metadata already
+            schema = schema.to_arrow_schema()
+
         if data is not None:
             data = _sanitize_data(
-                data, schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+                data,
+                schema,
+                metadata=None,
+                on_bad_vectors=on_bad_vectors,
+                fill_value=fill_value,
             )
         else:
             if schema is None:
@@ -109,6 +258,7 @@ class RemoteDBConnection(DBConnection):
         )
         return RemoteTable(self, name)
 
+    @override
     def drop_table(self, name: str):
         """Drop a table from the database.
 
@@ -122,3 +272,8 @@ class RemoteDBConnection(DBConnection):
                 f"/v1/table/{name}/drop/",
             )
         )
+
+    async def close(self):
+        """Close the connection to the database."""
+        self._loop.close()
+        await self._client.close()

python/lancedb/remote/table.py

@@ -13,7 +13,7 @@
 
 import uuid
 from functools import cached_property
-from typing import Optional, Union
+from typing import Dict, Optional, Union
 
 import pyarrow as pa
 from lance import json_to_schema
@@ -22,6 +22,7 @@ from lancedb.common import DATA, VEC, VECTOR_COLUMN_NAME
 
 from ..query import LanceVectorQueryBuilder
 from ..table import Query, Table, _sanitize_data
+from ..util import value_to_sql
 from .arrow import to_ipc_binary
 from .client import ARROW_STREAM_CONTENT_TYPE
 from .db import RemoteDBConnection
@@ -37,34 +38,87 @@ class RemoteTable(Table):
 
     @cached_property
     def schema(self) -> pa.Schema:
-        """Return the schema of the table."""
+        """The [Arrow Schema](https://arrow.apache.org/docs/python/api/datatypes.html#)
+        of this Table
+
+        """
         resp = self._conn._loop.run_until_complete(
             self._conn._client.post(f"/v1/table/{self._name}/describe/")
         )
         schema = json_to_schema(resp["schema"])
         return schema
 
+    @property
+    def version(self) -> int:
+        """Get the current version of the table"""
+        resp = self._conn._loop.run_until_complete(
+            self._conn._client.post(f"/v1/table/{self._name}/describe/")
+        )
+        return resp["version"]
+
     def to_arrow(self) -> pa.Table:
-        """Return the table as an Arrow table."""
+        """to_arrow() is not supported on the LanceDB cloud"""
         raise NotImplementedError("to_arrow() is not supported on the LanceDB cloud")
 
     def to_pandas(self):
-        """Return the table as a Pandas DataFrame.
-
-        Intercept `to_arrow()` for better error message.
-        """
+        """to_pandas() is not supported on the LanceDB cloud"""
         return NotImplementedError("to_pandas() is not supported on the LanceDB cloud")
 
+    def create_scalar_index(self, *args, **kwargs):
+        """Creates a scalar index"""
+        return NotImplementedError(
+            "create_scalar_index() is not supported on the LanceDB cloud"
+        )
+
     def create_index(
         self,
         metric="L2",
-        num_partitions=256,
-        num_sub_vectors=96,
         vector_column_name: str = VECTOR_COLUMN_NAME,
-        replace: bool = True,
-        accelerator: Optional[str] = None,
+        index_cache_size: Optional[int] = None,
     ):
-        raise NotImplementedError
+        """Create an index on the table.
+        Currently, the only parameters that matter are
+        the metric and the vector column name.
+
+        Parameters
+        ----------
+        metric : str
+            The metric to use for the index. Default is "L2".
+        vector_column_name : str
+            The name of the vector column. Default is "vector".
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> import uuid
+        >>> from lancedb.schema import vector
+        >>> db = lancedb.connect("db://...", api_key="...", region="...") # doctest: +SKIP
+        >>> table_name = uuid.uuid4().hex
+        >>> schema = pa.schema(
+        ...     [
+        ...             pa.field("id", pa.uint32(), False),
+        ...            pa.field("vector", vector(128), False),
+        ...             pa.field("s", pa.string(), False),
+        ...     ]
+        ... )
+        >>> table = db.create_table( # doctest: +SKIP
+        ...     table_name, # doctest: +SKIP
+        ...     schema=schema, # doctest: +SKIP
+        ... )
+        >>> table.create_index("L2", "vector") # doctest: +SKIP
+        """
+        index_type = "vector"
+
+        data = {
+            "column": vector_column_name,
+            "index_type": index_type,
+            "metric_type": metric,
+            "index_cache_size": index_cache_size,
+        }
+        resp = self._conn._loop.run_until_complete(
+            self._conn._client.post(f"/v1/table/{self._name}/create_index/", data=data)
+        )
+        return resp
 
     def add(
         self,
@@ -73,6 +127,28 @@ class RemoteTable(Table):
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
     ) -> int:
+        """Add more data to the [Table](Table). It has the same API signature as the OSS version.
+
+        Parameters
+        ----------
+        data: DATA
+            The data to insert into the table. Acceptable types are:
+
+            - dict or list-of-dict
+
+            - pandas.DataFrame
+
+            - pyarrow.Table or pyarrow.RecordBatch
+        mode: str
+            The mode to use when writing the data. Valid values are
+            "append" and "overwrite".
+        on_bad_vectors: str, default "error"
+            What to do if any of the vectors are not the same size or contains NaNs.
+            One of "error", "drop", "fill".
+        fill_value: float, default 0.
+            The value to use when filling vectors. Only used if on_bad_vectors="fill".
+
+        """
         data = _sanitize_data(
             data,
             self.schema,
@@ -96,17 +172,173 @@ class RemoteTable(Table):
     def search(
         self, query: Union[VEC, str], vector_column_name: str = VECTOR_COLUMN_NAME
     ) -> LanceVectorQueryBuilder:
+        """Create a search query to find the nearest neighbors
+        of the given query vector. We currently support [vector search][search]
+
+        All query options are defined in [Query][lancedb.query.Query].
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> db = lancedb.connect("db://...", api_key="...", region="...") # doctest: +SKIP
+        >>> data = [
+        ...    {"original_width": 100, "caption": "bar", "vector": [0.1, 2.3, 4.5]},
+        ...    {"original_width": 2000, "caption": "foo",  "vector": [0.5, 3.4, 1.3]},
+        ...    {"original_width": 3000, "caption": "test", "vector": [0.3, 6.2, 2.6]}
+        ... ]
+        >>> table = db.create_table("my_table", data) # doctest: +SKIP
+        >>> query = [0.4, 1.4, 2.4]
+        >>> (table.search(query, vector_column_name="vector") # doctest: +SKIP
+        ...     .where("original_width > 1000", prefilter=True) # doctest: +SKIP
+        ...     .select(["caption", "original_width"]) # doctest: +SKIP
+        ...     .limit(2) # doctest: +SKIP
+        ...     .to_pandas()) # doctest: +SKIP
+          caption  original_width           vector  _distance # doctest: +SKIP
+        0     foo            2000  [0.5, 3.4, 1.3]   5.220000 # doctest: +SKIP
+        1    test            3000  [0.3, 6.2, 2.6]  23.089996 # doctest: +SKIP
+
+        Parameters
+        ----------
+        query: list/np.ndarray/str/PIL.Image.Image, default None
+            The targetted vector to search for.
+
+            - *default None*.
+            Acceptable types are: list, np.ndarray, PIL.Image.Image
+
+            - If None then the select/where/limit clauses are applied to filter
+            the table
+        vector_column_name: str
+            The name of the vector column to search.
+            *default "vector"*
+
+        Returns
+        -------
+        LanceQueryBuilder
+            A query builder object representing the query.
+            Once executed, the query returns
+
+            - selected columns
+
+            - the vector
+
+            - and also the "_distance" column which is the distance between the query
+            vector and the returned vector.
+        """
         return LanceVectorQueryBuilder(self, query, vector_column_name)
 
     def _execute_query(self, query: Query) -> pa.Table:
-        if query.prefilter:
-            raise NotImplementedError("Cloud support for prefiltering is coming soon")
         result = self._conn._client.query(self._name, query)
         return self._conn._loop.run_until_complete(result).to_arrow()
 
     def delete(self, predicate: str):
-        """Delete rows from the table."""
+        """Delete rows from the table.
+
+        This can be used to delete a single row, many rows, all rows, or
+        sometimes no rows (if your predicate matches nothing).
+
+        Parameters
+        ----------
+        predicate: str
+            The SQL where clause to use when deleting rows.
+
+            - For example, 'x = 2' or 'x IN (1, 2, 3)'.
+
+            The filter must not be empty, or it will error.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> data = [
+        ...    {"x": 1, "vector": [1, 2]},
+        ...    {"x": 2, "vector": [3, 4]},
+        ...    {"x": 3, "vector": [5, 6]}
+        ... ]
+        >>> db = lancedb.connect("db://...", api_key="...", region="...") # doctest: +SKIP
+        >>> table = db.create_table("my_table", data) # doctest: +SKIP
+        >>> table.search([10,10]).to_pandas() # doctest: +SKIP
+           x      vector  _distance # doctest: +SKIP
+        0  3  [5.0, 6.0]       41.0 # doctest: +SKIP
+        1  2  [3.0, 4.0]       85.0 # doctest: +SKIP
+        2  1  [1.0, 2.0]      145.0 # doctest: +SKIP
+        >>> table.delete("x = 2") # doctest: +SKIP
+        >>> table.search([10,10]).to_pandas() # doctest: +SKIP
+           x      vector  _distance # doctest: +SKIP
+        0  3  [5.0, 6.0]       41.0 # doctest: +SKIP
+        1  1  [1.0, 2.0]      145.0 # doctest: +SKIP
+
+        If you have a list of values to delete, you can combine them into a
+        stringified list and use the `IN` operator:
+
+        >>> to_remove = [1, 3] # doctest: +SKIP
+        >>> to_remove = ", ".join([str(v) for v in to_remove]) # doctest: +SKIP
+        >>> table.delete(f"x IN ({to_remove})") # doctest: +SKIP
+        >>> table.search([10,10]).to_pandas() # doctest: +SKIP
+           x      vector  _distance # doctest: +SKIP
+        0  2  [3.0, 4.0]       85.0 # doctest: +SKIP
+        """
         payload = {"predicate": predicate}
         self._conn._loop.run_until_complete(
             self._conn._client.post(f"/v1/table/{self._name}/delete/", data=payload)
         )
+
+    def update(
+        self,
+        where: Optional[str] = None,
+        values: Optional[dict] = None,
+        *,
+        values_sql: Optional[Dict[str, str]] = None,
+    ):
+        """
+        This can be used to update zero to all rows depending on how many
+        rows match the where clause.
+
+        Parameters
+        ----------
+        where: str, optional
+            The SQL where clause to use when updating rows. For example, 'x = 2'
+            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
+        values: dict, optional
+            The values to update. The keys are the column names and the values
+            are the values to set.
+        values_sql: dict, optional
+            The values to update, expressed as SQL expression strings. These can
+            reference existing columns. For example, {"x": "x + 1"} will increment
+            the x column by 1.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> data = [
+        ...    {"x": 1, "vector": [1, 2]},
+        ...    {"x": 2, "vector": [3, 4]},
+        ...    {"x": 3, "vector": [5, 6]}
+        ... ]
+        >>> db = lancedb.connect("db://...", api_key="...", region="...") # doctest: +SKIP
+        >>> table = db.create_table("my_table", data) # doctest: +SKIP
+        >>> table.to_pandas() # doctest: +SKIP
+           x      vector # doctest: +SKIP
+        0  1  [1.0, 2.0] # doctest: +SKIP
+        1  2  [3.0, 4.0] # doctest: +SKIP
+        2  3  [5.0, 6.0] # doctest: +SKIP
+        >>> table.update(where="x = 2", values={"vector": [10, 10]}) # doctest: +SKIP
+        >>> table.to_pandas() # doctest: +SKIP
+           x        vector # doctest: +SKIP
+        0  1    [1.0, 2.0] # doctest: +SKIP
+        1  3    [5.0, 6.0] # doctest: +SKIP
+        2  2  [10.0, 10.0] # doctest: +SKIP
+
+        """
+        if values is not None and values_sql is not None:
+            raise ValueError("Only one of values or values_sql can be provided")
+        if values is None and values_sql is None:
+            raise ValueError("Either values or values_sql must be provided")
+
+        if values is not None:
+            updates = [[k, value_to_sql(v)] for k, v in values.items()]
+        else:
+            updates = [[k, v] for k, v in values_sql.items()]
+
+        payload = {"predicate": where, "updates": updates}
+        self._conn._loop.run_until_complete(
+            self._conn._client.post(f"/v1/table/{self._name}/update/", data=payload)
+        )

python/lancedb/table.py

@@ -16,26 +16,30 @@ from __future__ import annotations
 import inspect
 import os
 from abc import ABC, abstractmethod
-from datetime import timedelta
 from functools import cached_property
-from typing import Any, Iterable, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
 
 import lance
 import numpy as np
 import pyarrow as pa
 import pyarrow.compute as pc
+import pyarrow.fs as pa_fs
 from lance import LanceDataset
-from lance.dataset import CleanupStats, ReaderLike
 from lance.vector import vec_to_table
 
 from .common import DATA, VEC, VECTOR_COLUMN_NAME
-from .embeddings import EmbeddingFunctionRegistry
-from .embeddings.functions import EmbeddingFunctionConfig
-from .pydantic import LanceModel
+from .embeddings import EmbeddingFunctionConfig, EmbeddingFunctionRegistry
+from .pydantic import LanceModel, model_to_dict
 from .query import LanceQueryBuilder, Query
-from .util import fs_from_uri, safe_import_pandas
+from .util import fs_from_uri, safe_import_pandas, value_to_sql, join_uri
 from .utils.events import register_event
 
+if TYPE_CHECKING:
+    from datetime import timedelta
+
+    from lance.dataset import CleanupStats, ReaderLike
+
+
 pd = safe_import_pandas()
 
 
@@ -50,8 +54,10 @@ def _sanitize_data(
         # convert to list of dict if data is a bunch of LanceModels
         if isinstance(data[0], LanceModel):
             schema = data[0].__class__.to_arrow_schema()
-            data = [dict(d) for d in data]
-        data = pa.Table.from_pylist(data)
+            data = [model_to_dict(d) for d in data]
+            data = pa.Table.from_pylist(data, schema=schema)
+        else:
+            data = pa.Table.from_pylist(data)
     elif isinstance(data, dict):
         data = vec_to_table(data)
     elif pd is not None and isinstance(data, pd.DataFrame):
@@ -87,7 +93,9 @@ def _append_vector_col(data: pa.Table, metadata: dict, schema: Optional[pa.Schem
     for vector_column, conf in functions.items():
         func = conf.function
         if vector_column not in data.column_names:
-            col_data = func.compute_source_embeddings(data[conf.source_column])
+            col_data = func.compute_source_embeddings_with_retry(
+                data[conf.source_column]
+            )
             if schema is not None:
                 dtype = schema.field(vector_column).type
             else:
@@ -150,13 +158,13 @@ class Table(ABC):
     @property
     @abstractmethod
     def schema(self) -> pa.Schema:
-        """The [Arrow Schema](https://arrow.apache.org/docs/python/api/datatypes.html#) of
-        this Table
+        """The [Arrow Schema](https://arrow.apache.org/docs/python/api/datatypes.html#)
+        of this Table
 
         """
         raise NotImplementedError
 
-    def to_pandas(self):
+    def to_pandas(self) -> "pd.DataFrame":
         """Return the table as a pandas DataFrame.
 
         Returns
@@ -183,6 +191,7 @@ class Table(ABC):
         vector_column_name: str = VECTOR_COLUMN_NAME,
         replace: bool = True,
         accelerator: Optional[str] = None,
+        index_cache_size: Optional[int] = None,
     ):
         """Create an index on the table.
 
@@ -192,20 +201,94 @@ class Table(ABC):
             The distance metric to use when creating the index.
             Valid values are "L2", "cosine", or "dot".
             L2 is euclidean distance.
-        num_partitions: int
+        num_partitions: int, default 256
             The number of IVF partitions to use when creating the index.
             Default is 256.
-        num_sub_vectors: int
+        num_sub_vectors: int, default 96
             The number of PQ sub-vectors to use when creating the index.
             Default is 96.
         vector_column_name: str, default "vector"
             The vector column name to create the index.
         replace: bool, default True
-            If True, replace the existing index if it exists.
-            If False, raise an error if duplicate index exists.
+            - If True, replace the existing index if it exists.
+
+            - If False, raise an error if duplicate index exists.
         accelerator: str, default None
             If set, use the given accelerator to create the index.
             Only support "cuda" for now.
+        index_cache_size : int, optional
+            The size of the index cache in number of entries. Default value is 256.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def create_scalar_index(
+        self,
+        column: str,
+        *,
+        replace: bool = True,
+    ):
+        """Create a scalar index on a column.
+
+        Scalar indices, like vector indices, can be used to speed up scans.  A scalar
+        index can speed up scans that contain filter expressions on the indexed column.
+        For example, the following scan will be faster if the column ``my_col`` has
+        a scalar index:
+
+        .. code-block:: python
+
+            import lancedb
+
+            db = lancedb.connect("/data/lance")
+            img_table = db.open_table("images")
+            my_df = img_table.search().where("my_col = 7", prefilter=True).to_pandas()
+
+        Scalar indices can also speed up scans containing a vector search and a
+        prefilter:
+
+        .. code-block::python
+
+            import lancedb
+
+            db = lancedb.connect("/data/lance")
+            img_table = db.open_table("images")
+            img_table.search([1, 2, 3, 4], vector_column_name="vector")
+                .where("my_col != 7", prefilter=True)
+                .to_pandas()
+
+        Scalar indices can only speed up scans for basic filters using
+        equality, comparison, range (e.g. ``my_col BETWEEN 0 AND 100``), and set
+        membership (e.g. `my_col IN (0, 1, 2)`)
+
+        Scalar indices can be used if the filter contains multiple indexed columns and
+        the filter criteria are AND'd or OR'd together
+        (e.g. ``my_col < 0 AND other_col> 100``)
+
+        Scalar indices may be used if the filter contains non-indexed columns but,
+        depending on the structure of the filter, they may not be usable.  For example,
+        if the column ``not_indexed`` does not have a scalar index then the filter
+        ``my_col = 0 OR not_indexed = 1`` will not be able to use any scalar index on
+        ``my_col``.
+
+        **Experimental API**
+
+        Parameters
+        ----------
+        column : str
+            The column to be indexed.  Must be a boolean, integer, float,
+            or string column.
+        replace : bool, default True
+            Replace the existing index if it exists.
+
+        Examples
+        --------
+
+        .. code-block:: python
+
+            import lance
+
+            dataset = lance.dataset("/tmp/images.lance")
+            dataset.create_scalar_index("category")
         """
         raise NotImplementedError
 
@@ -221,8 +304,14 @@ class Table(ABC):
 
         Parameters
         ----------
-        data: list-of-dict, dict, pd.DataFrame
-            The data to insert into the table.
+        data: DATA
+            The data to insert into the table. Acceptable types are:
+
+            - dict or list-of-dict
+
+            - pandas.DataFrame
+
+            - pyarrow.Table or pyarrow.RecordBatch
         mode: str
             The mode to use when writing the data. Valid values are
             "append" and "overwrite".
@@ -243,31 +332,70 @@ class Table(ABC):
         query_type: str = "auto",
     ) -> LanceQueryBuilder:
         """Create a search query to find the nearest neighbors
-        of the given query vector.
+        of the given query vector. We currently support [vector search][search]
+        and [full-text search][experimental-full-text-search].
+
+        All query options are defined in [Query][lancedb.query.Query].
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> db = lancedb.connect("./.lancedb")
+        >>> data = [
+        ...    {"original_width": 100, "caption": "bar", "vector": [0.1, 2.3, 4.5]},
+        ...    {"original_width": 2000, "caption": "foo",  "vector": [0.5, 3.4, 1.3]},
+        ...    {"original_width": 3000, "caption": "test", "vector": [0.3, 6.2, 2.6]}
+        ... ]
+        >>> table = db.create_table("my_table", data)
+        >>> query = [0.4, 1.4, 2.4]
+        >>> (table.search(query, vector_column_name="vector")
+        ...     .where("original_width > 1000", prefilter=True)
+        ...     .select(["caption", "original_width"])
+        ...     .limit(2)
+        ...     .to_pandas())
+          caption  original_width           vector  _distance
+        0     foo            2000  [0.5, 3.4, 1.3]   5.220000
+        1    test            3000  [0.3, 6.2, 2.6]  23.089996
 
         Parameters
         ----------
-        query: str, list, np.ndarray, PIL.Image.Image, default None
-            The query to search for. If None then
-            the select/where/limit clauses are applied to filter
+        query: list/np.ndarray/str/PIL.Image.Image, default None
+            The targetted vector to search for.
+
+            - *default None*.
+            Acceptable types are: list, np.ndarray, PIL.Image.Image
+
+            - If None then the select/where/limit clauses are applied to filter
             the table
-        vector_column_name: str, default "vector"
+        vector_column_name: str
             The name of the vector column to search.
-        query_type: str, default "auto"
-            "vector", "fts", or "auto"
-            If "auto" then the query type is inferred from the query;
-            If `query` is a list/np.ndarray then the query type is "vector";
-            If `query` is a PIL.Image.Image then either do vector search
-            or raise an error if no corresponding embedding function is found.
-            If `query` is a string, then the query type is "vector" if the
+            *default "vector"*
+        query_type: str
+            *default "auto"*.
+            Acceptable types are: "vector", "fts", or "auto"
+
+            - If "auto" then the query type is inferred from the query;
+
+                - If `query` is a list/np.ndarray then the query type is
+                "vector";
+
+                - If `query` is a PIL.Image.Image then either do vector search,
+                or raise an error if no corresponding embedding function is found.
+
+            - If `query` is a string, then the query type is "vector" if the
             table has embedding functions else the query type is "fts"
 
         Returns
         -------
         LanceQueryBuilder
             A query builder object representing the query.
-            Once executed, the query returns selected columns, the vector,
-            and also the "_distance" column which is the distance between the query
+            Once executed, the query returns
+
+            - selected columns
+
+            - the vector
+
+            - and also the "_distance" column which is the distance between the query
             vector and the returned vector.
         """
         raise NotImplementedError
@@ -286,14 +414,19 @@ class Table(ABC):
         Parameters
         ----------
         where: str
-            The SQL where clause to use when deleting rows. For example, 'x = 2'
-            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
+            The SQL where clause to use when deleting rows.
+
+            - For example, 'x = 2' or 'x IN (1, 2, 3)'.
+
+            The filter must not be empty, or it will error.
 
         Examples
         --------
         >>> import lancedb
         >>> data = [
-        ...   {"x": 1, "vector": [1, 2]}, {"x": 2, "vector": [3, 4]}, {"x": 3, "vector": [5, 6]}
+        ...    {"x": 1, "vector": [1, 2]},
+        ...    {"x": 2, "vector": [3, 4]},
+        ...    {"x": 3, "vector": [5, 6]}
         ... ]
         >>> db = lancedb.connect("./.lancedb")
         >>> table = db.create_table("my_table", data)
@@ -322,6 +455,62 @@ class Table(ABC):
         """
         raise NotImplementedError
 
+    @abstractmethod
+    def update(
+        self,
+        where: Optional[str] = None,
+        values: Optional[dict] = None,
+        *,
+        values_sql: Optional[Dict[str, str]] = None,
+    ):
+        """
+        This can be used to update zero to all rows depending on how many
+        rows match the where clause. If no where clause is provided, then
+        all rows will be updated.
+
+        Either `values` or `values_sql` must be provided. You cannot provide
+        both.
+
+        Parameters
+        ----------
+        where: str, optional
+            The SQL where clause to use when updating rows. For example, 'x = 2'
+            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
+        values: dict, optional
+            The values to update. The keys are the column names and the values
+            are the values to set.
+        values_sql: dict, optional
+            The values to update, expressed as SQL expression strings. These can
+            reference existing columns. For example, {"x": "x + 1"} will increment
+            the x column by 1.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> import pandas as pd
+        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
+        >>> db = lancedb.connect("./.lancedb")
+        >>> table = db.create_table("my_table", data)
+        >>> table.to_pandas()
+           x      vector
+        0  1  [1.0, 2.0]
+        1  2  [3.0, 4.0]
+        2  3  [5.0, 6.0]
+        >>> table.update(where="x = 2", values={"vector": [10, 10]})
+        >>> table.to_pandas()
+           x        vector
+        0  1    [1.0, 2.0]
+        1  3    [5.0, 6.0]
+        2  2  [10.0, 10.0]
+        >>> table.update(values_sql={"x": "x + 1"})
+        >>> table.to_pandas()
+           x        vector
+        0  2    [1.0, 2.0]
+        1  4    [5.0, 6.0]
+        2  3  [10.0, 10.0]
+        """
+        raise NotImplementedError
+
 
 class LanceTable(Table):
     """
@@ -378,7 +567,8 @@ class LanceTable(Table):
         --------
         >>> import lancedb
         >>> db = lancedb.connect("./.lancedb")
-        >>> table = db.create_table("my_table", [{"vector": [1.1, 0.9], "type": "vector"}])
+        >>> table = db.create_table("my_table",
+        ...    [{"vector": [1.1, 0.9], "type": "vector"}])
         >>> table.version
         2
         >>> table.to_pandas()
@@ -425,7 +615,8 @@ class LanceTable(Table):
         --------
         >>> import lancedb
         >>> db = lancedb.connect("./.lancedb")
-        >>> table = db.create_table("my_table", [{"vector": [1.1, 0.9], "type": "vector"}])
+        >>> table = db.create_table("my_table", [
+        ...     {"vector": [1.1, 0.9], "type": "vector"}])
         >>> table.version
         2
         >>> table.to_pandas()
@@ -488,7 +679,7 @@ class LanceTable(Table):
 
     @property
     def _dataset_uri(self) -> str:
-        return os.path.join(self._conn.uri, f"{self.name}.lance")
+        return join_uri(self._conn.uri, f"{self.name}.lance")
 
     def create_index(
         self,
@@ -498,6 +689,7 @@ class LanceTable(Table):
         vector_column_name=VECTOR_COLUMN_NAME,
         replace: bool = True,
         accelerator: Optional[str] = None,
+        index_cache_size: Optional[int] = None,
     ):
         """Create an index on the table."""
         self._dataset.create_index(
@@ -508,11 +700,17 @@ class LanceTable(Table):
             num_sub_vectors=num_sub_vectors,
             replace=replace,
             accelerator=accelerator,
+            index_cache_size=index_cache_size,
         )
         self._reset_dataset()
         register_event("create_index")
 
-    def create_fts_index(self, field_names: Union[str, List[str]]):
+    def create_scalar_index(self, column: str, *, replace: bool = True):
+        self._dataset.create_scalar_index(column, index_type="BTREE", replace=replace)
+
+    def create_fts_index(
+        self, field_names: Union[str, List[str]], *, replace: bool = False
+    ):
         """Create a full-text search index on the table.
 
         Warning - this API is highly experimental and is highly likely to change
@@ -522,17 +720,31 @@ class LanceTable(Table):
         ----------
         field_names: str or list of str
             The name(s) of the field to index.
+        replace: bool, default False
+            If True, replace the existing index if it exists. Note that this is
+            not yet an atomic operation; the index will be temporarily
+            unavailable while the new index is being created.
         """
         from .fts import create_index, populate_index
 
         if isinstance(field_names, str):
             field_names = [field_names]
+
+        fs, path = fs_from_uri(self._get_fts_index_path())
+        index_exists = fs.get_file_info(path).type != pa_fs.FileType.NotFound
+        if index_exists:
+            if not replace:
+                raise ValueError(
+                    f"Index already exists. Use replace=True to overwrite."
+                )
+            fs.delete_dir(path)
+
         index = create_index(self._get_fts_index_path(), field_names)
         populate_index(index, self, field_names)
         register_event("create_fts_index")
 
     def _get_fts_index_path(self):
-        return os.path.join(self._dataset_uri, "_indices", "tantivy")
+        return join_uri(self._dataset_uri, "_indices", "tantivy")
 
     @cached_property
     def _dataset(self) -> LanceDataset:
@@ -670,14 +882,39 @@ class LanceTable(Table):
         query_type: str = "auto",
     ) -> LanceQueryBuilder:
         """Create a search query to find the nearest neighbors
-        of the given query vector.
+        of the given query vector. We currently support [vector search][search]
+        and [full-text search][search].
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> db = lancedb.connect("./.lancedb")
+        >>> data = [
+        ...    {"original_width": 100, "caption": "bar", "vector": [0.1, 2.3, 4.5]},
+        ...    {"original_width": 2000, "caption": "foo",  "vector": [0.5, 3.4, 1.3]},
+        ...    {"original_width": 3000, "caption": "test", "vector": [0.3, 6.2, 2.6]}
+        ... ]
+        >>> table = db.create_table("my_table", data)
+        >>> query = [0.4, 1.4, 2.4]
+        >>> (table.search(query, vector_column_name="vector")
+        ...     .where("original_width > 1000", prefilter=True)
+        ...     .select(["caption", "original_width"])
+        ...     .limit(2)
+        ...     .to_pandas())
+          caption  original_width           vector  _distance
+        0     foo            2000  [0.5, 3.4, 1.3]   5.220000
+        1    test            3000  [0.3, 6.2, 2.6]  23.089996
 
         Parameters
         ----------
-        query: str, list, np.ndarray, a PIL Image or None
-            The query to search for. If None then
-            the select/where/limit clauses are applied to filter
-            the table
+        query: list/np.ndarray/str/PIL.Image.Image, default None
+            The targetted vector to search for.
+
+            - *default None*.
+            Acceptable types are: list, np.ndarray, PIL.Image.Image
+
+            - If None then the select/[where][sql]/limit clauses are applied
+            to filter the table
         vector_column_name: str, default "vector"
             The name of the vector column to search.
         query_type: str, default "auto"
@@ -686,7 +923,7 @@ class LanceTable(Table):
             If `query` is a list/np.ndarray then the query type is "vector";
             If `query` is a PIL.Image.Image then either do vector search
             or raise an error if no corresponding embedding function is found.
-            If the query is a string, then the query type is "vector" if the
+            If the `query` is a string, then the query type is "vector" if the
             table has embedding functions, else the query type is "fts"
 
         Returns
@@ -697,7 +934,7 @@ class LanceTable(Table):
             and also the "_distance" column which is the distance between the query
             vector and the returned vector.
         """
-        register_event("search")
+        register_event("search_table")
         return LanceQueryBuilder.create(
             self, query, query_type, vector_column_name=vector_column_name
         )
@@ -721,7 +958,9 @@ class LanceTable(Table):
         --------
         >>> import lancedb
         >>> data = [
-        ...   {"x": 1, "vector": [1, 2]}, {"x": 2, "vector": [3, 4]}, {"x": 3, "vector": [5, 6]}
+        ...    {"x": 1, "vector": [1, 2]},
+        ...    {"x": 2, "vector": [3, 4]},
+        ...    {"x": 3, "vector": [5, 6]}
         ... ]
         >>> db = lancedb.connect("./.lancedb")
         >>> table = db.create_table("my_table", data)
@@ -741,7 +980,8 @@ class LanceTable(Table):
             The data to insert into the table.
             At least one of `data` or `schema` must be provided.
         schema: pa.Schema or LanceModel, optional
-            The schema of the table. If not provided, the schema is inferred from the data.
+            The schema of the table. If not provided,
+            the schema is inferred from the data.
             At least one of `data` or `schema` must be provided.
         mode: str, default "create"
             The mode to use when writing the data. Valid values are
@@ -812,35 +1052,45 @@ class LanceTable(Table):
         file_info = fs.get_file_info(path)
         if file_info.type != pa.fs.FileType.Directory:
             raise FileNotFoundError(
-                f"Table {name} does not exist. Please first call db.create_table({name}, data)"
+                f"Table {name} does not exist."
+                f"Please first call db.create_table({name}, data)"
             )
+        register_event("open_table")
+
         return tbl
 
     def delete(self, where: str):
         self._dataset.delete(where)
 
-    def update(self, where: str, values: dict):
+    def update(
+        self,
+        where: Optional[str] = None,
+        values: Optional[dict] = None,
+        *,
+        values_sql: Optional[Dict[str, str]] = None,
+    ):
         """
-        EXPERIMENTAL: Update rows in the table (not threadsafe).
-
         This can be used to update zero to all rows depending on how many
         rows match the where clause.
 
         Parameters
         ----------
-        where: str
+        where: str, optional
             The SQL where clause to use when updating rows. For example, 'x = 2'
             or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
-        values: dict
+        values: dict, optional
             The values to update. The keys are the column names and the values
             are the values to set.
+        values_sql: dict, optional
+            The values to update, expressed as SQL expression strings. These can
+            reference existing columns. For example, {"x": "x + 1"} will increment
+            the x column by 1.
 
         Examples
         --------
         >>> import lancedb
-        >>> data = [
-        ...   {"x": 1, "vector": [1, 2]}, {"x": 2, "vector": [3, 4]}, {"x": 3, "vector": [5, 6]}
-        ... ]
+        >>> import pandas as pd
+        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
         >>> db = lancedb.connect("./.lancedb")
         >>> table = db.create_table("my_table", data)
         >>> table.to_pandas()
@@ -856,29 +1106,20 @@ class LanceTable(Table):
         2  2  [10.0, 10.0]
 
         """
-        orig_data = self._dataset.to_table(filter=where).combine_chunks()
-        if len(orig_data) == 0:
-            return
-        for col, val in values.items():
-            i = orig_data.column_names.index(col)
-            if i < 0:
-                raise ValueError(f"Column {col} does not exist")
-            orig_data = orig_data.set_column(
-                i, col, pa.array([val] * len(orig_data), type=orig_data[col].type)
-            )
-        self.delete(where)
-        self.add(orig_data, mode="append")
+        if values is not None and values_sql is not None:
+            raise ValueError("Only one of values or values_sql can be provided")
+        if values is None and values_sql is None:
+            raise ValueError("Either values or values_sql must be provided")
+
+        if values is not None:
+            values_sql = {k: value_to_sql(v) for k, v in values.items()}
+
+        self.to_lance().update(values_sql, where)
         self._reset_dataset()
         register_event("update")
 
     def _execute_query(self, query: Query) -> pa.Table:
         ds = self.to_lance()
-        if query.prefilter:
-            for idx in ds.list_indices():
-                if query.vector_column in idx["fields"]:
-                    raise NotImplementedError(
-                        "Prefiltering for indexed vector column is coming soon."
-                    )
         return ds.to_table(
             columns=query.columns,
             filter=query.filter,
@@ -1020,7 +1261,8 @@ def _sanitize_vector_column(
     # ChunkedArray is annoying to work with, so we combine chunks here
     vec_arr = data[vector_column_name].combine_chunks()
     if pa.types.is_list(data[vector_column_name].type):
-        # if it's a variable size list array we make sure the dimensions are all the same
+        # if it's a variable size list array,
+        # we make sure the dimensions are all the same
         has_jagged_ndims = len(vec_arr.values) % len(data) != 0
         if has_jagged_ndims:
             data = _sanitize_jagged(

python/lancedb/util.py

@@ -12,9 +12,13 @@
 #  limitations under the License.
 
 import os
-from typing import Tuple
+from datetime import date, datetime
+from functools import singledispatch
+import pathlib
+from typing import Tuple, Union
 from urllib.parse import urlparse
 
+import numpy as np
 import pyarrow.fs as pa_fs
 
 
@@ -59,6 +63,12 @@ def get_uri_location(uri: str) -> str:
     str: Location part of the URL, without scheme
     """
     parsed = urlparse(uri)
+    if len(parsed.scheme) == 1:
+        # Windows drive names are parsed as the scheme
+        # e.g. "c:\path" -> ParseResult(scheme="c", netloc="", path="/path", ...)
+        # So we add special handling here for schemes that are a single character
+        return uri
+
     if not parsed.netloc:
         return parsed.path
     else:
@@ -81,6 +91,29 @@ def fs_from_uri(uri: str) -> Tuple[pa_fs.FileSystem, str]:
     return pa_fs.FileSystem.from_uri(uri)
 
 
+def join_uri(base: Union[str, pathlib.Path], *parts: str) -> str:
+    """
+    Join a URI with multiple parts, handles both local and remote paths
+
+    Parameters
+    ----------
+    base : str
+        The base URI
+    parts : str
+        The parts to join to the base URI, each separated by the
+        appropriate path separator for the URI scheme and OS
+    """
+    if isinstance(base, pathlib.Path):
+        return base.joinpath(*parts)
+    base = str(base)
+    if get_uri_scheme(base) == "file":
+        # using pathlib for local paths make this windows compatible
+        # `get_uri_scheme` returns `file` for windows drive names (e.g. `c:\path`)
+        return str(pathlib.Path(base, *parts))
+    # for remote paths, just use os.path.join
+    return "/".join([p.rstrip("/") for p in [base, *parts]])
+
+
 def safe_import_pandas():
     try:
         import pandas as pd
@@ -88,3 +121,53 @@ def safe_import_pandas():
         return pd
     except ImportError:
         return None
+
+
+@singledispatch
+def value_to_sql(value):
+    raise NotImplementedError("SQL conversion is not implemented for this type")
+
+
+@value_to_sql.register(str)
+def _(value: str):
+    return f"'{value}'"
+
+
+@value_to_sql.register(int)
+def _(value: int):
+    return str(value)
+
+
+@value_to_sql.register(float)
+def _(value: float):
+    return str(value)
+
+
+@value_to_sql.register(bool)
+def _(value: bool):
+    return str(value).upper()
+
+
+@value_to_sql.register(type(None))
+def _(value: type(None)):
+    return "NULL"
+
+
+@value_to_sql.register(datetime)
+def _(value: datetime):
+    return f"'{value.isoformat()}'"
+
+
+@value_to_sql.register(date)
+def _(value: date):
+    return f"'{value.isoformat()}'"
+
+
+@value_to_sql.register(list)
+def _(value: list):
+    return "[" + ", ".join(map(value_to_sql, value)) + "]"
+
+
+@value_to_sql.register(np.ndarray)
+def _(value: np.ndarray):
+    return value_to_sql(value.tolist())

python/lancedb/utils/events.py

@@ -64,8 +64,10 @@ class _Events:
         Initializes the Events object with default values for events, rate_limit, and metadata.
         """
         self.events = []  # events list
-        self.max_events = 25  # max events to store in memory
-        self.rate_limit = 60.0  # rate limit (seconds)
+        self.throttled_event_names = ["search_table"]
+        self.throttled_events = set()
+        self.max_events = 5  # max events to store in memory
+        self.rate_limit = 60.0 * 5  # rate limit (seconds)
         self.time = 0.0
 
         if is_git_dir():
@@ -112,18 +114,21 @@ class _Events:
             return
         if (
             len(self.events) < self.max_events
-        ):  # Events list limited to 25 events (drop any events past this)
+        ):  # Events list limited to self.max_events (drop any events past this)
             params.update(self.metadata)
-            self.events.append(
-                {
-                    "event": event_name,
-                    "properties": params,
-                    "timestamp": datetime.datetime.now(
-                        tz=datetime.timezone.utc
-                    ).isoformat(),
-                    "distinct_id": CONFIG["uuid"],
-                }
-            )
+            event = {
+                "event": event_name,
+                "properties": params,
+                "timestamp": datetime.datetime.now(
+                    tz=datetime.timezone.utc
+                ).isoformat(),
+                "distinct_id": CONFIG["uuid"],
+            }
+            if event_name not in self.throttled_event_names:
+                self.events.append(event)
+            elif event_name not in self.throttled_events:
+                self.throttled_events.add(event_name)
+                self.events.append(event)
 
         # Check rate limit
         t = time.time()
@@ -135,7 +140,6 @@ class _Events:
             "distinct_id": CONFIG["uuid"],  # posthog needs this to accepts the event
             "batch": self.events,
         }
-
         # POST equivalent to requests.post(self.url, json=data).
         # threaded request is used to avoid blocking, retries are disabled, and verbose is disabled
         # to avoid any possible disruption in the console.
@@ -150,6 +154,7 @@ class _Events:
 
         # Flush & Reset
         self.events = []
+        self.throttled_events = set()
         self.time = t
 
 

python/lancedb/utils/sentry_log.py

@@ -63,7 +63,8 @@ def set_sentry():
         """
         if "exc_info" in hint:
             exc_type, exc_value, tb = hint["exc_info"]
-            if "out of memory" in str(exc_value).lower():
+            ignored_errors = ["out of memory", "no space left on device", "testing"]
+            if any(error in str(exc_value).lower() for error in ignored_errors):
                 return None
 
         if is_git_dir():
@@ -97,7 +98,7 @@ def set_sentry():
             dsn="https://c63ef8c64e05d1aa1a96513361f3ca2f@o4505950840946688.ingest.sentry.io/4505950933614592",
             debug=False,
             include_local_variables=False,
-            traces_sample_rate=1.0,
+            traces_sample_rate=0.5,
             environment="production",  # 'dev' or 'production'
             before_send=before_send,
             ignore_errors=[KeyboardInterrupt, FileNotFoundError, bdb.BdbQuit],

python/pyproject.toml

@@ -1,12 +1,12 @@
 [project]
 name = "lancedb"
-version = "0.3.1"
+version = "0.4.1"
 dependencies = [
     "deprecation",
-    "pylance==0.8.5",
+    "pylance==0.9.1",
     "ratelimiter~=1.0",
     "retry>=0.9.2",
-    "tqdm>=4.1.0",
+    "tqdm>=4.27.0",
     "aiohttp",
     "pydantic>=1.10",
     "attrs>=21.3.0",
@@ -14,7 +14,8 @@ dependencies = [
     "cachetools",
     "pyyaml>=6.0",
     "click>=8.1.7",
-    "requests>=2.31.0"
+    "requests>=2.31.0",
+    "overrides>=0.7"
 ]
 description = "lancedb"
 authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }]
@@ -52,7 +53,7 @@ tests = ["pandas>=1.4", "pytest", "pytest-mock", "pytest-asyncio", "requests"]
 dev = ["ruff", "pre-commit", "black"]
 docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings[python]"]
 clip = ["torch", "pillow", "open-clip"]
-embeddings = ["openai", "sentence-transformers", "torch", "pillow", "open-clip", "cohere"]
+embeddings = ["openai", "sentence-transformers", "torch", "pillow", "open-clip-torch", "cohere", "InstructorEmbedding"]
 
 [project.scripts]
 lancedb = "lancedb.cli.cli:cli"
@@ -64,6 +65,9 @@ build-backend = "setuptools.build_meta"
 [tool.isort]
 profile = "black"
 
+[tool.ruff]
+select = ["F", "E", "W", "I", "G", "TCH", "PERF"]
+
 [tool.pytest.ini_options]
 addopts = "--strict-markers"
 markers = [

python/tests/test_db.py

@@ -129,7 +129,7 @@ def test_ingest_iterator(tmp_path):
                 [
                     PydanticSchema(vector=[3.1, 4.1], item="foo", price=10.0),
                     PydanticSchema(vector=[5.9, 26.5], item="bar", price=20.0),
-                ]
+                ],
                 # TODO: test pydict separately. it is unique column number and names contraint
             ]
 
@@ -150,6 +150,21 @@ def test_ingest_iterator(tmp_path):
     run_tests(PydanticSchema)
 
 
+def test_table_names(tmp_path):
+    db = lancedb.connect(tmp_path)
+    data = pd.DataFrame(
+        {
+            "vector": [[3.1, 4.1], [5.9, 26.5]],
+            "item": ["foo", "bar"],
+            "price": [10.0, 20.0],
+        }
+    )
+    db.create_table("test2", data=data)
+    db.create_table("test1", data=data)
+    db.create_table("test3", data=data)
+    assert db.table_names() == ["test1", "test2", "test3"]
+
+
 def test_create_mode(tmp_path):
     db = lancedb.connect(tmp_path)
     data = pd.DataFrame(
@@ -286,4 +301,29 @@ def test_replace_index(tmp_path):
         num_partitions=2,
         num_sub_vectors=4,
         replace=True,
+        index_cache_size=10,
     )
+
+
+def test_prefilter_with_index(tmp_path):
+    db = lancedb.connect(uri=tmp_path)
+    data = [
+        {"vector": np.random.rand(128), "item": "foo", "price": float(i)}
+        for i in range(1000)
+    ]
+    sample_key = data[100]["vector"]
+    table = db.create_table(
+        "test",
+        data,
+    )
+    table.create_index(
+        num_partitions=2,
+        num_sub_vectors=4,
+    )
+    table = (
+        table.search(sample_key)
+        .where("price == 500", prefilter=True)
+        .limit(5)
+        .to_arrow()
+    )
+    assert table.num_rows == 1

python/tests/test_embeddings.py

@@ -15,13 +15,16 @@ import sys
 import lance
 import numpy as np
 import pyarrow as pa
+import pytest
 
-from lancedb.conftest import MockTextEmbeddingFunction
+import lancedb
+from lancedb.conftest import MockRateLimitedEmbeddingFunction, MockTextEmbeddingFunction
 from lancedb.embeddings import (
     EmbeddingFunctionConfig,
     EmbeddingFunctionRegistry,
     with_embeddings,
 )
+from lancedb.pydantic import LanceModel, Vector
 
 
 def mock_embed_func(input_data):
@@ -83,3 +86,29 @@ def test_embedding_function(tmp_path):
     expected = func.compute_query_embeddings("hello world")
 
     assert np.allclose(actual, expected)
+
+
+def test_embedding_function_rate_limit(tmp_path):
+    def _get_schema_from_model(model):
+        class Schema(LanceModel):
+            text: str = model.SourceField()
+            vector: Vector(model.ndims()) = model.VectorField()
+
+        return Schema
+
+    db = lancedb.connect(tmp_path)
+    registry = EmbeddingFunctionRegistry.get_instance()
+    model = registry.get("test-rate-limited").create(max_retries=0)
+    schema = _get_schema_from_model(model)
+    table = db.create_table("test", schema=schema, mode="overwrite")
+    table.add([{"text": "hello world"}])
+    with pytest.raises(Exception):
+        table.add([{"text": "hello world"}])
+    assert len(table) == 1
+
+    model = registry.get("test-rate-limited").create()
+    schema = _get_schema_from_model(model)
+    table = db.create_table("test", schema=schema, mode="overwrite")
+    table.add([{"text": "hello world"}])
+    table.add([{"text": "hello world"}])
+    assert len(table) == 2

python/tests/test_embeddings_slow.py

@@ -19,7 +19,7 @@ import pytest
 import requests
 
 import lancedb
-from lancedb.embeddings import EmbeddingFunctionRegistry
+from lancedb.embeddings import get_registry
 from lancedb.pydantic import LanceModel, Vector
 
 # These are integration tests for embedding functions.
@@ -31,12 +31,15 @@ from lancedb.pydantic import LanceModel, Vector
 @pytest.mark.parametrize("alias", ["sentence-transformers", "openai"])
 def test_sentence_transformer(alias, tmp_path):
     db = lancedb.connect(tmp_path)
-    registry = EmbeddingFunctionRegistry.get_instance()
-    func = registry.get(alias).create()
+    registry = get_registry()
+    func = registry.get(alias).create(max_retries=0)
+    func2 = registry.get(alias).create(max_retries=0)
 
     class Words(LanceModel):
         text: str = func.SourceField()
+        text2: str = func2.SourceField()
         vector: Vector(func.ndims()) = func.VectorField()
+        vector2: Vector(func2.ndims()) = func2.VectorField()
 
     table = db.create_table("words", schema=Words)
     table.add(
@@ -50,7 +53,16 @@ def test_sentence_transformer(alias, tmp_path):
                     "foo",
                     "bar",
                     "baz",
-                ]
+                ],
+                "text2": [
+                    "to be or not to be",
+                    "that is the question",
+                    "for whether tis nobler",
+                    "in the mind to suffer",
+                    "the slings and arrows",
+                    "of outrageous fortune",
+                    "or to take arms",
+                ],
             }
         )
     )
@@ -62,6 +74,13 @@ def test_sentence_transformer(alias, tmp_path):
     expected = table.search(vec).limit(1).to_pydantic(Words)[0]
     assert actual.text == expected.text
     assert actual.text == "hello world"
+    assert not np.allclose(actual.vector, actual.vector2)
+
+    actual = (
+        table.search(query, vector_column_name="vector2").limit(1).to_pydantic(Words)[0]
+    )
+    assert actual.text != "hello world"
+    assert not np.allclose(actual.vector, actual.vector2)
 
 
 @pytest.mark.slow
@@ -69,7 +88,7 @@ def test_openclip(tmp_path):
     from PIL import Image
 
     db = lancedb.connect(tmp_path)
-    registry = EmbeddingFunctionRegistry.get_instance()
+    registry = get_registry()
     func = registry.get("open-clip").create()
 
     class Images(LanceModel):
@@ -132,9 +151,9 @@ def test_openclip(tmp_path):
 )  # also skip if cohere not installed
 def test_cohere_embedding_function():
     cohere = (
-        EmbeddingFunctionRegistry.get_instance()
+        get_registry()
         .get("cohere")
-        .create(name="embed-multilingual-v2.0")
+        .create(name="embed-multilingual-v2.0", max_retries=0)
     )
 
     class TextModel(LanceModel):
@@ -147,3 +166,19 @@ def test_cohere_embedding_function():
 
     tbl.add(df)
     assert len(tbl.to_pandas()["vector"][0]) == cohere.ndims()
+
+
+@pytest.mark.slow
+def test_instructor_embedding(tmp_path):
+    model = get_registry().get("instructor").create()
+
+    class TextModel(LanceModel):
+        text: str = model.SourceField()
+        vector: Vector(model.ndims()) = model.VectorField()
+
+    df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
+    db = lancedb.connect(tmp_path)
+    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+    tbl.add(df)
+    assert len(tbl.to_pandas()["vector"][0]) == model.ndims()

python/tests/test_fts.py

@@ -43,7 +43,15 @@ def table(tmp_path) -> ldb.table.LanceTable:
         for _ in range(100)
     ]
     table = db.create_table(
-        "test", data=pd.DataFrame({"vector": vectors, "text": text, "text2": text})
+        "test",
+        data=pd.DataFrame(
+            {
+                "vector": vectors,
+                "text": text,
+                "text2": text,
+                "nested": [{"text": t} for t in text],
+            }
+        ),
     )
     return table
 
@@ -75,6 +83,24 @@ def test_create_index_from_table(tmp_path, table):
     assert len(df) == 10
     assert "text" in df.columns
 
+    # Check whether it can be updated
+    table.add(
+        [
+            {
+                "vector": np.random.randn(128),
+                "text": "gorilla",
+                "text2": "gorilla",
+                "nested": {"text": "gorilla"},
+            }
+        ]
+    )
+
+    with pytest.raises(ValueError, match="already exists"):
+        table.create_fts_index("text")
+
+    table.create_fts_index("text", replace=True)
+    assert len(table.search("gorilla").limit(1).to_pandas()) == 1
+
 
 def test_create_index_multiple_columns(tmp_path, table):
     table.create_fts_index(["text", "text2"])
@@ -89,3 +115,9 @@ def test_empty_rs(tmp_path, table, mocker):
     mocker.patch("lancedb.fts.search_index", return_value=([], []))
     df = table.search("puppy").limit(10).to_pandas()
     assert len(df) == 0
+
+
+def test_nested_schema(tmp_path, table):
+    table.create_fts_index("nested.text")
+    rs = table.search("puppy").limit(10).to_list()
+    assert len(rs) == 10

python/tests/test_pydantic.py

@@ -14,6 +14,7 @@
 
 import json
 import sys
+from datetime import date, datetime
 from typing import List, Optional
 
 import pyarrow as pa
@@ -40,10 +41,18 @@ def test_pydantic_to_arrow():
         li: List[int]
         opt: Optional[str] = None
         st: StructModel
+        dt: date
+        dtt: datetime
         # d: dict
 
     m = TestModel(
-        id=1, s="hello", vec=[1.0, 2.0, 3.0], li=[2, 3, 4], st=StructModel(a="a", b=1.0)
+        id=1,
+        s="hello",
+        vec=[1.0, 2.0, 3.0],
+        li=[2, 3, 4],
+        st=StructModel(a="a", b=1.0),
+        dt=date.today(),
+        dtt=datetime.now(),
     )
 
     schema = pydantic_to_schema(TestModel)
@@ -62,6 +71,8 @@ def test_pydantic_to_arrow():
                 ),
                 False,
             ),
+            pa.field("dt", pa.date32(), False),
+            pa.field("dtt", pa.timestamp("us"), False),
         ]
     )
     assert schema == expect_schema
@@ -79,10 +90,18 @@ def test_pydantic_to_arrow_py38():
         li: List[int]
         opt: Optional[str] = None
         st: StructModel
+        dt: date
+        dtt: datetime
         # d: dict
 
     m = TestModel(
-        id=1, s="hello", vec=[1.0, 2.0, 3.0], li=[2, 3, 4], st=StructModel(a="a", b=1.0)
+        id=1,
+        s="hello",
+        vec=[1.0, 2.0, 3.0],
+        li=[2, 3, 4],
+        st=StructModel(a="a", b=1.0),
+        dt=date.today(),
+        dtt=datetime.now(),
     )
 
     schema = pydantic_to_schema(TestModel)
@@ -101,6 +120,8 @@ def test_pydantic_to_arrow_py38():
                 ),
                 False,
             ),
+            pa.field("dt", pa.date32(), False),
+            pa.field("dtt", pa.timestamp("us"), False),
         ]
     )
     assert schema == expect_schema