[python] Bump version: 0.3.5 → 0.3.6

pass vector column name to remote as well.

`vector_column` is already part of `Query` just declearing it as part to
`remote.VectorQuery` as well

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.19
+current_version = 0.3.11
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True

.github/workflows/node.yml

@@ -9,6 +9,11 @@ on:
       - node/**
       - rust/ffi/node/**
       - .github/workflows/node.yml
+      - docker-compose.yml
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 env:
   # Disable full debug symbol generation to speed up CI build and keep memory down
@@ -107,3 +112,56 @@ jobs:
     - name: Test
       run: |
         npm run test
+  aws-integtest:
+    timeout-minutes: 45
+    runs-on: "ubuntu-22.04"
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
+    env:
+      AWS_ACCESS_KEY_ID: ACCESSKEY
+      AWS_SECRET_ACCESS_KEY: SECRETKEY
+      AWS_DEFAULT_REGION: us-west-2
+      # this one is for s3
+      AWS_ENDPOINT: http://localhost:4566
+      # this one is for dynamodb
+      DYNAMODB_ENDPOINT: http://localhost:4566
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 18
+        cache: 'npm'
+        cache-dependency-path: node/package-lock.json
+    - name: start local stack
+      run: docker compose -f ../docker-compose.yml up -d --wait
+    - name: create s3
+      run: aws s3 mb s3://lancedb-integtest --endpoint $AWS_ENDPOINT
+    - name: create ddb
+      run: |
+        aws dynamodb create-table \
+          --table-name lancedb-integtest \
+          --attribute-definitions '[{"AttributeName": "base_uri", "AttributeType": "S"}, {"AttributeName": "version", "AttributeType": "N"}]' \
+          --key-schema '[{"AttributeName": "base_uri", "KeyType": "HASH"}, {"AttributeName": "version", "KeyType": "RANGE"}]' \
+          --provisioned-throughput '{"ReadCapacityUnits": 10, "WriteCapacityUnits": 10}' \
+          --endpoint-url $DYNAMODB_ENDPOINT
+    - uses: Swatinem/rust-cache@v2
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y protobuf-compiler libssl-dev
+    - name: Build
+      run: |
+        npm ci
+        npm run tsc
+        npm run build
+        npm run pack-build
+        npm install --no-save ./dist/lancedb-vectordb-*.tgz
+        # Remove index.node to test with dependency installed
+        rm index.node
+    - name: Test
+      run: npm run integration-test

.github/workflows/npm-publish.yml

@@ -38,13 +38,17 @@ jobs:
             node/vectordb-*.tgz
 
   node-macos:
-    runs-on: macos-12
+    strategy:
+      matrix:
+        config:
+          - arch: x86_64-apple-darwin
+            runner: macos-13
+          - arch: aarch64-apple-darwin
+            # xlarge is implicitly arm64.
+            runner: macos-13-xlarge
+    runs-on: ${{ matrix.config.runner }}
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64-apple-darwin, aarch64-apple-darwin]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -54,11 +58,8 @@ jobs:
         run: |
           cd node
           npm ci
-      - name: Install rustup target
-        if: ${{ matrix.target == 'aarch64-apple-darwin' }}
-        run: rustup target add aarch64-apple-darwin
       - name: Build MacOS native node modules
-        run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
+        run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }}
       - name: Upload Darwin Artifacts
         uses: actions/upload-artifact@v3
         with:
@@ -66,6 +67,7 @@ jobs:
           path: |
             node/dist/lancedb-vectordb-darwin*.tgz
          
+
   node-linux:
     name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu
     runs-on: ${{ matrix.config.runner }}

.github/workflows/python.yml

@@ -8,6 +8,11 @@ on:
     paths:
       - python/**
       - .github/workflows/python.yml
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   linux:
     timeout-minutes: 30
@@ -32,18 +37,19 @@ jobs:
       run: |
         pip install -e .[tests]
         pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock black isort
-    - name: Black
-      run: black --check --diff --no-color --quiet .
-    - name: isort
-      run: isort --check --diff --quiet .
+        pip install pytest pytest-mock ruff
+    - name: Lint
+      run: ruff format --check .
     - name: Run tests
-      run: pytest -x -v --durations=30 tests
+      run: pytest -m "not slow" -x -v --durations=30 tests
     - name: doctest
       run: pytest --doctest-modules lancedb
   mac:
     timeout-minutes: 30
-    runs-on: "macos-12"
+    strategy:
+      matrix:
+        mac-runner: [ "macos-13", "macos-13-xlarge" ]
+    runs-on: "${{ matrix.mac-runner }}"
     defaults:
       run:
         shell: bash
@@ -62,7 +68,31 @@ jobs:
         pip install -e .[tests]
         pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
         pip install pytest pytest-mock black
-    - name: Black
-      run: black --check --diff --no-color --quiet .
     - name: Run tests
-      run: pytest -x -v --durations=30 tests
+      run: pytest -m "not slow" -x -v --durations=30 tests
+  pydantic1x:
+    timeout-minutes: 30
+    runs-on: "ubuntu-22.04"
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+    - name: Install lancedb
+      run: |
+        pip install "pydantic<2"
+        pip install -e .[tests]
+        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+        pip install pytest pytest-mock
+    - name: Run tests
+      run: pytest -m "not slow" -x -v --durations=30 tests
+    - name: doctest
+      run: pytest --doctest-modules lancedb

.github/workflows/rust.yml

@@ -10,6 +10,10 @@ on:
       - rust/**
       - .github/workflows/rust.yml
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   # This env var is used by Swatinem/rust-cache@v2 for the cache
   # key, so we set it to make sure it is always consistent.
@@ -44,8 +48,11 @@ jobs:
     - name: Run tests
       run: cargo test --all-features
   macos:
-    runs-on: macos-12
     timeout-minutes: 30
+    strategy:
+      matrix:
+        mac-runner: [ "macos-13", "macos-13-xlarge" ]
+    runs-on: "${{ matrix.mac-runner }}"
     defaults:
       run:
         shell: bash

.github/workflows/trigger-vectordb-recipes.yml

@@ -0,0 +1,26 @@
+name: Trigger vectordb-recipers workflow
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    paths:
+      - .github/workflows/trigger-vectordb-recipes.yml
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Trigger vectordb-recipes workflow
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.VECTORDB_RECIPES_ACTION_TOKEN }}
+          script: |
+            const result = await github.rest.actions.createWorkflowDispatch({
+                owner: 'lancedb',
+                repo: 'vectordb-recipes',
+                workflow_id: 'examples-test.yml',
+                ref: 'main'
+            });
+            console.log(result);

Cargo.toml

@@ -1,17 +1,28 @@
 [workspace]
-members = [
-    "rust/vectordb",
-    "rust/ffi/node"
-]
+members = ["rust/ffi/node", "rust/vectordb"]
+# Python package needs to be built by maturin.
+exclude = ["python"]
 resolver = "2"
 
 [workspace.dependencies]
-lance = "=0.5.9"
-arrow-array = "42.0"
-arrow-data = "42.0"
-arrow-schema = "42.0"
-arrow-ipc = "42.0"
-half = { "version" = "=2.2.1", default-features = false }
-object_store = "0.6.1"
+lance = { "version" = "=0.8.20", "features" = ["dynamodb"] }
+lance-index = { "version" = "=0.8.20" }
+lance-linalg = { "version" = "=0.8.20" }
+lance-testing = { "version" = "=0.8.20" }
+# Note that this one does not include pyarrow
+arrow = { version = "47.0.0", optional = false }
+arrow-array = "47.0"
+arrow-data = "47.0"
+arrow-ipc = "47.0"
+arrow-ord = "47.0"
+arrow-schema = "47.0"
+arrow-arith = "47.0"
+arrow-cast = "47.0"
+chrono = "0.4.23"
+half = { "version" = "=2.3.1", default-features = false, features = [
+    "num-traits",
+] }
+log = "0.4"
+object_store = "0.7.1"
 snafu = "0.7.4"
-
+url = "2"

README.md

@@ -5,10 +5,11 @@
 
 **Developer-friendly, serverless vector database for AI applications**
 
-<a href="https://lancedb.github.io/lancedb/">Documentation</a> •
-<a href="https://blog.lancedb.com/">Blog</a> •
-<a href="https://discord.gg/zMM32dvNtd">Discord</a> •
-<a href="https://twitter.com/lancedb">Twitter</a>
+<a href='https://github.com/lancedb/vectordb-recipes/tree/main' target="_blank"><img alt='LanceDB' src='https://img.shields.io/badge/VectorDB_Recipes-100000?style=for-the-badge&logo=LanceDB&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
+<a href='https://lancedb.github.io/lancedb/' target="_blank"><img alt='lancdb' src='https://img.shields.io/badge/DOCS-100000?style=for-the-badge&logo=lancdb&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
+[![Medium](https://img.shields.io/badge/Medium-12100E?style=for-the-badge&logo=medium&logoColor=white)](https://blog.lancedb.com/) 
+[![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/zMM32dvNtd) 
+[![Twitter](https://img.shields.io/badge/Twitter-%231DA1F2.svg?style=for-the-badge&logo=Twitter&logoColor=white)](https://twitter.com/lancedb)
 
 </p>
 
@@ -33,6 +34,8 @@ The key features of LanceDB include:
 
 * Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure.
 
+* GPU support in building vector index(*).
+
 * Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lanecdb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
 
 LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
@@ -52,8 +55,7 @@ const table = await db.createTable('vectors',
       [{ id: 1, vector: [0.1, 0.2], item: "foo", price: 10 },
        { id: 2, vector: [1.1, 1.2], item: "bar", price: 50 }])
 
-const query = table.search([0.1, 0.3]);
-query.limit = 20;
+const query = table.search([0.1, 0.3]).limit(2);
 const results = await query.execute();
 ```
 
@@ -70,7 +72,7 @@ db = lancedb.connect(uri)
 table = db.create_table("my_table",
                          data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
                                {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-result = table.search([100, 100]).limit(2).to_df()
+result = table.search([100, 100]).limit(2).to_pandas()
 ```
 
 ## Blogs, Tutorials & Videos

ci/build_macos_artifacts.sh

@@ -1,6 +1,7 @@
 # Builds the macOS artifacts (node binaries).
 # Usage: ./ci/build_macos_artifacts.sh [target]
 # Targets supported: x86_64-apple-darwin aarch64-apple-darwin
+set -e
 
 prebuild_rust() {
     # Building here for the sake of easier debugging.

docker-compose.yml

@@ -0,0 +1,18 @@
+version: "3.9"
+services:
+  localstack:
+    image: localstack/localstack:0.14
+    ports:
+      - 4566:4566
+    environment:
+      - SERVICES=s3,dynamodb
+      - DEBUG=1
+      - LS_LOG=trace
+      - DOCKER_HOST=unix:///var/run/docker.sock
+      - AWS_ACCESS_KEY_ID=ACCESSKEY
+      - AWS_SECRET_ACCESS_KEY=SECRETKEY
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:4566/health" ]
+      interval: 5s
+      retries: 3
+      start_period: 10s

docs/README.md

@@ -0,0 +1,26 @@
+# LanceDB Documentation
+
+LanceDB docs are deployed to https://lancedb.github.io/lancedb/.
+
+Docs is built and deployed automatically by [Github Actions](.github/workflows/docs.yml)
+whenever a commit is pushed to the `main` branch. So it is possible for the docs to show
+unreleased features.
+
+## Building the docs
+
+### Setup
+1. Install LanceDB. From LanceDB repo root: `pip install -e python`
+2. Install dependencies. From LanceDB repo root: `pip install -r docs/requirements.txt`
+3. Make sure you have node and npm setup
+4. Make sure protobuf and libssl are installed
+
+### Building node module and create markdown files
+
+See [Javascript docs README](docs/src/javascript/README.md)
+
+### Build docs
+From LanceDB repo root:
+
+Run: `PYTHONPATH=. mkdocs build -f docs/mkdocs.yml`
+
+If successful, you should see a `docs/site` directory that you can verify locally.

docs/mkdocs.yml

@@ -1,5 +1,7 @@
 site_name: LanceDB Docs
+site_url: https://lancedb.github.io/lancedb/
 repo_url: https://github.com/lancedb/lancedb
+edit_uri: https://github.com/lancedb/lancedb/tree/main/docs/src
 repo_name: lancedb/lancedb
 docs_dir: src
 
@@ -10,6 +12,17 @@ theme:
   features:
     - content.code.copy
     - content.tabs.link
+    - content.action.edit
+    - toc.follow
+    - toc.integrate
+    - navigation.top
+    - navigation.tabs
+    - navigation.tabs.sticky
+    - navigation.footer
+    - navigation.tracking
+    - navigation.instant
+    - navigation.indexes
+    - navigation.expand
   icon:
     repo: fontawesome/brands/github
   custom_dir: overrides
@@ -25,7 +38,7 @@ plugins:
           docstring_style: numpy
         rendering:
           heading_level: 4
-          show_source: false
+          show_source: true
           show_symbol_type_in_heading: true
           show_signature_annotations: true
           show_root_heading: true
@@ -53,11 +66,65 @@ markdown_extensions:
 - md_in_html
 
 nav:
-- Home: index.md
+- Home:
+  - 🏢 Home: index.md
+  - 💡 Basics: basic.md
+  - 📚 Guides:
+    - Create Ingest Update Delete: guides/tables.md
+    - Vector Search: search.md
+    - SQL filters: sql.md
+    - Indexing: ann_indexes.md
+    - Versioning & Reproducibility: notebooks/reproducibility.ipynb
+  - 🧬 Embeddings:
+    - embeddings/index.md
+    - Ingest Embedding Functions: embeddings/embedding_functions.md
+    - Available Functions: embeddings/default_embedding_functions.md
+    - Create Custom Embedding Functions: embeddings/api.md
+    - Example - Multi-lingual semantic search: notebooks/multi_lingual_example.ipynb
+    - Example - MultiModal CLIP Embeddings: notebooks/DisappearingEmbeddingFunction.ipynb
+  - 🔍 Python full-text search: fts.md
+  - 🔌 Integrations:
+    - integrations/index.md
+    - Pandas and PyArrow: python/arrow.md
+    - DuckDB: python/duckdb.md
+    - LangChain 🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
+    - LangChain JS/TS 🔗: https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb
+    - LlamaIndex 🦙: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
+    - Pydantic: python/pydantic.md
+    - Voxel51: integrations/voxel51.md
+    - PromptTools: integrations/prompttools.md
+  - 🐍 Python examples:
+    - examples/index.md
+    - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
+    - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
+    - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
+    - Example - Calculate CLIP Embeddings with Roboflow Inference: examples/image_embeddings_roboflow.md
+    - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
+    - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
+  - 🌐 Javascript examples:
+    - Examples: examples/index_js.md
+    - Serverless Website Chatbot: examples/serverless_website_chatbot.md
+    - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
+    - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
+  - ⚙️ CLI & Config: cli_config.md
+
 - Basics: basic.md
-- Embeddings: embedding.md
+- Guides:
+  - Create Ingest Update Delete: guides/tables.md
+  - Vector Search: search.md
+  - SQL filters: sql.md
+  - Indexing: ann_indexes.md
+  - Versioning & Reproducibility: notebooks/reproducibility.ipynb
+- Embeddings:
+  - embeddings/index.md
+  - Ingest Embedding Functions: embeddings/embedding_functions.md
+  - Available Functions: embeddings/default_embedding_functions.md
+  - Create Custom Embedding Functions: embeddings/api.md
+  - Example - Multi-lingual semantic search: notebooks/multi_lingual_example.ipynb
+  - Example - MultiModal CLIP Embeddings: notebooks/DisappearingEmbeddingFunction.ipynb
 - Python full-text search: fts.md
 - Integrations:
+  - integrations/index.md
   - Pandas and PyArrow: python/arrow.md
   - DuckDB: python/duckdb.md
   - LangChain 🦜️🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
@@ -65,23 +132,29 @@ nav:
   - LlamaIndex 🦙: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
   - Pydantic: python/pydantic.md
   - Voxel51: integrations/voxel51.md
+  - PromptTools: integrations/prompttools.md
 - Python examples:
+  - examples/index.md
   - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
   - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
   - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
   - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
   - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
 - Javascript examples:
+  - examples/index_js.md
   - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
+  - Serverless Chatbot from any website: examples/serverless_website_chatbot.md
   - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
-
-- References:
-  - Vector Search: search.md
-  - SQL filters: sql.md
-  - Indexing: ann_indexes.md
 - API references:
-  - Python API: python/python.md
+  - OSS Python API: python/python.md
+  - SaaS Python API: python/saas-python.md
   - Javascript API: javascript/modules.md
+- LanceDB Cloud↗: https://noteforms.com/forms/lancedb-mailing-list-cloud-kty1o5?notionforms=1&utm_source=notionforms
 
 extra_css:
   - styles/global.css
+
+extra:
+  analytics:
+    provider: google
+    property: G-B7NFM40W74

docs/src/ann_indexes.md

@@ -6,7 +6,7 @@ LanceDB provides many parameters to fine-tune the index's size, the speed of que
 
 Currently, LanceDB does *not* automatically create the ANN index.
 LanceDB has optimized code for KNN as well. For many use-cases, datasets under 100K vectors won't require index creation at all.
-If you can live with <100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
+If you can live with < 100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
 
 In the future we will look to automatically create and configure the ANN index.
 
@@ -68,6 +68,44 @@ a single PQ code.
   <figcaption>IVF_PQ index with <code>num_partitions=2, num_sub_vectors=4</code></figcaption>
 </figure>
 
+### Use GPU to build vector index
+
+Lance Python SDK has experimental GPU support for creating IVF index.
+Using GPU for index creation requires [PyTorch>2.0](https://pytorch.org/) being installed.
+
+You can specify the GPU device to train IVF partitions via
+
+- **accelerator**: Specify to ``cuda`` or ``mps`` (on Apple Silicon) to enable GPU training.
+
+=== "Linux"
+
+     <!-- skip-test -->
+     ``` { .python .copy }
+     # Create index using CUDA on Nvidia GPUs.
+     tbl.create_index(
+          num_partitions=256,
+          num_sub_vectors=96,
+          accelerator="cuda"
+     )
+     ```
+
+=== "Macos"
+
+     <!-- skip-test -->
+     ```python
+     # Create index using MPS on Apple Silicon.
+     tbl.create_index(
+          num_partitions=256,
+          num_sub_vectors=96,
+          accelerator="mps"
+     )
+     ```
+
+Trouble shootings:
+
+If you see ``AssertionError: Torch not compiled with CUDA enabled``, you need to [install
+PyTorch with CUDA support](https://pytorch.org/get-started/locally/).
+
 
 ## Querying an ANN Index
 
@@ -91,10 +129,10 @@ There are a couple of parameters that can be used to fine-tune the search:
          .limit(2) \
          .nprobes(20) \
          .refine_factor(10) \
-         .to_df()
+         .to_pandas()
      ```
      ```
-                                              vector       item       score
+                                              vector       item       _distance
      0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
      1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
      ```
@@ -109,9 +147,8 @@ There are a couple of parameters that can be used to fine-tune the search:
          .execute()
      ```
 
-The search will return the data requested in addition to the score of each item.
+The search will return the data requested in addition to the distance of each item.
 
-**Note:** The score is the distance between the query vector and the element. A lower number means that the result is more relevant.
 
 ### Filtering (where clause)
 
@@ -119,7 +156,7 @@ You can further filter the elements returned by a search using a where clause.
 
 === "Python"
      ```python
-     tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_df()
+     tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
      ```
 
 === "Javascript"
@@ -136,10 +173,10 @@ You can select the columns returned by the query using a select clause.
 
 === "Python"
      ```python
-     tbl.search(np.random.random((1536))).select(["vector"]).to_df()
+     tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
      ```
      ```
-     vector      score
+        vector                                             _distance
      0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
      1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
      ...
@@ -155,28 +192,28 @@ You can select the columns returned by the query using a select clause.
 
 ## FAQ
 
-### When is it necessary to create an ANN vector index.
+### When is it necessary to create an ANN vector index?
 
-`LanceDB` has manually tuned SIMD code for computing vector distances.
-In our benchmarks, computing 100K pairs of 1K dimension vectors only take less than 20ms.
-For small dataset (<100K rows) or the applications which can accept 100ms latency, vector indices are usually not necessary.
+`LanceDB` has manually-tuned SIMD code for computing vector distances.
+In our benchmarks, computing 100K pairs of 1K dimension vectors takes **less than 20ms**.
+For small datasets (< 100K rows) or applications that can accept 100ms latency, vector indices are usually not necessary.
 
 For large-scale or higher dimension vectors, it is beneficial to create vector index.
 
-### How big is my index, and how many memory will it take.
+### How big is my index, and how many memory will it take?
 
-In LanceDB, all vector indices are disk-based, meaning that when responding to a vector query, only the relevant pages from the index file are loaded from disk and cached in memory. Additionally, each sub-vector is usually encoded into 1 byte PQ code.
+In LanceDB, all vector indices are **disk-based**, meaning that when responding to a vector query, only the relevant pages from the index file are loaded from disk and cached in memory. Additionally, each sub-vector is usually encoded into 1 byte PQ code.
 
 For example, with a 1024-dimension dataset, if we choose `num_sub_vectors=64`, each sub-vector has `1024 / 64 = 16` float32 numbers.
 Product quantization can lead to approximately `16 * sizeof(float32) / 1 = 64` times of space reduction.
 
-### How to choose `num_partitions` and `num_sub_vectors` for `IVF_PQ` index.
+### How to choose `num_partitions` and `num_sub_vectors` for `IVF_PQ` index?
 
 `num_partitions` is used to decide how many partitions the first level `IVF` index uses.
 Higher number of partitions could lead to more efficient I/O during queries and better accuracy, but it takes much more time to train.
 On `SIFT-1M` dataset, our benchmark shows that keeping each partition 1K-4K rows lead to a good latency / recall.
 
-`num_sub_vectors` decides how many Product Quantization code to generate on each vector. Because
-Product Quantization is a lossy compression of the original vector, the more `num_sub_vectors` usually results to
-less space distortion, and thus yield better accuracy. However, similarly, more `num_sub_vectors` causes heavier I/O and
-more PQ computation, thus, higher latency. `dimension / num_sub_vectors` should be aligned with 8 for better SIMD efficiency.
+`num_sub_vectors` specifies how many Product Quantization (PQ) short codes to generate on each vector. Because
+PQ is a lossy compression of the original vector, a higher `num_sub_vectors` usually results in
+less space distortion, and thus yields better accuracy. However, a higher `num_sub_vectors` also causes heavier I/O and
+more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.

docs/src/basic.md

@@ -79,6 +79,18 @@ We'll cover the basics of using LanceDB on your local machine in this section.
 
 ??? info "Under the hood, LanceDB is converting the input data into an Apache Arrow table and persisting it to disk in [Lance format](https://www.github.com/lancedb/lance)."
 
+### Creating an empty table
+
+Sometimes you may not have the data to insert into the table at creation time.
+In this case, you can create an empty table and specify the schema.
+
+=== "Python"
+      ```python
+      import pyarrow as pa
+      schema = pa.schema([pa.field("vector", pa.list_(pa.float32(), list_size=2))])
+      tbl = db.create_table("empty_table", schema=schema)
+      ```
+
 ## How to open an existing table
 
 Once created, you can open a table using the following code:
@@ -111,9 +123,15 @@ After a table has been created, you can always add more data to it using
 
 === "Python"
       ```python
-      df = pd.DataFrame([{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
-                        {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}])
-      tbl.add(df)
+
+      # Option 1: Add a list of dicts to a table
+      data = [{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
+            {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}]
+      tbl.add(data)
+
+      # Option 2: Add a pandas DataFrame to a table
+      df = pd.DataFrame(data)
+      tbl.add(data)
       ```
 
 === "Javascript"
@@ -122,6 +140,22 @@ After a table has been created, you can always add more data to it using
               {vector: [9.5, 56.2], item: "buzz", price: 200.0}])
       ```
 
+## How to search for (approximate) nearest neighbors
+
+Once you've embedded the query, you can find its nearest neighbors using the following code:
+
+=== "Python"
+      ```python
+      tbl.search([100, 100]).limit(2).to_pandas()
+      ```
+
+      This returns a pandas DataFrame with the results.
+
+=== "Javascript"
+      ```javascript
+      const query = await tbl.search([100, 100]).limit(2).execute();
+      ```
+
 ## How to delete rows from a table
 
 Use the `delete()` method on tables to delete rows from a table. To choose
@@ -151,24 +185,34 @@ To see what expressions are supported, see the [SQL filters](sql.md) section.
 
       Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
 
-## How to search for (approximate) nearest neighbors
+## How to remove a table
 
-Once you've embedded the query, you can find its nearest neighbors using the following code:
+Use the `drop_table()` method on the database to remove a table.
 
 === "Python"
       ```python
-      tbl.search([100, 100]).limit(2).to_df()
+      db.drop_table("my_table")
       ```
 
-      This returns a pandas DataFrame with the results.
+This permanently removes the table and is not recoverable, unlike deleting rows.
+By default, if the table does not exist an exception is raised. To suppress this,
+you can pass in `ignore_missing=True`.
 
-=== "Javascript"
-      ```javascript
-      const query = await tbl.search([100, 100]).limit(2).execute();
-      ```
 
 ## What's next
 
 This section covered the very basics of the LanceDB API.
 LanceDB supports many additional features when creating indices to speed up search and options for search.
 These are contained in the next section of the documentation.
+
+## Note: Bundling vectorDB apps with webpack
+Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying on Vercel.
+```javascript
+/** @type {import('next').NextConfig} */
+module.exports = ({
+  webpack(config) {
+    config.externals.push({ vectordb: 'vectordb' })
+    return config;
+  }
+})
+```

docs/src/cli_config.md

@@ -0,0 +1,37 @@
+
+## LanceDB CLI
+Once lanceDB is installed, you can access the CLI using `lancedb` command on the console
+```
+lancedb
+```
+This lists out all the various command-line options available. You can get the usage or help for a particular command
+```
+lancedb {command} --help
+```
+
+## LanceDB config
+LanceDB uses a global config file to store certain settings. These settings are configurable using the lanceDB cli.
+To view your config settings, you can use:
+```
+lancedb config
+```
+These config parameters can be tuned using the cli.
+```
+lancedb {config_name} --{argument}
+```
+
+## LanceDB Opt-in Diagnostics
+When enabled, LanceDB will send anonymous events to help us improve LanceDB. These diagnostics are used only for error reporting and no data is collected. Error & stats allow us to automate certain aspects of bug reporting, prioritization of fixes and feature requests.
+These diagnostics are opt-in and can be enabled or disabled using the `lancedb diagnostics` command. These are enabled by default.
+Get usage help.
+```
+lancedb diagnostics --help
+```
+Disable diagnostics
+```
+lancedb diagnostics --disabled
+```
+Enable diagnostics
+```
+lancedb diagnostics --enabled
+```

docs/src/embeddings/api.md

@@ -0,0 +1,213 @@
+To use your own custom embedding function, you need to follow these 2 simple steps.
+1. Create your embedding function by implementing the `EmbeddingFunction` interface
+2. Register your embedding function in the global `EmbeddingFunctionRegistry`.
+
+Let us see how this looks like in action.
+
+![](../assets/embeddings_api.png)
+
+
+`EmbeddingFunction` & `EmbeddingFunctionRegistry` handle low-level details for serializing schema and model information as metadata. To build a custom embdding function, you don't need to worry about those details and simply focus on setting up the model.
+
+## `TextEmbeddingFunction` Interface
+
+There is another optional layer of abstraction provided in form of `TextEmbeddingFunction`. You can use this if your model isn't multi-modal in nature and only operates on text. In such case both source and vector fields will have the same pathway for vectorization, so you simply just need to setup the model and rest is handled by `TextEmbeddingFunction`. You can read more about the class and its attributes in the class reference.
+
+
+Let's implement `SentenceTransformerEmbeddings` class. All you need to do is implement the `generate_embeddings()` and `ndims` function to handle the input types you expect and register the class in the global `EmbeddingFunctionRegistry`
+
+```python
+from lancedb.embeddings import register
+
+@register("sentence-transformers")
+class SentenceTransformerEmbeddings(TextEmbeddingFunction):
+    name: str = "all-MiniLM-L6-v2"
+    # set more default instance vars like device, etc.
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._ndims = None
+    
+    def generate_embeddings(self, texts):
+        return self._embedding_model().encode(list(texts), ...).tolist()
+
+    def ndims(self):
+        if self._ndims is None:
+            self._ndims = len(self.generate_embeddings("foo")[0])
+        return self._ndims
+
+    @cached(cache={}) 
+    def _embedding_model(self):
+        return sentence_transformers.SentenceTransformer(name)
+
+```
+
+This is a stripped down version of our implementation of `SentenceTransformerEmbeddings` that removes certain optimizations and defaul settings.
+
+Now you can use this embedding function to create your table schema and that's it! you can then ingest data and run queries without manually vectorizing the inputs.
+
+```python
+from lancedb.pydantic import LanceModel, Vector
+
+registry = EmbeddingFunctionRegistry.get_instance()
+stransformer = registry.get("sentence-transformers").create()
+
+class TextModelSchema(LanceModel):
+    vector: Vector(stransformer.ndims) = stransformer.VectorField()
+    text: str = stransformer.SourceField()
+
+tbl = db.create_table("table", schema=TextModelSchema)
+
+tbl.add(pd.DataFrame({"text": ["halo", "world"]}))
+result = tbl.search("world").limit(5)
+```
+
+NOTE:
+
+You can always implement the `EmbeddingFunction` interface directly if you want or need to, `TextEmbeddingFunction` just makes it much simpler and faster for you to do so, by setting up the boiler plat for text-specific use case
+
+## Multi-modal embedding function example
+You can also use the `EmbeddingFunction` interface to implement more complex workflows such as multi-modal embedding function support. LanceDB implements `OpenClipEmeddingFunction` class that suppports multi-modal seach. Here's the implementation that you can use as a reference to build your own multi-modal embedding functions.
+
+```python
+@register("open-clip")
+class OpenClipEmbeddings(EmbeddingFunction):
+    name: str = "ViT-B-32"
+    pretrained: str = "laion2b_s34b_b79k"
+    device: str = "cpu"
+    batch_size: int = 64
+    normalize: bool = True
+    _model = PrivateAttr()
+    _preprocess = PrivateAttr()
+    _tokenizer = PrivateAttr()
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        open_clip = self.safe_import("open_clip", "open-clip") # EmbeddingFunction util to import external libs and raise if not found
+        model, _, preprocess = open_clip.create_model_and_transforms(
+            self.name, pretrained=self.pretrained
+        )
+        model.to(self.device)
+        self._model, self._preprocess = model, preprocess
+        self._tokenizer = open_clip.get_tokenizer(self.name)
+        self._ndims = None
+
+    def ndims(self):
+        if self._ndims is None:
+            self._ndims = self.generate_text_embeddings("foo").shape[0]
+        return self._ndims
+
+    def compute_query_embeddings(
+        self, query: Union[str, "PIL.Image.Image"], *args, **kwargs
+    ) -> List[np.ndarray]:
+        """
+        Compute the embeddings for a given user query
+
+        Parameters
+        ----------
+        query : Union[str, PIL.Image.Image]
+            The query to embed. A query can be either text or an image.
+        """
+        if isinstance(query, str):
+            return [self.generate_text_embeddings(query)]
+        else:
+            PIL = self.safe_import("PIL", "pillow")
+            if isinstance(query, PIL.Image.Image):
+                return [self.generate_image_embedding(query)]
+            else:
+                raise TypeError("OpenClip supports str or PIL Image as query")
+
+    def generate_text_embeddings(self, text: str) -> np.ndarray:
+        torch = self.safe_import("torch")
+        text = self.sanitize_input(text)
+        text = self._tokenizer(text)
+        text.to(self.device)
+        with torch.no_grad():
+            text_features = self._model.encode_text(text.to(self.device))
+            if self.normalize:
+                text_features /= text_features.norm(dim=-1, keepdim=True)
+            return text_features.cpu().numpy().squeeze()
+
+    def sanitize_input(self, images: IMAGES) -> Union[List[bytes], np.ndarray]:
+        """
+        Sanitize the input to the embedding function.
+        """
+        if isinstance(images, (str, bytes)):
+            images = [images]
+        elif isinstance(images, pa.Array):
+            images = images.to_pylist()
+        elif isinstance(images, pa.ChunkedArray):
+            images = images.combine_chunks().to_pylist()
+        return images
+
+    def compute_source_embeddings(
+        self, images: IMAGES, *args, **kwargs
+    ) -> List[np.array]:
+        """
+        Get the embeddings for the given images
+        """
+        images = self.sanitize_input(images)
+        embeddings = []
+        for i in range(0, len(images), self.batch_size):
+            j = min(i + self.batch_size, len(images))
+            batch = images[i:j]
+            embeddings.extend(self._parallel_get(batch))
+        return embeddings
+
+    def _parallel_get(self, images: Union[List[str], List[bytes]]) -> List[np.ndarray]:
+        """
+        Issue concurrent requests to retrieve the image data
+        """
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [
+                executor.submit(self.generate_image_embedding, image)
+                for image in images
+            ]
+            return [future.result() for future in futures]
+
+    def generate_image_embedding(
+        self, image: Union[str, bytes, "PIL.Image.Image"]
+    ) -> np.ndarray:
+        """
+        Generate the embedding for a single image
+
+        Parameters
+        ----------
+        image : Union[str, bytes, PIL.Image.Image]
+            The image to embed. If the image is a str, it is treated as a uri.
+            If the image is bytes, it is treated as the raw image bytes.
+        """
+        torch = self.safe_import("torch")
+        # TODO handle retry and errors for https
+        image = self._to_pil(image)
+        image = self._preprocess(image).unsqueeze(0)
+        with torch.no_grad():
+            return self._encode_and_normalize_image(image)
+
+    def _to_pil(self, image: Union[str, bytes]):
+        PIL = self.safe_import("PIL", "pillow")
+        if isinstance(image, bytes):
+            return PIL.Image.open(io.BytesIO(image))
+        if isinstance(image, PIL.Image.Image):
+            return image
+        elif isinstance(image, str):
+            parsed = urlparse.urlparse(image)
+            # TODO handle drive letter on windows.
+            if parsed.scheme == "file":
+                return PIL.Image.open(parsed.path)
+            elif parsed.scheme == "":
+                return PIL.Image.open(image if os.name == "nt" else parsed.path)
+            elif parsed.scheme.startswith("http"):
+                return PIL.Image.open(io.BytesIO(url_retrieve(image)))
+            else:
+                raise NotImplementedError("Only local and http(s) urls are supported")
+
+    def _encode_and_normalize_image(self, image_tensor: "torch.Tensor"):
+        """
+        encode a single image tensor and optionally normalize the output
+        """
+        image_features = self._model.encode_image(image_tensor)
+        if self.normalize:
+            image_features /= image_features.norm(dim=-1, keepdim=True)
+        return image_features.cpu().numpy().squeeze()
+```

docs/src/embeddings/default_embedding_functions.md

@@ -0,0 +1,208 @@
+There are various Embedding functions available out of the box with lancedb. We're working on supporting other popular embedding APIs.
+
+## Text Embedding Functions
+Here are the text embedding functions registered by default.
+Embedding functions have inbuilt rate limit handler wrapper for source and query embedding function calls that retry with exponential standoff. 
+Each `EmbeddingFunction` implementation automatically takes `max_retries` as an argument which has the deafult value of 7.
+
+### Sentence Transformers
+Here are the parameters that you can set when registering a `sentence-transformers` object, and their default values:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"all-MiniLM-L6-v2"` | The name of the model. |
+| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
+| `normalize` | `bool` | `True` | Whether to normalize the input text before feeding it to the model. |
+
+
+```python
+db = lancedb.connect("/tmp/db")
+registry = EmbeddingFunctionRegistry.get_instance()
+func = registry.get("sentence-transformers").create(device="cpu")
+
+class Words(LanceModel):
+    text: str = func.SourceField()
+    vector: Vector(func.ndims()) = func.VectorField()
+
+table = db.create_table("words", schema=Words)
+table.add(
+    [
+        {"text": "hello world"}
+        {"text": "goodbye world"}
+    ]
+)
+
+query = "greetings"
+actual = table.search(query).limit(1).to_pydantic(Words)[0]
+print(actual.text)
+```
+
+### OpenAIEmbeddings
+LanceDB has OpenAI embeddings function in the registry by default. It is registered as `openai` and here are the parameters that you can customize when creating the instances
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"text-embedding-ada-002"` | The name of the model. |
+
+
+
+```python
+db = lancedb.connect("/tmp/db")
+registry = EmbeddingFunctionRegistry.get_instance()
+func = registry.get("openai").create()
+
+class Words(LanceModel):
+    text: str = func.SourceField()
+    vector: Vector(func.ndims()) = func.VectorField()
+
+table = db.create_table("words", schema=Words)
+table.add(
+    [
+        {"text": "hello world"}
+        {"text": "goodbye world"}
+    ]
+    )
+
+query = "greetings"
+actual = table.search(query).limit(1).to_pydantic(Words)[0]
+print(actual.text)
+```
+
+### Instructor Embeddings
+Instructor is an instruction-finetuned text embedding model that can generate text embeddings tailored to any task (e.g., classification, retrieval, clustering, text evaluation, etc.) and domains (e.g., science, finance, etc.) by simply providing the task instruction, without any finetuning
+
+If you want to calculate customized embeddings for specific sentences, you may follow the unified template to write instructions:
+
+                          Represent the `domain` `text_type` for `task_objective`:
+
+* `domain` is optional, and it specifies the domain of the text, e.g., science, finance, medicine, etc.
+* `text_type` is required, and it specifies the encoding unit, e.g., sentence, document, paragraph, etc.
+* `task_objective` is optional, and it specifies the objective of embedding, e.g., retrieve a document, classify the sentence, etc.
+
+More information about the model can be found here - https://github.com/xlang-ai/instructor-embedding
+
+| Argument | Type | Default | Description |
+|---|---|---|---|
+| `name` | `str` | "hkunlp/instructor-base" | The name of the model to use |
+| `batch_size` | `int` | `32` | The batch size to use when generating embeddings |
+| `device` | `str` | `"cpu"` | The device to use when generating embeddings |
+| `show_progress_bar` | `bool` | `True` | Whether to show a progress bar when generating embeddings |
+| `normalize_embeddings` | `bool` | `True` | Whether to normalize the embeddings |
+| `quantize` | `bool` | `False` | Whether to quantize the model |
+| `source_instruction` | `str` | `"represent the docuement for retreival"` | The instruction for the source column |
+| `query_instruction` | `str` | `"represent the document for retreiving the most similar documents"` | The instruction for the query |
+
+
+
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry, InstuctorEmbeddingFunction
+
+instructor = get_registry().get("instructor").create(
+                            source_instruction="represent the docuement for retreival",
+                            query_instruction="represent the document for retreiving the most similar documents"
+                            )
+
+class Schema(LanceModel):
+    vector: Vector(instructor.ndims()) = instructor.VectorField()
+    text: str = instructor.SourceField()
+
+db = lancedb.connect("~/.lancedb")
+tbl = db.create_table("test", schema=Schema, mode="overwrite")
+
+texts = [{"text": "Capitalism has been dominant in the Western world since the end of feudalism, but most feel[who?] that..."},
+        {"text": "The disparate impact theory is especially controversial under the Fair Housing Act because the Act..."},
+        {"text": "Disparate impact in United States labor law refers to practices in employment, housing, and other areas that.."}]
+
+tbl.add(texts)
+```
+
+## Multi-modal embedding functions
+Multi-modal embedding functions allow you query your table using both images and text.
+
+### OpenClipEmbeddings
+We support CLIP model embeddings using the open souce alternbative, open-clip which support various customizations. It is registered as `open-clip` and supports following customizations.
+
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"ViT-B-32"` | The name of the model. |
+| `pretrained` | `str` | `"laion2b_s34b_b79k"` | The name of the pretrained model to load. |
+| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
+| `batch_size` | `int` | `64` | The number of images to process in a batch. |
+| `normalize` | `bool` | `True` | Whether to normalize the input images before feeding them to the model. |
+
+
+This embedding function supports ingesting images as both bytes and urls. You can query them using both test and other images.
+
+NOTE:
+LanceDB supports ingesting images directly from accessible links.
+
+
+```python
+
+db = lancedb.connect(tmp_path)
+registry = EmbeddingFunctionRegistry.get_instance()
+func = registry.get("open-clip").create()
+
+class Images(LanceModel):
+    label: str
+    image_uri: str = func.SourceField() # image uri as the source
+    image_bytes: bytes = func.SourceField() # image bytes as the source
+    vector: Vector(func.ndims()) = func.VectorField() # vector column 
+    vec_from_bytes: Vector(func.ndims()) = func.VectorField() # Another vector column 
+
+table = db.create_table("images", schema=Images)
+labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
+uris = [
+    "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
+    "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
+    "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
+    "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
+    "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
+    "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
+]
+# get each uri as bytes
+image_bytes = [requests.get(uri).content for uri in uris]
+table.add(
+    [{"label": labels, "image_uri": uris, "image_bytes": image_bytes}]
+)
+```
+Now we can search using text from both the default vector column and the custom vector column
+```python
+
+# text search
+actual = table.search("man's best friend").limit(1).to_pydantic(Images)[0]
+print(actual.label) # prints "dog"
+
+frombytes = (
+    table.search("man's best friend", vector_column_name="vec_from_bytes")
+    .limit(1)
+    .to_pydantic(Images)[0]
+)
+print(frombytes.label)
+
+```
+
+Because we're using a multi-modal embedding function, we can also search using images
+
+```python
+# image search
+query_image_uri = "http://farm1.staticflickr.com/200/467715466_ed4a31801f_z.jpg"
+image_bytes = requests.get(query_image_uri).content
+query_image = Image.open(io.BytesIO(image_bytes))
+actual = table.search(query_image).limit(1).to_pydantic(Images)[0]
+print(actual.label == "dog")
+
+# image search using a custom vector column
+other = (
+    table.search(query_image, vector_column_name="vec_from_bytes")
+    .limit(1)
+    .to_pydantic(Images)[0]
+)
+print(actual.label)
+
+```
+
+If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue.

docs/src/embeddings/embedding_functions.md

@@ -0,0 +1,95 @@
+Representing multi-modal data as vector embeddings is becoming a standard practice. Embedding functions themselves be thought of as a part of the processing pipeline that each request(input) has to be passed through. After initial setup these components are not expected to change for a particular project. 
+
+This is main motivation behind our new embedding functions API, that allow you simply set it up once and the table remembers it, effectively making the **embedding functions disappear in the background** so you don't have to worry about modelling and simply focus on the DB aspects of VectorDB.
+
+
+You can simply follow these steps and forget about the details of your embedding functions as long as you don't intend to change it.
+
+### Step 1 - Define the embedding function
+We have some pre-defined embedding functions in the global registry with more coming soon. Here's let's an implementation of CLIP as example.
+```
+registry = EmbeddingFunctionRegistry.get_instance()
+clip = registry.get("open-clip").create()
+
+```
+You can also define your own embedding function by implementing the `EmbeddingFunction` abstract base interface. It subclasses PyDantic Model which can be utilized to write complex schemas simply as we'll see next!
+
+### Step 2 - Define the Data Model or Schema
+Our embedding function from the previous section abstracts away all the details about the models and dimensions required to define the schema. You can simply set a feild as **source** or **vector** column. Here's how
+
+```python
+class Pets(LanceModel):
+    vector: Vector(clip.ndims) = clip.VectorField()
+    image_uri: str = clip.SourceField()
+
+```
+`VectorField` tells LanceDB to use the clip embedding function to generate query embeddings for `vector` column & `SourceField` tells that when adding data, automatically use the embedding function to encode `image_uri`.
+
+
+### Step 3 - Create LanceDB Table
+Now that we have chosen/defined our embedding function and the schema, we can create the table
+
+```python
+db = lancedb.connect("~/lancedb")
+table = db.create_table("pets", schema=Pets)
+
+```
+That's it! We have ingested all the information needed to embed source and query inputs. We can now forget about the model and dimension details and start to build or VectorDB
+
+### Step 4 - Ingest lots of data and run vector search!
+Now you can just add the data and it'll be vectorized automatically
+
+```python
+table.add([{"image_uri": u} for u in uris])
+```
+
+Our OpenCLIP query embedding function support querying via both text and images. 
+
+```python
+result = table.search("dog")
+```
+
+Let's query an image
+
+```python
+p = Path("path/to/images/samoyed_100.jpg")
+query_image = Image.open(p)
+table.search(query_image)
+
+```
+### Rate limit Handling
+`EmbeddingFunction` class wraps the calls for source and query embedding generation inside a rate limit handler that retries the requests with exponential backoff after successive failures. By default the maximum retires is set to 7. You can tune it by setting it to a different number or disable it by setting it to 0.
+Example
+----
+
+```python
+clip = registry.get("open-clip").create() # Defaults to 7 max retries
+clip = registry.get("open-clip").create(max_retries=10) # Increase max retries to 10
+clip = registry.get("open-clip").create(max_retries=0) # Retries disabled
+````
+
+NOTE:
+Embedding functions can also fail due to other errors that have nothing to do with rate limits. This is why the error is also logged.
+
+### A little fun with PyDantic
+LanceDB is integrated with PyDantic. Infact we've used the integration in the above example to define the schema. It is also being used behing the scene by the embdding function API to ingest useful information as table metadata.
+You can also use it for adding utility operations in the schema. For example, in our multi-modal example, you can search images using text or another image. Let us define a utility function to plot the image.
+```python
+class Pets(LanceModel):
+    vector: Vector(clip.ndims) = clip.VectorField()
+    image_uri: str = clip.SourceField()
+
+    @property
+    def image(self):
+        return Image.open(self.image_uri)
+```
+Now, you can covert your search results to pydantic model and use this property.
+
+```python
+rs = table.search(query_image).limit(3).to_pydantic(Pets)
+rs[2].image
+```
+
+![](../assets/dog_clip_output.png)
+
+Now that you've the basic idea about LanceDB embedding function, let us now dive deeper into the API that you can use to implement your own embedding functions!

docs/src/embeddings/index.md

@@ -1,13 +1,20 @@
-# Embedding Functions
+# Embedding
 
-Embeddings are high dimensional floating-point vector representations of your data or query.
-Anything can be embedded using some embedding model or function.
-For a given embedding function, the output will always have the same number of dimensions.
+Embeddings are high dimensional floating-point vector representations of your data or query. Anything can be embedded using some embedding model or function. Position of embedding in a high dimensional vector space has semantic significance to a degree that depends on the type of modal and training. These embeddings when projected in a 2-D space generally group similar entities close-by forming groups.
 
-## Creating an embedding function
+![](../assets/embedding_intro.png)
 
-Any function that takes as input a batch (list) of data and outputs a batch (list) of embeddings
-can be used by LanceDB as an embedding function. The input and output batch sizes should be the same.
+# Creating an embedding function
+
+LanceDB supports 2 major ways of vectorizing your data, explicit and implicit.
+
+1. By manually embedding the data before ingesting in the table
+2. By automatically embedding the data and query as they come, by ingesting embedding function information in the table itself! Covered in [Next Section](embedding_functions.md)
+
+Whatever workflow you prefer, we have the tools to support you.
+## Explicit Vectorization
+
+In this workflow, you can create your embedding function and vectorize your data using lancedb's `with_embedding` function. Let's look at some examples.
 
 ### HuggingFace example
 
@@ -66,7 +73,7 @@ You can also use an external API like OpenAI to generate embeddings
     to generate embeddings for each row.
 
     Say if you have a pandas DataFrame with a `text` column that you want to be embedded,
-    you can use the [with_embeddings](https://lancedb.github.io/lancedb/python/#lancedb.embeddings.with_embeddings)
+    you can use the [with_embeddings](https://lancedb.github.io/lancedb/python/python/#lancedb.embeddings.with_embeddings)
     function to generate embeddings and add create a combined pyarrow table:
 
 
@@ -118,7 +125,7 @@ belong in the same latent space and your results will be nonsensical.
      ```python
      query = "What's the best pizza topping?"
      query_vector = embed_func([query])[0]
-     tbl.search(query_vector).limit(10).to_df()
+     tbl.search(query_vector).limit(10).to_pandas()
      ```
 
      The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
@@ -134,9 +141,9 @@ belong in the same latent space and your results will be nonsensical.
      The above snippet returns an array of records with the 10 closest vectors to the query.
 
 
-## Roadmap
+## Implicit vectorization / Ingesting embedding functions
+Representing multi-modal data as vector embeddings is becoming a standard practice. Embedding functions themselves be thought of as a part of the processing pipeline that each request(input) has to be passed through. After initial setup these components are not expected to change for a particular project. 
 
-In the near future, we'll be integrating the embedding functions deeper into LanceDB<br/>.
-The goal is that you just have to configure the function once when you create the table,
-and then you'll never have to deal with embeddings / vectors after that unless you want to.
-We'll also integrate more popular models and APIs.
+This is main motivation behind our new embedding functions API, that allow you simply set it up once and the table remembers it, effectively making the **embedding functions disappear in the background** so you don't have to worry about modelling and simply focus on the DB aspects of VectorDB.
+
+Learn more in the Next Section

docs/src/examples/image_embeddings_roboflow.md

@@ -0,0 +1,165 @@
+# How to Load Image Embeddings into LanceDB
+
+With the rise of Large Multimodal Models (LMMs) such as [GPT-4 Vision](https://blog.roboflow.com/gpt-4-vision/), the need for storing image embeddings is growing. The most effective way to store text and image embeddings is in a vector database such as LanceDB. Vector databases are a special kind of data store that enables efficient search over stored embeddings. 
+
+[CLIP](https://blog.roboflow.com/openai-clip/), a multimodal model developed by OpenAI, is commonly used to calculate image embeddings. These embeddings can then be used with a vector database to build a semantic search engine that you can query using images or text. For example, you could use LanceDB and CLIP embeddings to build a search engine for a database of folders.
+
+In this guide, we are going to show you how to use Roboflow Inference to load image embeddings into LanceDB. Without further ado, let’s get started!
+
+## Step #1: Install Roboflow Inference
+
+[Roboflow Inference](https://inference.roboflow.com) enables you to run state-of-the-art computer vision models with minimal configuration. Inference supports a range of models, from fine-tuned object detection, classification, and segmentation models to foundation models like CLIP. We will use Inference to calculate CLIP image embeddings.
+
+Inference provides a HTTP API through which you can run vision models.
+
+Inference powers the Roboflow hosted API, and is available as an open source utility. In this guide, we are going to run Inference locally, which enables you to calculate CLIP embeddings on your own hardware. We will also show you how to use the hosted Roboflow CLIP API, which is ideal if you need to scale and do not want to manage a system for calculating embeddings.
+
+To get started, first install the Inference CLI:
+
+```
+pip install inference-cli
+```
+
+Next, install Docker. Refer to the official Docker installation instructions for your operating system to get Docker set up. Once Docker is ready, you can start Inference using the following command:
+
+```
+inference server start
+```
+
+An Inference server will start running at ‘http://localhost:9001’.
+
+## Step #2: Set Up a LanceDB Vector Database
+
+Now that we have Inference running, we can set up a LanceDB vector database. You can run LanceDB in JavaScript and Python. For this guide, we will use the Python API. But, you can take the HTTP requests we make below and change them to JavaScript if required.
+
+For this guide, we are going to search the [COCO 128 dataset](https://universe.roboflow.com/team-roboflow/coco-128), which contains a wide range of objects. The variability in objects present in this dataset makes it a good dataset to demonstrate the capabilities of vector search. If you want to use this dataset, you can download [COCO 128 from Roboflow Universe](https://universe.roboflow.com/team-roboflow/coco-128). With that said, you can search whatever folder of images you want.
+
+Once you have a dataset ready, install LanceDB with the following command:
+
+```
+pip install lancedb
+```
+
+We also need to install a specific commit of `tantivy`, a dependency of the LanceDB full text search engine we will use later in this guide:
+
+```
+pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+```
+
+Create a new Python file and add the following code:
+
+```python
+import cv2
+import supervision as sv
+import requests
+
+import lancedb
+
+db = lancedb.connect("./embeddings")
+
+IMAGE_DIR = "images/"
+API_KEY = os.environ.get("ROBOFLOW_API_KEY")
+SERVER_URL = "http://localhost:9001"
+
+results = []
+
+for i, image in enumerate(os.listdir(IMAGE_DIR)):
+    infer_clip_payload = {
+        #Images can be provided as urls or as base64 encoded strings
+        "image": {
+            "type": "base64",
+            "value": base64.b64encode(open(IMAGE_DIR + image, "rb").read()).decode("utf-8"),
+        },
+    }
+
+    res = requests.post(
+        f"{SERVER_URL}/clip/embed_image?api_key={API_KEY}",
+        json=infer_clip_payload,
+    )
+
+    embeddings = res.json()['embeddings']
+
+    print("Calculated embedding for image: ", image)
+
+    image = {"vector": embeddings[0], "name": os.path.join(IMAGE_DIR, image)}
+
+    results.append(image)
+
+tbl = db.create_table("images", data=results)
+
+tbl.create_fts_index("name")
+```
+
+To use the code above, you will need a Roboflow API key. [Learn how to retrieve a Roboflow API key](https://docs.roboflow.com/api-reference/authentication#retrieve-an-api-key). Run the following command to set up your API key in your environment:
+
+```
+export ROBOFLOW_API_KEY=""
+```
+
+Replace the `IMAGE_DIR` value with the folder in which you are storing the images for which you want to calculate embeddings. If you want to use the Roboflow CLIP API to calculate embeddings, replace the `SERVER_URL` value with `https://infer.roboflow.com`.
+
+Run the script above to create a new LanceDB database. This database will be stored on your local machine. The database will be called `embeddings` and the table will be called `images`.
+
+The script above calculates all embeddings for a folder then creates a new table. To add additional images, use the following code:
+
+```python
+def make_batches():
+    for i in range(5):
+        yield [
+                {"vector": [3.1, 4.1], "name": "image1.png"},
+                {"vector": [5.9, 26.5], "name": "image2.png"}
+            ]
+
+tbl = db.open_table("images")
+tbl.add(make_batches())
+```
+
+Replacing the `make_batches()` function with code to load embeddings for images.
+
+## Step #3: Run a Search Query
+
+We are now ready to run a search query. To run a search query, we need a text embedding that represents a text query. We can use this embedding to search our LanceDB database for an entry.
+
+Let’s calculate a text embedding for the query “cat”, then run a search query:
+
+```python
+infer_clip_payload = {
+    "text": "cat",
+}
+
+res = requests.post(
+    f"{SERVER_URL}/clip/embed_text?api_key={API_KEY}",
+    json=infer_clip_payload,
+)
+
+embeddings = res.json()['embeddings']
+
+df = tbl.search(embeddings[0]).limit(3).to_list()
+
+print("Results:")
+
+for i in df:
+    print(i["name"])
+```
+
+This code will search for the three images most closely related to the prompt “cat”. The names of the most similar three images will be printed to the console. Here are the three top results:
+
+```
+dataset/images/train/000000000650_jpg.rf.1b74ba165c5a3513a3211d4a80b69e1c.jpg
+dataset/images/train/000000000138_jpg.rf.af439ef1c55dd8a4e4b142d186b9c957.jpg
+dataset/images/train/000000000165_jpg.rf.eae14d5509bf0c9ceccddbb53a5f0c66.jpg
+```
+
+Let’s open the top image:
+
+![Cat](https://media.roboflow.com/cat_lancedb.jpg)
+
+The top image was a cat. Our search was successful.
+
+## Conclusion
+
+LanceDB is a vector database that you can use to store and efficiently search your image embeddings. You can use Roboflow Inference, a scalable computer vision inference server, to calculate CLIP embeddings that you can store in LanceDB.
+
+You can use Inference and LanceDB together to build a range of applications with image embeddings, from a media search engine to a retrieval-augmented generation pipeline for use with LMMs.
+
+To learn more about Inference and its capabilities, refer to the Inference documentation.

docs/src/examples/index.md

@@ -0,0 +1,23 @@
+# Examples
+
+Here are some of the examples, projects and applications using LanceDB python library. Some examples are covered in detail in the next sections. You can find more on [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes)
+
+| Example | Interactive Envs | Scripts  |
+|-------- | ---------------- | ------   |
+| | | |
+| [Youtube transcript search bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/youtube_bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/main.py)|
+| [Langchain: Code Docs QA bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Code-Documentation-QA-Bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/main.py) |
+| [AI Agents: Reducing Hallucination](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/reducing_hallucinations_ai_agents/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/main.py)|
+| [Multimodal CLIP: DiffusionDB](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_clip/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_clip/main.py)  |
+| [Multimodal CLIP: Youtube videos](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_video_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_video_search/main.py)  |
+| [Movie Recommender](https://github.com/lancedb/vectordb-recipes/tree/main/examples/movie-recommender/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/movie-recommender/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/movie-recommender/main.py)  |
+| [Audio Search](https://github.com/lancedb/vectordb-recipes/tree/main/examples/audio_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/audio_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/audio_search/main.py)  |
+| [Multimodal Image + Text Search](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_search/main.py)  |
+| [Evaluating Prompts with Prompttools](https://github.com/lancedb/vectordb-recipes/tree/main/examples/prompttools-eval-prompts/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/prompttools-eval-prompts/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> |  |
+
+## Projects & Applications powered by LanceDB
+
+| Project Name                                        | Description                                                                                                          | Screenshot                                |
+|-----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------|-------------------------------------------|
+| [YOLOExplorer](https://github.com/lancedb/yoloexplorer) | Iterate on your YOLO / CV datasets using SQL, Vector semantic search, and more within seconds                  | ![YOLOExplorer](https://github.com/lancedb/vectordb-recipes/assets/15766192/ae513a29-8f15-4e0b-99a1-ccd8272b6131) |
+| [Website Chatbot (Deployable Vercel Template)](https://github.com/lancedb/lancedb-vercel-chatbot) | Create a chatbot from the sitemap of any website/docs of your choice. Built using vectorDB serverless native javascript package. | ![Chatbot](../assets/vercel-template.gif)    |

docs/src/examples/index_js.md

@@ -0,0 +1,19 @@
+# Examples
+
+Here are some of the examples, projects and applications using vectordb native javascript library.
+Some examples are covered in detail in the next sections. You can find more on [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes)
+
+| Example | Scripts  |
+|-------- | ------   |
+| | |
+| [Youtube transcript search bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/index.js)|
+| [Langchain: Code Docs QA bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/index.js)|
+| [AI Agents: Reducing Hallucination](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/index.js)|
+| [TransformersJS Embedding example](https://github.com/lancedb/vectordb-recipes/tree/main/examples/js-transformers/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/js-transformers/index.js)  |
+
+## Projects & Applications
+
+| Project Name                                        | Description                                                                                                          | Screenshot                                |
+|-----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------|-------------------------------------------|
+| [YOLOExplorer](https://github.com/lancedb/yoloexplorer) | Iterate on your YOLO / CV datasets using SQL, Vector semantic search, and more within seconds                  | ![YOLOExplorer](https://github.com/lancedb/vectordb-recipes/assets/15766192/ae513a29-8f15-4e0b-99a1-ccd8272b6131) |
+| [Website Chatbot (Deployable Vercel Template)](https://github.com/lancedb/lancedb-vercel-chatbot) | Create a chatbot from the sitemap of any website/docs of your choice. Built using vectorDB serverless native javascript package. | ![Chatbot](../assets/vercel-template.gif)    |

docs/src/examples/serverless_lancedb_with_s3_and_lambda.md

@@ -80,14 +80,14 @@ def handler(event, context):
     # Shape of SIFT is (128,1M), d=float32
     query_vector = np.array(event['query_vector'], dtype=np.float32)
 
-    rs = table.search(query_vector).limit(2).to_df()
+    rs = table.search(query_vector).limit(2).to_list()
 
     return {
         "statusCode": status_code,
         "headers": {
             "Content-Type": "application/json"
         },
-        "body": rs.to_json()
+        "body": json.dumps(rs)
     }
 ``` 
 

docs/src/examples/serverless_website_chatbot.md

@@ -0,0 +1,61 @@
+# LanceDB Chatbot - Vercel Next.js Template
+Use an AI chatbot with website context retrieved from a vector store like LanceDB. LanceDB is lightweight and can be embedded directly into Next.js, with data stored on-prem.
+
+## One click deploy on Vercel
+[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Flancedb%2Flancedb-vercel-chatbot&env=OPENAI_API_KEY&envDescription=OpenAI%20API%20Key%20for%20chat%20completion.&project-name=lancedb-vercel-chatbot&repository-name=lancedb-vercel-chatbot&demo-title=LanceDB%20Chatbot%20Demo&demo-description=Demo%20website%20chatbot%20with%20LanceDB.&demo-url=https%3A%2F%2Flancedb.vercel.app&demo-image=https%3A%2F%2Fi.imgur.com%2FazVJtvr.png)
+
+![Demo website landing page](../assets/vercel-template.gif)
+
+## Development
+
+First, rename `.env.example` to `.env.local`, and fill out `OPENAI_API_KEY` with your OpenAI API key. You can get one [here](https://openai.com/blog/openai-api).
+
+Run the development server:
+
+```bash
+npm run dev
+# or
+yarn dev
+# or
+pnpm dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
+
+## Learn More
+
+To learn more about LanceDB or Next.js, take a look at the following resources:
+
+- [LanceDB Documentation](https://lancedb.github.io/lancedb/) - learn about LanceDB, the developer-friendly serverless vector database.
+- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
+- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
+
+## LanceDB on Next.js and Vercel
+
+FYI: these configurations have been pre-implemented in this template.
+
+Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying on Vercel.
+```js
+/** @type {import('next').NextConfig} */
+module.exports = ({
+  webpack(config) {
+    config.externals.push({ vectordb: 'vectordb' })
+    return config;
+  }
+})
+```
+
+To deploy on Vercel, we need to make sure that the NodeJS runtime static file analysis for Vercel can find the binary, since LanceDB uses dynamic imports by default. We can do this by modifying `package.json` in the `scripts` section.
+```json
+{
+  ...
+  "scripts": {
+    ...
+    "vercel-build": "sed -i 's/nativeLib = require(`@lancedb\\/vectordb-\\${currentTarget()}`);/nativeLib = require(`@lancedb\\/vectordb-linux-x64-gnu`);/' node_modules/vectordb/native.js && next build",
+    ...
+  },
+  ...
+}
+```

docs/src/examples/transformerjs_embedding_search_nodejs.md

@@ -99,7 +99,7 @@ Output of `results`:
     id: 5,
     text: 'Banana',
     type: 'fruit',
-    score: 0.4919965863227844
+    _distance: 0.4919965863227844
   },
   {
     vector: Float32Array(384) [
@@ -111,7 +111,7 @@ Output of `results`:
     id: 1,
     text: 'Cherry',
     type: 'fruit',
-    score: 0.5540297031402588
+    _distance: 0.5540297031402588
   }
 ]
 ```

docs/src/examples/youtube_transcript_bot.md

@@ -7,7 +7,7 @@
 
 <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/youtube_bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab">
 
-Scripts - [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](./examples/youtube_bot/main.py)  [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](./examples/youtube_bot/index.js)
+Scripts - [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipesexamples/youtube_bot/main.py)  [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/examples/youtube_bot/index.js)
 
 
 This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb)

docs/src/fts.md

@@ -6,17 +6,19 @@ to make this available for JS as well.
 
 ## Installation
 
-To use full text search, you must install optional dependency tantivy-py:
+To use full text search, you must install the dependency `tantivy-py`:
 
-# tantivy 0.19.2
-pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+# tantivy 0.20.1
+```sh
+pip install tantivy==0.20.1
+```
 
 
 ## Quickstart
 
 Assume:
 1. `table` is a LanceDB Table
-2. `text` is the name of the Table column that we want to index
+2. `text` is the name of the `Table` column that we want to index
 
 For example,
 
@@ -41,7 +43,13 @@ table.create_fts_index("text")
 To search:
 
 ```python
-df = table.search("puppy").limit(10).select(["text"]).to_df()
+table.search("puppy").limit(10).select(["text"]).to_list()
+```
+
+Which returns a list of dictionaries:
+
+```python
+[{'text': 'Frodo was a happy puppy', 'score': 0.6931471824645996}]
 ```
 
 LanceDB automatically looks for an FTS index if the input is str.

docs/src/guides/tables.md

@@ -0,0 +1,408 @@
+ <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/tables_guide.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+A Table is a collection of Records in a LanceDB Database. You can follow along on colab!
+
+## Creating a LanceDB Table
+
+=== "Python"
+    ### LanceDB Connection
+
+    ```python
+    import lancedb
+    db = lancedb.connect("./.lancedb")
+    ```
+
+    LanceDB allows ingesting data from various sources - `dict`, `list[dict]`, `pd.DataFrame`, `pa.Table` or a `Iterator[pa.RecordBatch]`. Let's take a look at some of the these.
+
+    ### From list of tuples or dictionaries
+
+    ```python
+    import lancedb
+
+    db = lancedb.connect("./.lancedb")
+
+    data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
+            {"vector": [0.2, 1.8], "lat": 40.1, "long": -74.1}]
+
+    db.create_table("my_table", data)
+
+    db["my_table"].head()
+    ```
+
+    !!! info "Note"
+        If the table already exists, LanceDB will raise an error by default. If you want to overwrite the table, you can pass in mode="overwrite" to the createTable function.
+
+        ```python
+        db.create_table("name", data, mode="overwrite")
+        ```
+
+
+    ### From pandas DataFrame
+
+    ```python
+    import pandas as pd
+
+    data = pd.DataFrame({
+        "vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],
+        "lat": [45.5, 40.1],
+        "long": [-122.7, -74.1]
+    })
+
+    db.create_table("table2", data)
+
+    db["table2"].head()
+    ```
+    !!! info "Note"
+        Data is converted to Arrow before being written to disk. For maximum control over how data is saved, either provide the PyArrow schema to convert to or else provide a PyArrow Table directly.
+
+    ```python
+    custom_schema = pa.schema([
+    pa.field("vector", pa.list_(pa.float32(), 4)),
+    pa.field("lat", pa.float32()),
+    pa.field("long", pa.float32())
+    ])
+
+    table = db.create_table("table3", data, schema=custom_schema)
+    ```
+
+    ### From PyArrow Tables
+    You can also create LanceDB tables directly from pyarrow tables
+
+    ```python
+    table = pa.Table.from_arrays(
+            [
+                pa.array([[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],
+                        pa.list_(pa.float32(), 4)),
+                pa.array(["foo", "bar"]),
+                pa.array([10.0, 20.0]),
+            ],
+            ["vector", "item", "price"],
+        )
+
+    db = lancedb.connect("db")
+
+    tbl = db.create_table("test1", table)
+    ```
+
+    ### From Pydantic Models
+    When you create an empty table without data, you must specify the table schema.
+    LanceDB supports creating tables by specifying a pyarrow schema or a specialized
+    pydantic model called `LanceModel`.
+
+    For example, the following Content model specifies a table with 5 columns:
+    movie_id, vector, genres, title, and imdb_id. When you create a table, you can
+    pass the class as the value of the `schema` parameter to `create_table`.
+    The `vector` column is a `Vector` type, which is a specialized pydantic type that
+    can be configured with the vector dimensions. It is also important to note that
+    LanceDB only understands subclasses of `lancedb.pydantic.LanceModel`
+    (which itself derives from `pydantic.BaseModel`).
+
+    ```python
+    from lancedb.pydantic import Vector, LanceModel
+
+    class Content(LanceModel):
+        movie_id: int
+        vector: Vector(128)
+        genres: str
+        title: str
+        imdb_id: int
+
+        @property
+        def imdb_url(self) -> str:
+            return f"https://www.imdb.com/title/tt{self.imdb_id}"
+
+    import pyarrow as pa
+    db = lancedb.connect("~/.lancedb")
+    table_name = "movielens_small"
+    table = db.create_table(table_name, schema=Content)
+    ```
+
+    ### Using Iterators / Writing Large Datasets
+
+    It is recommended to use itertators to add large datasets in batches when creating your table in one go. This does not create multiple versions of your dataset unlike manually adding batches using `table.add()`
+
+    LanceDB additionally supports pyarrow's `RecordBatch` Iterators or other generators producing supported data types.
+
+    Here's an example using using `RecordBatch` iterator for creating tables.
+
+    ```python
+    import pyarrow as pa
+
+    def make_batches():
+        for i in range(5):
+            yield pa.RecordBatch.from_arrays(
+                [
+                    pa.array([[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],
+                            pa.list_(pa.float32(), 4)),
+                    pa.array(["foo", "bar"]),
+                    pa.array([10.0, 20.0]),
+                ],
+                ["vector", "item", "price"],
+            )
+
+    schema = pa.schema([
+        pa.field("vector", pa.list_(pa.float32(), 4)),
+        pa.field("item", pa.utf8()),
+        pa.field("price", pa.float32()),
+    ])
+
+    db.create_table("table4", make_batches(), schema=schema)
+    ```
+
+    You can also use iterators of other types like Pandas dataframe or Pylists directly in the above example.
+
+    ## Creating Empty Table
+    You can also create empty tables in python. Initialize it with schema and later ingest data into it.
+
+    ```python
+    import lancedb
+    import pyarrow as pa
+
+    schema = pa.schema(
+      [
+          pa.field("vector", pa.list_(pa.float32(), 2)),
+          pa.field("item", pa.string()),
+          pa.field("price", pa.float32()),
+      ])
+    tbl = db.create_table("table5", schema=schema)
+    data = [
+        {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0},
+    ]
+    tbl.add(data=data)
+    ```
+
+    You can also use Pydantic to specify the schema
+
+    ```python
+    import lancedb
+    from lancedb.pydantic import LanceModel, vector
+
+    class Model(LanceModel):
+          vector: Vector(2)
+
+    tbl = db.create_table("table5", schema=Model.to_arrow_schema())
+    ```
+
+=== "Javascript/Typescript"
+
+    ### VectorDB Connection
+
+    ```javascript
+    const lancedb = require("vectordb");
+
+    const uri = "data/sample-lancedb";
+    const db = await lancedb.connect(uri);
+    ```
+
+    ### Creating a Table
+
+    You can create a LanceDB table in javascript using an array of records.
+
+    ```javascript
+    data
+    const tb = await db.createTable("my_table",
+                  data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+    ```
+
+    !!! info "Note"
+    If the table already exists, LanceDB will raise an error by default. If you want to overwrite the table, you need to specify the `WriteMode` in the createTable function.
+
+    ```javascript
+    const table = await con.createTable(tableName, data, { writeMode: WriteMode.Overwrite })
+    ```
+
+## Open existing tables
+
+If you forget the name of your table, you can always get a listing of all table names:
+
+
+=== "Python"
+    ### Get a list of existing Tables
+
+    ```python
+    print(db.table_names())
+    ```
+=== "Javascript/Typescript"
+
+    ```javascript
+    console.log(await db.tableNames());
+    ```
+
+Then, you can open any existing tables
+
+=== "Python"
+
+    ```python
+    tbl = db.open_table("my_table")
+    ```
+=== "Javascript/Typescript"
+
+    ```javascript
+    const tbl = await db.openTable("my_table");
+    ```
+
+## Adding to a Table
+After a table has been created, you can always add more data to it using
+
+=== "Python"
+    You can add any of the valid data structures accepted by LanceDB table, i.e, `dict`, `list[dict]`, `pd.DataFrame`, or a `Iterator[pa.RecordBatch]`. Here are some examples.
+
+    ### Adding Pandas DataFrame
+
+    ```python
+    df = pd.DataFrame({
+        "vector": [[1.3, 1.4], [9.5, 56.2]], "item": ["fizz", "buzz"], "price": [100.0, 200.0]
+    })
+    tbl.add(df)
+    ```
+
+    You can also add a large dataset batch in one go using Iterator of any supported data types.
+
+    ### Adding to table using Iterator
+
+    ```python
+    def make_batches():
+        for i in range(5):
+            yield [
+                    {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                    {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}
+                ]
+    tbl.add(make_batches())
+    ```
+
+    The other arguments accepted:
+
+    | Name | Type | Description | Default |
+    |---|---|---|---|
+    | data | DATA | The data to insert into the table. | required |
+    | mode | str | The mode to use when writing the data. Valid values are "append" and "overwrite". | append |
+    | on_bad_vectors | str | What to do if any of the vectors are not the same size or contains NaNs. One of "error", "drop", "fill". | drop |
+    | fill value | float | The value to use when filling vectors: Only used if on_bad_vectors="fill". | 0.0 |
+
+
+=== "Javascript/Typescript"
+
+    ```javascript
+    await tbl.add([{vector: [1.3, 1.4], item: "fizz", price: 100.0},
+        {vector: [9.5, 56.2], item: "buzz", price: 200.0}])
+    ```
+
+## Deleting from a Table
+
+Use the `delete()` method on tables to delete rows from a table. To choose which rows to delete, provide a filter that matches on the metadata columns. This can delete any number of rows that match the filter.
+
+=== "Python"
+
+    ```python
+    tbl.delete('item = "fizz"')
+    ```
+
+    ### Deleting row with specific column value
+
+    ```python
+    import lancedb
+
+    data = [{"x": 1, "vector": [1, 2]},
+            {"x": 2, "vector": [3, 4]},
+            {"x": 3, "vector": [5, 6]}]
+    db = lancedb.connect("./.lancedb")
+    table = db.create_table("my_table", data)
+    table.to_pandas()
+    #   x      vector
+    # 0  1  [1.0, 2.0]
+    # 1  2  [3.0, 4.0]
+    # 2  3  [5.0, 6.0]
+
+    table.delete("x = 2")
+    table.to_pandas()
+    #   x      vector
+    # 0  1  [1.0, 2.0]
+    # 1  3  [5.0, 6.0]
+    ```
+
+    ### Delete from a list of values
+
+    ```python
+    to_remove = [1, 5]
+    to_remove = ", ".join(str(v) for v in to_remove)
+
+    table.delete(f"x IN ({to_remove})")
+    table.to_pandas()
+    #   x      vector
+    # 0  3  [5.0, 6.0]
+    ```
+
+=== "Javascript/Typescript"
+
+    ```javascript
+    await tbl.delete('item = "fizz"')
+    ```
+
+    ### Deleting row with specific column value
+
+    ```javascript
+    const con = await lancedb.connect("./.lancedb")
+    const data = [
+      {id: 1, vector: [1, 2]},
+      {id: 2, vector: [3, 4]},
+      {id: 3, vector: [5, 6]},
+    ];
+    const tbl = await con.createTable("my_table", data)
+    await tbl.delete("id = 2")
+    await tbl.countRows() // Returns 2
+    ```
+
+    ### Delete from a list of values
+
+    ```javascript
+    const to_remove = [1, 5];
+    await tbl.delete(`id IN (${to_remove.join(",")})`)
+    await tbl.countRows() // Returns 1
+    ```
+
+### Updating a Table [Experimental]
+EXPERIMENTAL: Update rows in the table (not threadsafe).
+
+This can be used to update zero to all rows depending on how many rows match the where clause.
+
+| Parameter | Type | Description |
+|---|---|---|
+| `where` | `str` | The SQL where clause to use when updating rows. For example, `'x = 2'` or `'x IN (1, 2, 3)'`. The filter must not be empty, or it will error. |
+| `values` | `dict` | The values to update. The keys are the column names and the values are the values to set. |
+
+
+=== "Python"
+
+    ```python
+    import lancedb
+    import pandas as pd
+
+    # Create a lancedb connection
+    db = lancedb.connect("./.lancedb")
+
+    # Create a table from a pandas DataFrame
+    data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
+    table = db.create_table("my_table", data)
+
+    # Update the table where x = 2
+    table.update(where="x = 2", values={"vector": [10, 10]})
+
+    # Get the updated table as a pandas DataFrame
+    df = table.to_pandas()
+
+    # Print the DataFrame
+    print(df)
+    ```
+
+    Output
+    ```shell
+        x  vector
+    0  1  [1.0, 2.0]
+    1  3  [5.0, 6.0]
+    2  2  [10.0, 10.0]
+    ```
+
+## What's Next?
+
+Learn how to Query your tables and create indices

docs/src/index.md

@@ -1,20 +1,23 @@
-# Welcome to LanceDB's Documentation
+# LanceDB
 
-LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrevial, filtering and management of embeddings.
+LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrieval, filtering and management of embeddings.
+
+![Illustration](/lancedb/assets/ecosystem-illustration.png)
 
 The key features of LanceDB include:
 
-* Production-scale vector search with no servers to manage.
-
 * Store, query and filter vectors, metadata and multi-modal data (text, images, videos, point clouds, and more).
 
-* Support for vector similarity search, full-text search and SQL.
+* Support for production-scale vector similarity search, full-text search and SQL, with no servers to manage.
 
 * Native Python and Javascript/Typescript support.
 
 * Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure.
 
-* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
+* Persisted on HDD, allowing scalability without breaking the bank.
+
+* Ingest your favorite data formats directly, like pandas DataFrames, Pydantic objects and more.
+
 
 LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
 
@@ -33,7 +36,7 @@ LanceDB's core is written in Rust 🦀 and is built using <a href="https://githu
       table = db.create_table("my_table",
                               data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
                                     {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      result = table.search([100, 100]).limit(2).to_df()
+      result = table.search([100, 100]).limit(2).to_list()
       ```
 
 === "Javascript"
@@ -64,9 +67,9 @@ LanceDB's core is written in Rust 🦀 and is built using <a href="https://githu
 
 ## Documentation Quick Links
 * [`Basic Operations`](basic.md) - basic functionality of LanceDB.
-* [`Embedding Functions`](embedding.md) - functions for working with embeddings.
+* [`Embedding Functions`](embeddings/index.md) - functions for working with embeddings.
 * [`Indexing`](ann_indexes.md) - create vector indexes to speed up queries.
 * [`Full text search`](fts.md) - [EXPERIMENTAL] full-text search API
 * [`Ecosystem Integrations`](python/integration.md) - integrating LanceDB with python data tooling ecosystem.
 * [`Python API Reference`](python/python.md) - detailed documentation for the LanceDB Python SDK.
-* [`Node API Reference`](javascript/modules.md) - detailed documentation for the LanceDB Python SDK.
+* [`Node API Reference`](javascript/modules.md) - detailed documentation for the LanceDB Node SDK.

docs/src/integrations/index.md

@@ -0,0 +1,21 @@
+# Integrations
+
+## Data Formats
+
+LanceDB supports ingesting from your favorite data tools.
+
+![Illustration](/lancedb/assets/ecosystem-illustration.png)
+
+
+## Tools
+
+LanceDB is integrated with most of the popular AI tools, with more coming soon.
+Get started using these examples and quick links.
+
+| Integrations | |
+|---|---:|
+| <h3> LlamaIndex </h3>LlamaIndex is a simple, flexible data framework for connecting custom data sources to large language models. Llama index integrates with LanceDB as the serverless VectorDB. <h3>[Lean More](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html) </h3> |<img src="../assets/llama-index.jpg" alt="image" width="150" height="auto">|
+| <h3>Langchain</h3>Langchain allows building applications with LLMs through composability <h3>[Lean More](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html) | <img src="../assets/langchain.png" alt="image" width="150" height="auto">|
+| <h3>Langchain TS</h3> Javascript bindings for Langchain. It integrates with LanceDB's serverless vectordb allowing you to build powerful AI applications through composibility using only serverless functions. <h3>[Learn More]( https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb) | <img src="../assets/langchain.png" alt="image" width="150" height="auto">|
+| <h3>Voxel51</h3>  It is an open source toolkit that enables you to build better computer vision workflows by improving the quality of your datasets and delivering insights about your models.<h3>[Learn More](./voxel51.md) | <img src="../assets/voxel.gif" alt="image" width="150" height="auto">|
+| <h3>PromptTools</h3>  Offers a set of free, open-source tools for testing and experimenting with models, prompts, and configurations. The core idea is to enable developers to evaluate prompts using familiar interfaces like code and notebooks. You can use it to experiment with different configurations of LanceDB, and test how LanceDB integrates with the LLM of your choice.<h3>[Learn More](./prompttools.md) | <img src="../assets/prompttools.jpeg" alt="image" width="150" height="auto">|

docs/src/integrations/prompttools.md

@@ -0,0 +1,7 @@
+
+[PromptTools](https://github.com/hegelai/prompttools) offers a set of free, open-source tools for testing and experimenting with models, prompts, and configurations. The core idea is to enable developers to evaluate prompts using familiar interfaces like code and notebooks. You can use it to experiment with different configurations of LanceDB, and test how LanceDB integrates with the LLM of your choice.
+
+[Evaluating Prompts with PromptTools](./examples/prompttools-eval-prompts/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/prompttools-eval-prompts/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>
+
+![Alt text](https://prompttools.readthedocs.io/en/latest/_images/demo.gif "a title")
+

docs/src/notebooks/code_qa_bot.ipynb

@@ -144,7 +144,7 @@
    "source": [
     "# Pre-processing and loading the documentation\n",
     "\n",
-    "Next, let's pre-process and load the documentation. To make sure we don't need to do this repeatedly if we were updating code, we're caching it using pickle so we can retrieve it again (this could take a few minutes to run the first time yyou do it). We'll also add some more metadata to the docs here such as the title and version of the code:"
+    "Next, let's pre-process and load the documentation. To make sure we don't need to do this repeatedly if we were updating code, we're caching it using pickle so we can retrieve it again (this could take a few minutes to run the first time you do it). We'll also add some more metadata to the docs here such as the title and version of the code:"
    ]
   },
   {
@@ -255,7 +255,7 @@
    "id": "28d93b85",
    "metadata": {},
    "source": [
-    "And thats it! We're all setup. The next step is to run some queries, let's try a few:"
+    "And that's it! We're all set up. The next step is to run some queries, let's try a few:"
    ]
   },
   {

docs/src/notebooks/multimodal_search.ipynb

@@ -19,11 +19,11 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
+      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip available: \u001B[0m\u001B[31;49m22.3.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m23.1.2\u001B[0m\n",
+      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpip install --upgrade pip\u001B[0m\n",
       "\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
+      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip available: \u001B[0m\u001B[31;49m22.3.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m23.1.2\u001B[0m\n",
+      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpip install --upgrade pip\u001B[0m\n"
      ]
     }
    ],
@@ -39,6 +39,7 @@
    "outputs": [],
    "source": [
     "import io\n",
+    "\n",
     "import PIL\n",
     "import duckdb\n",
     "import lancedb"
@@ -158,18 +159,18 @@
     "        \"db = lancedb.connect('~/datasets/demo')\\n\"\n",
     "        \"tbl = db.open_table('diffusiondb')\\n\\n\"\n",
     "        f\"embedding = embed_func('{query}')\\n\"\n",
-    "        \"tbl.search(embedding).limit(9).to_df()\"\n",
+    "        \"tbl.search(embedding).limit(9).to_pandas()\"\n",
     "    )\n",
-    "    return (_extract(tbl.search(emb).limit(9).to_df()), code)\n",
+    "    return (_extract(tbl.search(emb).limit(9).to_pandas()), code)\n",
     "\n",
     "def find_image_keywords(query):\n",
     "    code = (\n",
     "        \"import lancedb\\n\"\n",
     "        \"db = lancedb.connect('~/datasets/demo')\\n\"\n",
     "        \"tbl = db.open_table('diffusiondb')\\n\\n\"\n",
-    "        f\"tbl.search('{query}').limit(9).to_df()\"\n",
+    "        f\"tbl.search('{query}').limit(9).to_pandas()\"\n",
     "    )\n",
-    "    return (_extract(tbl.search(query).limit(9).to_df()), code)\n",
+    "    return (_extract(tbl.search(query).limit(9).to_pandas()), code)\n",
     "\n",
     "def find_image_sql(query):\n",
     "    code = (\n",

docs/src/notebooks/tables_guide.ipynb

@@ -0,0 +1,825 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "d24eb4c6-e246-44ca-ba7c-6eae7923bd4c",
+   "metadata": {},
+   "source": [
+    "## LanceDB Tables\n",
+    "A Table is a collection of Records in a LanceDB Database.\n",
+    "\n",
+    "![illustration](../assets/ecosystem-illustration.png)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 50,
+   "id": "c1b4e34b-a49c-471d-a343-a5940bb5138a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install lancedb -qq"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "4e5a8d07-d9a1-48c1-913a-8e0629289579",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import lancedb\n",
+    "db = lancedb.connect(\"./.lancedb\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "66fb93d5-3551-406b-99b2-488442d61d06",
+   "metadata": {},
+   "source": [
+    "LanceDB allows ingesting data from various sources - `dict`, `list[dict]`, `pd.DataFrame`, `pa.Table` or a `Iterator[pa.RecordBatch]`. Let's take a look at some of the these.\n",
+    "\n",
+    " ### From list of tuples or dictionaries"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "5df12f66-8d99-43ad-8d0b-22189ec0a6b9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "pyarrow.Table\n",
+       "vector: fixed_size_list<item: float>[2]\n",
+       "  child 0, item: float\n",
+       "lat: double\n",
+       "long: double\n",
+       "----\n",
+       "vector: [[[1.1,1.2],[0.2,1.8]]]\n",
+       "lat: [[45.5,40.1]]\n",
+       "long: [[-122.7,-74.1]]"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import lancedb\n",
+    "\n",
+    "db = lancedb.connect(\"./.lancedb\")\n",
+    "\n",
+    "data = [{\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7},\n",
+    "        {\"vector\": [0.2, 1.8], \"lat\": 40.1, \"long\": -74.1}]\n",
+    "\n",
+    "db.create_table(\"my_table\", data)\n",
+    "\n",
+    "db[\"my_table\"].head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "10ce802f-1a10-49ee-8ee3-a9bfb302d86c",
+   "metadata": {},
+   "source": [
+    "## From pandas DataFrame\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "f4d87ae9-0ccb-48eb-b31d-bb8f2370e47e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "pyarrow.Table\n",
+       "vector: fixed_size_list<item: float>[2]\n",
+       "  child 0, item: float\n",
+       "lat: double\n",
+       "long: double\n",
+       "----\n",
+       "vector: [[[1.1,1.2],[0.2,1.8]]]\n",
+       "lat: [[45.5,40.1]]\n",
+       "long: [[-122.7,-74.1]]"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data = [\n",
+    "    {\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7},\n",
+    "    {\"vector\": [0.2, 1.8], \"lat\": 40.1, \"long\": -74.1},\n",
+    "]\n",
+    "\n",
+    "db.create_table(\"table2\", data)\n",
+    "\n",
+    "db[\"table2\"].head() "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4be81469-5b57-4f78-9c72-3938c0378d9d",
+   "metadata": {},
+   "source": [
+    "Data is converted to Arrow before being written to disk. For maximum control over how data is saved, either provide the PyArrow schema to convert to or else provide a PyArrow Table directly.\n",
+    "  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "25f34bcf-fca0-4431-8601-eac95d1bd347",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "vector: fixed_size_list<item: float>[2]\n",
+       "  child 0, item: float\n",
+       "lat: float\n",
+       "long: float"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pyarrow as pa\n",
+    "\n",
+    "custom_schema = pa.schema([\n",
+    "pa.field(\"vector\", pa.list_(pa.float32(), 2)),\n",
+    "pa.field(\"lat\", pa.float32()),\n",
+    "pa.field(\"long\", pa.float32())\n",
+    "])\n",
+    "\n",
+    "table = db.create_table(\"table3\", data, schema=custom_schema, mode=\"overwrite\")\n",
+    "table.schema"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4df51925-7ca2-4005-9c72-38b3d26240c6",
+   "metadata": {},
+   "source": [
+    "### From PyArrow Tables\n",
+    "\n",
+    "You can also create LanceDB tables directly from pyarrow tables"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "90a880f6-be43-4c9d-ba65-0b05197c0f6f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "vector: fixed_size_list<item: float>[2]\n",
+       "  child 0, item: float\n",
+       "item: string\n",
+       "price: double"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "table = pa.Table.from_arrays(\n",
+    "        [\n",
+    "            pa.array([[3.1, 4.1], [5.9, 26.5]],\n",
+    "                    pa.list_(pa.float32(), 2)),\n",
+    "            pa.array([\"foo\", \"bar\"]),\n",
+    "            pa.array([10.0, 20.0]),\n",
+    "        ],\n",
+    "        [\"vector\", \"item\", \"price\"],\n",
+    "    )\n",
+    "\n",
+    "db = lancedb.connect(\"db\")\n",
+    "\n",
+    "tbl = db.create_table(\"test1\", table, mode=\"overwrite\")\n",
+    "tbl.schema"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0f36c51c-d902-449d-8292-700e53990c32",
+   "metadata": {},
+   "source": [
+    "### From Pydantic Models\n",
+    "\n",
+    "LanceDB supports to create Apache Arrow Schema from a Pydantic BaseModel."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "d81121d7-e4b7-447c-a48c-974b6ebb464a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "movie_id: int64 not null\n",
+       "vector: fixed_size_list<item: float>[128] not null\n",
+       "  child 0, item: float\n",
+       "genres: string not null\n",
+       "title: string not null\n",
+       "imdb_id: int64 not null"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from lancedb.pydantic import Vector, LanceModel\n",
+    "\n",
+    "class Content(LanceModel):\n",
+    "    movie_id: int\n",
+    "    vector: Vector(128)\n",
+    "    genres: str\n",
+    "    title: str\n",
+    "    imdb_id: int\n",
+    "        \n",
+    "    @property\n",
+    "    def imdb_url(self) -> str:\n",
+    "        return f\"https://www.imdb.com/title/tt{self.imdb_id}\"\n",
+    "\n",
+    "import pyarrow as pa\n",
+    "db = lancedb.connect(\"~/.lancedb\")\n",
+    "table_name = \"movielens_small\"\n",
+    "table = db.create_table(table_name, schema=Content)\n",
+    "table.schema"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "860e1f77-e860-46a9-98b7-b2979092ccd6",
+   "metadata": {},
+   "source": [
+    "### Using Iterators / Writing Large Datasets\n",
+    "\n",
+    "It is recommended to use itertators to add large datasets in batches when creating your table in one go. This does not create multiple versions of your dataset unlike manually adding batches using `table.add()`\n",
+    "\n",
+    "LanceDB additionally supports pyarrow's `RecordBatch` Iterators or other generators producing supported data types.\n",
+    "\n",
+    "## Here's an example using using `RecordBatch` iterator for creating tables."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "bc247142-4e3c-41a2-b94c-8e00d2c2a508",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "LanceTable(table4)"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pyarrow as pa\n",
+    "\n",
+    "def make_batches():\n",
+    "    for i in range(5):\n",
+    "        yield pa.RecordBatch.from_arrays(\n",
+    "            [\n",
+    "                pa.array([[3.1, 4.1], [5.9, 26.5]],\n",
+    "                        pa.list_(pa.float32(), 2)),\n",
+    "                pa.array([\"foo\", \"bar\"]),\n",
+    "                pa.array([10.0, 20.0]),\n",
+    "            ],\n",
+    "            [\"vector\", \"item\", \"price\"],\n",
+    "        )\n",
+    "\n",
+    "schema = pa.schema([\n",
+    "    pa.field(\"vector\", pa.list_(pa.float32(), 2)),\n",
+    "    pa.field(\"item\", pa.utf8()),\n",
+    "    pa.field(\"price\", pa.float32()),\n",
+    "])\n",
+    "\n",
+    "db.create_table(\"table4\", make_batches(), schema=schema)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "94f7dd2b-bae4-4bdf-8534-201437c31027",
+   "metadata": {},
+   "source": [
+    "### Using pandas `DataFrame` Iterator and Pydantic Schema\n",
+    "\n",
+    "You can set the schema via pyarrow schema object or using Pydantic object"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "25ad3523-e0c9-4c28-b3df-38189c4e0e5f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "vector: fixed_size_list<item: float>[2] not null\n",
+       "  child 0, item: float\n",
+       "item: string not null\n",
+       "price: double not null"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pyarrow as pa\n",
+    "import pandas as pd\n",
+    "\n",
+    "class PydanticSchema(LanceModel):\n",
+    "    vector: Vector(2)\n",
+    "    item: str\n",
+    "    price: float\n",
+    "\n",
+    "def make_batches():\n",
+    "    for i in range(5):\n",
+    "        yield pd.DataFrame(\n",
+    "                {\n",
+    "                    \"vector\": [[3.1, 4.1], [1, 1]],\n",
+    "                    \"item\": [\"foo\", \"bar\"],\n",
+    "                    \"price\": [10.0, 20.0],\n",
+    "                })\n",
+    "\n",
+    "tbl = db.create_table(\"table5\", make_batches(), schema=PydanticSchema)\n",
+    "tbl.schema"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4aa955e9-fcd0-4c99-b644-f218f3bb3f1a",
+   "metadata": {},
+   "source": [
+    "## Creating Empty Table\n",
+    "\n",
+    "You can create an empty table by just passing the schema and later add to it using `table.add()`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "2814173a-eacc-4dd8-a64d-6312b44582cc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import lancedb\n",
+    "from lancedb.pydantic import LanceModel, Vector\n",
+    "\n",
+    "class Model(LanceModel):\n",
+    "      vector: Vector(2)\n",
+    "\n",
+    "tbl = db.create_table(\"table6\", schema=Model.to_arrow_schema())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1d1b0f5c-a1d9-459f-8614-8376b6f577e1",
+   "metadata": {},
+   "source": [
+    "## Open Existing Tables\n",
+    "\n",
+    "If you forget the name of your table, you can always get a listing of all table names:\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "df9e13c0-41f6-437f-9dfa-2fd71d3d9c45",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['table6', 'table4', 'table5', 'movielens_small']"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "db.table_names()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "9343f5ad-6024-42ee-ac2f-6c1471df8679",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>vector</th>\n",
+       "      <th>item</th>\n",
+       "      <th>price</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>[3.1, 4.1]</td>\n",
+       "      <td>foo</td>\n",
+       "      <td>10.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>[5.9, 26.5]</td>\n",
+       "      <td>bar</td>\n",
+       "      <td>20.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>[3.1, 4.1]</td>\n",
+       "      <td>foo</td>\n",
+       "      <td>10.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>[5.9, 26.5]</td>\n",
+       "      <td>bar</td>\n",
+       "      <td>20.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>[3.1, 4.1]</td>\n",
+       "      <td>foo</td>\n",
+       "      <td>10.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>[5.9, 26.5]</td>\n",
+       "      <td>bar</td>\n",
+       "      <td>20.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>[3.1, 4.1]</td>\n",
+       "      <td>foo</td>\n",
+       "      <td>10.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>[5.9, 26.5]</td>\n",
+       "      <td>bar</td>\n",
+       "      <td>20.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>[3.1, 4.1]</td>\n",
+       "      <td>foo</td>\n",
+       "      <td>10.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>[5.9, 26.5]</td>\n",
+       "      <td>bar</td>\n",
+       "      <td>20.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        vector item  price\n",
+       "0   [3.1, 4.1]  foo   10.0\n",
+       "1  [5.9, 26.5]  bar   20.0\n",
+       "2   [3.1, 4.1]  foo   10.0\n",
+       "3  [5.9, 26.5]  bar   20.0\n",
+       "4   [3.1, 4.1]  foo   10.0\n",
+       "5  [5.9, 26.5]  bar   20.0\n",
+       "6   [3.1, 4.1]  foo   10.0\n",
+       "7  [5.9, 26.5]  bar   20.0\n",
+       "8   [3.1, 4.1]  foo   10.0\n",
+       "9  [5.9, 26.5]  bar   20.0"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tbl = db.open_table(\"table4\")\n",
+    "tbl.to_pandas()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5019246f-12e3-4f78-88a8-9f4939802c76",
+   "metadata": {},
+   "source": [
+    "## Adding to table\n",
+    "After a table has been created, you can always add more data to it using\n",
+    "\n",
+    "You can add any of the valid data structures accepted by LanceDB table, i.e, `dict`, `list[dict]`, `pd.DataFrame`, or a `Iterator[pa.RecordBatch]`. Here are some examples."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "8a56250f-73a1-4c26-a6ad-5c7a0ce3a9ab",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = [\n",
+    "        {\"vector\": [1.3, 1.4], \"item\": \"fizz\", \"price\": 100.0},\n",
+    "        {\"vector\": [9.5, 56.2], \"item\": \"buzz\", \"price\": 200.0}\n",
+    "]\n",
+    "tbl.add(data)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9985f6ee-67e1-45a9-b233-94e3d121ecbf",
+   "metadata": {},
+   "source": [
+    "You can also add a large dataset batch in one go using Iterator of supported data types\n",
+    "\n",
+    "### Adding via Iterator\n",
+    "\n",
+    "here, we'll use pandas DataFrame Iterator"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "030c7057-b98e-4e2f-be14-b8c1f927f83c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def make_batches():\n",
+    "    for i in range(5):\n",
+    "        yield [\n",
+    "                  {\"vector\": [3.1, 4.1], \"item\": \"foo\", \"price\": 10.0},\n",
+    "                  {\"vector\": [1, 1], \"item\": \"bar\", \"price\": 20.0},\n",
+    "              ]\n",
+    "tbl.add(make_batches())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b8316d5d-0a23-4675-b0ee-178711db873a",
+   "metadata": {},
+   "source": [
+    "## Deleting from a Table\n",
+    "\n",
+    "Use the `delete()` method on tables to delete rows from a table. To choose which rows to delete, provide a filter that matches on the metadata columns. This can delete any number of rows that match the filter, like:\n",
+    "\n",
+    "\n",
+    "```python\n",
+    "tbl.delete('item = \"fizz\"')\n",
+    "```\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "e7a17de2-08d2-41b7-bd05-f63d1045ab1f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "32\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "17"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "print(len(tbl))\n",
+    "      \n",
+    "tbl.delete(\"price = 20.0\")\n",
+    "      \n",
+    "len(tbl)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "74ac180b-5432-4c14-b1a8-22c35ac83af8",
+   "metadata": {},
+   "source": [
+    "### Delete from a list of values"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "id": "fe3310bd-08f4-4a22-a63b-b3127d22f9f7",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "         vector  item  price\n",
+      "0    [3.1, 4.1]   foo   10.0\n",
+      "1    [3.1, 4.1]   foo   10.0\n",
+      "2    [3.1, 4.1]   foo   10.0\n",
+      "3    [3.1, 4.1]   foo   10.0\n",
+      "4    [3.1, 4.1]   foo   10.0\n",
+      "5    [1.3, 1.4]  fizz  100.0\n",
+      "6   [9.5, 56.2]  buzz  200.0\n",
+      "7    [3.1, 4.1]   foo   10.0\n",
+      "8    [3.1, 4.1]   foo   10.0\n",
+      "9    [3.1, 4.1]   foo   10.0\n",
+      "10   [3.1, 4.1]   foo   10.0\n",
+      "11   [3.1, 4.1]   foo   10.0\n",
+      "12   [3.1, 4.1]   foo   10.0\n",
+      "13   [3.1, 4.1]   foo   10.0\n",
+      "14   [3.1, 4.1]   foo   10.0\n",
+      "15   [3.1, 4.1]   foo   10.0\n",
+      "16   [3.1, 4.1]   foo   10.0\n"
+     ]
+    },
+    {
+     "ename": "OSError",
+     "evalue": "LanceError(IO): Error during planning: column foo does not exist",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[30], line 4\u001b[0m\n\u001b[1;32m      2\u001b[0m to_remove \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mstr\u001b[39m(v) \u001b[38;5;28;01mfor\u001b[39;00m v \u001b[38;5;129;01min\u001b[39;00m to_remove)\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28mprint\u001b[39m(tbl\u001b[38;5;241m.\u001b[39mto_pandas())\n\u001b[0;32m----> 4\u001b[0m \u001b[43mtbl\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mitem IN (\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mto_remove\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m)\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      5\u001b[0m tbl\u001b[38;5;241m.\u001b[39mto_pandas()\n",
+      "File \u001b[0;32m~/Documents/lancedb/lancedb/python/lancedb/table.py:610\u001b[0m, in \u001b[0;36mLanceTable.delete\u001b[0;34m(self, where)\u001b[0m\n\u001b[1;32m    609\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdelete\u001b[39m(\u001b[38;5;28mself\u001b[39m, where: \u001b[38;5;28mstr\u001b[39m):\n\u001b[0;32m--> 610\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dataset\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete\u001b[49m\u001b[43m(\u001b[49m\u001b[43mwhere\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/Documents/lancedb/lancedb/env/lib/python3.11/site-packages/lance/dataset.py:489\u001b[0m, in \u001b[0;36mLanceDataset.delete\u001b[0;34m(self, predicate)\u001b[0m\n\u001b[1;32m    487\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(predicate, pa\u001b[38;5;241m.\u001b[39mcompute\u001b[38;5;241m.\u001b[39mExpression):\n\u001b[1;32m    488\u001b[0m     predicate \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(predicate)\n\u001b[0;32m--> 489\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_ds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpredicate\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[0;31mOSError\u001b[0m: LanceError(IO): Error during planning: column foo does not exist"
+     ]
+    }
+   ],
+   "source": [
+    "to_remove = [\"foo\", \"buzz\"]\n",
+    "to_remove = \", \".join(str(v) for v in to_remove)\n",
+    "print(tbl.to_pandas())\n",
+    "tbl.delete(f\"item IN ({to_remove})\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "id": "87d5bc21-847f-4c81-b56e-f6dbe5d05aac",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.DataFrame(\n",
+    "                    {\n",
+    "                        \"vector\": [[3.1, 4.1], [1, 1]],\n",
+    "                        \"item\": [\"foo\", \"bar\"],\n",
+    "                        \"price\": [10.0, 20.0],\n",
+    "                    })\n",
+    "\n",
+    "tbl = db.create_table(\"table7\", data=df, mode=\"overwrite\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "id": "9cba4519-eb3a-4941-ab7e-873d762e750f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "to_remove = [10.0, 20.0]\n",
+    "to_remove = \", \".join(str(v) for v in to_remove)\n",
+    "\n",
+    "tbl.delete(f\"price IN ({to_remove})\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "id": "5bdc9801-d5ed-4871-92d0-88b27108e788",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>vector</th>\n",
+       "      <th>item</th>\n",
+       "      <th>price</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "Empty DataFrame\n",
+       "Columns: [vector, item, price]\n",
+       "Index: []"
+      ]
+     },
+     "execution_count": 46,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tbl.to_pandas()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "752d33d4-ce1c-48e5-90d2-c85f0982182d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/src/notebooks/youtube_transcript_search.ipynb

@@ -27,11 +27,11 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.1\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
+      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.0\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m23.1.1\u001B[0m\n",
+      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpip install --upgrade pip\u001B[0m\n",
       "\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.1\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
+      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.0\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m23.1.1\u001B[0m\n",
+      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpip install --upgrade pip\u001B[0m\n"
      ]
     }
    ],
@@ -184,7 +184,7 @@
     "df = (contextualize(data.to_pandas())\n",
     "      .groupby(\"title\").text_col(\"text\")\n",
     "      .window(20).stride(4)\n",
-    "      .to_df())\n",
+    "      .to_pandas())\n",
     "df.head(1)"
    ]
   },
@@ -603,7 +603,7 @@
    "outputs": [],
    "source": [
     "# Use LanceDB to get top 3 most relevant context\n",
-    "context = tbl.search(emb).limit(3).to_df()"
+    "context = tbl.search(emb).limit(3).to_pandas()"
    ]
   },
   {

docs/src/python/arrow.md

@@ -39,7 +39,6 @@ to lazily generate data:
 
 from typing import Iterable
 import pyarrow as pa
-import lancedb
 
 def make_batches() -> Iterable[pa.RecordBatch]:
     for i in range(5):
@@ -74,12 +73,12 @@ table = db.open_table("pd_table")
 
 query_vector = [100, 100]
 # Pandas DataFrame
-df = table.search(query_vector).limit(1).to_df()
+df = table.search(query_vector).limit(1).to_pandas()
 print(df)
 ```
 
 ```
-    vector     item  price        score
+    vector     item  price    _distance
 0  [5.9, 26.5]  bar   20.0  14257.05957
 ```
 
@@ -89,12 +88,12 @@ If you have more complex criteria, you can always apply the filter to the result
 ```python
 
 # Apply the filter via LanceDB
-results = table.search([100, 100]).where("price < 15").to_df()
+results = table.search([100, 100]).where("price < 15").to_pandas()
 assert len(results) == 1
 assert results["item"].iloc[0] == "foo"
 
 # Apply the filter via Pandas
-df = results = table.search([100, 100]).to_df()
+df = results = table.search([100, 100]).to_pandas()
 results = df[df.price < 15]
 assert len(results) == 1
 assert results["item"].iloc[0] == "foo"

docs/src/python/duckdb.md

@@ -11,15 +11,13 @@ pip install duckdb lancedb
 We will re-use [the dataset created previously](./arrow.md):
 
 ```python
-import pandas as pd
 import lancedb
 
 db = lancedb.connect("data/sample-lancedb")
-data = pd.DataFrame({
-    "vector": [[3.1, 4.1], [5.9, 26.5]],
-    "item": ["foo", "bar"],
-    "price": [10.0, 20.0]
-})
+data = [
+    {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+    {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}
+]
 table = db.create_table("pd_table", data=data)
 arrow_table = table.to_arrow()
 ```

docs/src/python/pydantic.md

@@ -3,7 +3,6 @@
 [Pydantic](https://docs.pydantic.dev/latest/) is a data validation library in Python.
 LanceDB integrates with Pydantic for schema inference, data ingestion, and query result casting.
 
-
 ## Schema
 
 LanceDB supports to create Apache Arrow Schema from a
@@ -14,10 +13,10 @@ via [pydantic_to_schema()](python.md##lancedb.pydantic.pydantic_to_schema) metho
 
 ## Vector Field
 
-LanceDB provides a [`vector(dim)`](python.md#lancedb.pydantic.vector) method to define a
+LanceDB provides a [`Vector(dim)`](python.md#lancedb.pydantic.Vector) method to define a
 vector Field in a Pydantic Model.
 
-::: lancedb.pydantic.vector
+::: lancedb.pydantic.Vector
 
 ## Type Conversion
 
@@ -34,4 +33,4 @@ Current supported type conversions:
 | `str`               | `pyarrow.utf8()`    |
 | `list`              | `pyarrow.List`    |
 | `BaseModel`         | `pyarrow.Struct`    |
-| `vector(n)`         | `pyarrow.FixedSizeList(float32, n)` |
+| `Vector(n)`         | `pyarrow.FixedSizeList(float32, n)` |

docs/src/python/python.md

@@ -22,13 +22,21 @@ pip install lancedb
 
 ::: lancedb.query.LanceQueryBuilder
 
-::: lancedb.query.LanceFtsQueryBuilder
-
 ## Embeddings
 
-::: lancedb.embeddings.with_embeddings
+::: lancedb.embeddings.registry.EmbeddingFunctionRegistry
 
-::: lancedb.embeddings.EmbeddingFunction
+::: lancedb.embeddings.base.EmbeddingFunction
+
+::: lancedb.embeddings.base.TextEmbeddingFunction
+
+::: lancedb.embeddings.sentence_transformers.SentenceTransformerEmbeddings
+
+::: lancedb.embeddings.openai.OpenAIEmbeddings
+
+::: lancedb.embeddings.open_clip.OpenClipEmbeddings
+
+::: lancedb.embeddings.with_embeddings
 
 ## Context
 
@@ -46,7 +54,7 @@ pip install lancedb
 
 ## Utilities
 
-::: lancedb.vector
+::: lancedb.schema.vector
 
 ## Integrations
 
@@ -56,4 +64,4 @@ pip install lancedb
 
 ::: lancedb.pydantic.vector
 
-
+::: lancedb.pydantic.LanceModel

docs/src/python/saas-python.md

@@ -0,0 +1,18 @@
+# LanceDB Python API Reference
+
+## Installation
+
+```shell
+pip install lancedb
+```
+
+## Connection
+
+::: lancedb.connect
+
+::: lancedb.remote.db.RemoteDBConnection
+
+## Table
+
+::: lancedb.remote.table.RemoteTable
+

docs/src/robots.txt

@@ -0,0 +1 @@
+User-agent: *

docs/src/scripts/posthog.js

@@ -0,0 +1,4 @@
+window.addEventListener("DOMContentLoaded", (event) => {
+    !function(t,e){var o,n,p,r;e.__SV||(window.posthog=e,e._i=[],e.init=function(i,s,a){function g(t,e){var o=e.split(".");2==o.length&&(t=t[o[0]],e=o[1]),t[e]=function(){t.push([e].concat(Array.prototype.slice.call(arguments,0)))}}(p=t.createElement("script")).type="text/javascript",p.async=!0,p.src=s.api_host+"/static/array.js",(r=t.getElementsByTagName("script")[0]).parentNode.insertBefore(p,r);var u=e;for(void 0!==a?u=e[a]=[]:a="posthog",u.people=u.people||[],u.toString=function(t){var e="posthog";return"posthog"!==a&&(e+="."+a),t||(e+=" (stub)"),e},u.people.toString=function(){return u.toString(1)+".people (stub)"},o="capture identify alias people.set people.set_once set_config register register_once unregister opt_out_capturing has_opted_out_capturing opt_in_capturing reset isFeatureEnabled onFeatureFlags getFeatureFlag getFeatureFlagPayload reloadFeatureFlags group updateEarlyAccessFeatureEnrollment getEarlyAccessFeatures getActiveMatchingSurveys getSurveys".split(" "),n=0;n<o.length;n++)g(u,o[n]);e._i.push([i,s,a])},e.__SV=1)}(document,window.posthog||[]);
+    posthog.init('phc_oENDjGgHtmIDrV6puUiFem2RB4JA8gGWulfdulmMdZP',{api_host:'https://app.posthog.com'})
+});

docs/src/search.md

@@ -4,7 +4,7 @@
 In a recommendation system or search engine, you can find similar products from
 the one you searched.
 In LLM and other AI applications,
-each data point can be [presented by the embeddings generated from some models](embedding.md),
+each data point can be [presented by the embeddings generated from some models](embeddings/index.md),
 it returns the most relevant features.
 
 A search in high-dimensional vector space, is to find `K-Nearest-Neighbors (KNN)` of the query vector.
@@ -25,8 +25,8 @@ Currently, we support the following metrics:
 
 ### Flat Search
 
-If LanceDB does not create a vector index, LanceDB would need to scan (`Flat Search`) the entire vector column
-and compute the distance for each vector in order to find the closest matches.
+If you do not create a vector index, LanceDB would need to exhaustively scan the entire vector column (via `Flat Search`)
+and compute the distance for *every* vector in order to find the closest matches. This is effectively a KNN search.
 
 
 <!-- Setup Code
@@ -67,7 +67,7 @@ await db_setup.createTable('my_vectors', data)
 
     df = tbl.search(np.random.random((1536))) \
         .limit(10) \
-        .to_df()
+        .to_list()
     ```
 
 === "JavaScript"
@@ -92,7 +92,7 @@ as well.
     df = tbl.search(np.random.random((1536))) \
         .metric("cosine") \
         .limit(10) \
-        .to_df()
+        .to_list()
     ```
 
 
@@ -110,7 +110,7 @@ as well.
 
 To accelerate vector retrievals, it is common to build vector indices.
 A vector index is a data structure specifically designed to efficiently organize and
-search vector data based on their similarity or distance metrics.
+search vector data based on their similarity via the chosen distance metric.
 By constructing a vector index, you can reduce the search space and avoid the need
 for brute-force scanning of the entire vector column.
 

docs/src/styles/global.css

@@ -4,3 +4,12 @@
     --md-text-font: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
     --md-code-font: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
 }
+
+.md-nav__item, .md-tabs__item {
+    font-size: large;
+}
+
+/* Maximum space for text block */
+.md-grid {
+    max-width: 90%;
+  }

docs/test/md_testing.js

@@ -2,18 +2,18 @@ const glob = require("glob");
 const fs = require("fs");
 const path = require("path");
 
-const excludedFiles = [
+const globString = "../src/**/*.md";
+const excludedGlobs = [
   "../src/fts.md",
   "../src/embedding.md",
-  "../src/examples/serverless_lancedb_with_s3_and_lambda.md",
-  "../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
-  "../src/examples/transformerjs_embedding_search_nodejs.md",
-  "../src/examples/youtube_transcript_bot_with_nodejs.md",
+  "../src/examples/*.md",
+  "../src/guides/tables.md",
+  "../src/embeddings/*.md",
 ];
+
 const nodePrefix = "javascript";
 const nodeFile = ".js";
 const nodeFolder = "node";
-const globString = "../src/**/*.md";
 const asyncPrefix = "(async () => {\n";
 const asyncSuffix = "})();";
 
@@ -32,6 +32,7 @@ function* yieldLines(lines, prefix, suffix) {
 }
 
 const files = glob.sync(globString, { recursive: true });
+const excludedFiles = glob.sync(excludedGlobs, { recursive: true });
 
 for (const file of files.filter((file) => !excludedFiles.includes(file))) {
   const lines = [];

docs/test/md_testing.py

@@ -2,40 +2,60 @@ import glob
 from typing import Iterator
 from pathlib import Path
 
-excluded_files = [
+glob_string = "../src/**/*.md"
+excluded_globs = [
     "../src/fts.md",
     "../src/embedding.md",
-    "../src/examples/serverless_lancedb_with_s3_and_lambda.md",
-    "../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
-    "../src/examples/youtube_transcript_bot_with_nodejs.md",
+    "../src/examples/*.md",
     "../src/integrations/voxel51.md",
+    "../src/guides/tables.md",
+    "../src/python/duckdb.md",
+    "../src/embeddings/*.md",
 ]
 
 python_prefix = "py"
 python_file = ".py"
 python_folder = "python"
-glob_string = "../src/**/*.md"
+
+files = glob.glob(glob_string, recursive=True)
+excluded_files = [
+    f
+    for excluded_glob in excluded_globs
+    for f in glob.glob(excluded_glob, recursive=True)
+]
+
 
 def yield_lines(lines: Iterator[str], prefix: str, suffix: str):
     in_code_block = False
     # Python code has strict indentation
     strip_length = 0
+    skip_test = False
     for line in lines:
+        if "skip-test" in line:
+            skip_test = True
         if line.strip().startswith(prefix + python_prefix):
             in_code_block = True
             strip_length = len(line) - len(line.lstrip())
         elif in_code_block and line.strip().startswith(suffix):
             in_code_block = False
+            if not skip_test:
                 yield "\n"
+            skip_test = False
         elif in_code_block:
+            if not skip_test:
                 yield line[strip_length:]
 
-for file in filter(lambda file: file not in excluded_files, glob.glob(glob_string, recursive=True)):
+for file in filter(lambda file: file not in excluded_files, files):
     with open(file, "r") as f:
         lines = list(yield_lines(iter(f), "```", "```"))
 
     if len(lines) > 0:
-        out_path = Path(python_folder) / Path(file).name.strip(".md") / (Path(file).name.strip(".md") + python_file)
+        print(lines)
+        out_path = (
+            Path(python_folder)
+            / Path(file).name.strip(".md")
+            / (Path(file).name.strip(".md") + python_file)
+        )
         print(out_path)
         out_path.parent.mkdir(exist_ok=True, parents=True)
         with open(out_path, "w") as out:

docs/test/requirements.txt

@@ -1,5 +1,8 @@
-lancedb @ git+https://github.com/lancedb/lancedb.git#egg=subdir&subdirectory=python
+-e ../../python
 numpy
 pandas
 pylance
 duckdb
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch
+

node/README.md

@@ -10,7 +10,7 @@ npm install vectordb
 
 This will download the appropriate native library for your platform. We currently
 support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
-yet support Windows or musl-based Linux (such as Alpine Linux).
+yet support musl-based Linux (such as Alpine Linux).
 
 ## Usage
 

node/examples/js-transformers/index.js

@@ -50,7 +50,7 @@ async function example() {
         { id: 5, text: 'Banana', type: 'fruit' }
     ]
 
-    const table = await db.createTable('food_table', data, "create", embed_fun)
+    const table = await db.createTable('food_table', data, embed_fun)
 
 
     // Query the table

node/examples/js-transformers/package.json

@@ -10,7 +10,7 @@
   "license": "Apache-2.0",
   "dependencies": {
     "@xenova/transformers": "^2.4.1",
-    "vectordb": "^0.1.12"
+    "vectordb": "file:../.."
   }
 
 }

node/native.js

@@ -12,26 +12,25 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-const { currentTarget } = require('@neon-rs/load');
+const { currentTarget } = require('@neon-rs/load')
 
-let nativeLib;
+let nativeLib
 
 try {
-    nativeLib = require(`@lancedb/vectordb-${currentTarget()}`);
-} catch (e) {
+  // When developing locally, give preference to the local built library
+  nativeLib = require('./index.node')
+} catch {
   try {
-        // Might be developing locally, so try that. But don't expose that error
-        // to the user.
-        nativeLib = require("./index.node");
-    } catch {
+    nativeLib = require(`@lancedb/vectordb-${currentTarget()}`)
+  } catch (e) {
     throw new Error(`vectordb: failed to load native library.
   You may need to run \`npm install @lancedb/vectordb-${currentTarget()}\`.
 
   If that does not work, please file a bug report at https://github.com/lancedb/lancedb/issues
       
-  Source error: ${e}`);
+  Source error: ${e}`)
   }
 }
 
 // Dynamic require for runtime.
-module.exports = nativeLib;
+module.exports = nativeLib

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.1.19",
+  "version": "0.3.11",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.1.19",
+      "version": "0.3.11",
       "cpu": [
         "x64",
         "arm64"
@@ -31,6 +31,7 @@
         "@types/node": "^18.16.2",
         "@types/sinon": "^10.0.15",
         "@types/temp": "^0.9.1",
+        "@types/uuid": "^9.0.3",
         "@typescript-eslint/eslint-plugin": "^5.59.1",
         "cargo-cp-artifact": "^0.1",
         "chai": "^4.3.7",
@@ -48,14 +49,15 @@
         "ts-node-dev": "^2.0.0",
         "typedoc": "^0.24.7",
         "typedoc-plugin-markdown": "^3.15.3",
-        "typescript": "*"
+        "typescript": "*",
+        "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.1.19",
-        "@lancedb/vectordb-darwin-x64": "0.1.19",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.1.19",
-        "@lancedb/vectordb-linux-x64-gnu": "0.1.19",
-        "@lancedb/vectordb-win32-x64-msvc": "0.1.19"
+        "@lancedb/vectordb-darwin-arm64": "0.3.11",
+        "@lancedb/vectordb-darwin-x64": "0.3.11",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.3.11",
+        "@lancedb/vectordb-linux-x64-gnu": "0.3.11",
+        "@lancedb/vectordb-win32-x64-msvc": "0.3.11"
       }
     },
     "node_modules/@apache-arrow/ts": {
@@ -314,66 +316,6 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
-    "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.1.19.tgz",
-      "integrity": "sha512-efQhJkBKvMNhjFq3Sw3/qHo9D9gb9UqiIr98n3STsbNxBQjMnWemXn91Ckl40siRG1O8qXcINW7Qs/EGmus+kg==",
-      "cpu": [
-        "arm64"
-      ],
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.1.19.tgz",
-      "integrity": "sha512-r6OZNVyemAssABz2w7CRhe7dyREwBEfTytn+ux1zzTnzsgMgDovCQ0rQ3WZcxWvcy7SFCxiemA9IP1b/lsb4tQ==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.1.19.tgz",
-      "integrity": "sha512-mL/hRmZp6Kw7hmGJBdOZfp/tTYiCdlOcs8DA/+nr2eiXERv0gIhyiKvr2P5DwbBmut3qXEkDalMHTo95BSdL2A==",
-      "cpu": [
-        "arm64"
-      ],
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.1.19.tgz",
-      "integrity": "sha512-AG0FHksbbr+cHVKPi4B8cmBtqb6T9E0uaK4kyZkXrX52/xtv9RYVZcykaB/tSSm0XNFPWWRnx9R8UqNZV/hxMA==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.1.19.tgz",
-      "integrity": "sha512-PDWZ2hvLVXH4Z4WIO1rsWY8ev3NpNm7aXlaey32P+l1Iz9Hia9+F2GBpp2UiEQKfvbk82ucAvBLRmpSsHY8Tlw==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "win32"
-      ]
-    },
     "node_modules/@neon-rs/cli": {
       "version": "0.0.160",
       "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.160.tgz",
@@ -596,6 +538,12 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/uuid": {
+      "version": "9.0.3",
+      "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.3.tgz",
+      "integrity": "sha512-taHQQH/3ZyI3zP8M/puluDEIEvtQHVYcC6y3N8ijFtAd28+Ey/G4sg1u2gB01S8MwybLOKAp9/yCMu/uR5l3Ug==",
+      "dev": true
+    },
     "node_modules/@typescript-eslint/eslint-plugin": {
       "version": "5.59.1",
       "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.59.1.tgz",
@@ -4451,6 +4399,15 @@
         "punycode": "^2.1.0"
       }
     },
+    "node_modules/uuid": {
+      "version": "9.0.0",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz",
+      "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==",
+      "dev": true,
+      "bin": {
+        "uuid": "dist/bin/uuid"
+      }
+    },
     "node_modules/v8-compile-cache-lib": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
@@ -4851,36 +4808,6 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
-    "@lancedb/vectordb-darwin-arm64": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.1.19.tgz",
-      "integrity": "sha512-efQhJkBKvMNhjFq3Sw3/qHo9D9gb9UqiIr98n3STsbNxBQjMnWemXn91Ckl40siRG1O8qXcINW7Qs/EGmus+kg==",
-      "optional": true
-    },
-    "@lancedb/vectordb-darwin-x64": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.1.19.tgz",
-      "integrity": "sha512-r6OZNVyemAssABz2w7CRhe7dyREwBEfTytn+ux1zzTnzsgMgDovCQ0rQ3WZcxWvcy7SFCxiemA9IP1b/lsb4tQ==",
-      "optional": true
-    },
-    "@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.1.19.tgz",
-      "integrity": "sha512-mL/hRmZp6Kw7hmGJBdOZfp/tTYiCdlOcs8DA/+nr2eiXERv0gIhyiKvr2P5DwbBmut3qXEkDalMHTo95BSdL2A==",
-      "optional": true
-    },
-    "@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.1.19.tgz",
-      "integrity": "sha512-AG0FHksbbr+cHVKPi4B8cmBtqb6T9E0uaK4kyZkXrX52/xtv9RYVZcykaB/tSSm0XNFPWWRnx9R8UqNZV/hxMA==",
-      "optional": true
-    },
-    "@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.1.19",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.1.19.tgz",
-      "integrity": "sha512-PDWZ2hvLVXH4Z4WIO1rsWY8ev3NpNm7aXlaey32P+l1Iz9Hia9+F2GBpp2UiEQKfvbk82ucAvBLRmpSsHY8Tlw==",
-      "optional": true
-    },
     "@neon-rs/cli": {
       "version": "0.0.160",
       "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.160.tgz",
@@ -5093,6 +5020,12 @@
         "@types/node": "*"
       }
     },
+    "@types/uuid": {
+      "version": "9.0.3",
+      "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.3.tgz",
+      "integrity": "sha512-taHQQH/3ZyI3zP8M/puluDEIEvtQHVYcC6y3N8ijFtAd28+Ey/G4sg1u2gB01S8MwybLOKAp9/yCMu/uR5l3Ug==",
+      "dev": true
+    },
     "@typescript-eslint/eslint-plugin": {
       "version": "5.59.1",
       "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.59.1.tgz",
@@ -7844,6 +7777,12 @@
         "punycode": "^2.1.0"
       }
     },
+    "uuid": {
+      "version": "9.0.0",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz",
+      "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==",
+      "dev": true
+    },
     "v8-compile-cache-lib": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.1.19",
+  "version": "0.3.11",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -9,7 +9,8 @@
     "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json",
     "build-release": "npm run build -- --release",
     "test": "npm run tsc && mocha -recursive dist/test",
-    "lint": "eslint src --ext .js,.ts",
+    "integration-test": "npm run tsc && mocha -recursive dist/integration_test",
+    "lint": "eslint native.js src --ext .js,.ts",
     "clean": "rm -rf node_modules *.node dist/",
     "pack-build": "neon pack-build",
     "check-npm": "printenv && which node && which npm && npm --version"
@@ -34,6 +35,7 @@
     "@types/node": "^18.16.2",
     "@types/sinon": "^10.0.15",
     "@types/temp": "^0.9.1",
+    "@types/uuid": "^9.0.3",
     "@typescript-eslint/eslint-plugin": "^5.59.1",
     "cargo-cp-artifact": "^0.1",
     "chai": "^4.3.7",
@@ -51,7 +53,8 @@
     "ts-node-dev": "^2.0.0",
     "typedoc": "^0.24.7",
     "typedoc-plugin-markdown": "^3.15.3",
-    "typescript": "*"
+    "typescript": "*",
+    "uuid": "^9.0.0"
   },
   "dependencies": {
     "@apache-arrow/ts": "^12.0.0",
@@ -78,10 +81,10 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.1.19",
-    "@lancedb/vectordb-darwin-x64": "0.1.19",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.1.19",
-    "@lancedb/vectordb-linux-x64-gnu": "0.1.19",
-    "@lancedb/vectordb-win32-x64-msvc": "0.1.19"
+    "@lancedb/vectordb-darwin-arm64": "0.3.11",
+    "@lancedb/vectordb-darwin-x64": "0.3.11",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.3.11",
+    "@lancedb/vectordb-linux-x64-gnu": "0.3.11",
+    "@lancedb/vectordb-win32-x64-msvc": "0.3.11"
   }
 }

node/src/arrow.ts

@@ -13,18 +13,19 @@
 // limitations under the License.
 
 import {
-  Field,
+  Field, type FixedSizeListBuilder,
   Float32,
-  List, type ListBuilder,
   makeBuilder,
   RecordBatchFileWriter,
-  Table, Utf8,
+  Utf8,
   type Vector,
-  vectorFromArray
+  FixedSizeList,
+  vectorFromArray, type Schema, Table as ArrowTable, RecordBatchStreamWriter
 } from 'apache-arrow'
 import { type EmbeddingFunction } from './index'
 
-export async function convertToTable<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Table> {
+// Converts an Array of records into an Arrow Table, optionally applying an embeddings function to it.
+export async function convertToTable<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<ArrowTable> {
   if (data.length === 0) {
     throw new Error('At least one record needs to be provided')
   }
@@ -34,8 +35,8 @@ export async function convertToTable<T> (data: Array<Record<string, unknown>>, e
 
   for (const columnsKey of columns) {
     if (columnsKey === 'vector') {
-      const listBuilder = newVectorListBuilder()
       const vectorSize = (data[0].vector as any[]).length
+      const listBuilder = newVectorBuilder(vectorSize)
       for (const datum of data) {
         if ((datum[columnsKey] as any[]).length !== vectorSize) {
           throw new Error(`Invalid vector size, expected ${vectorSize}`)
@@ -52,9 +53,7 @@ export async function convertToTable<T> (data: Array<Record<string, unknown>>, e
 
       if (columnsKey === embeddings?.sourceColumn) {
         const vectors = await embeddings.embed(values as T[])
-        const listBuilder = newVectorListBuilder()
-        vectors.map(v => listBuilder.append(v))
-        records.vector = listBuilder.finish().toVector()
+        records.vector = vectorFromArray(vectors, newVectorType(vectors[0].length))
       }
 
       if (typeof values[0] === 'string') {
@@ -66,20 +65,73 @@ export async function convertToTable<T> (data: Array<Record<string, unknown>>, e
     }
   }
 
-  return new Table(records)
+  return new ArrowTable(records)
 }
 
 // Creates a new Arrow ListBuilder that stores a Vector column
-function newVectorListBuilder (): ListBuilder<Float32, any> {
-  const children = new Field<Float32>('item', new Float32())
-  const list = new List(children)
+function newVectorBuilder (dim: number): FixedSizeListBuilder<Float32> {
   return makeBuilder({
-    type: list
+    type: newVectorType(dim)
   })
 }
 
+// Creates the Arrow Type for a Vector column with dimension `dim`
+function newVectorType (dim: number): FixedSizeList<Float32> {
+  // Somewhere we always default to have the elements nullable, so we need to set it to true
+  // otherwise we often get schema mismatches because the stored data always has schema with nullable elements
+  const children = new Field<Float32>('item', new Float32(), true)
+  return new FixedSizeList(dim, children)
+}
+
+// Converts an Array of records into Arrow IPC format
 export async function fromRecordsToBuffer<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Buffer> {
   const table = await convertToTable(data, embeddings)
   const writer = RecordBatchFileWriter.writeAll(table)
   return Buffer.from(await writer.toUint8Array())
 }
+
+// Converts an Array of records into Arrow IPC stream format
+export async function fromRecordsToStreamBuffer<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Buffer> {
+  const table = await convertToTable(data, embeddings)
+  const writer = RecordBatchStreamWriter.writeAll(table)
+  return Buffer.from(await writer.toUint8Array())
+}
+
+// Converts an Arrow Table into Arrow IPC format
+export async function fromTableToBuffer<T> (table: ArrowTable, embeddings?: EmbeddingFunction<T>): Promise<Buffer> {
+  if (embeddings !== undefined) {
+    const source = table.getChild(embeddings.sourceColumn)
+
+    if (source === null) {
+      throw new Error(`The embedding source column ${embeddings.sourceColumn} was not found in the Arrow Table`)
+    }
+
+    const vectors = await embeddings.embed(source.toArray() as T[])
+    const column = vectorFromArray(vectors, newVectorType(vectors[0].length))
+    table = table.assign(new ArrowTable({ vector: column }))
+  }
+  const writer = RecordBatchFileWriter.writeAll(table)
+  return Buffer.from(await writer.toUint8Array())
+}
+
+// Converts an Arrow Table into Arrow IPC stream format
+export async function fromTableToStreamBuffer<T> (table: ArrowTable, embeddings?: EmbeddingFunction<T>): Promise<Buffer> {
+  if (embeddings !== undefined) {
+    const source = table.getChild(embeddings.sourceColumn)
+
+    if (source === null) {
+      throw new Error(`The embedding source column ${embeddings.sourceColumn} was not found in the Arrow Table`)
+    }
+
+    const vectors = await embeddings.embed(source.toArray() as T[])
+    const column = vectorFromArray(vectors, newVectorType(vectors[0].length))
+    table = table.assign(new ArrowTable({ vector: column }))
+  }
+  const writer = RecordBatchStreamWriter.writeAll(table)
+  return Buffer.from(await writer.toUint8Array())
+}
+
+// Creates an empty Arrow Table
+export function createEmptyTable (schema: Schema): ArrowTable {
+  return new ArrowTable(schema)
+}

node/src/index.ts

@@ -13,17 +13,18 @@
 // limitations under the License.
 
 import {
-  RecordBatchFileWriter,
-  type Table as ArrowTable
+  type Schema,
+  Table as ArrowTable
 } from 'apache-arrow'
-import { fromRecordsToBuffer } from './arrow'
+import { createEmptyTable, fromRecordsToBuffer, fromTableToBuffer } from './arrow'
 import type { EmbeddingFunction } from './embedding/embedding_function'
 import { RemoteConnection } from './remote'
 import { Query } from './query'
 import { isEmbeddingFunction } from './embedding/embedding_function'
+import { type Literal, toSQL } from './util'
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete } = require('../native.js')
+const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete, tableUpdate, tableCleanupOldVersions, tableCompactFiles, tableListIndices, tableIndexStats } = require('../native.js')
 
 export { Query }
 export type { EmbeddingFunction }
@@ -42,6 +43,8 @@ export interface ConnectionOptions {
 
   awsCredentials?: AwsCredentials
 
+  awsRegion?: string
+
   // API key for the remote connections
   apiKey?: string
   // Region to connect
@@ -51,6 +54,40 @@ export interface ConnectionOptions {
   hostOverride?: string
 }
 
+function getAwsArgs (opts: ConnectionOptions): any[] {
+  const callArgs = []
+  const awsCredentials = opts.awsCredentials
+  if (awsCredentials !== undefined) {
+    callArgs.push(awsCredentials.accessKeyId)
+    callArgs.push(awsCredentials.secretKey)
+    callArgs.push(awsCredentials.sessionToken)
+  } else {
+    callArgs.push(undefined)
+    callArgs.push(undefined)
+    callArgs.push(undefined)
+  }
+
+  callArgs.push(opts.awsRegion)
+  return callArgs
+}
+
+export interface CreateTableOptions<T> {
+  // Name of Table
+  name: string
+
+  // Data to insert into the Table
+  data?: Array<Record<string, unknown>> | ArrowTable | undefined
+
+  // Optional Arrow Schema for this table
+  schema?: Schema | undefined
+
+  // Optional embedding function used to create embeddings
+  embeddingFunction?: EmbeddingFunction<T> | undefined
+
+  // WriteOptions for this operation
+  writeOptions?: WriteOptions | undefined
+}
+
 /**
  * Connect to a LanceDB instance at the given URI
  * @param uri The uri of the database.
@@ -97,6 +134,17 @@ export interface Connection {
    */
   openTable<T>(name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
 
+  /**
+   * Creates a new Table, optionally initializing it with new data.
+   *
+   * @param {string} name - The name of the table.
+   * @param data - Array of Records to be inserted into the table
+   * @param schema - An Arrow Schema that describe this table columns
+   * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
+   * @param {WriteOptions} writeOptions - The write options to use when creating the table.
+   */
+  createTable<T> ({ name, data, schema, embeddingFunction, writeOptions }: CreateTableOptions<T>): Promise<Table<T>>
+
   /**
    * Creates a new Table and initialize it with new data.
    *
@@ -132,8 +180,6 @@ export interface Connection {
    */
   createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>, options: WriteOptions): Promise<Table<T>>
 
-  createTableArrow(name: string, table: ArrowTable): Promise<Table>
-
   /**
    * Drop an existing table.
    * @param name The name of the table to drop.
@@ -215,22 +261,104 @@ export interface Table<T = number[]> {
    * ```
    */
   delete: (filter: string) => Promise<void>
+
+  /**
+   * Update rows in this table.
+   *
+   * This can be used to update a single row, many rows, all rows, or
+   * sometimes no rows (if your predicate matches nothing).
+   *
+   * @param args see {@link UpdateArgs} and {@link UpdateSqlArgs} for more details
+   *
+   * @examples
+   *
+   * ```ts
+   * const con = await lancedb.connect("./.lancedb")
+   * const data = [
+   *    {id: 1, vector: [3, 3], name: 'Ye'},
+   *    {id: 2, vector: [4, 4], name: 'Mike'},
+   * ];
+   * const tbl = await con.createTable("my_table", data)
+   *
+   * await tbl.update({
+   *   filter: "id = 2",
+   *   updates: { vector: [2, 2], name: "Michael" },
+   * })
+   *
+   * let results = await tbl.search([1, 1]).execute();
+   * // Returns [
+   * //   {id: 2, vector: [2, 2], name: 'Michael'}
+   * //   {id: 1, vector: [3, 3], name: 'Ye'}
+   * // ]
+   * ```
+   *
+   */
+  update: (args: UpdateArgs | UpdateSqlArgs) => Promise<void>
+
+  /**
+   * List the indicies on this table.
+   */
+  listIndices: () => Promise<VectorIndex[]>
+
+  /**
+   * Get statistics about an index.
+   */
+  indexStats: (indexUuid: string) => Promise<IndexStats>
+}
+
+export interface UpdateArgs {
+  /**
+   * A filter in the same format used by a sql WHERE clause. The filter may be empty,
+   * in which case all rows will be updated.
+   */
+  where?: string
+
+  /**
+   * A key-value map of updates. The keys are the column names, and the values are the
+   * new values to set
+   */
+  values: Record<string, Literal>
+}
+
+export interface UpdateSqlArgs {
+  /**
+   * A filter in the same format used by a sql WHERE clause. The filter may be empty,
+   * in which case all rows will be updated.
+   */
+  where?: string
+
+  /**
+   * A key-value map of updates. The keys are the column names, and the values are the
+   * new values to set as SQL expressions.
+   */
+  valuesSql: Record<string, string>
+}
+
+export interface VectorIndex {
+  columns: string[]
+  name: string
+  uuid: string
+}
+
+export interface IndexStats {
+  numIndexedRows: number | null
+  numUnindexedRows: number | null
 }
 
 /**
  * A connection to a LanceDB database.
  */
 export class LocalConnection implements Connection {
-  private readonly _options: ConnectionOptions
+  private readonly _options: () => ConnectionOptions
   private readonly _db: any
 
   constructor (db: any, options: ConnectionOptions) {
-    this._options = options
+    this._options = () => options
     this._db = db
   }
 
   get uri (): string {
-    return this._options.uri
+    return this._options().uri
   }
 
   /**
@@ -256,15 +384,16 @@ export class LocalConnection implements Connection {
   async openTable<T> (name: string, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
   async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
   async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
-    const tbl = await databaseOpenTable.call(this._db, name)
+    const tbl = await databaseOpenTable.call(this._db, name, ...getAwsArgs(this._options()))
     if (embeddings !== undefined) {
-      return new LocalTable(tbl, name, this._options, embeddings)
+      return new LocalTable(tbl, name, this._options(), embeddings)
     } else {
-      return new LocalTable(tbl, name, this._options)
+      return new LocalTable(tbl, name, this._options())
     }
   }
 
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, optsOrEmbedding?: WriteOptions | EmbeddingFunction<T>, opt?: WriteOptions): Promise<Table<T>> {
+  async createTable<T> (name: string | CreateTableOptions<T>, data?: Array<Record<string, unknown>>, optsOrEmbedding?: WriteOptions | EmbeddingFunction<T>, opt?: WriteOptions): Promise<Table<T>> {
+    if (typeof name === 'string') {
       let writeOptions: WriteOptions = new DefaultWriteOptions()
       if (opt !== undefined && isWriteOptions(opt)) {
         writeOptions = opt
@@ -276,28 +405,45 @@ export class LocalConnection implements Connection {
       if (optsOrEmbedding !== undefined && isEmbeddingFunction(optsOrEmbedding)) {
         embeddings = optsOrEmbedding
       }
-    const createArgs = [this._db, name, await fromRecordsToBuffer(data, embeddings), writeOptions.writeMode?.toString()]
-    if (this._options.awsCredentials !== undefined) {
-      createArgs.push(this._options.awsCredentials.accessKeyId)
-      createArgs.push(this._options.awsCredentials.secretKey)
-      if (this._options.awsCredentials.sessionToken !== undefined) {
-        createArgs.push(this._options.awsCredentials.sessionToken)
+      return await this.createTableImpl({ name, data, embeddingFunction: embeddings, writeOptions })
     }
+    return await this.createTableImpl(name)
   }
 
-    const tbl = await tableCreate.call(...createArgs)
+  private async createTableImpl<T> ({ name, data, schema, embeddingFunction, writeOptions = new DefaultWriteOptions() }: {
+    name: string
+    data?: Array<Record<string, unknown>> | ArrowTable | undefined
+    schema?: Schema | undefined
+    embeddingFunction?: EmbeddingFunction<T> | undefined
+    writeOptions?: WriteOptions | undefined
+  }): Promise<Table<T>> {
+    let buffer: Buffer
 
-    if (embeddings !== undefined) {
-      return new LocalTable(tbl, name, this._options, embeddings)
+    function isEmpty (data: Array<Record<string, unknown>> | ArrowTable<any>): boolean {
+      if (data instanceof ArrowTable) {
+        return data.data.length === 0
+      }
+      return data.length === 0
+    }
+
+    if ((data === undefined) || isEmpty(data)) {
+      if (schema === undefined) {
+        throw new Error('Either data or schema needs to defined')
+      }
+      buffer = await fromTableToBuffer(createEmptyTable(schema))
+    } else if (data instanceof ArrowTable) {
+      buffer = await fromTableToBuffer(data, embeddingFunction)
     } else {
-      return new LocalTable(tbl, name, this._options)
-    }
+      // data is Array<Record<...>>
+      buffer = await fromRecordsToBuffer(data, embeddingFunction)
     }
 
-  async createTableArrow (name: string, table: ArrowTable): Promise<Table> {
-    const writer = RecordBatchFileWriter.writeAll(table)
-    await tableCreate.call(this._db, name, Buffer.from(await writer.toUint8Array()))
-    return await this.openTable(name)
+    const tbl = await tableCreate.call(this._db, name, buffer, writeOptions?.writeMode?.toString(), ...getAwsArgs(this._options()))
+    if (embeddingFunction !== undefined) {
+      return new LocalTable(tbl, name, this._options(), embeddingFunction)
+    } else {
+      return new LocalTable(tbl, name, this._options())
+    }
   }
 
   /**
@@ -310,10 +456,10 @@ export class LocalConnection implements Connection {
 }
 
 export class LocalTable<T = number[]> implements Table<T> {
-  private readonly _tbl: any
+  private _tbl: any
   private readonly _name: string
   private readonly _embeddings?: EmbeddingFunction<T>
-  private readonly _options: ConnectionOptions
+  private readonly _options: () => ConnectionOptions
 
   constructor (tbl: any, name: string, options: ConnectionOptions)
   /**
@@ -327,7 +473,7 @@ export class LocalTable<T = number[]> implements Table<T> {
     this._tbl = tbl
     this._name = name
     this._embeddings = embeddings
-    this._options = options
+    this._options = () => options
   }
 
   get name (): string {
@@ -342,6 +488,16 @@ export class LocalTable<T = number[]> implements Table<T> {
     return new Query(query, this._tbl, this._embeddings)
   }
 
+  /**
+   * Creates a filter query to find all rows matching the specified criteria
+   * @param value The filter criteria (like SQL where clause syntax)
+   */
+  filter (value: string): Query<T> {
+    return new Query(undefined, this._tbl, this._embeddings).filter(value)
+  }
+
+  where = this.filter
+
   /**
    * Insert records into this Table.
    *
@@ -349,15 +505,12 @@ export class LocalTable<T = number[]> implements Table<T> {
    * @return The number of rows added to the table
    */
   async add (data: Array<Record<string, unknown>>): Promise<number> {
-    const callArgs = [this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Append.toString()]
-    if (this._options.awsCredentials !== undefined) {
-      callArgs.push(this._options.awsCredentials.accessKeyId)
-      callArgs.push(this._options.awsCredentials.secretKey)
-      if (this._options.awsCredentials.sessionToken !== undefined) {
-        callArgs.push(this._options.awsCredentials.sessionToken)
-      }
-    }
-    return tableAdd.call(...callArgs)
+    return tableAdd.call(
+      this._tbl,
+      await fromRecordsToBuffer(data, this._embeddings),
+      WriteMode.Append.toString(),
+      ...getAwsArgs(this._options())
+    ).then((newTable: any) => { this._tbl = newTable })
   }
 
   /**
@@ -367,15 +520,12 @@ export class LocalTable<T = number[]> implements Table<T> {
    * @return The number of rows added to the table
    */
   async overwrite (data: Array<Record<string, unknown>>): Promise<number> {
-    const callArgs = [this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Overwrite.toString()]
-    if (this._options.awsCredentials !== undefined) {
-      callArgs.push(this._options.awsCredentials.accessKeyId)
-      callArgs.push(this._options.awsCredentials.secretKey)
-      if (this._options.awsCredentials.sessionToken !== undefined) {
-        callArgs.push(this._options.awsCredentials.sessionToken)
-      }
-    }
-    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Overwrite.toString())
+    return tableAdd.call(
+      this._tbl,
+      await fromRecordsToBuffer(data, this._embeddings),
+      WriteMode.Overwrite.toString(),
+      ...getAwsArgs(this._options())
+    ).then((newTable: any) => { this._tbl = newTable })
   }
 
   /**
@@ -384,7 +534,7 @@ export class LocalTable<T = number[]> implements Table<T> {
    * @param indexParams The parameters of this Index, @see VectorIndexParams.
    */
   async createIndex (indexParams: VectorIndexParams): Promise<any> {
-    return tableCreateVectorIndex.call(this._tbl, indexParams)
+    return tableCreateVectorIndex.call(this._tbl, indexParams).then((newTable: any) => { this._tbl = newTable })
   }
 
   /**
@@ -400,8 +550,146 @@ export class LocalTable<T = number[]> implements Table<T> {
    * @param filter A filter in the same format used by a sql WHERE clause.
    */
   async delete (filter: string): Promise<void> {
-    return tableDelete.call(this._tbl, filter)
+    return tableDelete.call(this._tbl, filter).then((newTable: any) => { this._tbl = newTable })
   }
+
+  /**
+   * Update rows in this table.
+   *
+   * @param args see {@link UpdateArgs} and {@link UpdateSqlArgs} for more details
+   *
+   * @returns
+   */
+  async update (args: UpdateArgs | UpdateSqlArgs): Promise<void> {
+    let filter: string | null
+    let updates: Record<string, string>
+
+    if ('valuesSql' in args) {
+      filter = args.where ?? null
+      updates = args.valuesSql
+    } else {
+      filter = args.where ?? null
+      updates = {}
+      for (const [key, value] of Object.entries(args.values)) {
+        updates[key] = toSQL(value)
+      }
+    }
+
+    return tableUpdate.call(this._tbl, filter, updates).then((newTable: any) => { this._tbl = newTable })
+  }
+
+  /**
+   * Clean up old versions of the table, freeing disk space.
+   *
+   * @param olderThan The minimum age in minutes of the versions to delete. If not
+   *                  provided, defaults to two weeks.
+   * @param deleteUnverified Because they may be part of an in-progress
+   *                  transaction, uncommitted files newer than 7 days old are
+   *                  not deleted by default. This means that failed transactions
+   *                  can leave around data that takes up disk space for up to
+   *                  7 days. You can override this safety mechanism by setting
+   *                 this option to `true`, only if you promise there are no
+   *                 in progress writes while you run this operation. Failure to
+   *                 uphold this promise can lead to corrupted tables.
+   * @returns
+   */
+  async cleanupOldVersions (olderThan?: number, deleteUnverified?: boolean): Promise<CleanupStats> {
+    return tableCleanupOldVersions.call(this._tbl, olderThan, deleteUnverified)
+      .then((res: { newTable: any, metrics: CleanupStats }) => {
+        this._tbl = res.newTable
+        return res.metrics
+      })
+  }
+
+  /**
+   * Run the compaction process on the table.
+   *
+   * This can be run after making several small appends to optimize the table
+   * for faster reads.
+   *
+   * @param options Advanced options configuring compaction. In most cases, you
+   *               can omit this arguments, as the default options are sensible
+   *               for most tables.
+   * @returns Metrics about the compaction operation.
+   */
+  async compactFiles (options?: CompactionOptions): Promise<CompactionMetrics> {
+    const optionsArg = options ?? {}
+    return tableCompactFiles.call(this._tbl, optionsArg)
+      .then((res: { newTable: any, metrics: CompactionMetrics }) => {
+        this._tbl = res.newTable
+        return res.metrics
+      })
+  }
+
+  async listIndices (): Promise<VectorIndex[]> {
+    return tableListIndices.call(this._tbl)
+  }
+
+  async indexStats (indexUuid: string): Promise<IndexStats> {
+    return tableIndexStats.call(this._tbl, indexUuid)
+  }
+}
+
+export interface CleanupStats {
+  /**
+   * The number of bytes removed from disk.
+   */
+  bytesRemoved: number
+  /**
+   * The number of old table versions removed.
+   */
+  oldVersions: number
+}
+
+export interface CompactionOptions {
+  /**
+   * The number of rows per fragment to target. Fragments that have fewer rows
+   * will be compacted into adjacent fragments to produce larger fragments.
+   * Defaults to 1024 * 1024.
+   */
+  targetRowsPerFragment?: number
+  /**
+   * The maximum number of rows per group. Defaults to 1024.
+   */
+  maxRowsPerGroup?: number
+  /**
+   * If true, fragments that have rows that are deleted may be compacted to
+   * remove the deleted rows. This can improve the performance of queries.
+   * Default is true.
+   */
+  materializeDeletions?: boolean
+  /**
+   * A number between 0 and 1, representing the proportion of rows that must be
+   * marked deleted before a fragment is a candidate for compaction to remove
+   * the deleted rows. Default is 10%.
+   */
+  materializeDeletionsThreshold?: number
+  /**
+   * The number of threads to use for compaction. If not provided, defaults to
+   * the number of cores on the machine.
+   */
+  numThreads?: number
+}
+
+export interface CompactionMetrics {
+  /**
+   * The number of fragments that were removed.
+   */
+  fragmentsRemoved: number
+  /**
+   * The number of new fragments that were created.
+   */
+  fragmentsAdded: number
+  /**
+   * The number of files that were removed. Each fragment may have more than one
+   * file.
+   */
+  filesRemoved: number
+  /**
+   * The number of files added. This is typically equal to the number of
+   * fragments added.
+   */
+  filesAdded: number
 }
 
 /// Config to build IVF_PQ index.

node/src/integration_test/test.ts

@@ -0,0 +1,180 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { describe } from 'mocha'
+import * as chai from 'chai'
+import * as chaiAsPromised from 'chai-as-promised'
+import { v4 as uuidv4 } from 'uuid'
+
+import * as lancedb from '../index'
+import { tmpdir } from 'os'
+import * as fs from 'fs'
+import * as path from 'path'
+
+const assert = chai.assert
+chai.use(chaiAsPromised)
+
+describe('LanceDB AWS Integration test', function () {
+  it('s3+ddb schema is processed correctly', async function () {
+    this.timeout(15000)
+
+    // WARNING: specifying engine is NOT a publicly supported feature in lancedb yet
+    // THE API WILL CHANGE
+    const conn = await lancedb.connect('s3://lancedb-integtest?engine=ddb&ddbTableName=lancedb-integtest')
+    const data = [{ vector: Array(128).fill(1.0) }]
+
+    const tableName = uuidv4()
+    let table = await conn.createTable(tableName, data, { writeMode: lancedb.WriteMode.Overwrite })
+
+    const futs = [table.add(data), table.add(data), table.add(data), table.add(data), table.add(data)]
+    await Promise.allSettled(futs)
+
+    table = await conn.openTable(tableName)
+    assert.equal(await table.countRows(), 6)
+  })
+})
+
+describe('LanceDB Mirrored Store Integration test', function () {
+  it('s3://...?mirroredStore=... param is processed correctly', async function () {
+    this.timeout(600000)
+
+    const dir = tmpdir()
+    console.log(dir)
+    const conn = await lancedb.connect(`s3://lancedb-integtest?mirroredStore=${dir}`)
+    const data = Array(200).fill({ vector: Array(128).fill(1.0), id: 0 })
+    data.push(...Array(200).fill({ vector: Array(128).fill(1.0), id: 1 }))
+    data.push(...Array(200).fill({ vector: Array(128).fill(1.0), id: 2 }))
+    data.push(...Array(200).fill({ vector: Array(128).fill(1.0), id: 3 }))
+
+    const tableName = uuidv4()
+
+    // try create table and check if it's mirrored
+    const t = await conn.createTable(tableName, data, { writeMode: lancedb.WriteMode.Overwrite })
+
+    const mirroredPath = path.join(dir, `${tableName}.lance`)
+    fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
+      if (err != null) throw err
+      // there should be three dirs
+      assert.equal(files.length, 3)
+      assert.isTrue(files[0].isDirectory())
+      assert.isTrue(files[1].isDirectory())
+
+      fs.readdir(path.join(mirroredPath, '_transactions'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].name.endsWith('.txn'))
+      })
+
+      fs.readdir(path.join(mirroredPath, '_versions'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].name.endsWith('.manifest'))
+      })
+
+      fs.readdir(path.join(mirroredPath, 'data'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].name.endsWith('.lance'))
+      })
+    })
+
+    // try create index and check if it's mirrored
+    await t.createIndex({ column: 'vector', type: 'ivf_pq' })
+
+    fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
+      if (err != null) throw err
+      // there should be four dirs
+      assert.equal(files.length, 4)
+      assert.isTrue(files[0].isDirectory())
+      assert.isTrue(files[1].isDirectory())
+      assert.isTrue(files[2].isDirectory())
+
+      // Two TXs now
+      fs.readdir(path.join(mirroredPath, '_transactions'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 2)
+        assert.isTrue(files[0].name.endsWith('.txn'))
+        assert.isTrue(files[1].name.endsWith('.txn'))
+      })
+
+      fs.readdir(path.join(mirroredPath, 'data'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].name.endsWith('.lance'))
+      })
+
+      fs.readdir(path.join(mirroredPath, '_indices'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].isDirectory())
+
+        fs.readdir(path.join(mirroredPath, '_indices', files[0].name), { withFileTypes: true }, (err, files) => {
+          if (err != null) throw err
+
+          assert.equal(files.length, 1)
+          assert.isTrue(files[0].isFile())
+          assert.isTrue(files[0].name.endsWith('.idx'))
+        })
+      })
+    })
+
+    // try delete and check if it's mirrored
+    await t.delete('id = 0')
+
+    fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
+      if (err != null) throw err
+      // there should be five dirs
+      assert.equal(files.length, 5)
+      assert.isTrue(files[0].isDirectory())
+      assert.isTrue(files[1].isDirectory())
+      assert.isTrue(files[2].isDirectory())
+      assert.isTrue(files[3].isDirectory())
+      assert.isTrue(files[4].isDirectory())
+
+      // Three TXs now
+      fs.readdir(path.join(mirroredPath, '_transactions'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 3)
+        assert.isTrue(files[0].name.endsWith('.txn'))
+        assert.isTrue(files[1].name.endsWith('.txn'))
+      })
+
+      fs.readdir(path.join(mirroredPath, 'data'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].name.endsWith('.lance'))
+      })
+
+      fs.readdir(path.join(mirroredPath, '_indices'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].isDirectory())
+
+        fs.readdir(path.join(mirroredPath, '_indices', files[0].name), { withFileTypes: true }, (err, files) => {
+          if (err != null) throw err
+
+          assert.equal(files.length, 1)
+          assert.isTrue(files[0].isFile())
+          assert.isTrue(files[0].name.endsWith('.idx'))
+        })
+      })
+
+      fs.readdir(path.join(mirroredPath, '_deletions'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].name.endsWith('.arrow'))
+      })
+    })
+  })
+})

node/src/query.ts

@@ -23,27 +23,29 @@ const { tableSearch } = require('../native.js')
  * A builder for nearest neighbor queries for LanceDB.
  */
 export class Query<T = number[]> {
-  private readonly _query: T
+  private readonly _query?: T
   private readonly _tbl?: any
   private _queryVector?: number[]
-  private _limit: number
+  private _limit?: number
   private _refineFactor?: number
   private _nprobes: number
   private _select?: string[]
   private _filter?: string
   private _metricType?: MetricType
+  private _prefilter: boolean
   protected readonly _embeddings?: EmbeddingFunction<T>
 
-  constructor (query: T, tbl?: any, embeddings?: EmbeddingFunction<T>) {
+  constructor (query?: T, tbl?: any, embeddings?: EmbeddingFunction<T>) {
     this._tbl = tbl
     this._query = query
-    this._limit = 10
+    this._limit = undefined
     this._nprobes = 20
     this._refineFactor = undefined
     this._select = undefined
     this._filter = undefined
     this._metricType = undefined
     this._embeddings = embeddings
+    this._prefilter = false
   }
 
   /***
@@ -102,17 +104,25 @@ export class Query<T = number[]> {
     return this
   }
 
+  prefilter (value: boolean): Query<T> {
+    this._prefilter = value
+    return this
+  }
+
   /**
      * Execute the query and return the results as an Array of Objects
      */
   async execute<T = Record<string, unknown>> (): Promise<T[]> {
+    if (this._query !== undefined) {
       if (this._embeddings !== undefined) {
         this._queryVector = (await this._embeddings.embed([this._query]))[0]
       } else {
         this._queryVector = this._query as number[]
       }
+    }
 
-    const buffer = await tableSearch.call(this._tbl, this)
+    const isElectron = this.isElectron()
+    const buffer = await tableSearch.call(this._tbl, this, isElectron)
     const data = tableFromIPC(buffer)
 
     return data.toArray().map((entry: Record<string, unknown>) => {
@@ -127,4 +137,14 @@ export class Query<T = number[]> {
       return newObject as unknown as T
     })
   }
+
+  // See https://github.com/electron/electron/issues/2288
+  private isElectron (): boolean {
+    try {
+      // eslint-disable-next-line no-prototype-builtins
+      return (process?.versions?.hasOwnProperty('electron') || navigator?.userAgent?.toLowerCase()?.includes(' electron'))
+    } catch (e) {
+      return false
+    }
+  }
 }

node/src/remote/client.ts

@@ -38,6 +38,7 @@ export class HttpLancedbClient {
     vector: number[],
     k: number,
     nprobes: number,
+    prefilter: boolean,
     refineFactor?: number,
     columns?: string[],
     filter?: string
@@ -50,7 +51,8 @@ export class HttpLancedbClient {
                 nprobes,
                 refineFactor,
                 columns,
-                filter
+                filter,
+                prefilter
               },
               {
                 headers: {
@@ -63,6 +65,9 @@ export class HttpLancedbClient {
               }
     ).catch((err) => {
       console.error('error: ', err)
+      if (err.response === undefined) {
+        throw new Error(`Network Error: ${err.message as string}`)
+      }
       return err.response
     })
     if (response.status !== 200) {
@@ -86,13 +91,55 @@ export class HttpLancedbClient {
       {
         headers: {
           'Content-Type': 'application/json',
-          'x-api-key': this._apiKey()
+          'x-api-key': this._apiKey(),
+          ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
         },
         params,
         timeout: 10000
       }
     ).catch((err) => {
       console.error('error: ', err)
+      if (err.response === undefined) {
+        throw new Error(`Network Error: ${err.message as string}`)
+      }
+      return err.response
+    })
+    if (response.status !== 200) {
+      const errorData = new TextDecoder().decode(response.data)
+      throw new Error(
+        `Server Error, status: ${response.status as number}, ` +
+        `message: ${response.statusText as string}: ${errorData}`
+      )
+    }
+    return response
+  }
+
+  /**
+   * Sent POST request.
+   */
+  public async post (
+    path: string,
+    data?: any,
+    params?: Record<string, string | number>,
+    content?: string | undefined
+  ): Promise<AxiosResponse> {
+    const response = await axios.post(
+        `${this._url}${path}`,
+        data,
+        {
+          headers: {
+            'Content-Type': content ?? 'application/json',
+            'x-api-key': this._apiKey(),
+            ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
+          },
+          params,
+          timeout: 30000
+        }
+    ).catch((err) => {
+      console.error('error: ', err)
+      if (err.response === undefined) {
+        throw new Error(`Network Error: ${err.message as string}`)
+      }
       return err.response
     })
     if (response.status !== 200) {

node/src/remote/index.ts

@@ -14,12 +14,18 @@
 
 import {
   type EmbeddingFunction, type Table, type VectorIndexParams, type Connection,
-  type ConnectionOptions
+  type ConnectionOptions, type CreateTableOptions, type VectorIndex,
+  type WriteOptions,
+  type IndexStats,
+  type UpdateArgs, type UpdateSqlArgs
 } from '../index'
 import { Query } from '../query'
 
-import { type Table as ArrowTable, Vector } from 'apache-arrow'
+import { Vector, Table as ArrowTable } from 'apache-arrow'
 import { HttpLancedbClient } from './client'
+import { isEmbeddingFunction } from '../embedding/embedding_function'
+import { createEmptyTable, fromRecordsToStreamBuffer, fromTableToStreamBuffer } from '../arrow'
+import { toSQL } from '../util'
 
 /**
  * Remote connection.
@@ -66,18 +72,64 @@ export class RemoteConnection implements Connection {
     }
   }
 
-  async createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table>
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
-    throw new Error('Not implemented')
+  async createTable<T> (nameOrOpts: string | CreateTableOptions<T>, data?: Array<Record<string, unknown>>, optsOrEmbedding?: WriteOptions | EmbeddingFunction<T>, opt?: WriteOptions): Promise<Table<T>> {
+    // Logic copied from LocatlConnection, refactor these to a base class + connectionImpl pattern
+    let schema
+    let embeddings: undefined | EmbeddingFunction<T>
+    let tableName: string
+    if (typeof nameOrOpts === 'string') {
+      if (optsOrEmbedding !== undefined && isEmbeddingFunction(optsOrEmbedding)) {
+        embeddings = optsOrEmbedding
+      }
+      tableName = nameOrOpts
+    } else {
+      schema = nameOrOpts.schema
+      embeddings = nameOrOpts.embeddingFunction
+      tableName = nameOrOpts.name
     }
 
-  async createTableArrow (name: string, table: ArrowTable): Promise<Table> {
-    throw new Error('Not implemented')
+    let buffer: Buffer
+
+    function isEmpty (data: Array<Record<string, unknown>> | ArrowTable<any>): boolean {
+      if (data instanceof ArrowTable) {
+        return data.data.length === 0
+      }
+      return data.length === 0
+    }
+
+    if ((data === undefined) || isEmpty(data)) {
+      if (schema === undefined) {
+        throw new Error('Either data or schema needs to defined')
+      }
+      buffer = await fromTableToStreamBuffer(createEmptyTable(schema))
+    } else if (data instanceof ArrowTable) {
+      buffer = await fromTableToStreamBuffer(data, embeddings)
+    } else {
+      // data is Array<Record<...>>
+      buffer = await fromRecordsToStreamBuffer(data, embeddings)
+    }
+
+    const res = await this._client.post(
+      `/v1/table/${tableName}/create/`,
+      buffer,
+      undefined,
+      'application/vnd.apache.arrow.stream'
+    )
+    if (res.status !== 200) {
+      throw new Error(`Server Error, status: ${res.status}, ` +
+      // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+        `message: ${res.statusText}: ${res.data}`)
+    }
+
+    if (embeddings === undefined) {
+      return new RemoteTable(this._client, tableName)
+    } else {
+      return new RemoteTable(this._client, tableName, embeddings)
+    }
   }
 
   async dropTable (name: string): Promise<void> {
-    throw new Error('Not implemented')
+    await this._client.post(`/v1/table/${name}/drop/`)
   }
 }
 
@@ -104,6 +156,7 @@ export class RemoteQuery<T = number[]> extends Query<T> {
       queryVector,
       (this as any)._limit,
       (this as any)._nprobes,
+      (this as any)._prefilter,
       (this as any)._refineFactor,
       (this as any)._select,
       (this as any)._filter
@@ -147,11 +200,39 @@ export class RemoteTable<T = number[]> implements Table<T> {
   }
 
   async add (data: Array<Record<string, unknown>>): Promise<number> {
-    throw new Error('Not implemented')
+    const buffer = await fromRecordsToStreamBuffer(data, this._embeddings)
+    const res = await this._client.post(
+      `/v1/table/${this._name}/insert/`,
+      buffer,
+      {
+        mode: 'append'
+      },
+      'application/vnd.apache.arrow.stream'
+    )
+    if (res.status !== 200) {
+      throw new Error(`Server Error, status: ${res.status}, ` +
+      // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+        `message: ${res.statusText}: ${res.data}`)
+    }
+    return data.length
   }
 
   async overwrite (data: Array<Record<string, unknown>>): Promise<number> {
-    throw new Error('Not implemented')
+    const buffer = await fromRecordsToStreamBuffer(data, this._embeddings)
+    const res = await this._client.post(
+      `/v1/table/${this._name}/insert/`,
+      buffer,
+      {
+        mode: 'overwrite'
+      },
+      'application/vnd.apache.arrow.stream'
+    )
+    if (res.status !== 200) {
+      throw new Error(`Server Error, status: ${res.status}, ` +
+      // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+        `message: ${res.statusText}: ${res.data}`)
+    }
+    return data.length
   }
 
   async createIndex (indexParams: VectorIndexParams): Promise<any> {
@@ -159,10 +240,48 @@ export class RemoteTable<T = number[]> implements Table<T> {
   }
 
   async countRows (): Promise<number> {
-    throw new Error('Not implemented')
+    const result = await this._client.post(`/v1/table/${this._name}/describe/`)
+    return result.data?.stats?.num_rows
   }
 
   async delete (filter: string): Promise<void> {
-    throw new Error('Not implemented')
+    await this._client.post(`/v1/table/${this._name}/delete/`, { predicate: filter })
+  }
+
+  async update (args: UpdateArgs | UpdateSqlArgs): Promise<void> {
+    let filter: string | null
+    let updates: Record<string, string>
+
+    if ('valuesSql' in args) {
+      filter = args.where ?? null
+      updates = args.valuesSql
+    } else {
+      filter = args.where ?? null
+      updates = {}
+      for (const [key, value] of Object.entries(args.values)) {
+        updates[key] = toSQL(value)
+      }
+    }
+    await this._client.post(`/v1/table/${this._name}/update/`, {
+      predicate: filter,
+      updates: Object.entries(updates).map(([key, value]) => [key, value])
+    })
+  }
+
+  async listIndices (): Promise<VectorIndex[]> {
+    const results = await this._client.post(`/v1/table/${this._name}/index/list/`)
+    return results.data.indexes?.map((index: any) => ({
+      columns: index.columns,
+      name: index.index_name,
+      uuid: index.index_uuid
+    }))
+  }
+
+  async indexStats (indexUuid: string): Promise<IndexStats> {
+    const results = await this._client.post(`/v1/table/${this._name}/index/${indexUuid}/stats/`)
+    return {
+      numIndexedRows: results.data.num_indexed_rows,
+      numUnindexedRows: results.data.num_unindexed_rows
+    }
   }
 }

node/src/test/io.ts

@@ -47,7 +47,9 @@ describe('LanceDB S3 client', function () {
         }
       }
       const table = await createTestDB(opts, 2, 20)
+      console.log(table)
       const con = await lancedb.connect(opts)
+      console.log(con)
       assert.equal(con.uri, opts.uri)
 
       const results = await table.search([0.1, 0.3]).limit(5).execute()
@@ -70,5 +72,5 @@ async function createTestDB (opts: ConnectionOptions, numDimensions: number = 2,
     data.push({ id: i + 1, name: `name_${i}`, price: i + 10, is_active: (i % 2 === 0), vector })
   }
 
-  return await con.createTable('vectors', data)
+  return await con.createTable('vectors_2', data)
 }

node/src/test/test.ts

@@ -18,7 +18,8 @@ import * as chai from 'chai'
 import * as chaiAsPromised from 'chai-as-promised'
 
 import * as lancedb from '../index'
-import { type AwsCredentials, type EmbeddingFunction, MetricType, Query, WriteMode, DefaultWriteOptions, isWriteOptions } from '../index'
+import { type AwsCredentials, type EmbeddingFunction, MetricType, Query, WriteMode, DefaultWriteOptions, isWriteOptions, type LocalTable } from '../index'
+import { FixedSizeList, Field, Int32, makeVector, Schema, Utf8, Table as ArrowTable, vectorFromArray, Float32 } from 'apache-arrow'
 
 const expect = chai.expect
 const assert = chai.assert
@@ -77,12 +78,31 @@ describe('LanceDB client', function () {
     })
 
     it('limits # of results', async function () {
-      const uri = await createTestDB()
+      const uri = await createTestDB(2, 100)
       const con = await lancedb.connect(uri)
       const table = await con.openTable('vectors')
-      const results = await table.search([0.1, 0.3]).limit(1).execute()
+      let results = await table.search([0.1, 0.3]).limit(1).execute()
       assert.equal(results.length, 1)
       assert.equal(results[0].id, 1)
+
+      // there is a default limit if unspecified
+      results = await table.search([0.1, 0.3]).execute()
+      assert.equal(results.length, 10)
+    })
+
+    it('uses a filter / where clause without vector search', async function () {
+      // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
+      const assertResults = (results: Array<Record<string, unknown>>) => {
+        assert.equal(results.length, 50)
+      }
+
+      const uri = await createTestDB(2, 100)
+      const con = await lancedb.connect(uri)
+      const table = (await con.openTable('vectors')) as LocalTable
+      let results = await table.filter('id % 2 = 0').execute()
+      assertResults(results)
+      results = await table.where('id % 2 = 0').execute()
+      assertResults(results)
     })
 
     it('uses a filter / where clause', async function () {
@@ -101,15 +121,29 @@ describe('LanceDB client', function () {
       assertResults(results)
     })
 
+    it('should correctly process prefilter/postfilter', async function () {
+      const uri = await createTestDB(16, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+      // post filter should return less than the limit
+      let results = await table.search(new Array(16).fill(0.1)).limit(10).filter('id >= 10').prefilter(false).execute()
+      assert.isTrue(results.length < 10)
+
+      // pre filter should return exactly the limit
+      results = await table.search(new Array(16).fill(0.1)).limit(10).filter('id >= 10').prefilter(true).execute()
+      assert.isTrue(results.length === 10)
+    })
+
     it('select only a subset of columns', async function () {
       const uri = await createTestDB()
       const con = await lancedb.connect(uri)
       const table = await con.openTable('vectors')
       const results = await table.search([0.1, 0.1]).select(['is_active']).execute()
       assert.equal(results.length, 2)
-      // vector and score are always returned
+      // vector and _distance are always returned
       assert.isDefined(results[0].vector)
-      assert.isDefined(results[0].score)
+      assert.isDefined(results[0]._distance)
       assert.isDefined(results[0].is_active)
 
       assert.isUndefined(results[0].id)
@@ -119,6 +153,45 @@ describe('LanceDB client', function () {
   })
 
   describe('when creating a new dataset', function () {
+    it('create an empty table', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const schema = new Schema(
+        [new Field('id', new Int32()), new Field('name', new Utf8())]
+      )
+      const table = await con.createTable({ name: 'vectors', schema })
+      assert.equal(table.name, 'vectors')
+      assert.deepEqual(await con.tableNames(), ['vectors'])
+    })
+
+    it('create a table with a empty data array', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const schema = new Schema(
+        [new Field('id', new Int32()), new Field('name', new Utf8())]
+      )
+      const table = await con.createTable({ name: 'vectors', schema, data: [] })
+      assert.equal(table.name, 'vectors')
+      assert.deepEqual(await con.tableNames(), ['vectors'])
+    })
+
+    it('create a table from an Arrow Table', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const i32s = new Int32Array(new Array<number>(10))
+      const i32 = makeVector(i32s)
+
+      const data = new ArrowTable({ vector: i32 })
+
+      const table = await con.createTable({ name: 'vectors', data })
+      assert.equal(table.name, 'vectors')
+      assert.equal(await table.countRows(), 10)
+      assert.deepEqual(await con.tableNames(), ['vectors'])
+    })
+
     it('creates a new table from javascript objects', async function () {
       const dir = await track().mkdir('lancejs')
       const con = await lancedb.connect(dir)
@@ -206,6 +279,46 @@ describe('LanceDB client', function () {
       assert.equal(await table.countRows(), 2)
     })
 
+    it('can update records in the table', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+
+      const table = await con.openTable('vectors')
+      assert.equal(await table.countRows(), 2)
+
+      await table.update({ where: 'price = 10', valuesSql: { price: '100' } })
+      const results = await table.search([0.1, 0.2]).execute()
+      assert.equal(results[0].price, 100)
+      assert.equal(results[1].price, 11)
+    })
+
+    it('can update the records using a literal value', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+
+      const table = await con.openTable('vectors')
+      assert.equal(await table.countRows(), 2)
+
+      await table.update({ where: 'price = 10', values: { price: 100 } })
+      const results = await table.search([0.1, 0.2]).execute()
+      assert.equal(results[0].price, 100)
+      assert.equal(results[1].price, 11)
+    })
+
+    it('can update every record in the table', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+
+      const table = await con.openTable('vectors')
+      assert.equal(await table.countRows(), 2)
+
+      await table.update({ valuesSql: { price: '100' } })
+      const results = await table.search([0.1, 0.2]).execute()
+
+      assert.equal(results[0].price, 100)
+      assert.equal(results[1].price, 100)
+    })
+
     it('can delete records from a table', async function () {
       const uri = await createTestDB()
       const con = await lancedb.connect(uri)
@@ -218,6 +331,37 @@ describe('LanceDB client', function () {
     })
   })
 
+  describe('when searching an empty dataset', function () {
+    it('should not fail', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const schema = new Schema(
+        [new Field('vector', new FixedSizeList(128, new Field('float32', new Float32())))]
+      )
+      const table = await con.createTable({ name: 'vectors', schema })
+      const result = await table.search(Array(128).fill(0.1)).execute()
+      assert.isEmpty(result)
+    })
+  })
+
+  describe('when searching an empty-after-delete dataset', function () {
+    it('should not fail', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const schema = new Schema(
+        [new Field('vector', new FixedSizeList(128, new Field('float32', new Float32())))]
+      )
+      const table = await con.createTable({ name: 'vectors', schema })
+      await table.add([{ vector: Array(128).fill(0.1) }])
+      // https://github.com/lancedb/lance/issues/1635
+      await table.delete('true')
+      const result = await table.search(Array(128).fill(0.1)).execute()
+      assert.isEmpty(result)
+    })
+  })
+
   describe('when creating a vector index', function () {
     it('overwrite all records in a table', async function () {
       const uri = await createTestDB(32, 300)
@@ -258,6 +402,24 @@ describe('LanceDB client', function () {
       const createIndex = table.createIndex({ type: 'ivf_pq', column: 'name', num_partitions: -1, max_iters: 2, num_sub_vectors: 2 })
       await expect(createIndex).to.be.rejectedWith('num_partitions: must be > 0')
     })
+
+    it('should be able to list index and stats', async function () {
+      const uri = await createTestDB(32, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+
+      const indices = await table.listIndices()
+      expect(indices).to.have.lengthOf(1)
+      expect(indices[0].name).to.equal('vector_idx')
+      expect(indices[0].uuid).to.not.be.equal(undefined)
+      expect(indices[0].columns).to.have.lengthOf(1)
+      expect(indices[0].columns[0]).to.equal('vector')
+
+      const stats = await table.indexStats(indices[0].uuid)
+      expect(stats.numIndexedRows).to.equal(300)
+      expect(stats.numUnindexedRows).to.equal(0)
+    }).timeout(50_000)
   })
 
   describe('when using a custom embedding function', function () {
@@ -291,6 +453,54 @@ describe('LanceDB client', function () {
       const results = await table.search('foo').execute()
       assert.equal(results.length, 2)
     })
+
+    it('should create embeddings for Arrow Table', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+      const embeddingFunction = new TextEmbedding('name')
+
+      const names = vectorFromArray(['foo', 'bar'], new Utf8())
+      const data = new ArrowTable({ name: names })
+
+      const table = await con.createTable({ name: 'vectors', data, embeddingFunction })
+      assert.equal(table.name, 'vectors')
+      const results = await table.search('foo').execute()
+      assert.equal(results.length, 2)
+    })
+  })
+})
+
+describe('Remote LanceDB client', function () {
+  describe('when the server is not reachable', function () {
+    it('produces a network error', async function () {
+      const con = await lancedb.connect({
+        uri: 'db://test-1234',
+        region: 'asdfasfasfdf',
+        apiKey: 'some-api-key'
+      })
+
+      // GET
+      try {
+        await con.tableNames()
+      } catch (err) {
+        expect(err).to.have.property('message', 'Network Error: getaddrinfo ENOTFOUND test-1234.asdfasfasfdf.api.lancedb.com')
+      }
+
+      // POST
+      try {
+        await con.createTable({ name: 'vectors', schema: new Schema([]) })
+      } catch (err) {
+        expect(err).to.have.property('message', 'Network Error: getaddrinfo ENOTFOUND test-1234.asdfasfasfdf.api.lancedb.com')
+      }
+
+      // Search
+      const table = await con.openTable('vectors')
+      try {
+        await table.search([0.1, 0.3]).execute()
+      } catch (err) {
+        expect(err).to.have.property('message', 'Network Error: getaddrinfo ENOTFOUND test-1234.asdfasfasfdf.api.lancedb.com')
+      }
+    })
   })
 })
 
@@ -362,3 +572,45 @@ describe('WriteOptions', function () {
     })
   })
 })
+
+describe('Compact and cleanup', function () {
+  it('can cleanup after compaction', async function () {
+    const dir = await track().mkdir('lancejs')
+    const con = await lancedb.connect(dir)
+
+    const data = [
+      { price: 10, name: 'foo', vector: [1, 2, 3] },
+      { price: 50, name: 'bar', vector: [4, 5, 6] }
+    ]
+    const table = await con.createTable('t1', data) as LocalTable
+
+    const newData = [
+      { price: 30, name: 'baz', vector: [7, 8, 9] }
+    ]
+    await table.add(newData)
+
+    const compactionMetrics = await table.compactFiles({
+      numThreads: 2
+    })
+    assert.equal(compactionMetrics.fragmentsRemoved, 2)
+    assert.equal(compactionMetrics.fragmentsAdded, 1)
+    assert.equal(await table.countRows(), 3)
+
+    await table.cleanupOldVersions()
+    assert.equal(await table.countRows(), 3)
+
+    // should have no effect, but this validates the arguments are parsed.
+    await table.compactFiles({
+      targetRowsPerFragment: 102410,
+      maxRowsPerGroup: 1024,
+      materializeDeletions: true,
+      materializeDeletionsThreshold: 0.5,
+      numThreads: 2
+    })
+
+    const cleanupMetrics = await table.cleanupOldVersions(0, true)
+    assert.isAtLeast(cleanupMetrics.bytesRemoved, 1)
+    assert.isAtLeast(cleanupMetrics.oldVersions, 1)
+    assert.equal(await table.countRows(), 3)
+  })
+})

node/src/test/util.ts

@@ -0,0 +1,45 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { toSQL } from '../util'
+import * as chai from 'chai'
+
+const expect = chai.expect
+
+describe('toSQL', function () {
+  it('should turn string to SQL expression', function () {
+    expect(toSQL('foo')).to.equal("'foo'")
+  })
+
+  it('should turn number to SQL expression', function () {
+    expect(toSQL(123)).to.equal('123')
+  })
+
+  it('should turn boolean to SQL expression', function () {
+    expect(toSQL(true)).to.equal('TRUE')
+  })
+
+  it('should turn null to SQL expression', function () {
+    expect(toSQL(null)).to.equal('NULL')
+  })
+
+  it('should turn Date to SQL expression', function () {
+    const date = new Date('05 October 2011 14:48 UTC')
+    expect(toSQL(date)).to.equal("'2011-10-05T14:48:00.000Z'")
+  })
+
+  it('should turn array to SQL expression', function () {
+    expect(toSQL(['foo', 'bar', true, 1])).to.equal("['foo', 'bar', TRUE, 1]")
+  })
+})

node/src/util.ts

@@ -0,0 +1,44 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+export type Literal = string | number | boolean | null | Date | Literal[]
+
+export function toSQL (value: Literal): string {
+  if (typeof value === 'string') {
+    return `'${value}'`
+  }
+
+  if (typeof value === 'number') {
+    return value.toString()
+  }
+
+  if (typeof value === 'boolean') {
+    return value ? 'TRUE' : 'FALSE'
+  }
+
+  if (value === null) {
+    return 'NULL'
+  }
+
+  if (value instanceof Date) {
+    return `'${value.toISOString()}'`
+  }
+
+  if (Array.isArray(value)) {
+    return `[${value.map(toSQL).join(', ')}]`
+  }
+
+  // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+  throw new Error(`Unsupported value type: ${typeof value} value: (${value})`)
+}

python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.16
+current_version = 0.3.6
 commit = True
 message = [python] Bump version: {current_version} → {new_version}
 tag = True

python/LICENSE

@@ -0,0 +1 @@
+../LICENSE

python/README.md

@@ -16,7 +16,7 @@ pip install lancedb
 import lancedb
 db = lancedb.connect('<PATH_TO_LANCEDB_DATASET>')
 table = db.open_table('my_table')
-results = table.search([0.1, 0.3]).limit(20).to_df()
+results = table.search([0.1, 0.3]).limit(20).to_list()
 print(results)
 ```
 

python/lancedb/__init__.py

@@ -11,18 +11,23 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import importlib.metadata
 from typing import Optional
 
-from .db import URI, DBConnection, LanceDBConnection
+__version__ = importlib.metadata.version("lancedb")
+
+from .common import URI
+from .db import DBConnection, LanceDBConnection
 from .remote.db import RemoteDBConnection
-from .schema import vector
+from .schema import vector  # noqa: F401
+from .utils import sentry_log  # noqa: F401
 
 
 def connect(
     uri: URI,
     *,
     api_key: Optional[str] = None,
-    region: str = "us-west-2",
+    region: str = "us-east-1",
     host_override: Optional[str] = None,
 ) -> DBConnection:
     """Connect to a LanceDB database.
@@ -31,9 +36,13 @@ def connect(
     ----------
     uri: str or Path
         The uri of the database.
-    api_token: str, optional
+    api_key: str, optional
         If presented, connect to LanceDB cloud.
         Otherwise, connect to a database on file system or cloud storage.
+    region: str, default "us-east-1"
+        The region to use for LanceDB Cloud.
+    host_override: str, optional
+        The override url for LanceDB Cloud.
 
     Examples
     --------

python/lancedb/cli/__init__.py

@@ -0,0 +1,12 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.

python/lancedb/cli/cli.py

@@ -0,0 +1,46 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import click
+
+from lancedb.utils import CONFIG
+
+
+@click.group()
+@click.version_option(help="LanceDB command line interface entry point")
+def cli():
+    "LanceDB command line interface"
+
+
+diagnostics_help = """
+Enable or disable LanceDB diagnostics. When enabled, LanceDB will send anonymous events to help us improve LanceDB.
+These diagnostics are used only for error reporting and no data is collected. You can find more about diagnosis on
+our docs: https://lancedb.github.io/lancedb/cli_config/
+"""
+
+
+@cli.command(help=diagnostics_help)
+@click.option("--enabled/--disabled", default=True)
+def diagnostics(enabled):
+    CONFIG.update({"diagnostics": True if enabled else False})
+    click.echo("LanceDB diagnostics is %s" % ("enabled" if enabled else "disabled"))
+
+
+@cli.command(help="Show current LanceDB configuration")
+def config():
+    # TODO: pretty print as table with colors and formatting
+    click.echo("Current LanceDB configuration:")
+    cfg = CONFIG.copy()
+    cfg.pop("uuid")  # Don't show uuid as it is not configurable
+    for item, amount in cfg.items():
+        click.echo("{} ({})".format(item, amount))

python/lancedb/conftest.py

@@ -1,7 +1,12 @@
 import os
+import time
+from typing import Any
 
+import numpy as np
 import pytest
 
+from .embeddings import EmbeddingFunctionRegistry, TextEmbeddingFunction
+
 # import lancedb so we don't have to in every example
 
 
@@ -14,3 +19,47 @@ def doctest_setup(monkeypatch, tmpdir):
     monkeypatch.setitem(os.environ, "COLUMNS", "80")
     # Work in a temporary directory
     monkeypatch.chdir(tmpdir)
+
+
+registry = EmbeddingFunctionRegistry.get_instance()
+
+
+@registry.register("test")
+class MockTextEmbeddingFunction(TextEmbeddingFunction):
+    """
+    Return the hash of the first 10 characters
+    """
+
+    def generate_embeddings(self, texts):
+        return [self._compute_one_embedding(row) for row in texts]
+
+    def _compute_one_embedding(self, row):
+        emb = np.array([float(hash(c)) for c in row[:10]])
+        emb /= np.linalg.norm(emb)
+        return emb
+
+    def ndims(self):
+        return 10
+
+
+class RateLimitedAPI:
+    rate_limit = 0.1  # 1 request per 0.1 second
+    last_request_time = 0
+
+    @staticmethod
+    def make_request():
+        current_time = time.time()
+
+        if current_time - RateLimitedAPI.last_request_time < RateLimitedAPI.rate_limit:
+            raise Exception("Rate limit exceeded. Please try again later.")
+
+        # Simulate a successful request
+        RateLimitedAPI.last_request_time = current_time
+        return "Request successful"
+
+
+@registry.register("test-rate-limited")
+class MockRateLimitedEmbeddingFunction(MockTextEmbeddingFunction):
+    def generate_embeddings(self, texts):
+        RateLimitedAPI.make_request()
+        return [self._compute_one_embedding(row) for row in texts]

python/lancedb/context.py

@@ -12,6 +12,9 @@
 #  limitations under the License.
 from __future__ import annotations
 
+import deprecation
+
+from . import __version__
 from .exceptions import MissingColumnError, MissingValueError
 from .util import safe_import_pandas
 
@@ -43,7 +46,7 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
     this how many tokens, but depending on the input data, it could be sentences,
     paragraphs, messages, etc.
 
-    >>> contextualize(data).window(3).stride(1).text_col('token').to_df()
+    >>> contextualize(data).window(3).stride(1).text_col('token').to_pandas()
                     token  document_id
     0     The quick brown            1
     1     quick brown fox            1
@@ -56,7 +59,7 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
     8          dog I love            1
     9   I love sandwiches            2
     10    love sandwiches            2
-    >>> contextualize(data).window(7).stride(1).min_window_size(7).text_col('token').to_df()
+    >>> contextualize(data).window(7).stride(1).min_window_size(7).text_col('token').to_pandas()
                                       token  document_id
     0   The quick brown fox jumped over the            1
     1  quick brown fox jumped over the lazy            1
@@ -68,7 +71,7 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
     ``stride`` determines how many rows to skip between each window start. This can
     be used to reduce the total number of windows generated.
 
-    >>> contextualize(data).window(4).stride(2).text_col('token').to_df()
+    >>> contextualize(data).window(4).stride(2).text_col('token').to_pandas()
                         token  document_id
     0     The quick brown fox            1
     2   brown fox jumped over            1
@@ -81,7 +84,9 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
     context windows that don't cross document boundaries. In this case, we can
     pass ``document_id`` as the group by.
 
-    >>> contextualize(data).window(4).stride(2).text_col('token').groupby('document_id').to_df()
+    >>> (contextualize(data)
+    ...     .window(4).stride(2).text_col('token').groupby('document_id')
+    ...     .to_pandas())
                        token  document_id
     0    The quick brown fox            1
     2  brown fox jumped over            1
@@ -89,18 +94,24 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
     6           the lazy dog            1
     9      I love sandwiches            2
 
-    ``min_window_size`` determines the minimum size of the  context windows that are generated
-    This can be used to trim the last few context windows which have size less than
-    ``min_window_size``. By default context windows of size 1 are skipped.
+    ``min_window_size`` determines the minimum size of the context windows
+    that are generated.This can be used to trim the last few context windows
+    which have size less than ``min_window_size``.
+    By default context windows of size 1 are skipped.
 
-    >>> contextualize(data).window(6).stride(3).text_col('token').groupby('document_id').to_df()
+    >>> (contextualize(data)
+    ...     .window(6).stride(3).text_col('token').groupby('document_id')
+    ...     .to_pandas())
                                  token  document_id
     0  The quick brown fox jumped over            1
     3     fox jumped over the lazy dog            1
     6                     the lazy dog            1
     9                I love sandwiches            2
 
-    >>> contextualize(data).window(6).stride(3).min_window_size(4).text_col('token').groupby('document_id').to_df()
+    >>> (contextualize(data)
+    ...     .window(6).stride(3).min_window_size(4).text_col('token')
+    ...     .groupby('document_id')
+    ...     .to_pandas())
                                  token  document_id
     0  The quick brown fox jumped over            1
     3     fox jumped over the lazy dog            1
@@ -110,7 +121,9 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
 
 
 class Contextualizer:
-    """Create context windows from a DataFrame. See [lancedb.context.contextualize][]."""
+    """Create context windows from a DataFrame.
+    See [lancedb.context.contextualize][].
+    """
 
     def __init__(self, raw_df):
         self._text_col = None
@@ -176,7 +189,16 @@ class Contextualizer:
         self._min_window_size = min_window_size
         return self
 
+    @deprecation.deprecated(
+        deprecated_in="0.3.1",
+        removed_in="0.4.0",
+        current_version=__version__,
+        details="Use to_pandas() instead",
+    )
     def to_df(self) -> "pd.DataFrame":
+        return self.to_pandas()
+
+    def to_pandas(self) -> "pd.DataFrame":
         """Create the context windows and return a DataFrame."""
         if pd is None:
             raise ImportError(

python/lancedb/db.py

@@ -14,24 +14,39 @@
 from __future__ import annotations
 
 import os
-from abc import ABC, abstractmethod
+from abc import abstractmethod
 from pathlib import Path
-from typing import Optional
+from typing import TYPE_CHECKING, Iterable, List, Optional, Union
 
 import pyarrow as pa
+from overrides import EnforceOverrides, override
 from pyarrow import fs
 
-from .common import DATA, URI
 from .table import LanceTable, Table
 from .util import fs_from_uri, get_uri_location, get_uri_scheme
 
+if TYPE_CHECKING:
+    from .common import DATA, URI
+    from .embeddings import EmbeddingFunctionConfig
+    from .pydantic import LanceModel
 
-class DBConnection(ABC):
+
+class DBConnection(EnforceOverrides):
     """An active LanceDB connection interface."""
 
     @abstractmethod
-    def table_names(self) -> list[str]:
-        """List all table names in the database."""
+    def table_names(
+        self, page_token: Optional[str] = None, limit: int = 10
+    ) -> Iterable[str]:
+        """List all table in this database
+
+        Parameters
+        ----------
+        page_token: str, optional
+            The token to use for pagination. If not present, start from the beginning.
+        limit: int, default 10
+            The size of the page to return.
+        """
         pass
 
     @abstractmethod
@@ -39,10 +54,11 @@ class DBConnection(ABC):
         self,
         name: str,
         data: Optional[DATA] = None,
-        schema: Optional[pa.Schema] = None,
+        schema: Optional[Union[pa.Schema, LanceModel]] = None,
         mode: str = "create",
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
+        embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
     ) -> Table:
         """Create a [Table][lancedb.table.Table] in the database.
 
@@ -50,12 +66,24 @@ class DBConnection(ABC):
         ----------
         name: str
             The name of the table.
-        data: list, tuple, dict, pd.DataFrame; optional
-            The data to initialize the table. User must provide at least one of `data` or `schema`.
-        schema: pyarrow.Schema; optional
-            The schema of the table.
+        data: The data to initialize the table, *optional*
+            User must provide at least one of `data` or `schema`.
+            Acceptable types are:
+
+            - dict or list-of-dict
+
+            - pandas.DataFrame
+
+            - pyarrow.Table or pyarrow.RecordBatch
+        schema: The schema of the table, *optional*
+            Acceptable types are:
+
+            - pyarrow.Schema
+
+            - [LanceModel][lancedb.pydantic.LanceModel]
         mode: str; default "create"
-            The mode to use when creating the table. Can be either "create" or "overwrite".
+            The mode to use when creating the table.
+            Can be either "create" or "overwrite".
             By default, if the table already exists, an exception is raised.
             If you want to overwrite the table, use mode="overwrite".
         on_bad_vectors: str, default "error"
@@ -148,14 +176,15 @@ class DBConnection(ABC):
         ...     for i in range(5):
         ...         yield pa.RecordBatch.from_arrays(
         ...             [
-        ...                 pa.array([[3.1, 4.1], [5.9, 26.5]]),
+        ...                 pa.array([[3.1, 4.1], [5.9, 26.5]],
+        ...                     pa.list_(pa.float32(), 2)),
         ...                 pa.array(["foo", "bar"]),
         ...                 pa.array([10.0, 20.0]),
         ...             ],
         ...             ["vector", "item", "price"],
         ...         )
         >>> schema=pa.schema([
-        ...     pa.field("vector", pa.list_(pa.float32())),
+        ...     pa.field("vector", pa.list_(pa.float32(), 2)),
         ...     pa.field("item", pa.utf8()),
         ...     pa.field("price", pa.float32()),
         ... ])
@@ -192,6 +221,13 @@ class DBConnection(ABC):
         """
         raise NotImplementedError
 
+    def drop_database(self):
+        """
+        Drop database
+        This is the same thing as dropping all the tables
+        """
+        raise NotImplementedError
+
 
 class LanceDBConnection(DBConnection):
     """
@@ -240,12 +276,15 @@ class LanceDBConnection(DBConnection):
     def uri(self) -> str:
         return self._uri
 
-    def table_names(self) -> list[str]:
-        """Get the names of all tables in the database.
+    @override
+    def table_names(
+        self, page_token: Optional[str] = None, limit: int = 10
+    ) -> Iterable[str]:
+        """Get the names of all tables in the database. The names are sorted.
 
         Returns
         -------
-        list of str
+        Iterator of str.
             A list of table names.
         """
         try:
@@ -265,6 +304,7 @@ class LanceDBConnection(DBConnection):
             for file_info in paths
             if file_info.extension == "lance"
         ]
+        tables.sort()
         return tables
 
     def __len__(self) -> int:
@@ -273,14 +313,16 @@ class LanceDBConnection(DBConnection):
     def __contains__(self, name: str) -> bool:
         return name in self.table_names()
 
+    @override
     def create_table(
         self,
         name: str,
         data: Optional[DATA] = None,
-        schema: pa.Schema = None,
+        schema: Optional[Union[pa.Schema, LanceModel]] = None,
         mode: str = "create",
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
+        embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
     ) -> LanceTable:
         """Create a table in the database.
 
@@ -299,9 +341,11 @@ class LanceDBConnection(DBConnection):
             mode=mode,
             on_bad_vectors=on_bad_vectors,
             fill_value=fill_value,
+            embedding_functions=embedding_functions,
         )
         return tbl
 
+    @override
     def open_table(self, name: str) -> LanceTable:
         """Open a table in the database.
 
@@ -316,6 +360,7 @@ class LanceDBConnection(DBConnection):
         """
         return LanceTable.open(self, name)
 
+    @override
     def drop_table(self, name: str, ignore_missing: bool = False):
         """Drop a table from the database.
 
@@ -333,3 +378,8 @@ class LanceDBConnection(DBConnection):
         except FileNotFoundError:
             if not ignore_missing:
                 raise
+
+    @override
+    def drop_database(self):
+        filesystem, path = fs_from_uri(self.uri)
+        filesystem.delete_dir(path)

python/lancedb/embeddings/__init__.py

@@ -0,0 +1,22 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# ruff: noqa: F401
+from .base import EmbeddingFunction, EmbeddingFunctionConfig, TextEmbeddingFunction
+from .cohere import CohereEmbeddingFunction
+from .instructor import InstructorEmbeddingFunction
+from .open_clip import OpenClipEmbeddings
+from .openai import OpenAIEmbeddings
+from .registry import EmbeddingFunctionRegistry, get_registry
+from .sentence_transformers import SentenceTransformerEmbeddings
+from .utils import with_embeddings

python/lancedb/embeddings/base.py

@@ -0,0 +1,181 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+import importlib
+from abc import ABC, abstractmethod
+from typing import List, Union
+
+import numpy as np
+import pyarrow as pa
+from pydantic import BaseModel, Field, PrivateAttr
+
+from .utils import TEXT, retry_with_exponential_backoff
+
+
+class EmbeddingFunction(BaseModel, ABC):
+    """
+    An ABC for embedding functions.
+
+    All concrete embedding functions must implement the following:
+    1. compute_query_embeddings() which takes a query and returns a list of embeddings
+    2. get_source_embeddings() which returns a list of embeddings for the source column
+    For text data, the two will be the same. For multi-modal data, the source column
+    might be images and the vector column might be text.
+    3. ndims method which returns the number of dimensions of the vector column
+    """
+
+    __slots__ = ("__weakref__",)  # pydantic 1.x compatibility
+    max_retries: int = (
+        7  # Setitng 0 disables retires. Maybe this should not be enabled by default,
+    )
+    _ndims: int = PrivateAttr()
+
+    @classmethod
+    def create(cls, **kwargs):
+        """
+        Create an instance of the embedding function
+        """
+        return cls(**kwargs)
+
+    @abstractmethod
+    def compute_query_embeddings(self, *args, **kwargs) -> List[np.array]:
+        """
+        Compute the embeddings for a given user query
+        """
+        pass
+
+    @abstractmethod
+    def compute_source_embeddings(self, *args, **kwargs) -> List[np.array]:
+        """
+        Compute the embeddings for the source column in the database
+        """
+        pass
+
+    def compute_query_embeddings_with_retry(self, *args, **kwargs) -> List[np.array]:
+        """
+        Compute the embeddings for a given user query with retries
+        """
+        return retry_with_exponential_backoff(
+            self.compute_query_embeddings, max_retries=self.max_retries
+        )(
+            *args,
+            **kwargs,
+        )
+
+    def compute_source_embeddings_with_retry(self, *args, **kwargs) -> List[np.array]:
+        """
+        Compute the embeddings for the source column in the database with retries
+        """
+        return retry_with_exponential_backoff(
+            self.compute_source_embeddings, max_retries=self.max_retries
+        )(*args, **kwargs)
+
+    def sanitize_input(self, texts: TEXT) -> Union[List[str], np.ndarray]:
+        """
+        Sanitize the input to the embedding function.
+        """
+        if isinstance(texts, str):
+            texts = [texts]
+        elif isinstance(texts, pa.Array):
+            texts = texts.to_pylist()
+        elif isinstance(texts, pa.ChunkedArray):
+            texts = texts.combine_chunks().to_pylist()
+        return texts
+
+    @classmethod
+    def safe_import(cls, module: str, mitigation=None):
+        """
+        Import the specified module. If the module is not installed,
+        raise an ImportError with a helpful message.
+
+        Parameters
+        ----------
+        module : str
+            The name of the module to import
+        mitigation : Optional[str]
+            The package(s) to install to mitigate the error.
+            If not provided then the module name will be used.
+        """
+        try:
+            return importlib.import_module(module)
+        except ImportError:
+            raise ImportError(f"Please install {mitigation or module}")
+
+    def safe_model_dump(self):
+        from ..pydantic import PYDANTIC_VERSION
+
+        if PYDANTIC_VERSION.major < 2:
+            return dict(self)
+        return self.model_dump()
+
+    @abstractmethod
+    def ndims(self):
+        """
+        Return the dimensions of the vector column
+        """
+        pass
+
+    def SourceField(self, **kwargs):
+        """
+        Creates a pydantic Field that can automatically annotate
+        the source column for this embedding function
+        """
+        return Field(json_schema_extra={"source_column_for": self}, **kwargs)
+
+    def VectorField(self, **kwargs):
+        """
+        Creates a pydantic Field that can automatically annotate
+        the target vector column for this embedding function
+        """
+        return Field(json_schema_extra={"vector_column_for": self}, **kwargs)
+
+    def __eq__(self, __value: object) -> bool:
+        if not hasattr(__value, "__dict__"):
+            return False
+        return vars(self) == vars(__value)
+
+    def __hash__(self) -> int:
+        return hash(frozenset(vars(self).items()))
+
+
+class EmbeddingFunctionConfig(BaseModel):
+    """
+    This model encapsulates the configuration for a embedding function
+    in a lancedb table. It holds the embedding function, the source column,
+    and the vector column
+    """
+
+    vector_column: str
+    source_column: str
+    function: EmbeddingFunction
+
+
+class TextEmbeddingFunction(EmbeddingFunction):
+    """
+    A callable ABC for embedding functions that take text as input
+    """
+
+    def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]:
+        return self.compute_source_embeddings(query, *args, **kwargs)
+
+    def compute_source_embeddings(self, texts: TEXT, *args, **kwargs) -> List[np.array]:
+        texts = self.sanitize_input(texts)
+        return self.generate_embeddings(texts)
+
+    @abstractmethod
+    def generate_embeddings(
+        self, texts: Union[List[str], np.ndarray]
+    ) -> List[np.array]:
+        """
+        Generate the embeddings for the given texts
+        """
+        pass

python/lancedb/embeddings/cohere.py

@@ -0,0 +1,91 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+from typing import ClassVar, List, Union
+
+import numpy as np
+
+from .base import TextEmbeddingFunction
+from .registry import register
+from .utils import api_key_not_found_help
+
+
+@register("cohere")
+class CohereEmbeddingFunction(TextEmbeddingFunction):
+    """
+    An embedding function that uses the Cohere API
+
+    https://docs.cohere.com/docs/multilingual-language-models
+
+    Parameters
+    ----------
+    name: str, default "embed-multilingual-v2.0"
+        The name of the model to use. See the Cohere documentation for
+        a list of available models.
+
+    Examples
+    --------
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import EmbeddingFunctionRegistry
+
+    cohere = EmbeddingFunctionRegistry
+        .get_instance()
+        .get("cohere")
+        .create(name="embed-multilingual-v2.0")
+
+    class TextModel(LanceModel):
+        text: str = cohere.SourceField()
+        vector: Vector(cohere.ndims()) =  cohere.VectorField()
+
+    data = [ { "text": "hello world" },
+            { "text": "goodbye world" }]
+
+    db = lancedb.connect("~/.lancedb")
+    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+    tbl.add(data)
+
+    """
+
+    name: str = "embed-multilingual-v2.0"
+    client: ClassVar = None
+
+    def ndims(self):
+        # TODO: fix hardcoding
+        return 768
+
+    def generate_embeddings(
+        self, texts: Union[List[str], np.ndarray]
+    ) -> List[np.array]:
+        """
+        Get the embeddings for the given texts
+
+        Parameters
+        ----------
+        texts: list[str] or np.ndarray (of str)
+            The texts to embed
+        """
+        # TODO retry, rate limit, token limit
+        self._init_client()
+        rs = CohereEmbeddingFunction.client.embed(texts=texts, model=self.name)
+
+        return [emb for emb in rs.embeddings]
+
+    def _init_client(self):
+        cohere = self.safe_import("cohere")
+        if CohereEmbeddingFunction.client is None:
+            if os.environ.get("COHERE_API_KEY") is None:
+                api_key_not_found_help("cohere")
+            CohereEmbeddingFunction.client = cohere.Client(os.environ["COHERE_API_KEY"])

python/lancedb/embeddings/instructor.py

@@ -0,0 +1,137 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from typing import List
+
+import numpy as np
+
+from .base import TextEmbeddingFunction
+from .registry import register
+from .utils import TEXT, weak_lru
+
+
+@register("instructor")
+class InstructorEmbeddingFunction(TextEmbeddingFunction):
+    """
+    An embedding function that uses the InstructorEmbedding library. Instructor models support multi-task learning, and can be used for a
+    variety of tasks, including text classification, sentence similarity, and document retrieval.
+    If you want to calculate customized embeddings for specific sentences, you may follow the unified template to write instructions:
+        "Represent the `domain` `text_type` for `task_objective`":
+
+        * domain is optional, and it specifies the domain of the text, e.g., science, finance, medicine, etc.
+        * text_type is required, and it specifies the encoding unit, e.g., sentence, document, paragraph, etc.
+        * task_objective is optional, and it specifies the objective of embedding, e.g., retrieve a document, classify the sentence, etc.
+
+    For example, if you want to calculate embeddings for a document, you may write the instruction as follows:
+        "Represent the document for retreival"
+
+    Parameters
+    ----------
+    name: str
+        The name of the model to use. Available models are listed at https://github.com/xlang-ai/instructor-embedding#model-list;
+        The default model is hkunlp/instructor-base
+    batch_size: int, default 32
+        The batch size to use when generating embeddings
+    device: str, default "cpu"
+        The device to use when generating embeddings
+    show_progress_bar: bool, default True
+        Whether to show a progress bar when generating embeddings
+    normalize_embeddings: bool, default True
+        Whether to normalize the embeddings
+    quantize: bool, default False
+        Whether to quantize the model
+    source_instruction: str, default "represent the docuement for retreival"
+        The instruction for the source column
+    query_instruction: str, default "represent the document for retreiving the most similar documents"
+        The instruction for the query
+
+    Examples
+    --------
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import get_registry, InstuctorEmbeddingFunction
+
+    instructor = get_registry().get("instructor").create(
+                                source_instruction="represent the docuement for retreival",
+                                query_instruction="represent the document for retreiving the most similar documents"
+                                )
+
+    class Schema(LanceModel):
+        vector: Vector(instructor.ndims()) = instructor.VectorField()
+        text: str = instructor.SourceField()
+
+    db = lancedb.connect("~/.lancedb")
+    tbl = db.create_table("test", schema=Schema, mode="overwrite")
+
+    texts = [{"text": "Capitalism has been dominant in the Western world since the end of feudalism, but most feel[who?] that..."},
+            {"text": "The disparate impact theory is especially controversial under the Fair Housing Act because the Act..."},
+            {"text": "Disparate impact in United States labor law refers to practices in employment, housing, and other areas that.."}]
+
+    tbl.add(texts)
+
+    """
+
+    name: str = "hkunlp/instructor-base"
+    batch_size: int = 32
+    device: str = "cpu"
+    show_progress_bar: bool = True
+    normalize_embeddings: bool = True
+    quantize: bool = False
+    # convert_to_numpy: bool = True # Hardcoding this as numpy can be ingested directly
+
+    source_instruction: str = "represent the document for retrieval"
+    query_instruction: str = (
+        "represent the document for retrieving the most similar documents"
+    )
+
+    @weak_lru(maxsize=1)
+    def ndims(self):
+        model = self.get_model()
+        return model.encode("foo").shape[0]
+
+    def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]:
+        return self.generate_embeddings([[self.query_instruction, query]])
+
+    def compute_source_embeddings(self, texts: TEXT, *args, **kwargs) -> List[np.array]:
+        texts = self.sanitize_input(texts)
+        texts_formatted = []
+        for text in texts:
+            texts_formatted.append([self.source_instruction, text])
+        return self.generate_embeddings(texts_formatted)
+
+    def generate_embeddings(self, texts: List) -> List:
+        model = self.get_model()
+        res = model.encode(
+            texts,
+            batch_size=self.batch_size,
+            show_progress_bar=self.show_progress_bar,
+            normalize_embeddings=self.normalize_embeddings,
+        ).tolist()
+        return res
+
+    @weak_lru(maxsize=1)
+    def get_model(self):
+        instructor_embedding = self.safe_import(
+            "InstructorEmbedding", "InstructorEmbedding"
+        )
+        torch = self.safe_import("torch", "torch")
+
+        model = instructor_embedding.INSTRUCTOR(self.name)
+        if self.quantize:
+            if (
+                "qnnpack" in torch.backends.quantized.supported_engines
+            ):  # fix for https://github.com/pytorch/pytorch/issues/29327
+                torch.backends.quantized.engine = "qnnpack"
+            model = torch.quantization.quantize_dynamic(
+                model, {torch.nn.Linear}, dtype=torch.qint8
+            )
+        return model

python/lancedb/embeddings/open_clip.py

@@ -0,0 +1,175 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+import concurrent.futures
+import io
+import os
+import urllib.parse as urlparse
+from typing import List, Union
+
+import numpy as np
+import pyarrow as pa
+from pydantic import PrivateAttr
+from tqdm import tqdm
+
+from .base import EmbeddingFunction
+from .registry import register
+from .utils import IMAGES, url_retrieve
+
+
+@register("open-clip")
+class OpenClipEmbeddings(EmbeddingFunction):
+    """
+    An embedding function that uses the OpenClip API
+    For multi-modal text-to-image search
+
+    https://github.com/mlfoundations/open_clip
+    """
+
+    name: str = "ViT-B-32"
+    pretrained: str = "laion2b_s34b_b79k"
+    device: str = "cpu"
+    batch_size: int = 64
+    normalize: bool = True
+    _model = PrivateAttr()
+    _preprocess = PrivateAttr()
+    _tokenizer = PrivateAttr()
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        open_clip = self.safe_import("open_clip", "open-clip")
+        model, _, preprocess = open_clip.create_model_and_transforms(
+            self.name, pretrained=self.pretrained
+        )
+        model.to(self.device)
+        self._model, self._preprocess = model, preprocess
+        self._tokenizer = open_clip.get_tokenizer(self.name)
+        self._ndims = None
+
+    def ndims(self):
+        if self._ndims is None:
+            self._ndims = self.generate_text_embeddings("foo").shape[0]
+        return self._ndims
+
+    def compute_query_embeddings(
+        self, query: Union[str, "PIL.Image.Image"], *args, **kwargs
+    ) -> List[np.ndarray]:
+        """
+        Compute the embeddings for a given user query
+
+        Parameters
+        ----------
+        query : Union[str, PIL.Image.Image]
+            The query to embed. A query can be either text or an image.
+        """
+        if isinstance(query, str):
+            return [self.generate_text_embeddings(query)]
+        else:
+            PIL = self.safe_import("PIL", "pillow")
+            if isinstance(query, PIL.Image.Image):
+                return [self.generate_image_embedding(query)]
+            else:
+                raise TypeError("OpenClip supports str or PIL Image as query")
+
+    def generate_text_embeddings(self, text: str) -> np.ndarray:
+        torch = self.safe_import("torch")
+        text = self.sanitize_input(text)
+        text = self._tokenizer(text)
+        text.to(self.device)
+        with torch.no_grad():
+            text_features = self._model.encode_text(text.to(self.device))
+            if self.normalize:
+                text_features /= text_features.norm(dim=-1, keepdim=True)
+            return text_features.cpu().numpy().squeeze()
+
+    def sanitize_input(self, images: IMAGES) -> Union[List[bytes], np.ndarray]:
+        """
+        Sanitize the input to the embedding function.
+        """
+        if isinstance(images, (str, bytes)):
+            images = [images]
+        elif isinstance(images, pa.Array):
+            images = images.to_pylist()
+        elif isinstance(images, pa.ChunkedArray):
+            images = images.combine_chunks().to_pylist()
+        return images
+
+    def compute_source_embeddings(
+        self, images: IMAGES, *args, **kwargs
+    ) -> List[np.array]:
+        """
+        Get the embeddings for the given images
+        """
+        images = self.sanitize_input(images)
+        embeddings = []
+        for i in range(0, len(images), self.batch_size):
+            j = min(i + self.batch_size, len(images))
+            batch = images[i:j]
+            embeddings.extend(self._parallel_get(batch))
+        return embeddings
+
+    def _parallel_get(self, images: Union[List[str], List[bytes]]) -> List[np.ndarray]:
+        """
+        Issue concurrent requests to retrieve the image data
+        """
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [
+                executor.submit(self.generate_image_embedding, image)
+                for image in images
+            ]
+            return [future.result() for future in tqdm(futures)]
+
+    def generate_image_embedding(
+        self, image: Union[str, bytes, "PIL.Image.Image"]
+    ) -> np.ndarray:
+        """
+        Generate the embedding for a single image
+
+        Parameters
+        ----------
+        image : Union[str, bytes, PIL.Image.Image]
+            The image to embed. If the image is a str, it is treated as a uri.
+            If the image is bytes, it is treated as the raw image bytes.
+        """
+        torch = self.safe_import("torch")
+        # TODO handle retry and errors for https
+        image = self._to_pil(image)
+        image = self._preprocess(image).unsqueeze(0)
+        with torch.no_grad():
+            return self._encode_and_normalize_image(image)
+
+    def _to_pil(self, image: Union[str, bytes]):
+        PIL = self.safe_import("PIL", "pillow")
+        if isinstance(image, bytes):
+            return PIL.Image.open(io.BytesIO(image))
+        if isinstance(image, PIL.Image.Image):
+            return image
+        elif isinstance(image, str):
+            parsed = urlparse.urlparse(image)
+            # TODO handle drive letter on windows.
+            if parsed.scheme == "file":
+                return PIL.Image.open(parsed.path)
+            elif parsed.scheme == "":
+                return PIL.Image.open(image if os.name == "nt" else parsed.path)
+            elif parsed.scheme.startswith("http"):
+                return PIL.Image.open(io.BytesIO(url_retrieve(image)))
+            else:
+                raise NotImplementedError("Only local and http(s) urls are supported")
+
+    def _encode_and_normalize_image(self, image_tensor: "torch.Tensor"):
+        """
+        encode a single image tensor and optionally normalize the output
+        """
+        image_features = self._model.encode_image(image_tensor.to(self.device))
+        if self.normalize:
+            image_features /= image_features.norm(dim=-1, keepdim=True)
+        return image_features.cpu().numpy().squeeze()

python/lancedb/embeddings/openai.py

@@ -0,0 +1,49 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from typing import List, Union
+
+import numpy as np
+
+from .base import TextEmbeddingFunction
+from .registry import register
+
+
+@register("openai")
+class OpenAIEmbeddings(TextEmbeddingFunction):
+    """
+    An embedding function that uses the OpenAI API
+
+    https://platform.openai.com/docs/guides/embeddings
+    """
+
+    name: str = "text-embedding-ada-002"
+
+    def ndims(self):
+        # TODO don't hardcode this
+        return 1536
+
+    def generate_embeddings(
+        self, texts: Union[List[str], np.ndarray]
+    ) -> List[np.array]:
+        """
+        Get the embeddings for the given texts
+
+        Parameters
+        ----------
+        texts: list[str] or np.ndarray (of str)
+            The texts to embed
+        """
+        # TODO retry, rate limit, token limit
+        openai = self.safe_import("openai")
+        rs = openai.Embedding.create(input=texts, model=self.name)["data"]
+        return [v["embedding"] for v in rs]

python/lancedb/embeddings/registry.py

@@ -0,0 +1,186 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+import json
+from typing import Dict, Optional
+
+from .base import EmbeddingFunction, EmbeddingFunctionConfig
+
+
+class EmbeddingFunctionRegistry:
+    """
+    This is a singleton class used to register embedding functions
+    and fetch them by name. It also handles serializing and deserializing.
+    You can implement your own embedding function by subclassing EmbeddingFunction
+    or TextEmbeddingFunction and registering it with the registry.
+
+    NOTE: Here TEXT is a type alias for Union[str, List[str], pa.Array, pa.ChunkedArray, np.ndarray]
+    Examples
+    --------
+    >>> registry = EmbeddingFunctionRegistry.get_instance()
+    >>> @registry.register("my-embedding-function")
+    ... class MyEmbeddingFunction(EmbeddingFunction):
+    ...     def ndims(self) -> int:
+    ...         return 128
+    ...
+    ...     def compute_query_embeddings(self, query: str, *args, **kwargs):
+    ...         return self.compute_source_embeddings(query, *args, **kwargs)
+    ...
+    ...     def compute_source_embeddings(self, texts, *args, **kwargs):
+    ...         return [np.random.rand(self.ndims()) for _ in range(len(texts))]
+    ...
+    >>> registry.get("my-embedding-function")
+    <class 'lancedb.embeddings.registry.MyEmbeddingFunction'>
+    """
+
+    @classmethod
+    def get_instance(cls):
+        return __REGISTRY__
+
+    def __init__(self):
+        self._functions = {}
+
+    def register(self, alias: str = None):
+        """
+        This creates a decorator that can be used to register
+        an EmbeddingFunction.
+
+        Parameters
+        ----------
+        alias : Optional[str]
+            a human friendly name for the embedding function. If not
+            provided, the class name will be used.
+        """
+
+        # This is a decorator for a class that inherits from BaseModel
+        # It adds the class to the registry
+        def decorator(cls):
+            if not issubclass(cls, EmbeddingFunction):
+                raise TypeError("Must be a subclass of EmbeddingFunction")
+            if cls.__name__ in self._functions:
+                raise KeyError(f"{cls.__name__} was already registered")
+            key = alias or cls.__name__
+            self._functions[key] = cls
+            cls.__embedding_function_registry_alias__ = alias
+            return cls
+
+        return decorator
+
+    def reset(self):
+        """
+        Reset the registry to its initial state
+        """
+        self._functions = {}
+
+    def get(self, name: str):
+        """
+        Fetch an embedding function class by name
+
+        Parameters
+        ----------
+        name : str
+            The name of the embedding function to fetch
+            Either the alias or the class name if no alias was provided
+            during registration
+        """
+        return self._functions[name]
+
+    def parse_functions(
+        self, metadata: Optional[Dict[bytes, bytes]]
+    ) -> Dict[str, "EmbeddingFunctionConfig"]:
+        """
+        Parse the metadata from an arrow table and
+        return a mapping of the vector column to the
+        embedding function and source column
+
+        Parameters
+        ----------
+        metadata : Optional[Dict[bytes, bytes]]
+            The metadata from an arrow table. Note that
+            the keys and values are bytes (pyarrow api)
+
+        Returns
+        -------
+        functions : dict
+            A mapping of vector column name to embedding function.
+            An empty dict is returned if input is None or does not
+            contain b"embedding_functions".
+        """
+        if metadata is None or b"embedding_functions" not in metadata:
+            return {}
+        serialized = metadata[b"embedding_functions"]
+        raw_list = json.loads(serialized.decode("utf-8"))
+        return {
+            obj["vector_column"]: EmbeddingFunctionConfig(
+                vector_column=obj["vector_column"],
+                source_column=obj["source_column"],
+                function=self.get(obj["name"])(**obj["model"]),
+            )
+            for obj in raw_list
+        }
+
+    def function_to_metadata(self, conf: "EmbeddingFunctionConfig"):
+        """
+        Convert the given embedding function and source / vector column configs
+        into a config dictionary that can be serialized into arrow metadata
+        """
+        func = conf.function
+        name = getattr(
+            func, "__embedding_function_registry_alias__", func.__class__.__name__
+        )
+        json_data = func.safe_model_dump()
+        return {
+            "name": name,
+            "model": json_data,
+            "source_column": conf.source_column,
+            "vector_column": conf.vector_column,
+        }
+
+    def get_table_metadata(self, func_list):
+        """
+        Convert a list of embedding functions and source / vector configs
+        into a config dictionary that can be serialized into arrow metadata
+        """
+        if func_list is None or len(func_list) == 0:
+            return None
+        json_data = [self.function_to_metadata(func) for func in func_list]
+        # Note that metadata dictionary values must be bytes
+        # so we need to json dump then utf8 encode
+        metadata = json.dumps(json_data, indent=2).encode("utf-8")
+        return {"embedding_functions": metadata}
+
+
+# Global instance
+__REGISTRY__ = EmbeddingFunctionRegistry()
+
+
+# @EmbeddingFunctionRegistry.get_instance().register(name) doesn't work in 3.8
+register = lambda name: EmbeddingFunctionRegistry.get_instance().register(name)
+
+
+def get_registry():
+    """
+    Utility function to get the global instance of the registry
+
+    Returns
+    -------
+    EmbeddingFunctionRegistry
+        The global registry instance
+
+    Examples
+    --------
+    from lancedb.embeddings import get_registry
+
+    registry = get_registry()
+    openai = registry.get("openai").create()
+    """
+    return __REGISTRY__.get_instance()

python/lancedb/embeddings/sentence_transformers.py

@@ -0,0 +1,89 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from typing import List, Union
+
+import numpy as np
+from cachetools import cached
+
+from .base import TextEmbeddingFunction
+from .registry import register
+from .utils import weak_lru
+
+
+@register("sentence-transformers")
+class SentenceTransformerEmbeddings(TextEmbeddingFunction):
+    """
+    An embedding function that uses the sentence-transformers library
+
+    https://huggingface.co/sentence-transformers
+    """
+
+    name: str = "all-MiniLM-L6-v2"
+    device: str = "cpu"
+    normalize: bool = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._ndims = None
+
+    @property
+    def embedding_model(self):
+        """
+        Get the sentence-transformers embedding model specified by the
+        name and device. This is cached so that the model is only loaded
+        once per process.
+        """
+        return self.get_embedding_model()
+
+    def ndims(self):
+        if self._ndims is None:
+            self._ndims = len(self.generate_embeddings("foo")[0])
+        return self._ndims
+
+    def generate_embeddings(
+        self, texts: Union[List[str], np.ndarray]
+    ) -> List[np.array]:
+        """
+        Get the embeddings for the given texts
+
+        Parameters
+        ----------
+        texts: list[str] or np.ndarray (of str)
+            The texts to embed
+        """
+        return self.embedding_model.encode(
+            list(texts),
+            convert_to_numpy=True,
+            normalize_embeddings=self.normalize,
+        ).tolist()
+
+    @weak_lru(maxsize=1)
+    def get_embedding_model(self):
+        """
+        Get the sentence-transformers embedding model specified by the
+        name and device. This is cached so that the model is only loaded
+        once per process.
+
+        Parameters
+        ----------
+        name : str
+            The name of the model to load
+        device : str
+            The device to load the model on
+
+        TODO: use lru_cache instead with a reasonable/configurable maxsize
+        """
+        sentence_transformers = self.safe_import(
+            "sentence_transformers", "sentence-transformers"
+        )
+        return sentence_transformers.SentenceTransformer(self.name, device=self.device)

python/lancedb/embeddings/utils.py

@@ -1,4 +1,4 @@
-#  Copyright 2023 LanceDB Developers
+#  Copyright (c) 2023. LanceDB Developers
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -11,19 +11,31 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import functools
 import math
+import random
+import socket
 import sys
-from typing import Callable, Union
+import time
+import urllib.error
+import weakref
+from typing import Callable, List, Union
 
 import numpy as np
 import pyarrow as pa
 from lance.vector import vec_to_table
 from retry import retry
 
-from .util import safe_import_pandas
+from ..util import safe_import_pandas
+from ..utils.general import LOGGER
 
 pd = safe_import_pandas()
+
 DATA = Union[pa.Table, "pd.DataFrame"]
+TEXT = Union[str, List[str], pa.Array, pa.ChunkedArray, np.ndarray]
+IMAGES = Union[
+    str, bytes, List[str], List[bytes], pa.Array, pa.ChunkedArray, np.ndarray
+]
 
 
 def with_embeddings(
@@ -58,7 +70,7 @@ def with_embeddings(
     pa.Table
         The input table with a new column called "vector" containing the embeddings.
     """
-    func = EmbeddingFunction(func)
+    func = FunctionWrapper(func)
     if wrap_api:
         func = func.retry().rate_limit()
     func = func.batch_size(batch_size)
@@ -71,7 +83,11 @@ def with_embeddings(
     return data.append_column("vector", table["vector"])
 
 
-class EmbeddingFunction:
+class FunctionWrapper:
+    """
+    A wrapper for embedding functions that adds rate limiting, retries, and batching.
+    """
+
     def __init__(self, func: Callable):
         self.func = func
         self.rate_limiter_kwargs = {}
@@ -148,3 +164,115 @@ class EmbeddingFunction:
             yield from tqdm(_chunker(arr), total=math.ceil(length / self._batch_size))
         else:
             yield from _chunker(arr)
+
+
+def weak_lru(maxsize=128):
+    """
+    LRU cache that keeps weak references to the objects it caches. Only caches the latest instance of the objects to make sure memory usage
+    is bounded.
+
+    Parameters
+    ----------
+    maxsize : int, default 128
+        The maximum number of objects to cache.
+
+    Returns
+    -------
+    Callable
+        A decorator that can be applied to a method.
+
+    Examples
+    --------
+    >>> class Foo:
+    ...     @weak_lru()
+    ...     def bar(self, x):
+    ...         return x
+    >>> foo = Foo()
+    >>> foo.bar(1)
+    1
+    >>> foo.bar(2)
+    2
+    >>> foo.bar(1)
+    1
+    """
+
+    def wrapper(func):
+        @functools.lru_cache(maxsize)
+        def _func(_self, *args, **kwargs):
+            return func(_self(), *args, **kwargs)
+
+        @functools.wraps(func)
+        def inner(self, *args, **kwargs):
+            return _func(weakref.ref(self), *args, **kwargs)
+
+        return inner
+
+    return wrapper
+
+
+def retry_with_exponential_backoff(
+    func,
+    initial_delay: float = 1,
+    exponential_base: float = 2,
+    jitter: bool = True,
+    max_retries: int = 7,
+    # errors: tuple = (),
+):
+    """Retry a function with exponential backoff.
+
+    Args:
+        func (function): The function to be retried.
+        initial_delay (float): Initial delay in seconds (default is 1).
+        exponential_base (float): The base for exponential backoff (default is 2).
+        jitter (bool): Whether to add jitter to the delay (default is True).
+        max_retries (int): Maximum number of retries (default is 10).
+        errors (tuple): Tuple of specific exceptions to retry on (default is (openai.error.RateLimitError,)).
+
+    Returns:
+        function: The decorated function.
+    """
+
+    def wrapper(*args, **kwargs):
+        num_retries = 0
+        delay = initial_delay
+
+        # Loop until a successful response or max_retries is hit or an exception is raised
+        while True:
+            try:
+                return func(*args, **kwargs)
+
+            # Currently retrying on all exceptions as there is no way to know the format of the error msgs used by different APIs
+            # We'll log the error and say that it is assumed that if this portion errors out, it's due to rate limit but the user
+            # should check the error message to be sure
+            except Exception as e:
+                num_retries += 1
+
+                if num_retries > max_retries:
+                    raise Exception(
+                        f"Maximum number of retries ({max_retries}) exceeded."
+                    )
+
+                delay *= exponential_base * (1 + jitter * random.random())
+                LOGGER.info(f"Retrying in {delay:.2f} seconds due to {e}")
+                time.sleep(delay)
+
+    return wrapper
+
+
+def url_retrieve(url: str):
+    """
+    Parameters
+    ----------
+    url: str
+        URL to download from
+    """
+    try:
+        with urllib.request.urlopen(url) as conn:
+            return conn.read()
+    except (socket.gaierror, urllib.error.URLError) as err:
+        raise ConnectionError("could not download {} due to {}".format(url, err))
+
+
+def api_key_not_found_help(provider):
+    LOGGER.error(f"Could not find API key for {provider}.")
+    raise ValueError(f"Please set the {provider.upper()}_API_KEY environment variable.")

python/lancedb/pydantic.py

@@ -19,6 +19,7 @@ import inspect
 import sys
 import types
 from abc import ABC, abstractmethod
+from datetime import date, datetime
 from typing import Any, Callable, Dict, Generator, List, Type, Union, _GenericAlias
 
 import numpy as np
@@ -26,6 +27,8 @@ import pyarrow as pa
 import pydantic
 import semver
 
+from .embeddings import EmbeddingFunctionRegistry
+
 PYDANTIC_VERSION = semver.Version.parse(pydantic.__version__)
 try:
     from pydantic_core import CoreSchema, core_schema
@@ -46,7 +49,19 @@ class FixedSizeListMixin(ABC):
         raise NotImplementedError
 
 
-def vector(
+def vector(dim: int, value_type: pa.DataType = pa.float32()):
+    # TODO: remove in future release
+    from warnings import warn
+
+    warn(
+        "lancedb.pydantic.vector() is deprecated, use lancedb.pydantic.Vector instead."
+        "This function will be removed in future release",
+        DeprecationWarning,
+    )
+    return Vector(dim, value_type)
+
+
+def Vector(
     dim: int, value_type: pa.DataType = pa.float32()
 ) -> Type[FixedSizeListMixin]:
     """Pydantic Vector Type.
@@ -65,12 +80,12 @@ def vector(
     --------
 
     >>> import pydantic
-    >>> from lancedb.pydantic import vector
+    >>> from lancedb.pydantic import Vector
     ...
     >>> class MyModel(pydantic.BaseModel):
     ...     id: int
     ...     url: str
-    ...     embeddings: vector(768)
+    ...     embeddings: Vector(768)
     >>> schema = pydantic_to_schema(MyModel)
     >>> assert schema == pa.schema([
     ...     pa.field("id", pa.int64(), False),
@@ -114,7 +129,7 @@ def vector(
         def validate(cls, v):
             if not isinstance(v, (list, range, np.ndarray)) or len(v) != dim:
                 raise TypeError("A list of numbers or numpy.ndarray is needed")
-            return v
+            return cls(v)
 
         if PYDANTIC_VERSION < (2, 0):
 
@@ -145,6 +160,10 @@ def _py_type_to_arrow_type(py_type: Type[Any]) -> pa.DataType:
         return pa.bool_()
     elif py_type == bytes:
         return pa.binary()
+    elif py_type == date:
+        return pa.date32()
+    elif py_type == datetime:
+        return pa.timestamp("us")
     raise TypeError(
         f"Converting Pydantic type to Arrow Type: unsupported type {py_type}"
     )
@@ -224,27 +243,18 @@ def pydantic_to_schema(model: Type[pydantic.BaseModel]) -> pa.Schema:
     >>> from typing import List, Optional
     >>> import pydantic
     >>> from lancedb.pydantic import pydantic_to_schema
-    ...
-    >>> class InnerModel(pydantic.BaseModel):
-    ...     a: str
-    ...     b: Optional[float]
-    >>>
     >>> class FooModel(pydantic.BaseModel):
     ...     id: int
-    ...     s: Optional[str] = None
+    ...     s: str
     ...     vec: List[float]
     ...     li: List[int]
-    ...     inner: InnerModel
+    ...
     >>> schema = pydantic_to_schema(FooModel)
     >>> assert schema == pa.schema([
     ...     pa.field("id", pa.int64(), False),
-    ...     pa.field("s", pa.utf8(), True),
+    ...     pa.field("s", pa.utf8(), False),
     ...     pa.field("vec", pa.list_(pa.float64()), False),
     ...     pa.field("li", pa.list_(pa.int64()), False),
-    ...     pa.field("inner", pa.struct([
-    ...         pa.field("a", pa.utf8(), False),
-    ...         pa.field("b", pa.float64(), True),
-    ...     ]), False),
     ... ])
     """
     fields = _pydantic_model_to_fields(model)
@@ -258,11 +268,11 @@ class LanceModel(pydantic.BaseModel):
     Examples
     --------
     >>> import lancedb
-    >>> from lancedb.pydantic import LanceModel, vector
+    >>> from lancedb.pydantic import LanceModel, Vector
     >>>
     >>> class TestModel(LanceModel):
     ...     name: str
-    ...     vector: vector(2)
+    ...     vector: Vector(2)
     ...
     >>> db = lancedb.connect("/tmp")
     >>> table = db.create_table("test", schema=TestModel.to_arrow_schema())
@@ -275,10 +285,83 @@ class LanceModel(pydantic.BaseModel):
 
     @classmethod
     def to_arrow_schema(cls):
-        return pydantic_to_schema(cls)
+        """
+        Get the Arrow Schema for this model.
+        """
+        schema = pydantic_to_schema(cls)
+        functions = cls.parse_embedding_functions()
+        if len(functions) > 0:
+            metadata = EmbeddingFunctionRegistry.get_instance().get_table_metadata(
+                functions
+            )
+            schema = schema.with_metadata(metadata)
+        return schema
 
     @classmethod
     def field_names(cls) -> List[str]:
+        """
+        Get the field names of this model.
+        """
+        return list(cls.safe_get_fields().keys())
+
+    @classmethod
+    def safe_get_fields(cls):
         if PYDANTIC_VERSION.major < 2:
-            return list(cls.__fields__.keys())
-        return list(cls.model_fields.keys())
+            return cls.__fields__
+        return cls.model_fields
+
+    @classmethod
+    def parse_embedding_functions(cls) -> List["EmbeddingFunctionConfig"]:
+        """
+        Parse the embedding functions from this model.
+        """
+        from .embeddings import EmbeddingFunctionConfig
+
+        vec_and_function = []
+        for name, field_info in cls.safe_get_fields().items():
+            func = get_extras(field_info, "vector_column_for")
+            if func is not None:
+                vec_and_function.append([name, func])
+
+        configs = []
+        for vec, func in vec_and_function:
+            for source, field_info in cls.safe_get_fields().items():
+                src_func = get_extras(field_info, "source_column_for")
+                if src_func is func:
+                    # note we can't use == here since the function is a pydantic
+                    # model so two instances of the same function are ==, so if you
+                    # have multiple vector columns from multiple sources, both will
+                    # be mapped to the same source column
+                    # GH594
+                    configs.append(
+                        EmbeddingFunctionConfig(
+                            source_column=source, vector_column=vec, function=func
+                        )
+                    )
+        return configs
+
+
+def get_extras(field_info: pydantic.fields.FieldInfo, key: str) -> Any:
+    """
+    Get the extra metadata from a Pydantic FieldInfo.
+    """
+    if PYDANTIC_VERSION.major >= 2:
+        return (field_info.json_schema_extra or {}).get(key)
+    return (field_info.field_info.extra or {}).get("json_schema_extra", {}).get(key)
+
+
+if PYDANTIC_VERSION.major < 2:
+
+    def model_to_dict(model: pydantic.BaseModel) -> Dict[str, Any]:
+        """
+        Convert a Pydantic model to a dictionary.
+        """
+        return model.dict()
+
+else:
+
+    def model_to_dict(model: pydantic.BaseModel) -> Dict[str, Any]:
+        """
+        Convert a Pydantic model to a dictionary.
+        """
+        return model.model_dump()

python/lancedb/query.py

@@ -13,21 +13,59 @@
 
 from __future__ import annotations
 
-from typing import List, Literal, Optional, Type, Union
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, List, Literal, Optional, Type, Union
 
+import deprecation
 import numpy as np
 import pyarrow as pa
 import pydantic
 
+from . import __version__
 from .common import VECTOR_COLUMN_NAME
-from .pydantic import LanceModel
 from .util import safe_import_pandas
 
+if TYPE_CHECKING:
+    from .pydantic import LanceModel
+
 pd = safe_import_pandas()
 
 
 class Query(pydantic.BaseModel):
-    """A Query"""
+    """The LanceDB Query
+
+    Attributes
+    ----------
+    vector : List[float]
+        the vector to search for
+    filter : Optional[str]
+        sql filter to refine the query with, optional
+    prefilter : bool
+        if True then apply the filter before vector search
+    k : int
+        top k results to return
+    metric : str
+        the distance metric between a pair of vectors,
+
+        can support L2 (default), Cosine and Dot.
+        [metric definitions][search]
+    columns : Optional[List[str]]
+        which columns to return in the results
+    nprobes : int
+        The number of probes used - optional
+
+        - A higher number makes search more accurate but also slower.
+
+        - See discussion in [Querying an ANN Index][querying-an-ann-index] for
+          tuning advice.
+    refine_factor : Optional[int]
+        Refine the results by reading extra elements and re-ranking them in memory - optional
+
+        - A higher number makes search more accurate but also slower.
+
+        - See discussion in [Querying an ANN Index][querying-an-ann-index] for
+          tuning advice.
+    """
 
     vector_column: str = VECTOR_COLUMN_NAME
 
@@ -37,6 +75,9 @@ class Query(pydantic.BaseModel):
     # sql filter to refine the query with
     filter: Optional[str] = None
 
+    # if True then apply the filter before vector search
+    prefilter: bool = False
+
     # top k results to return
     k: int
 
@@ -54,44 +95,143 @@ class Query(pydantic.BaseModel):
     refine_factor: Optional[int] = None
 
 
-class LanceQueryBuilder:
-    """
-    A builder for nearest neighbor queries for LanceDB.
-
-    Examples
-    --------
-    >>> import lancedb
-    >>> data = [{"vector": [1.1, 1.2], "b": 2},
-    ...         {"vector": [0.5, 1.3], "b": 4},
-    ...         {"vector": [0.4, 0.4], "b": 6},
-    ...         {"vector": [0.4, 0.4], "b": 10}]
-    >>> db = lancedb.connect("./.lancedb")
-    >>> table = db.create_table("my_table", data=data)
-    >>> (table.search([0.4, 0.4])
-    ...       .metric("cosine")
-    ...       .where("b < 10")
-    ...       .select(["b"])
-    ...       .limit(2)
-    ...       .to_df())
-       b      vector  score
-    0  6  [0.4, 0.4]    0.0
+class LanceQueryBuilder(ABC):
+    """Build LanceDB query based on specific query type:
+    vector or full text search.
     """
 
-    def __init__(
-        self,
+    @classmethod
+    def create(
+        cls,
         table: "lancedb.table.Table",
-        query: Union[np.ndarray, str],
-        vector_column: str = VECTOR_COLUMN_NAME,
-    ):
-        self._metric = "L2"
-        self._nprobes = 20
-        self._refine_factor = None
+        query: Optional[Union[np.ndarray, str, "PIL.Image.Image"]],
+        query_type: str,
+        vector_column_name: str,
+    ) -> LanceQueryBuilder:
+        if query is None:
+            return LanceEmptyQueryBuilder(table)
+
+        # convert "auto" query_type to "vector" or "fts"
+        # and convert the query to vector if needed
+        query, query_type = cls._resolve_query(
+            table, query, query_type, vector_column_name
+        )
+
+        if isinstance(query, str):
+            # fts
+            return LanceFtsQueryBuilder(table, query)
+
+        if isinstance(query, list):
+            query = np.array(query, dtype=np.float32)
+        elif isinstance(query, np.ndarray):
+            query = query.astype(np.float32)
+        else:
+            raise TypeError(f"Unsupported query type: {type(query)}")
+
+        return LanceVectorQueryBuilder(table, query, vector_column_name)
+
+    @classmethod
+    def _resolve_query(cls, table, query, query_type, vector_column_name):
+        # If query_type is fts, then query must be a string.
+        # otherwise raise TypeError
+        if query_type == "fts":
+            if not isinstance(query, str):
+                raise TypeError(f"'fts' queries must be a string: {type(query)}")
+            return query, query_type
+        elif query_type == "vector":
+            if not isinstance(query, (list, np.ndarray)):
+                conf = table.embedding_functions.get(vector_column_name)
+                if conf is not None:
+                    query = conf.function.compute_query_embeddings_with_retry(query)[0]
+                else:
+                    msg = f"No embedding function for {vector_column_name}"
+                    raise ValueError(msg)
+            return query, query_type
+        elif query_type == "auto":
+            if isinstance(query, (list, np.ndarray)):
+                return query, "vector"
+            else:
+                conf = table.embedding_functions.get(vector_column_name)
+                if conf is not None:
+                    query = conf.function.compute_query_embeddings_with_retry(query)[0]
+                    return query, "vector"
+                else:
+                    return query, "fts"
+        else:
+            raise ValueError(
+                f"Invalid query_type, must be 'vector', 'fts', or 'auto': {query_type}"
+            )
+
+    def __init__(self, table: "lancedb.table.Table"):
         self._table = table
-        self._query = query
         self._limit = 10
         self._columns = None
         self._where = None
-        self._vector_column = vector_column
+
+    @deprecation.deprecated(
+        deprecated_in="0.3.1",
+        removed_in="0.4.0",
+        current_version=__version__,
+        details="Use to_pandas() instead",
+    )
+    def to_df(self) -> "pd.DataFrame":
+        """
+        *Deprecated alias for `to_pandas()`. Please use `to_pandas()` instead.*
+
+        Execute the query and return the results as a pandas DataFrame.
+        In addition to the selected columns, LanceDB also returns a vector
+        and also the "_distance" column which is the distance between the query
+        vector and the returned vector.
+        """
+        return self.to_pandas()
+
+    def to_pandas(self) -> "pd.DataFrame":
+        """
+        Execute the query and return the results as a pandas DataFrame.
+        In addition to the selected columns, LanceDB also returns a vector
+        and also the "_distance" column which is the distance between the query
+        vector and the returned vector.
+        """
+        return self.to_arrow().to_pandas()
+
+    @abstractmethod
+    def to_arrow(self) -> pa.Table:
+        """
+        Execute the query and return the results as an
+        [Apache Arrow Table](https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table).
+
+        In addition to the selected columns, LanceDB also returns a vector
+        and also the "_distance" column which is the distance between the query
+        vector and the returned vectors.
+        """
+        raise NotImplementedError
+
+    def to_list(self) -> List[dict]:
+        """
+        Execute the query and return the results as a list of dictionaries.
+
+        Each list entry is a dictionary with the selected column names as keys,
+        or all table columns if `select` is not called. The vector and the "_distance"
+        fields are returned whether or not they're explicitly selected.
+        """
+        return self.to_arrow().to_pylist()
+
+    def to_pydantic(self, model: Type[LanceModel]) -> List[LanceModel]:
+        """Return the table as a list of pydantic models.
+
+        Parameters
+        ----------
+        model: Type[LanceModel]
+            The pydantic model to use.
+
+        Returns
+        -------
+        List[LanceModel]
+        """
+        return [
+            model(**{k: v for k, v in row.items() if k in model.field_names()})
+            for row in self.to_arrow().to_pylist()
+        ]
 
     def limit(self, limit: int) -> LanceQueryBuilder:
         """Set the maximum number of results to return.
@@ -125,13 +265,20 @@ class LanceQueryBuilder:
         self._columns = columns
         return self
 
-    def where(self, where: str) -> LanceQueryBuilder:
+    def where(self, where: str, prefilter: bool = False) -> LanceQueryBuilder:
         """Set the where clause.
 
         Parameters
         ----------
         where: str
-            The where clause.
+            The where clause which is a valid SQL where clause. See
+            `Lance filter pushdown <https://lancedb.github.io/lance/read_and_write.html#filter-push-down>`_
+            for valid SQL expressions.
+        prefilter: bool, default False
+            If True, apply the filter before vector search, otherwise the
+            filter is applied on the result of vector search.
+            This feature is **EXPERIMENTAL** and may be removed and modified
+            without warning in the future.
 
         Returns
         -------
@@ -139,9 +286,46 @@ class LanceQueryBuilder:
             The LanceQueryBuilder object.
         """
         self._where = where
+        self._prefilter = prefilter
         return self
 
-    def metric(self, metric: Literal["L2", "cosine"]) -> LanceQueryBuilder:
+
+class LanceVectorQueryBuilder(LanceQueryBuilder):
+    """
+    Examples
+    --------
+    >>> import lancedb
+    >>> data = [{"vector": [1.1, 1.2], "b": 2},
+    ...         {"vector": [0.5, 1.3], "b": 4},
+    ...         {"vector": [0.4, 0.4], "b": 6},
+    ...         {"vector": [0.4, 0.4], "b": 10}]
+    >>> db = lancedb.connect("./.lancedb")
+    >>> table = db.create_table("my_table", data=data)
+    >>> (table.search([0.4, 0.4])
+    ...       .metric("cosine")
+    ...       .where("b < 10")
+    ...       .select(["b"])
+    ...       .limit(2)
+    ...       .to_pandas())
+       b      vector  _distance
+    0  6  [0.4, 0.4]        0.0
+    """
+
+    def __init__(
+        self,
+        table: "lancedb.table.Table",
+        query: Union[np.ndarray, list, "PIL.Image.Image"],
+        vector_column: str = VECTOR_COLUMN_NAME,
+    ):
+        super().__init__(table)
+        self._query = query
+        self._metric = "L2"
+        self._nprobes = 20
+        self._refine_factor = None
+        self._vector_column = vector_column
+        self._prefilter = False
+
+    def metric(self, metric: Literal["L2", "cosine"]) -> LanceVectorQueryBuilder:
         """Set the distance metric to use.
 
         Parameters
@@ -151,19 +335,19 @@ class LanceQueryBuilder:
 
         Returns
         -------
-        LanceQueryBuilder
+        LanceVectorQueryBuilder
             The LanceQueryBuilder object.
         """
         self._metric = metric
         return self
 
-    def nprobes(self, nprobes: int) -> LanceQueryBuilder:
+    def nprobes(self, nprobes: int) -> LanceVectorQueryBuilder:
         """Set the number of probes to use.
 
         Higher values will yield better recall (more likely to find vectors if
         they exist) at the expense of latency.
 
-        See discussion in [Querying an ANN Index][../querying-an-ann-index] for
+        See discussion in [Querying an ANN Index][querying-an-ann-index] for
         tuning advice.
 
         Parameters
@@ -173,13 +357,13 @@ class LanceQueryBuilder:
 
         Returns
         -------
-        LanceQueryBuilder
+        LanceVectorQueryBuilder
             The LanceQueryBuilder object.
         """
         self._nprobes = nprobes
         return self
 
-    def refine_factor(self, refine_factor: int) -> LanceQueryBuilder:
+    def refine_factor(self, refine_factor: int) -> LanceVectorQueryBuilder:
         """Set the refine factor to use, increasing the number of vectors sampled.
 
         As an example, a refine factor of 2 will sample 2x as many vectors as
@@ -195,35 +379,26 @@ class LanceQueryBuilder:
 
         Returns
         -------
-        LanceQueryBuilder
+        LanceVectorQueryBuilder
             The LanceQueryBuilder object.
         """
         self._refine_factor = refine_factor
         return self
 
-    def to_df(self) -> "pd.DataFrame":
-        """
-        Execute the query and return the results as a pandas DataFrame.
-        In addition to the selected columns, LanceDB also returns a vector
-        and also the "score" column which is the distance between the query
-        vector and the returned vector.
-        """
-
-        return self.to_arrow().to_pandas()
-
     def to_arrow(self) -> pa.Table:
         """
         Execute the query and return the results as an
         [Apache Arrow Table](https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table).
 
         In addition to the selected columns, LanceDB also returns a vector
-        and also the "score" column which is the distance between the query
+        and also the "_distance" column which is the distance between the query
         vector and the returned vectors.
         """
         vector = self._query if isinstance(self._query, list) else self._query.tolist()
         query = Query(
             vector=vector,
             filter=self._where,
+            prefilter=self._prefilter,
             k=self._limit,
             metric=self._metric,
             columns=self._columns,
@@ -233,25 +408,38 @@ class LanceQueryBuilder:
         )
         return self._table._execute_query(query)
 
-    def to_pydantic(self, model: Type[LanceModel]) -> List[LanceModel]:
-        """Return the table as a list of pydantic models.
+    def where(self, where: str, prefilter: bool = False) -> LanceVectorQueryBuilder:
+        """Set the where clause.
 
         Parameters
         ----------
-        model: Type[LanceModel]
-            The pydantic model to use.
+        where: str
+            The where clause which is a valid SQL where clause. See
+            `Lance filter pushdown <https://lancedb.github.io/lance/read_and_write.html#filter-push-down>`_
+            for valid SQL expressions.
+        prefilter: bool, default False
+            If True, apply the filter before vector search, otherwise the
+            filter is applied on the result of vector search.
+            This feature is **EXPERIMENTAL** and may be removed and modified
+            without warning in the future.
 
         Returns
         -------
-        List[LanceModel]
+        LanceQueryBuilder
+            The LanceQueryBuilder object.
         """
-        return [
-            model(**{k: v for k, v in row.items() if k in model.field_names()})
-            for row in self.to_arrow().to_pylist()
-        ]
+        self._where = where
+        self._prefilter = prefilter
+        return self
 
 
 class LanceFtsQueryBuilder(LanceQueryBuilder):
+    """A builder for full text search for LanceDB."""
+
+    def __init__(self, table: "lancedb.table.Table", query: str):
+        super().__init__(table)
+        self._query = query
+
     def to_arrow(self) -> pa.Table:
         try:
             import tantivy
@@ -275,3 +463,13 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
         output_tbl = self._table.to_lance().take(row_ids, columns=self._columns)
         output_tbl = output_tbl.append_column("score", scores)
         return output_tbl
+
+
+class LanceEmptyQueryBuilder(LanceQueryBuilder):
+    def to_arrow(self) -> pa.Table:
+        ds = self._table.to_lance()
+        return ds.to_table(
+            columns=self._columns,
+            filter=self._where,
+            limit=self._limit,
+        )

python/lancedb/remote/__init__.py

@@ -14,10 +14,12 @@
 import abc
 from typing import List, Optional
 
-import attr
+import attrs
 import pyarrow as pa
 from pydantic import BaseModel
 
+from lancedb.common import VECTOR_COLUMN_NAME
+
 __all__ = ["LanceDBClient", "VectorQuery", "VectorQueryResult"]
 
 
@@ -43,8 +45,10 @@ class VectorQuery(BaseModel):
 
     refine_factor: Optional[int] = None
 
+    vector_column: str = VECTOR_COLUMN_NAME
 
-@attr.define
+
+@attrs.define
 class VectorQueryResult:
     # for now the response is directly seralized into a pandas dataframe
     tbl: pa.Table

python/lancedb/remote/client.py

@@ -13,10 +13,10 @@
 
 
 import functools
-from typing import Any, Callable, Dict, Optional, Union
+from typing import Any, Callable, Dict, Iterable, Optional, Union
 
 import aiohttp
-import attr
+import attrs
 import pyarrow as pa
 from pydantic import BaseModel
 
@@ -43,14 +43,14 @@ async def _read_ipc(resp: aiohttp.ClientResponse) -> pa.Table:
         return reader.read_all()
 
 
-@attr.define(slots=False)
+@attrs.define(slots=False)
 class RestfulLanceDBClient:
     db_name: str
     region: str
     api_key: Credential
-    host_override: Optional[str] = attr.field(default=None)
+    host_override: Optional[str] = attrs.field(default=None)
 
-    closed: bool = attr.field(default=False, init=False)
+    closed: bool = attrs.field(default=False, init=False)
 
     @functools.cached_property
     def session(self) -> aiohttp.ClientSession:
@@ -97,7 +97,12 @@ class RestfulLanceDBClient:
         """Send a GET request and returns the deserialized response payload."""
         if isinstance(params, BaseModel):
             params: Dict[str, Any] = params.dict(exclude_none=True)
-        async with self.session.get(uri, params=params, headers=self.headers) as resp:
+        async with self.session.get(
+            uri,
+            params=params,
+            headers=self.headers,
+            timeout=aiohttp.ClientTimeout(total=30),
+        ) as resp:
             await self._check_status(resp)
             return await resp.json()
 
@@ -109,6 +114,7 @@ class RestfulLanceDBClient:
         params: Optional[Dict[str, Any]] = None,
         content_type: Optional[str] = None,
         deserialize: Callable = lambda resp: resp.json(),
+        request_id: Optional[str] = None,
     ) -> Dict[str, Any]:
         """Send a POST request and returns the deserialized response payload.
 
@@ -117,6 +123,8 @@ class RestfulLanceDBClient:
         uri : str
             The uri to send the POST request to.
         data: Union[Dict[str, Any], BaseModel]
+        request_id: Optional[str]
+            Optional client side request id to be sent in the request headers.
 
         """
         if isinstance(data, BaseModel):
@@ -129,10 +137,13 @@ class RestfulLanceDBClient:
         headers = self.headers.copy()
         if content_type is not None:
             headers["content-type"] = content_type
+        if request_id is not None:
+            headers["x-request-id"] = request_id
         async with self.session.post(
             uri,
             headers=headers,
             params=params,
+            timeout=aiohttp.ClientTimeout(total=30),
             **req_kwargs,
         ) as resp:
             resp: aiohttp.ClientResponse = resp
@@ -140,9 +151,13 @@ class RestfulLanceDBClient:
             return await deserialize(resp)
 
     @_check_not_closed
-    async def list_tables(self):
+    async def list_tables(
+        self, limit: int, page_token: Optional[str] = None
+    ) -> Iterable[str]:
         """List all tables in the database."""
-        json = await self.get("/v1/table/", {})
+        if page_token is None:
+            page_token = ""
+        json = await self.get("/v1/table/", {"limit": limit, "page_token": page_token})
         return json["tables"]
 
     @_check_not_closed

python/lancedb/remote/db.py

@@ -12,18 +12,23 @@
 #  limitations under the License.
 
 import asyncio
+import inspect
+import logging
 import uuid
-from typing import List, Optional
+from typing import Iterable, List, Optional, Union
 from urllib.parse import urlparse
 
 import pyarrow as pa
+from overrides import override
 
-from lancedb.common import DATA
-from lancedb.db import DBConnection
-from lancedb.table import Table, _sanitize_data
-
+from ..common import DATA
+from ..db import DBConnection
+from ..embeddings import EmbeddingFunctionConfig
+from ..pydantic import LanceModel
+from ..table import Table, _sanitize_data
 from .arrow import to_ipc_binary
 from .client import ARROW_STREAM_CONTENT_TYPE, RestfulLanceDBClient
+from .errors import LanceDBClientError
 
 
 class RemoteDBConnection(DBConnection):
@@ -51,13 +56,37 @@ class RemoteDBConnection(DBConnection):
             self._loop = asyncio.get_event_loop()
 
     def __repr__(self) -> str:
-        return f"RemoveConnect(name={self.db_name})"
+        return f"RemoteConnect(name={self.db_name})"
 
-    def table_names(self) -> List[str]:
-        """List the names of all tables in the database."""
-        result = self._loop.run_until_complete(self._client.list_tables())
-        return result
+    @override
+    def table_names(
+        self, page_token: Optional[str] = None, limit: int = 10
+    ) -> Iterable[str]:
+        """List the names of all tables in the database.
 
+        Parameters
+        ----------
+        page_token: str
+            The last token to start the new page.
+        limit: int, default 10
+            The maximum number of tables to return for each page.
+
+        Returns
+        -------
+        An iterator of table names.
+        """
+        while True:
+            result = self._loop.run_until_complete(
+                self._client.list_tables(limit, page_token)
+            )
+            if len(result) > 0:
+                page_token = result[len(result) - 1]
+            else:
+                break
+            for item in result:
+                yield item
+
+    @override
     def open_table(self, name: str) -> Table:
         """Open a Lance Table in the database.
 
@@ -72,23 +101,142 @@ class RemoteDBConnection(DBConnection):
         """
         from .table import RemoteTable
 
-        # TODO: check if table exists
-
+        # check if table exists
+        try:
+            self._loop.run_until_complete(
+                self._client.post(f"/v1/table/{name}/describe/")
+            )
+        except LanceDBClientError as err:
+            if str(err).startswith("Not found"):
+                logging.error(
+                    f"Table {name} does not exist. "
+                    f"Please first call db.create_table({name}, data)"
+                )
         return RemoteTable(self, name)
 
+    @override
     def create_table(
         self,
         name: str,
         data: DATA = None,
-        schema: pa.Schema = None,
+        schema: Optional[Union[pa.Schema, LanceModel]] = None,
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
+        embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
     ) -> Table:
+        """Create a [Table][lancedb.table.Table] in the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+        data: The data to initialize the table, *optional*
+            User must provide at least one of `data` or `schema`.
+            Acceptable types are:
+
+            - dict or list-of-dict
+
+            - pandas.DataFrame
+
+            - pyarrow.Table or pyarrow.RecordBatch
+        schema: The schema of the table, *optional*
+            Acceptable types are:
+
+            - pyarrow.Schema
+
+            - [LanceModel][lancedb.pydantic.LanceModel]
+        on_bad_vectors: str, default "error"
+            What to do if any of the vectors are not the same size or contains NaNs.
+            One of "error", "drop", "fill".
+        fill_value: float
+            The value to use when filling vectors. Only used if on_bad_vectors="fill".
+
+        Returns
+        -------
+        LanceTable
+            A reference to the newly created table.
+
+        !!! note
+
+            The vector index won't be created by default.
+            To create the index, call the `create_index` method on the table.
+
+        Examples
+        --------
+
+        Can create with list of tuples or dictionaries:
+
+        >>> import lancedb
+        >>> db = lancedb.connect("db://...", api_key="...", region="...") # doctest: +SKIP
+        >>> data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
+        ...         {"vector": [0.2, 1.8], "lat": 40.1, "long":  -74.1}]
+        >>> db.create_table("my_table", data) # doctest: +SKIP
+        LanceTable(my_table)
+
+        You can also pass a pandas DataFrame:
+
+        >>> import pandas as pd
+        >>> data = pd.DataFrame({
+        ...    "vector": [[1.1, 1.2], [0.2, 1.8]],
+        ...    "lat": [45.5, 40.1],
+        ...    "long": [-122.7, -74.1]
+        ... })
+        >>> db.create_table("table2", data) # doctest: +SKIP
+        LanceTable(table2)
+
+        >>> custom_schema = pa.schema([
+        ...   pa.field("vector", pa.list_(pa.float32(), 2)),
+        ...   pa.field("lat", pa.float32()),
+        ...   pa.field("long", pa.float32())
+        ... ])
+        >>> db.create_table("table3", data, schema = custom_schema) # doctest: +SKIP
+        LanceTable(table3)
+
+        It is also possible to create an table from `[Iterable[pa.RecordBatch]]`:
+
+        >>> import pyarrow as pa
+        >>> def make_batches():
+        ...     for i in range(5):
+        ...         yield pa.RecordBatch.from_arrays(
+        ...             [
+        ...                 pa.array([[3.1, 4.1], [5.9, 26.5]],
+        ...                     pa.list_(pa.float32(), 2)),
+        ...                 pa.array(["foo", "bar"]),
+        ...                 pa.array([10.0, 20.0]),
+        ...             ],
+        ...             ["vector", "item", "price"],
+        ...         )
+        >>> schema=pa.schema([
+        ...     pa.field("vector", pa.list_(pa.float32(), 2)),
+        ...     pa.field("item", pa.utf8()),
+        ...     pa.field("price", pa.float32()),
+        ... ])
+        >>> db.create_table("table4", make_batches(), schema=schema) # doctest: +SKIP
+        LanceTable(table4)
+
+        """
         if data is None and schema is None:
             raise ValueError("Either data or schema must be provided.")
+        if embedding_functions is not None:
+            raise NotImplementedError(
+                "embedding_functions is not supported for remote databases."
+                "Please vote https://github.com/lancedb/lancedb/issues/626 "
+                "for this feature."
+            )
+
+        if inspect.isclass(schema) and issubclass(schema, LanceModel):
+            # convert LanceModel to pyarrow schema
+            # note that it's possible this contains
+            # embedding function metadata already
+            schema = schema.to_arrow_schema()
+
         if data is not None:
             data = _sanitize_data(
-                data, schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+                data,
+                schema,
+                metadata=None,
+                on_bad_vectors=on_bad_vectors,
+                fill_value=fill_value,
             )
         else:
             if schema is None:
@@ -104,8 +252,28 @@ class RemoteDBConnection(DBConnection):
             self._client.post(
                 f"/v1/table/{name}/create/",
                 data=data,
-                params={"request_id": request_id},
+                request_id=request_id,
                 content_type=ARROW_STREAM_CONTENT_TYPE,
             )
         )
         return RemoteTable(self, name)
+
+    @override
+    def drop_table(self, name: str):
+        """Drop a table from the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+        """
+        self._loop.run_until_complete(
+            self._client.post(
+                f"/v1/table/{name}/drop/",
+            )
+        )
+
+    async def close(self):
+        """Close the connection to the database."""
+        self._loop.close()
+        await self._client.close()

python/lancedb/remote/table.py

@@ -13,15 +13,16 @@
 
 import uuid
 from functools import cached_property
-from typing import Union
+from typing import Dict, Optional, Union
 
 import pyarrow as pa
 from lance import json_to_schema
 
 from lancedb.common import DATA, VEC, VECTOR_COLUMN_NAME
 
-from ..query import LanceQueryBuilder
+from ..query import LanceVectorQueryBuilder
 from ..table import Query, Table, _sanitize_data
+from ..util import value_to_sql
 from .arrow import to_ipc_binary
 from .client import ARROW_STREAM_CONTENT_TYPE
 from .db import RemoteDBConnection
@@ -37,25 +38,81 @@ class RemoteTable(Table):
 
     @cached_property
     def schema(self) -> pa.Schema:
-        """Return the schema of the table."""
+        """The [Arrow Schema](https://arrow.apache.org/docs/python/api/datatypes.html#)
+        of this Table
+
+        """
         resp = self._conn._loop.run_until_complete(
             self._conn._client.post(f"/v1/table/{self._name}/describe/")
         )
         schema = json_to_schema(resp["schema"])
         return schema
 
+    @property
+    def version(self) -> int:
+        """Get the current version of the table"""
+        resp = self._conn._loop.run_until_complete(
+            self._conn._client.post(f"/v1/table/{self._name}/describe/")
+        )
+        return resp["version"]
+
     def to_arrow(self) -> pa.Table:
-        raise NotImplementedError
+        """to_arrow() is not supported on the LanceDB cloud"""
+        raise NotImplementedError("to_arrow() is not supported on the LanceDB cloud")
+
+    def to_pandas(self):
+        """to_pandas() is not supported on the LanceDB cloud"""
+        return NotImplementedError("to_pandas() is not supported on the LanceDB cloud")
 
     def create_index(
         self,
         metric="L2",
-        num_partitions=256,
-        num_sub_vectors=96,
         vector_column_name: str = VECTOR_COLUMN_NAME,
-        replace: bool = True,
+        index_cache_size: Optional[int] = None,
     ):
-        raise NotImplementedError
+        """Create an index on the table.
+        Currently, the only parameters that matter are
+        the metric and the vector column name.
+
+        Parameters
+        ----------
+        metric : str
+            The metric to use for the index. Default is "L2".
+        vector_column_name : str
+            The name of the vector column. Default is "vector".
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> import uuid
+        >>> from lancedb.schema import vector
+        >>> db = lancedb.connect("db://...", api_key="...", region="...") # doctest: +SKIP
+        >>> table_name = uuid.uuid4().hex
+        >>> schema = pa.schema(
+        ...     [
+        ...             pa.field("id", pa.uint32(), False),
+        ...            pa.field("vector", vector(128), False),
+        ...             pa.field("s", pa.string(), False),
+        ...     ]
+        ... )
+        >>> table = db.create_table( # doctest: +SKIP
+        ...     table_name, # doctest: +SKIP
+        ...     schema=schema, # doctest: +SKIP
+        ... )
+        >>> table.create_index("L2", "vector") # doctest: +SKIP
+        """
+        index_type = "vector"
+
+        data = {
+            "column": vector_column_name,
+            "index_type": index_type,
+            "metric_type": metric,
+            "index_cache_size": index_cache_size,
+        }
+        resp = self._conn._loop.run_until_complete(
+            self._conn._client.post(f"/v1/table/{self._name}/create_index/", data=data)
+        )
+        return resp
 
     def add(
         self,
@@ -64,8 +121,34 @@ class RemoteTable(Table):
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
     ) -> int:
+        """Add more data to the [Table](Table). It has the same API signature as the OSS version.
+
+        Parameters
+        ----------
+        data: DATA
+            The data to insert into the table. Acceptable types are:
+
+            - dict or list-of-dict
+
+            - pandas.DataFrame
+
+            - pyarrow.Table or pyarrow.RecordBatch
+        mode: str
+            The mode to use when writing the data. Valid values are
+            "append" and "overwrite".
+        on_bad_vectors: str, default "error"
+            What to do if any of the vectors are not the same size or contains NaNs.
+            One of "error", "drop", "fill".
+        fill_value: float, default 0.
+            The value to use when filling vectors. Only used if on_bad_vectors="fill".
+
+        """
         data = _sanitize_data(
-            data, self.schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+            data,
+            self.schema,
+            metadata=None,
+            on_bad_vectors=on_bad_vectors,
+            fill_value=fill_value,
         )
         payload = to_ipc_binary(data)
 
@@ -81,13 +164,175 @@ class RemoteTable(Table):
         )
 
     def search(
-        self, query: Union[VEC, str], vector_column: str = VECTOR_COLUMN_NAME
-    ) -> LanceQueryBuilder:
-        return LanceQueryBuilder(self, query, vector_column)
+        self, query: Union[VEC, str], vector_column_name: str = VECTOR_COLUMN_NAME
+    ) -> LanceVectorQueryBuilder:
+        """Create a search query to find the nearest neighbors
+        of the given query vector. We currently support [vector search][search]
+
+        All query options are defined in [Query][lancedb.query.Query].
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> db = lancedb.connect("db://...", api_key="...", region="...") # doctest: +SKIP
+        >>> data = [
+        ...    {"original_width": 100, "caption": "bar", "vector": [0.1, 2.3, 4.5]},
+        ...    {"original_width": 2000, "caption": "foo",  "vector": [0.5, 3.4, 1.3]},
+        ...    {"original_width": 3000, "caption": "test", "vector": [0.3, 6.2, 2.6]}
+        ... ]
+        >>> table = db.create_table("my_table", data) # doctest: +SKIP
+        >>> query = [0.4, 1.4, 2.4]
+        >>> (table.search(query, vector_column_name="vector") # doctest: +SKIP
+        ...     .where("original_width > 1000", prefilter=True) # doctest: +SKIP
+        ...     .select(["caption", "original_width"]) # doctest: +SKIP
+        ...     .limit(2) # doctest: +SKIP
+        ...     .to_pandas()) # doctest: +SKIP
+          caption  original_width           vector  _distance # doctest: +SKIP
+        0     foo            2000  [0.5, 3.4, 1.3]   5.220000 # doctest: +SKIP
+        1    test            3000  [0.3, 6.2, 2.6]  23.089996 # doctest: +SKIP
+
+        Parameters
+        ----------
+        query: list/np.ndarray/str/PIL.Image.Image, default None
+            The targetted vector to search for.
+
+            - *default None*.
+            Acceptable types are: list, np.ndarray, PIL.Image.Image
+
+            - If None then the select/where/limit clauses are applied to filter
+            the table
+        vector_column_name: str
+            The name of the vector column to search.
+            *default "vector"*
+
+        Returns
+        -------
+        LanceQueryBuilder
+            A query builder object representing the query.
+            Once executed, the query returns
+
+            - selected columns
+
+            - the vector
+
+            - and also the "_distance" column which is the distance between the query
+            vector and the returned vector.
+        """
+        return LanceVectorQueryBuilder(self, query, vector_column_name)
 
     def _execute_query(self, query: Query) -> pa.Table:
         result = self._conn._client.query(self._name, query)
         return self._conn._loop.run_until_complete(result).to_arrow()
 
     def delete(self, predicate: str):
-        raise NotImplementedError
+        """Delete rows from the table.
+
+        This can be used to delete a single row, many rows, all rows, or
+        sometimes no rows (if your predicate matches nothing).
+
+        Parameters
+        ----------
+        predicate: str
+            The SQL where clause to use when deleting rows.
+
+            - For example, 'x = 2' or 'x IN (1, 2, 3)'.
+
+            The filter must not be empty, or it will error.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> data = [
+        ...    {"x": 1, "vector": [1, 2]},
+        ...    {"x": 2, "vector": [3, 4]},
+        ...    {"x": 3, "vector": [5, 6]}
+        ... ]
+        >>> db = lancedb.connect("db://...", api_key="...", region="...") # doctest: +SKIP
+        >>> table = db.create_table("my_table", data) # doctest: +SKIP
+        >>> table.search([10,10]).to_pandas() # doctest: +SKIP
+           x      vector  _distance # doctest: +SKIP
+        0  3  [5.0, 6.0]       41.0 # doctest: +SKIP
+        1  2  [3.0, 4.0]       85.0 # doctest: +SKIP
+        2  1  [1.0, 2.0]      145.0 # doctest: +SKIP
+        >>> table.delete("x = 2") # doctest: +SKIP
+        >>> table.search([10,10]).to_pandas() # doctest: +SKIP
+           x      vector  _distance # doctest: +SKIP
+        0  3  [5.0, 6.0]       41.0 # doctest: +SKIP
+        1  1  [1.0, 2.0]      145.0 # doctest: +SKIP
+
+        If you have a list of values to delete, you can combine them into a
+        stringified list and use the `IN` operator:
+
+        >>> to_remove = [1, 3] # doctest: +SKIP
+        >>> to_remove = ", ".join([str(v) for v in to_remove]) # doctest: +SKIP
+        >>> table.delete(f"x IN ({to_remove})") # doctest: +SKIP
+        >>> table.search([10,10]).to_pandas() # doctest: +SKIP
+           x      vector  _distance # doctest: +SKIP
+        0  2  [3.0, 4.0]       85.0 # doctest: +SKIP
+        """
+        payload = {"predicate": predicate}
+        self._conn._loop.run_until_complete(
+            self._conn._client.post(f"/v1/table/{self._name}/delete/", data=payload)
+        )
+
+    def update(
+        self,
+        where: Optional[str] = None,
+        values: Optional[dict] = None,
+        *,
+        values_sql: Optional[Dict[str, str]] = None,
+    ):
+        """
+        This can be used to update zero to all rows depending on how many
+        rows match the where clause.
+
+        Parameters
+        ----------
+        where: str, optional
+            The SQL where clause to use when updating rows. For example, 'x = 2'
+            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
+        values: dict, optional
+            The values to update. The keys are the column names and the values
+            are the values to set.
+        values_sql: dict, optional
+            The values to update, expressed as SQL expression strings. These can
+            reference existing columns. For example, {"x": "x + 1"} will increment
+            the x column by 1.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> data = [
+        ...    {"x": 1, "vector": [1, 2]},
+        ...    {"x": 2, "vector": [3, 4]},
+        ...    {"x": 3, "vector": [5, 6]}
+        ... ]
+        >>> db = lancedb.connect("db://...", api_key="...", region="...") # doctest: +SKIP
+        >>> table = db.create_table("my_table", data) # doctest: +SKIP
+        >>> table.to_pandas() # doctest: +SKIP
+           x      vector # doctest: +SKIP
+        0  1  [1.0, 2.0] # doctest: +SKIP
+        1  2  [3.0, 4.0] # doctest: +SKIP
+        2  3  [5.0, 6.0] # doctest: +SKIP
+        >>> table.update(where="x = 2", values={"vector": [10, 10]}) # doctest: +SKIP
+        >>> table.to_pandas() # doctest: +SKIP
+           x        vector # doctest: +SKIP
+        0  1    [1.0, 2.0] # doctest: +SKIP
+        1  3    [5.0, 6.0] # doctest: +SKIP
+        2  2  [10.0, 10.0] # doctest: +SKIP
+
+        """
+        if values is not None and values_sql is not None:
+            raise ValueError("Only one of values or values_sql can be provided")
+        if values is None and values_sql is None:
+            raise ValueError("Either values or values_sql must be provided")
+
+        if values is not None:
+            updates = [[k, value_to_sql(v)] for k, v in values.items()]
+        else:
+            updates = [[k, v] for k, v in values_sql.items()]
+
+        payload = {"predicate": where, "updates": updates}
+        self._conn._loop.run_until_complete(
+            self._conn._client.post(f"/v1/table/{self._name}/update/", data=payload)
+        )

python/lancedb/table.py

@@ -13,10 +13,11 @@
 
 from __future__ import annotations
 
+import inspect
 import os
 from abc import ABC, abstractmethod
 from functools import cached_property
-from typing import Iterable, List, Union
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
 
 import lance
 import numpy as np
@@ -26,38 +27,99 @@ from lance import LanceDataset
 from lance.vector import vec_to_table
 
 from .common import DATA, VEC, VECTOR_COLUMN_NAME
-from .pydantic import LanceModel
-from .query import LanceFtsQueryBuilder, LanceQueryBuilder, Query
-from .util import fs_from_uri, safe_import_pandas
+from .embeddings import EmbeddingFunctionConfig, EmbeddingFunctionRegistry
+from .pydantic import LanceModel, model_to_dict
+from .query import LanceQueryBuilder, Query
+from .util import fs_from_uri, safe_import_pandas, value_to_sql
+from .utils.events import register_event
+
+if TYPE_CHECKING:
+    from datetime import timedelta
+
+    from lance.dataset import CleanupStats, ReaderLike
+
 
 pd = safe_import_pandas()
 
 
-def _sanitize_data(data, schema, on_bad_vectors, fill_value):
+def _sanitize_data(
+    data,
+    schema: Optional[pa.Schema],
+    metadata: Optional[dict],
+    on_bad_vectors: str,
+    fill_value: Any,
+):
     if isinstance(data, list):
         # convert to list of dict if data is a bunch of LanceModels
         if isinstance(data[0], LanceModel):
             schema = data[0].__class__.to_arrow_schema()
-            data = [dict(d) for d in data]
+            data = [model_to_dict(d) for d in data]
+            data = pa.Table.from_pylist(data, schema=schema)
+        else:
             data = pa.Table.from_pylist(data)
-        data = _sanitize_schema(
-            data, schema=schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
-        )
-    if isinstance(data, dict):
+    elif isinstance(data, dict):
         data = vec_to_table(data)
-    if pd is not None and isinstance(data, pd.DataFrame):
-        data = pa.Table.from_pandas(data)
+    elif pd is not None and isinstance(data, pd.DataFrame):
+        data = pa.Table.from_pandas(data, preserve_index=False)
+        # Do not serialize Pandas metadata
+        meta = data.schema.metadata if data.schema.metadata is not None else {}
+        meta = {k: v for k, v in meta.items() if k != b"pandas"}
+        data = data.replace_schema_metadata(meta)
+
+    if isinstance(data, pa.Table):
+        if metadata:
+            data = _append_vector_col(data, metadata, schema)
+            metadata.update(data.schema.metadata or {})
+            data = data.replace_schema_metadata(metadata)
         data = _sanitize_schema(
             data, schema=schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
         )
-    if not isinstance(data, (pa.Table, Iterable)):
+    elif isinstance(data, Iterable):
+        data = _to_record_batch_generator(
+            data, schema, metadata, on_bad_vectors, fill_value
+        )
+    else:
         raise TypeError(f"Unsupported data type: {type(data)}")
     return data
 
 
+def _append_vector_col(data: pa.Table, metadata: dict, schema: Optional[pa.Schema]):
+    """
+    Use the embedding function to automatically embed the source column and add the
+    vector column to the table.
+    """
+    functions = EmbeddingFunctionRegistry.get_instance().parse_functions(metadata)
+    for vector_column, conf in functions.items():
+        func = conf.function
+        if vector_column not in data.column_names:
+            col_data = func.compute_source_embeddings_with_retry(
+                data[conf.source_column]
+            )
+            if schema is not None:
+                dtype = schema.field(vector_column).type
+            else:
+                dtype = pa.list_(pa.float32(), len(col_data[0]))
+            data = data.append_column(
+                pa.field(vector_column, type=dtype), pa.array(col_data, type=dtype)
+            )
+    return data
+
+
+def _to_record_batch_generator(
+    data: Iterable, schema, metadata, on_bad_vectors, fill_value
+):
+    for batch in data:
+        if not isinstance(batch, pa.RecordBatch):
+            table = _sanitize_data(batch, schema, metadata, on_bad_vectors, fill_value)
+            for batch in table.to_batches():
+                yield batch
+        else:
+            yield batch
+
+
 class Table(ABC):
     """
-    A [Table](Table) is a collection of Records in a LanceDB [Database](Database).
+    A Table is a collection of Records in a LanceDB Database.
 
     Examples
     --------
@@ -83,8 +145,8 @@ class Table(ABC):
 
     Can query the table with [Table.search][lancedb.table.Table.search].
 
-    >>> table.search([0.4, 0.4]).select(["b"]).to_df()
-       b      vector  score
+    >>> table.search([0.4, 0.4]).select(["b"]).to_pandas()
+       b      vector  _distance
     0  4  [0.5, 1.3]       0.82
     1  2  [1.1, 1.2]       1.13
 
@@ -92,15 +154,16 @@ class Table(ABC):
     [Table.create_index][lancedb.table.Table.create_index].
     """
 
+    @property
     @abstractmethod
     def schema(self) -> pa.Schema:
-        """Return the [Arrow Schema](https://arrow.apache.org/docs/python/api/datatypes.html#) of
-        this [Table](Table)
+        """The [Arrow Schema](https://arrow.apache.org/docs/python/api/datatypes.html#)
+        of this Table
 
         """
         raise NotImplementedError
 
-    def to_pandas(self):
+    def to_pandas(self) -> "pd.DataFrame":
         """Return the table as a pandas DataFrame.
 
         Returns
@@ -126,6 +189,8 @@ class Table(ABC):
         num_sub_vectors=96,
         vector_column_name: str = VECTOR_COLUMN_NAME,
         replace: bool = True,
+        accelerator: Optional[str] = None,
+        index_cache_size: Optional[int] = None,
     ):
         """Create an index on the table.
 
@@ -135,17 +200,23 @@ class Table(ABC):
             The distance metric to use when creating the index.
             Valid values are "L2", "cosine", or "dot".
             L2 is euclidean distance.
-        num_partitions: int
+        num_partitions: int, default 256
             The number of IVF partitions to use when creating the index.
             Default is 256.
-        num_sub_vectors: int
+        num_sub_vectors: int, default 96
             The number of PQ sub-vectors to use when creating the index.
             Default is 96.
         vector_column_name: str, default "vector"
             The vector column name to create the index.
         replace: bool, default True
-            If True, replace the existing index if it exists.
-            If False, raise an error if duplicate index exists.
+            - If True, replace the existing index if it exists.
+
+            - If False, raise an error if duplicate index exists.
+        accelerator: str, default None
+            If set, use the given accelerator to create the index.
+            Only support "cuda" for now.
+        index_cache_size : int, optional
+            The size of the index cache in number of entries. Default value is 256.
         """
         raise NotImplementedError
 
@@ -161,8 +232,14 @@ class Table(ABC):
 
         Parameters
         ----------
-        data: list-of-dict, dict, pd.DataFrame
-            The data to insert into the table.
+        data: DATA
+            The data to insert into the table. Acceptable types are:
+
+            - dict or list-of-dict
+
+            - pandas.DataFrame
+
+            - pyarrow.Table or pyarrow.RecordBatch
         mode: str
             The mode to use when writing the data. Valid values are
             "append" and "overwrite".
@@ -177,24 +254,76 @@ class Table(ABC):
 
     @abstractmethod
     def search(
-        self, query: Union[VEC, str], vector_column: str = VECTOR_COLUMN_NAME
+        self,
+        query: Optional[Union[VEC, str, "PIL.Image.Image"]] = None,
+        vector_column_name: str = VECTOR_COLUMN_NAME,
+        query_type: str = "auto",
     ) -> LanceQueryBuilder:
         """Create a search query to find the nearest neighbors
-        of the given query vector.
+        of the given query vector. We currently support [vector search][search]
+        and [full-text search][experimental-full-text-search].
+
+        All query options are defined in [Query][lancedb.query.Query].
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> db = lancedb.connect("./.lancedb")
+        >>> data = [
+        ...    {"original_width": 100, "caption": "bar", "vector": [0.1, 2.3, 4.5]},
+        ...    {"original_width": 2000, "caption": "foo",  "vector": [0.5, 3.4, 1.3]},
+        ...    {"original_width": 3000, "caption": "test", "vector": [0.3, 6.2, 2.6]}
+        ... ]
+        >>> table = db.create_table("my_table", data)
+        >>> query = [0.4, 1.4, 2.4]
+        >>> (table.search(query, vector_column_name="vector")
+        ...     .where("original_width > 1000", prefilter=True)
+        ...     .select(["caption", "original_width"])
+        ...     .limit(2)
+        ...     .to_pandas())
+          caption  original_width           vector  _distance
+        0     foo            2000  [0.5, 3.4, 1.3]   5.220000
+        1    test            3000  [0.3, 6.2, 2.6]  23.089996
 
         Parameters
         ----------
-        query: list, np.ndarray
-            The query vector.
-        vector_column: str, default "vector"
+        query: list/np.ndarray/str/PIL.Image.Image, default None
+            The targetted vector to search for.
+
+            - *default None*.
+            Acceptable types are: list, np.ndarray, PIL.Image.Image
+
+            - If None then the select/where/limit clauses are applied to filter
+            the table
+        vector_column_name: str
             The name of the vector column to search.
+            *default "vector"*
+        query_type: str
+            *default "auto"*.
+            Acceptable types are: "vector", "fts", or "auto"
+
+            - If "auto" then the query type is inferred from the query;
+
+                - If `query` is a list/np.ndarray then the query type is
+                "vector";
+
+                - If `query` is a PIL.Image.Image then either do vector search,
+                or raise an error if no corresponding embedding function is found.
+
+            - If `query` is a string, then the query type is "vector" if the
+            table has embedding functions else the query type is "fts"
 
         Returns
         -------
         LanceQueryBuilder
             A query builder object representing the query.
-            Once executed, the query returns selected columns, the vector,
-            and also the "score" column which is the distance between the query
+            Once executed, the query returns
+
+            - selected columns
+
+            - the vector
+
+            - and also the "_distance" column which is the distance between the query
             vector and the returned vector.
         """
         raise NotImplementedError
@@ -213,14 +342,20 @@ class Table(ABC):
         Parameters
         ----------
         where: str
-            The SQL where clause to use when deleting rows. For example, 'x = 2'
-            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
+            The SQL where clause to use when deleting rows.
+
+            - For example, 'x = 2' or 'x IN (1, 2, 3)'.
+
+            The filter must not be empty, or it will error.
 
         Examples
         --------
         >>> import lancedb
-        >>> import pandas as pd
-        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
+        >>> data = [
+        ...    {"x": 1, "vector": [1, 2]},
+        ...    {"x": 2, "vector": [3, 4]},
+        ...    {"x": 3, "vector": [5, 6]}
+        ... ]
         >>> db = lancedb.connect("./.lancedb")
         >>> table = db.create_table("my_table", data)
         >>> table.to_pandas()
@@ -261,10 +396,11 @@ class LanceTable(Table):
         self.name = name
         self._version = version
 
-    def _reset_dataset(self):
+    def _reset_dataset(self, version=None):
         try:
             if "_dataset" in self.__dict__:
                 del self.__dict__["_dataset"]
+            self._version = version
         except AttributeError:
             pass
 
@@ -290,7 +426,9 @@ class LanceTable(Table):
     def checkout(self, version: int):
         """Checkout a version of the table. This is an in-place operation.
 
-        This allows viewing previous versions of the table.
+        This allows viewing previous versions of the table. If you wish to
+        keep writing to the dataset starting from an old version, then use
+        the `restore` function.
 
         Parameters
         ----------
@@ -301,16 +439,17 @@ class LanceTable(Table):
         --------
         >>> import lancedb
         >>> db = lancedb.connect("./.lancedb")
-        >>> table = db.create_table("my_table", [{"vector": [1.1, 0.9], "type": "vector"}])
+        >>> table = db.create_table("my_table",
+        ...    [{"vector": [1.1, 0.9], "type": "vector"}])
         >>> table.version
-        1
+        2
         >>> table.to_pandas()
                vector    type
         0  [1.1, 0.9]  vector
         >>> table.add([{"vector": [0.5, 0.2], "type": "vector"}])
         >>> table.version
-        2
-        >>> table.checkout(1)
+        3
+        >>> table.checkout(2)
         >>> table.to_pandas()
                vector    type
         0  [1.1, 0.9]  vector
@@ -318,7 +457,66 @@ class LanceTable(Table):
         max_ver = max([v["version"] for v in self._dataset.versions()])
         if version < 1 or version > max_ver:
             raise ValueError(f"Invalid version {version}")
-        self._version = version
+        self._reset_dataset(version=version)
+
+        try:
+            # Accessing the property updates the cached value
+            _ = self._dataset
+        except Exception as e:
+            if "not found" in str(e):
+                raise ValueError(
+                    f"Version {version} no longer exists. Was it cleaned up?"
+                )
+            else:
+                raise e
+
+    def restore(self, version: int = None):
+        """Restore a version of the table. This is an in-place operation.
+
+        This creates a new version where the data is equivalent to the
+        specified previous version. Data is not copied (as of python-v0.2.1).
+
+        Parameters
+        ----------
+        version : int, default None
+            The version to restore. If unspecified then restores the currently
+            checked out version. If the currently checked out version is the
+            latest version then this is a no-op.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> db = lancedb.connect("./.lancedb")
+        >>> table = db.create_table("my_table", [
+        ...     {"vector": [1.1, 0.9], "type": "vector"}])
+        >>> table.version
+        2
+        >>> table.to_pandas()
+               vector    type
+        0  [1.1, 0.9]  vector
+        >>> table.add([{"vector": [0.5, 0.2], "type": "vector"}])
+        >>> table.version
+        3
+        >>> table.restore(2)
+        >>> table.to_pandas()
+               vector    type
+        0  [1.1, 0.9]  vector
+        >>> len(table.list_versions())
+        4
+        """
+        max_ver = max([v["version"] for v in self._dataset.versions()])
+        if version is None:
+            version = self.version
+        elif version < 1 or version > max_ver:
+            raise ValueError(f"Invalid version {version}")
+        else:
+            self.checkout(version)
+
+        if version == max_ver:
+            # no-op if restoring the latest version
+            return
+
+        self._dataset.restore()
         self._reset_dataset()
 
     def __len__(self):
@@ -362,6 +560,8 @@ class LanceTable(Table):
         num_sub_vectors=96,
         vector_column_name=VECTOR_COLUMN_NAME,
         replace: bool = True,
+        accelerator: Optional[str] = None,
+        index_cache_size: Optional[int] = None,
     ):
         """Create an index on the table."""
         self._dataset.create_index(
@@ -371,8 +571,11 @@ class LanceTable(Table):
             num_partitions=num_partitions,
             num_sub_vectors=num_sub_vectors,
             replace=replace,
+            accelerator=accelerator,
+            index_cache_size=index_cache_size,
         )
         self._reset_dataset()
+        register_event("create_index")
 
     def create_fts_index(self, field_names: Union[str, List[str]]):
         """Create a full-text search index on the table.
@@ -391,6 +594,7 @@ class LanceTable(Table):
             field_names = [field_names]
         index = create_index(self._get_fts_index_path(), field_names)
         populate_index(index, self, field_names)
+        register_event("create_fts_index")
 
     def _get_fts_index_path(self):
         return os.path.join(self._dataset_uri, "_indices", "tantivy")
@@ -411,6 +615,9 @@ class LanceTable(Table):
         fill_value: float = 0.0,
     ):
         """Add data to the table.
+        If vector columns are missing and the table
+        has embedding functions, then the vector columns
+        are automatically computed and added.
 
         Parameters
         ----------
@@ -432,43 +639,158 @@ class LanceTable(Table):
         """
         # TODO: manage table listing and metadata separately
         data = _sanitize_data(
-            data, self.schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+            data,
+            self.schema,
+            metadata=self.schema.metadata,
+            on_bad_vectors=on_bad_vectors,
+            fill_value=fill_value,
         )
-        lance.write_dataset(data, self._dataset_uri, mode=mode)
+        lance.write_dataset(data, self._dataset_uri, schema=self.schema, mode=mode)
         self._reset_dataset()
+        register_event("add")
 
-    def search(
-        self, query: Union[VEC, str], vector_column_name=VECTOR_COLUMN_NAME
-    ) -> LanceQueryBuilder:
-        """Create a search query to find the nearest neighbors
-        of the given query vector.
+    def merge(
+        self,
+        other_table: Union[LanceTable, ReaderLike],
+        left_on: str,
+        right_on: Optional[str] = None,
+        schema: Optional[Union[pa.Schema, LanceModel]] = None,
+    ):
+        """Merge another table into this table.
+
+        Performs a left join, where the dataset is the left side and other_table
+        is the right side. Rows existing in the dataset but not on the left will
+        be filled with null values, unless Lance doesn't support null values for
+        some types, in which case an error will be raised. The only overlapping
+        column allowed is the join column. If other overlapping columns exist,
+        an error will be raised.
 
         Parameters
         ----------
-        query: list, np.ndarray
-            The query vector.
+        other_table: LanceTable or Reader-like
+            The data to be merged. Acceptable types are:
+            - Pandas DataFrame, Pyarrow Table, Dataset, Scanner,
+            Iterator[RecordBatch], or RecordBatchReader
+            - LanceTable
+        left_on: str
+            The name of the column in the dataset to join on.
+        right_on: str or None
+            The name of the column in other_table to join on. If None, defaults to
+            left_on.
+        schema: pa.Schema or LanceModel, optional
+            The schema of the other_table.
+            If not provided, the schema is inferred from the data.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> import pyarrow as pa
+        >>> df = pa.table({'x': [1, 2, 3], 'y': ['a', 'b', 'c']})
+        >>> db = lancedb.connect("./.lancedb")
+        >>> table = db.create_table("dataset", df)
+        >>> table.to_pandas()
+           x  y
+        0  1  a
+        1  2  b
+        2  3  c
+        >>> new_df = pa.table({'x': [1, 2, 3], 'z': ['d', 'e', 'f']})
+        >>> table.merge(new_df, 'x')
+        >>> table.to_pandas()
+           x  y  z
+        0  1  a  d
+        1  2  b  e
+        2  3  c  f
+        """
+        if isinstance(schema, LanceModel):
+            schema = schema.to_arrow_schema()
+        if isinstance(other_table, LanceTable):
+            other_table = other_table.to_lance()
+        if isinstance(other_table, LanceDataset):
+            other_table = other_table.to_table()
+        self._dataset.merge(
+            other_table, left_on=left_on, right_on=right_on, schema=schema
+        )
+        self._reset_dataset()
+        register_event("merge")
+
+    @cached_property
+    def embedding_functions(self) -> dict:
+        """
+        Get the embedding functions for the table
+
+        Returns
+        -------
+        funcs: dict
+            A mapping of the vector column to the embedding function
+            or empty dict if not configured.
+        """
+        return EmbeddingFunctionRegistry.get_instance().parse_functions(
+            self.schema.metadata
+        )
+
+    def search(
+        self,
+        query: Optional[Union[VEC, str, "PIL.Image.Image"]] = None,
+        vector_column_name: str = VECTOR_COLUMN_NAME,
+        query_type: str = "auto",
+    ) -> LanceQueryBuilder:
+        """Create a search query to find the nearest neighbors
+        of the given query vector. We currently support [vector search][search]
+        and [full-text search][search].
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> db = lancedb.connect("./.lancedb")
+        >>> data = [
+        ...    {"original_width": 100, "caption": "bar", "vector": [0.1, 2.3, 4.5]},
+        ...    {"original_width": 2000, "caption": "foo",  "vector": [0.5, 3.4, 1.3]},
+        ...    {"original_width": 3000, "caption": "test", "vector": [0.3, 6.2, 2.6]}
+        ... ]
+        >>> table = db.create_table("my_table", data)
+        >>> query = [0.4, 1.4, 2.4]
+        >>> (table.search(query, vector_column_name="vector")
+        ...     .where("original_width > 1000", prefilter=True)
+        ...     .select(["caption", "original_width"])
+        ...     .limit(2)
+        ...     .to_pandas())
+          caption  original_width           vector  _distance
+        0     foo            2000  [0.5, 3.4, 1.3]   5.220000
+        1    test            3000  [0.3, 6.2, 2.6]  23.089996
+
+        Parameters
+        ----------
+        query: list/np.ndarray/str/PIL.Image.Image, default None
+            The targetted vector to search for.
+
+            - *default None*.
+            Acceptable types are: list, np.ndarray, PIL.Image.Image
+
+            - If None then the select/[where][sql]/limit clauses are applied
+            to filter the table
         vector_column_name: str, default "vector"
             The name of the vector column to search.
+        query_type: str, default "auto"
+            "vector", "fts", or "auto"
+            If "auto" then the query type is inferred from the query;
+            If `query` is a list/np.ndarray then the query type is "vector";
+            If `query` is a PIL.Image.Image then either do vector search
+            or raise an error if no corresponding embedding function is found.
+            If the `query` is a string, then the query type is "vector" if the
+            table has embedding functions, else the query type is "fts"
 
         Returns
         -------
         LanceQueryBuilder
             A query builder object representing the query.
             Once executed, the query returns selected columns, the vector,
-            and also the "score" column which is the distance between the query
+            and also the "_distance" column which is the distance between the query
             vector and the returned vector.
         """
-        if isinstance(query, str):
-            # fts
-            return LanceFtsQueryBuilder(self, query, vector_column_name)
-
-        if isinstance(query, list):
-            query = np.array(query)
-        if isinstance(query, np.ndarray):
-            query = query.astype(np.float32)
-        else:
-            raise TypeError(f"Unsupported query type: {type(query)}")
-        return LanceQueryBuilder(self, query, vector_column_name)
+        register_event("search_table")
+        return LanceQueryBuilder.create(
+            self, query, query_type, vector_column_name=vector_column_name
+        )
 
     @classmethod
     def create(
@@ -480,6 +802,7 @@ class LanceTable(Table):
         mode="create",
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
+        embedding_functions: List[EmbeddingFunctionConfig] = None,
     ):
         """
         Create a new table.
@@ -487,8 +810,11 @@ class LanceTable(Table):
         Examples
         --------
         >>> import lancedb
-        >>> import pandas as pd
-        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
+        >>> data = [
+        ...    {"x": 1, "vector": [1, 2]},
+        ...    {"x": 2, "vector": [3, 4]},
+        ...    {"x": 3, "vector": [5, 6]}
+        ... ]
         >>> db = lancedb.connect("./.lancedb")
         >>> table = db.create_table("my_table", data)
         >>> table.to_pandas()
@@ -506,8 +832,9 @@ class LanceTable(Table):
         data: list-of-dict, dict, pd.DataFrame, default None
             The data to insert into the table.
             At least one of `data` or `schema` must be provided.
-        schema: dict, optional
-            The schema of the table. If not provided, the schema is inferred from the data.
+        schema: pa.Schema or LanceModel, optional
+            The schema of the table. If not provided,
+            the schema is inferred from the data.
             At least one of `data` or `schema` must be provided.
         mode: str, default "create"
             The mode to use when writing the data. Valid values are
@@ -517,18 +844,59 @@ class LanceTable(Table):
             One of "error", "drop", "fill".
         fill_value: float, default 0.
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
+        embedding_functions: list of EmbeddingFunctionModel, default None
+            The embedding functions to use when creating the table.
         """
         tbl = LanceTable(db, name)
+        if inspect.isclass(schema) and issubclass(schema, LanceModel):
+            # convert LanceModel to pyarrow schema
+            # note that it's possible this contains
+            # embedding function metadata already
+            schema = schema.to_arrow_schema()
+
+        metadata = None
+        if embedding_functions is not None:
+            # If we passed in embedding functions explicitly
+            # then we'll override any schema metadata that
+            # may was implicitly specified by the LanceModel schema
+            registry = EmbeddingFunctionRegistry.get_instance()
+            metadata = registry.get_table_metadata(embedding_functions)
+
         if data is not None:
             data = _sanitize_data(
-                data, schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+                data,
+                schema,
+                metadata=metadata,
+                on_bad_vectors=on_bad_vectors,
+                fill_value=fill_value,
             )
-        else:
+
         if schema is None:
+            if data is None:
                 raise ValueError("Either data or schema must be provided")
-            data = pa.Table.from_pylist([], schema=schema)
-        lance.write_dataset(data, tbl._dataset_uri, schema=schema, mode=mode)
-        return LanceTable(db, name)
+            elif hasattr(data, "schema"):
+                schema = data.schema
+            elif isinstance(data, Iterable):
+                if metadata:
+                    raise TypeError(
+                        (
+                            "Persistent embedding functions not yet "
+                            "supported for generator data input"
+                        )
+                    )
+
+        if metadata:
+            schema = schema.with_metadata(metadata)
+
+        empty = pa.Table.from_pylist([], schema=schema)
+        lance.write_dataset(empty, tbl._dataset_uri, schema=schema, mode=mode)
+        table = LanceTable(db, name)
+
+        if data is not None:
+            table.add(data)
+
+        register_event("create_table")
+        return table
 
     @classmethod
     def open(cls, db, name):
@@ -537,18 +905,78 @@ class LanceTable(Table):
         file_info = fs.get_file_info(path)
         if file_info.type != pa.fs.FileType.Directory:
             raise FileNotFoundError(
-                f"Table {name} does not exist. Please first call db.create_table({name}, data)"
+                f"Table {name} does not exist."
+                f"Please first call db.create_table({name}, data)"
             )
+        register_event("open_table")
+
         return tbl
 
     def delete(self, where: str):
         self._dataset.delete(where)
 
+    def update(
+        self,
+        where: Optional[str] = None,
+        values: Optional[dict] = None,
+        *,
+        values_sql: Optional[Dict[str, str]] = None,
+    ):
+        """
+        This can be used to update zero to all rows depending on how many
+        rows match the where clause.
+
+        Parameters
+        ----------
+        where: str, optional
+            The SQL where clause to use when updating rows. For example, 'x = 2'
+            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
+        values: dict, optional
+            The values to update. The keys are the column names and the values
+            are the values to set.
+        values_sql: dict, optional
+            The values to update, expressed as SQL expression strings. These can
+            reference existing columns. For example, {"x": "x + 1"} will increment
+            the x column by 1.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> import pandas as pd
+        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
+        >>> db = lancedb.connect("./.lancedb")
+        >>> table = db.create_table("my_table", data)
+        >>> table.to_pandas()
+           x      vector
+        0  1  [1.0, 2.0]
+        1  2  [3.0, 4.0]
+        2  3  [5.0, 6.0]
+        >>> table.update(where="x = 2", values={"vector": [10, 10]})
+        >>> table.to_pandas()
+           x        vector
+        0  1    [1.0, 2.0]
+        1  3    [5.0, 6.0]
+        2  2  [10.0, 10.0]
+
+        """
+        if values is not None and values_sql is not None:
+            raise ValueError("Only one of values or values_sql can be provided")
+        if values is None and values_sql is None:
+            raise ValueError("Either values or values_sql must be provided")
+
+        if values is not None:
+            values_sql = {k: value_to_sql(v) for k, v in values.items()}
+
+        self.to_lance().update(values_sql, where)
+        self._reset_dataset()
+        register_event("update")
+
     def _execute_query(self, query: Query) -> pa.Table:
         ds = self.to_lance()
         return ds.to_table(
             columns=query.columns,
             filter=query.filter,
+            prefilter=query.prefilter,
             nearest={
                 "column": query.vector_column,
                 "q": query.vector,
@@ -559,6 +987,48 @@ class LanceTable(Table):
             },
         )
 
+    def cleanup_old_versions(
+        self,
+        older_than: Optional[timedelta] = None,
+        *,
+        delete_unverified: bool = False,
+    ) -> CleanupStats:
+        """
+        Clean up old versions of the table, freeing disk space.
+
+        Parameters
+        ----------
+        older_than: timedelta, default None
+            The minimum age of the version to delete. If None, then this defaults
+            to two weeks.
+        delete_unverified: bool, default False
+            Because they may be part of an in-progress transaction, files newer
+            than 7 days old are not deleted by default. If you are sure that
+            there are no in-progress transactions, then you can set this to True
+            to delete all files older than `older_than`.
+
+        Returns
+        -------
+        CleanupStats
+            The stats of the cleanup operation, including how many bytes were
+            freed.
+        """
+        return self.to_lance().cleanup_old_versions(
+            older_than, delete_unverified=delete_unverified
+        )
+
+    def compact_files(self, *args, **kwargs):
+        """
+        Run the compaction process on the table.
+
+        This can be run after making several small appends to optimize the table
+        for faster reads.
+
+        Arguments are passed onto :meth:`lance.dataset.DatasetOptimizer.compact_files`.
+        For most cases, the default should be fine.
+        """
+        return self.to_lance().optimize.compact_files(*args, **kwargs)
+
 
 def _sanitize_schema(
     data: pa.Table,
@@ -586,16 +1056,30 @@ def _sanitize_schema(
             return data
         # cast the columns to the expected types
         data = data.combine_chunks()
+        for field in schema:
+            # TODO: we're making an assumption that fixed size list of 10 or more
+            # is a vector column. This is definitely a bit hacky.
+            likely_vector_col = (
+                pa.types.is_fixed_size_list(field.type)
+                and pa.types.is_float32(field.type.value_type)
+                and field.type.list_size >= 10
+            )
+            is_default_vector_col = field.name == VECTOR_COLUMN_NAME
+            if field.name in data.column_names and (
+                likely_vector_col or is_default_vector_col
+            ):
                 data = _sanitize_vector_column(
                     data,
-            vector_column_name=VECTOR_COLUMN_NAME,
+                    vector_column_name=field.name,
                     on_bad_vectors=on_bad_vectors,
                     fill_value=fill_value,
                 )
         return pa.Table.from_arrays(
             [data[name] for name in schema.names], schema=schema
         )
+
     # just check the vector column
+    if VECTOR_COLUMN_NAME in data.column_names:
         return _sanitize_vector_column(
             data,
             vector_column_name=VECTOR_COLUMN_NAME,
@@ -603,6 +1087,8 @@ def _sanitize_schema(
             fill_value=fill_value,
         )
 
+    return data
+
 
 def _sanitize_vector_column(
     data: pa.Table,
@@ -625,12 +1111,11 @@ def _sanitize_vector_column(
     fill_value: float, default 0.0
         The value to use when filling vectors. Only used if on_bad_vectors="fill".
     """
-    if vector_column_name not in data.column_names:
-        raise ValueError(f"Missing vector column: {vector_column_name}")
     # ChunkedArray is annoying to work with, so we combine chunks here
     vec_arr = data[vector_column_name].combine_chunks()
     if pa.types.is_list(data[vector_column_name].type):
-        # if it's a variable size list array we make sure the dimensions are all the same
+        # if it's a variable size list array,
+        # we make sure the dimensions are all the same
         has_jagged_ndims = len(vec_arr.values) % len(data) != 0
         if has_jagged_ndims:
             data = _sanitize_jagged(

python/lancedb/util.py

@@ -12,9 +12,12 @@
 #  limitations under the License.
 
 import os
+from datetime import date, datetime
+from functools import singledispatch
 from typing import Tuple
 from urllib.parse import urlparse
 
+import numpy as np
 import pyarrow.fs as pa_fs
 
 
@@ -70,7 +73,11 @@ def fs_from_uri(uri: str) -> Tuple[pa_fs.FileSystem, str]:
     Get a PyArrow FileSystem from a URI, handling extra environment variables.
     """
     if get_uri_scheme(uri) == "s3":
-        fs = pa_fs.S3FileSystem(endpoint_override=os.environ.get("AWS_ENDPOINT"))
+        fs = pa_fs.S3FileSystem(
+            endpoint_override=os.environ.get("AWS_ENDPOINT"),
+            request_timeout=30,
+            connect_timeout=30,
+        )
         path = get_uri_location(uri)
         return fs, path
 
@@ -84,3 +91,53 @@ def safe_import_pandas():
         return pd
     except ImportError:
         return None
+
+
+@singledispatch
+def value_to_sql(value):
+    raise NotImplementedError("SQL conversion is not implemented for this type")
+
+
+@value_to_sql.register(str)
+def _(value: str):
+    return f"'{value}'"
+
+
+@value_to_sql.register(int)
+def _(value: int):
+    return str(value)
+
+
+@value_to_sql.register(float)
+def _(value: float):
+    return str(value)
+
+
+@value_to_sql.register(bool)
+def _(value: bool):
+    return str(value).upper()
+
+
+@value_to_sql.register(type(None))
+def _(value: type(None)):
+    return "NULL"
+
+
+@value_to_sql.register(datetime)
+def _(value: datetime):
+    return f"'{value.isoformat()}'"
+
+
+@value_to_sql.register(date)
+def _(value: date):
+    return f"'{value.isoformat()}'"
+
+
+@value_to_sql.register(list)
+def _(value: list):
+    return "[" + ", ".join(map(value_to_sql, value)) + "]"
+
+
+@value_to_sql.register(np.ndarray)
+def _(value: np.ndarray):
+    return value_to_sql(value.tolist())