Bump to v0.1.10-python

Support connect with remote database, and implement Search API

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.9
+current_version = 0.1.10
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True

.github/workflows/docs.yml

@@ -39,6 +39,28 @@ jobs:
         run: |
           python -m pip install -e .
           python -m pip install -r ../docs/requirements.txt
+      - name: Set up node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'npm'
+          cache-dependency-path: node/package-lock.json
+      - uses: Swatinem/rust-cache@v2
+      - name: Install node dependencies
+        working-directory: node
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Build node
+        working-directory: node
+        run: |
+          npm ci
+          npm run build
+          npm run tsc
+      - name: Create markdown files
+        working-directory: node
+        run: |
+          npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
       - name: Build docs
         run: |
           PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
@@ -50,4 +72,4 @@ jobs:
           path: "docs/site"
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v1
+        uses: actions/deploy-pages@v1

.github/workflows/docs_test.yml

@@ -0,0 +1,93 @@
+name: Documentation Code Testing
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - docs/**
+      - .github/workflows/docs_test.yml
+  pull_request:
+    paths:
+      - docs/**
+      - .github/workflows/docs_test.yml
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+env:
+  # Disable full debug symbol generation to speed up CI build and keep memory down
+  # "1" means line tables only, which is useful for panic tracebacks.
+  RUSTFLAGS: "-C debuginfo=1"
+  RUST_BACKTRACE: "1"
+
+jobs:
+  test-python:
+    name: Test doc python code
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-minor-version: [ "11" ]
+        os: ["ubuntu-22.04"]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.${{ matrix.python-minor-version }}
+        cache: "pip"
+        cache-dependency-path: "docs/test/requirements.txt"
+    - name: Build Python
+      working-directory: docs/test
+      run:
+        python -m pip install -r requirements.txt
+    - name: Create test files
+      run: |
+        cd docs/test
+        python md_testing.py
+    - name: Test
+      run: |
+        cd docs/test/python
+        for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
+  test-node:
+    name: Test doc nodejs code
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        node-version: [ "18" ]
+        os: ["ubuntu-22.04"]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - name: Set up Node
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+    - name: Install dependecies needed for ubuntu
+      if: ${{ matrix.os == 'ubuntu-22.04' }}
+      run: |
+        sudo apt install -y protobuf-compiler libssl-dev
+    - name: Install node dependencies
+      run: |
+        cd docs/test
+        npm install 
+    - name: Rust cache
+      uses: swatinem/rust-cache@v2
+    - name: Install LanceDB
+      run: |
+        cd docs/test/node_modules/vectordb 
+        npm ci
+        npm run build
+        npm run tsc
+    - name: Create test files
+      run: |
+        cd docs/test
+        node md_testing.js
+    - name: Test
+      run: |
+        cd docs/test/node
+        for d in *; do cd "$d"; echo "$d".js; node "$d".js; cd ..; done

.github/workflows/python.yml

@@ -61,6 +61,8 @@ jobs:
       run: |
         pip install -e .
         pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock
+        pip install pytest pytest-mock black
+    - name: Black
+      run: black --check --diff --no-color --quiet .
     - name: Run tests
-      run: pytest -x -v --durations=30 tests
+      run: pytest -x -v --durations=30 tests

.gitignore

@@ -3,6 +3,7 @@
 *.egg-info
 **/__pycache__
 .DS_Store
+venv
 
 rust/target
 rust/Cargo.lock
@@ -30,3 +31,4 @@ node/examples/**/dist
 ## Rust
 target
 
+Cargo.lock

.pre-commit-config.yaml

@@ -8,4 +8,14 @@ repos:
 -   repo: https://github.com/psf/black
     rev: 22.12.0
     hooks:
-    -   id: black
+    -   id: black
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.0.277
+    hooks:
+    - id: ruff
+-   repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+    - id: isort
+      name: isort (python)

Cargo.toml

@@ -4,3 +4,11 @@ members = [
     "rust/ffi/node"
 ]
 resolver = "2"
+
+[workspace.dependencies]
+lance = "0.5.3"
+arrow-array = "40.0"
+arrow-data = "40.0"
+arrow-schema = "40.0"
+arrow-ipc = "40.0"
+object_store = "0.6.1"

README.md

@@ -65,7 +65,7 @@ pip install lancedb
 ```python
 import lancedb
 
-uri = "/tmp/lancedb"
+uri = "data/sample-lancedb"
 db = lancedb.connect(uri)
 table = db.create_table("my_table",
                          data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},

docs/mkdocs.yml

@@ -38,6 +38,7 @@ plugins:
 
 markdown_extensions:
 - admonition
+- footnotes
 - pymdownx.superfences
 - pymdownx.details
 - pymdownx.highlight:
@@ -66,6 +67,7 @@ nav:
   - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
 - References:
   - Vector Search: search.md
+  - SQL filters: sql.md
   - Indexing: ann_indexes.md
 - API references:
   - Python API: python/python.md

docs/src/ann_indexes.md

@@ -23,7 +23,7 @@ In the future we will look to automatically create and configure the ANN index.
 
      # Create 10,000 sample vectors
      data = [{"vector": row, "item": f"item {i}"}
-        for i, row in enumerate(np.random.random((10_000, 768)).astype('float32'))]
+        for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]
 
      # Add the vectors to a table
      tbl = db.create_table("my_vectors", data=data)
@@ -41,8 +41,8 @@ In the future we will look to automatically create and configure the ANN index.
      for (let i = 0; i < 10_000; i++) {
          data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
      }
-     const table = await db.createTable('vectors', data)
-     await table.create_index({ type: 'ivf_pq', column: 'vector', num_partitions: 256, num_sub_vectors: 96 })
+     const table = await db.createTable('my_vectors', data)
+     await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 256, num_sub_vectors: 96 })
      ```
 
 Since `create_index` has a training step, it can take a few minutes to finish for large tables. You can control the index
@@ -73,12 +73,13 @@ There are a couple of parameters that can be used to fine-tune the search:
 
 === "Python"
      ```python
-     tbl.search(np.random.random((768))) \
+     tbl.search(np.random.random((1536))) \
          .limit(2) \
          .nprobes(20) \
          .refine_factor(10) \
          .to_df()
-
+     ```
+     ```
                                               vector       item       score
      0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
      1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
@@ -86,8 +87,8 @@ There are a couple of parameters that can be used to fine-tune the search:
 
 === "Javascript"
      ```javascript
-     const results = await table
-         .search(Array(768).fill(1.2))
+     const results_1 = await table
+         .search(Array(1536).fill(1.2))
          .limit(2)
          .nprobes(20)
          .refineFactor(10)
@@ -104,14 +105,14 @@ You can further filter the elements returned by a search using a where clause.
 
 === "Python"
      ```python
-     tbl.search(np.random.random((768))).where("item != 'item 1141'").to_df()
+     tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_df()
      ```
 
 === "Javascript"
      ```javascript
-     const results = await table
+     const results_2 = await table
          .search(Array(1536).fill(1.2))
-         .where("item != 'item 1141'")
+         .where("id != '1141'")
          .execute()
      ```
 
@@ -121,7 +122,9 @@ You can select the columns returned by the query using a select clause.
 
 === "Python"
      ```python
-     tbl.search(np.random.random((768))).select(["vector"]).to_df()
+     tbl.search(np.random.random((1536))).select(["vector"]).to_df()
+     ```
+     ```
      vector      score
      0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
      1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
@@ -130,7 +133,7 @@ You can select the columns returned by the query using a select clause.
 
 === "Javascript"
      ```javascript
-     const results = await table
+     const results_3 = await table
          .search(Array(1536).fill(1.2))
          .select(["id"])
          .execute()

docs/src/basic.md

@@ -23,7 +23,7 @@ We'll cover the basics of using LanceDB on your local machine in this section.
 === "Python"
       ```python
       import lancedb
-      uri = "~/.lancedb"
+      uri = "data/sample-lancedb"
       db = lancedb.connect(uri)
       ```
 
@@ -35,7 +35,7 @@ We'll cover the basics of using LanceDB on your local machine in this section.
       ```javascript
       const lancedb = require("vectordb");
 
-      const uri = "~./lancedb";
+      const uri = "data/sample-lancedb";
       const db = await lancedb.connect(uri);
       ```
       
@@ -102,7 +102,7 @@ Once created, you can open a table using the following code:
       If you forget the name of your table, you can always get a listing of all table names:
 
       ```javascript
-      console.log(db.tableNames());
+      console.log(await db.tableNames());
       ```
 
 ## How to add data to a table
@@ -118,7 +118,7 @@ After a table has been created, you can always add more data to it using
 
 === "Javascript"
       ```javascript
-      await tbl.add([vector: [1.3, 1.4], item: "fizz", price: 100.0},
+      await tbl.add([{vector: [1.3, 1.4], item: "fizz", price: 100.0},
               {vector: [9.5, 56.2], item: "buzz", price: 200.0}])
       ```
 

docs/src/embedding.md

@@ -98,7 +98,7 @@ You can also use an external API like OpenAI to generate embeddings
      embededings for your data.
 
       ```javascript
-      const db = await lancedb.connect("/tmp/lancedb");
+      const db = await lancedb.connect("data/sample-lancedb");
       const data = [
         { text: 'pepperoni'  },
         { text: 'pineapple' }

docs/src/examples/modal_langchain.py

@@ -79,10 +79,7 @@ def qanda_langchain(query):
     download_docs()
     docs = store_docs()
 
-    text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=1000,
-        chunk_overlap=200,
-    )
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200,)
     documents = text_splitter.split_documents(docs)
     embeddings = OpenAIEmbeddings()
 

docs/src/fts.md

@@ -18,6 +18,20 @@ Assume:
 1. `table` is a LanceDB Table
 2. `text` is the name of the Table column that we want to index
 
+For example,
+
+```python
+import lancedb
+
+uri = "data/sample-lancedb"
+db = lancedb.connect(uri)
+
+table = db.create_table("my_table",
+            data=[{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
+                  {"vector": [5.9, 26.5], "text": "There are several kittens playing"}])
+
+```
+
 To create the index:
 
 ```python

docs/src/index.md

@@ -28,7 +28,7 @@ LanceDB's core is written in Rust 🦀 and is built using <a href="https://githu
       ```python
       import lancedb
 
-      uri = "/tmp/lancedb"
+      uri = "data/sample-lancedb"
       db = lancedb.connect(uri)
       table = db.create_table("my_table",
                               data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
@@ -44,7 +44,7 @@ LanceDB's core is written in Rust 🦀 and is built using <a href="https://githu
       ```javascript
       const lancedb = require("vectordb");
 
-      const uri = "/tmp/lancedb";
+      const uri = "data/sample-lancedb";
       const db = await lancedb.connect(uri);
       const table = await db.createTable("my_table", 
             [{ id: 1, vector: [3.1, 4.1], item: "foo", price: 10.0 },

docs/src/integrations.md

@@ -6,11 +6,11 @@ Built on top of Apache Arrow, `LanceDB` is easy to integrate with the Python eco
 
 First, we need to connect to a `LanceDB` database.
 
-``` py
+```py
 
 import lancedb
 
-db = lancedb.connect("/tmp/lancedb")
+db = lancedb.connect("data/sample-lancedb")
 ```
 
 And write a `Pandas DataFrame` to LanceDB directly.
@@ -79,7 +79,7 @@ We will re-use the dataset created previously
 ```python
 import lancedb
 
-db = lancedb.connect("/tmp/lancedb")
+db = lancedb.connect("data/sample-lancedb")
 table = db.open_table("pd_table")
 arrow_table = table.to_arrow()
 ```
@@ -87,8 +87,12 @@ arrow_table = table.to_arrow()
 `DuckDB` can directly query the `arrow_table`:
 
 ```python
-In [15]: duckdb.query("SELECT * FROM t")
-Out[15]:
+import duckdb 
+
+duckdb.query("SELECT * FROM arrow_table")
+```
+
+```
 ┌─────────────┬─────────┬────────┐
 │   vector    │  item   │ price  │
 │   float[]   │ varchar │ double │
@@ -96,8 +100,12 @@ Out[15]:
 │ [3.1, 4.1]  │ foo     │   10.0 │
 │ [5.9, 26.5] │ bar     │   20.0 │
 └─────────────┴─────────┴────────┘
+```
+```python
+duckdb.query("SELECT mean(price) FROM arrow_table")
+```
 
-In [16]: duckdb.query("SELECT mean(price) FROM t")
+```
 Out[16]:
 ┌─────────────┐
 │ mean(price) │

docs/src/javascript/README.md

@@ -16,9 +16,11 @@ npm install vectordb
 
 ```javascript
 const lancedb = require('vectordb');
-const db = lancedb.connect('<PATH_TO_LANCEDB_DATASET>');
-const table = await db.openTable('my_table');
-const query = await table.search([0.1, 0.3]).setLimit(20).execute();
+const db = await lancedb.connect('data/sample-lancedb');
+const table = await db.createTable("my_table",
+      [{ id: 1, vector: [0.1, 1.0], item: "foo", price: 10.0 },
+      { id: 2, vector: [3.9, 0.5], item: "bar", price: 20.0 }])
+const results = await table.search([0.1, 0.3]).limit(20).execute();
 console.log(results);
 ```
 
@@ -26,12 +28,6 @@ The [examples](./examples) folder contains complete examples.
 
 ## Development
 
-The LanceDB javascript is built with npm:
-
-```bash
-npm run tsc
-```
-
 Run the tests with
 
 ```bash

docs/src/javascript/classes/Connection.md

@@ -1,211 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / Connection
-
-# Class: Connection
-
-A connection to a LanceDB database.
-
-## Table of contents
-
-### Constructors
-
-- [constructor](Connection.md#constructor)
-
-### Properties
-
-- [\_db](Connection.md#_db)
-- [\_uri](Connection.md#_uri)
-
-### Accessors
-
-- [uri](Connection.md#uri)
-
-### Methods
-
-- [createTable](Connection.md#createtable)
-- [createTableArrow](Connection.md#createtablearrow)
-- [openTable](Connection.md#opentable)
-- [tableNames](Connection.md#tablenames)
-
-## Constructors
-
-### constructor
-
-• **new Connection**(`db`, `uri`)
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `db` | `any` |
-| `uri` | `string` |
-
-#### Defined in
-
-[index.ts:46](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L46)
-
-## Properties
-
-### \_db
-
-• `Private` `Readonly` **\_db**: `any`
-
-#### Defined in
-
-[index.ts:44](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L44)
-
-___
-
-### \_uri
-
-• `Private` `Readonly` **\_uri**: `string`
-
-#### Defined in
-
-[index.ts:43](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L43)
-
-## Accessors
-
-### uri
-
-• `get` **uri**(): `string`
-
-#### Returns
-
-`string`
-
-#### Defined in
-
-[index.ts:51](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L51)
-
-## Methods
-
-### createTable
-
-▸ **createTable**(`name`, `data`): `Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-Creates a new Table and initialize it with new data.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-#### Defined in
-
-[index.ts:91](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L91)
-
-▸ **createTable**<`T`\>(`name`, `data`, `embeddings`): `Promise`<[`Table`](Table.md)<`T`\>\>
-
-Creates a new Table and initialize it with new data.
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`T`\>\>
-
-#### Defined in
-
-[index.ts:99](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L99)
-
-___
-
-### createTableArrow
-
-▸ **createTableArrow**(`name`, `table`): `Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` |
-| `table` | `Table`<`any`\> |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-#### Defined in
-
-[index.ts:109](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L109)
-
-___
-
-### openTable
-
-▸ **openTable**(`name`): `Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-Open a table in the database.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-#### Defined in
-
-[index.ts:67](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L67)
-
-▸ **openTable**<`T`\>(`name`, `embeddings`): `Promise`<[`Table`](Table.md)<`T`\>\>
-
-Open a table in the database.
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`T`\>\>
-
-#### Defined in
-
-[index.ts:74](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L74)
-
-___
-
-### tableNames
-
-▸ **tableNames**(): `Promise`<`string`[]\>
-
-Get the names of all tables in the database.
-
-#### Returns
-
-`Promise`<`string`[]\>
-
-#### Defined in
-
-[index.ts:58](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L58)

docs/src/javascript/classes/LocalConnection.md

@@ -0,0 +1,294 @@
+[vectordb](../README.md) / [Exports](../modules.md) / LocalConnection
+
+# Class: LocalConnection
+
+A connection to a LanceDB database.
+
+## Implements
+
+- [`Connection`](../interfaces/Connection.md)
+
+## Table of contents
+
+### Constructors
+
+- [constructor](LocalConnection.md#constructor)
+
+### Properties
+
+- [\_db](LocalConnection.md#_db)
+- [\_uri](LocalConnection.md#_uri)
+
+### Accessors
+
+- [uri](LocalConnection.md#uri)
+
+### Methods
+
+- [createTable](LocalConnection.md#createtable)
+- [createTableArrow](LocalConnection.md#createtablearrow)
+- [dropTable](LocalConnection.md#droptable)
+- [openTable](LocalConnection.md#opentable)
+- [tableNames](LocalConnection.md#tablenames)
+
+## Constructors
+
+### constructor
+
+• **new LocalConnection**(`db`, `uri`)
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `db` | `any` |
+| `uri` | `string` |
+
+#### Defined in
+
+[index.ts:132](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L132)
+
+## Properties
+
+### \_db
+
+• `Private` `Readonly` **\_db**: `any`
+
+#### Defined in
+
+[index.ts:130](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L130)
+
+___
+
+### \_uri
+
+• `Private` `Readonly` **\_uri**: `string`
+
+#### Defined in
+
+[index.ts:129](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L129)
+
+## Accessors
+
+### uri
+
+• `get` **uri**(): `string`
+
+#### Returns
+
+`string`
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[uri](../interfaces/Connection.md#uri)
+
+#### Defined in
+
+[index.ts:137](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L137)
+
+## Methods
+
+### createTable
+
+▸ **createTable**(`name`, `data`, `mode?`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+Creates a new Table and initialize it with new data.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
+| `mode?` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[createTable](../interfaces/Connection.md#createtable)
+
+#### Defined in
+
+[index.ts:177](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L177)
+
+▸ **createTable**(`name`, `data`, `mode`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `data` | `Record`<`string`, `unknown`\>[] |
+| `mode` | [`WriteMode`](../enums/WriteMode.md) |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Implementation of
+
+Connection.createTable
+
+#### Defined in
+
+[index.ts:178](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L178)
+
+▸ **createTable**<`T`\>(`name`, `data`, `mode`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+Creates a new Table and initialize it with new data.
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
+| `mode` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Implementation of
+
+Connection.createTable
+
+#### Defined in
+
+[index.ts:188](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L188)
+
+___
+
+### createTableArrow
+
+▸ **createTableArrow**(`name`, `table`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `table` | `Table`<`any`\> |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[createTableArrow](../interfaces/Connection.md#createtablearrow)
+
+#### Defined in
+
+[index.ts:201](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L201)
+
+___
+
+### dropTable
+
+▸ **dropTable**(`name`): `Promise`<`void`\>
+
+Drop an existing table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table to drop. |
+
+#### Returns
+
+`Promise`<`void`\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[dropTable](../interfaces/Connection.md#droptable)
+
+#### Defined in
+
+[index.ts:211](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L211)
+
+___
+
+### openTable
+
+▸ **openTable**(`name`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+Open a table in the database.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[openTable](../interfaces/Connection.md#opentable)
+
+#### Defined in
+
+[index.ts:153](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L153)
+
+▸ **openTable**<`T`\>(`name`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+Open a table in the database.
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Implementation of
+
+Connection.openTable
+
+#### Defined in
+
+[index.ts:160](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L160)
+
+___
+
+### tableNames
+
+▸ **tableNames**(): `Promise`<`string`[]\>
+
+Get the names of all tables in the database.
+
+#### Returns
+
+`Promise`<`string`[]\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[tableNames](../interfaces/Connection.md#tablenames)
+
+#### Defined in
+
+[index.ts:144](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L144)

docs/src/javascript/classes/LocalTable.md

@@ -0,0 +1,289 @@
+[vectordb](../README.md) / [Exports](../modules.md) / LocalTable
+
+# Class: LocalTable<T\>
+
+A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
+
+## Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+## Implements
+
+- [`Table`](../interfaces/Table.md)<`T`\>
+
+## Table of contents
+
+### Constructors
+
+- [constructor](LocalTable.md#constructor)
+
+### Properties
+
+- [\_embeddings](LocalTable.md#_embeddings)
+- [\_name](LocalTable.md#_name)
+- [\_tbl](LocalTable.md#_tbl)
+
+### Accessors
+
+- [name](LocalTable.md#name)
+
+### Methods
+
+- [add](LocalTable.md#add)
+- [countRows](LocalTable.md#countrows)
+- [createIndex](LocalTable.md#createindex)
+- [delete](LocalTable.md#delete)
+- [overwrite](LocalTable.md#overwrite)
+- [search](LocalTable.md#search)
+
+## Constructors
+
+### constructor
+
+• **new LocalTable**<`T`\>(`tbl`, `name`)
+
+#### Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `tbl` | `any` |
+| `name` | `string` |
+
+#### Defined in
+
+[index.ts:221](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L221)
+
+• **new LocalTable**<`T`\>(`tbl`, `name`, `embeddings`)
+
+#### Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `tbl` | `any` |  |
+| `name` | `string` |  |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use when interacting with this table |
+
+#### Defined in
+
+[index.ts:227](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L227)
+
+## Properties
+
+### \_embeddings
+
+• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
+
+#### Defined in
+
+[index.ts:219](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L219)
+
+___
+
+### \_name
+
+• `Private` `Readonly` **\_name**: `string`
+
+#### Defined in
+
+[index.ts:218](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L218)
+
+___
+
+### \_tbl
+
+• `Private` `Readonly` **\_tbl**: `any`
+
+#### Defined in
+
+[index.ts:217](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L217)
+
+## Accessors
+
+### name
+
+• `get` **name**(): `string`
+
+#### Returns
+
+`string`
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[name](../interfaces/Table.md#name)
+
+#### Defined in
+
+[index.ts:234](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L234)
+
+## Methods
+
+### add
+
+▸ **add**(`data`): `Promise`<`number`\>
+
+Insert records into this Table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+
+#### Returns
+
+`Promise`<`number`\>
+
+The number of rows added to the table
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[add](../interfaces/Table.md#add)
+
+#### Defined in
+
+[index.ts:252](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L252)
+
+___
+
+### countRows
+
+▸ **countRows**(): `Promise`<`number`\>
+
+Returns the number of rows in this table.
+
+#### Returns
+
+`Promise`<`number`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[countRows](../interfaces/Table.md#countrows)
+
+#### Defined in
+
+[index.ts:278](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L278)
+
+___
+
+### createIndex
+
+▸ **createIndex**(`indexParams`): `Promise`<`any`\>
+
+Create an ANN index on this Table vector index.
+
+**`See`**
+
+VectorIndexParams.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `indexParams` | `IvfPQIndexConfig` | The parameters of this Index, |
+
+#### Returns
+
+`Promise`<`any`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[createIndex](../interfaces/Table.md#createindex)
+
+#### Defined in
+
+[index.ts:271](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L271)
+
+___
+
+### delete
+
+▸ **delete**(`filter`): `Promise`<`void`\>
+
+Delete rows from this table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `filter` | `string` | A filter in the same format used by a sql WHERE clause. |
+
+#### Returns
+
+`Promise`<`void`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[delete](../interfaces/Table.md#delete)
+
+#### Defined in
+
+[index.ts:287](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L287)
+
+___
+
+### overwrite
+
+▸ **overwrite**(`data`): `Promise`<`number`\>
+
+Insert records into this Table, replacing its contents.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+
+#### Returns
+
+`Promise`<`number`\>
+
+The number of rows added to the table
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[overwrite](../interfaces/Table.md#overwrite)
+
+#### Defined in
+
+[index.ts:262](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L262)
+
+___
+
+### search
+
+▸ **search**(`query`): [`Query`](Query.md)<`T`\>
+
+Creates a search query to find the nearest neighbors of the given search term
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `query` | `T` | The query search term |
+
+#### Returns
+
+[`Query`](Query.md)<`T`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[search](../interfaces/Table.md#search)
+
+#### Defined in
+
+[index.ts:242](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L242)

docs/src/javascript/classes/OpenAIEmbeddingFunction.md

@@ -40,7 +40,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/openai.ts#L21)
+[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L21)
 
 ## Properties
 
@@ -50,7 +50,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/openai.ts#L19)
+[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L19)
 
 ___
 
@@ -60,7 +60,7 @@ ___
 
 #### Defined in
 
-[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/openai.ts#L18)
+[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L18)
 
 ___
 
@@ -76,7 +76,7 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/openai.ts#L50)
+[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L50)
 
 ## Methods
 
@@ -102,4 +102,4 @@ Creates a vector representation for the given values.
 
 #### Defined in
 
-[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/openai.ts#L38)
+[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L38)

docs/src/javascript/classes/Query.md

@@ -18,7 +18,6 @@ A builder for nearest neighbor queries for LanceDB.
 
 ### Properties
 
-- [\_columns](Query.md#_columns)
 - [\_embeddings](Query.md#_embeddings)
 - [\_filter](Query.md#_filter)
 - [\_limit](Query.md#_limit)
@@ -27,7 +26,9 @@ A builder for nearest neighbor queries for LanceDB.
 - [\_query](Query.md#_query)
 - [\_queryVector](Query.md#_queryvector)
 - [\_refineFactor](Query.md#_refinefactor)
+- [\_select](Query.md#_select)
 - [\_tbl](Query.md#_tbl)
+- [where](Query.md#where)
 
 ### Methods
 
@@ -37,6 +38,7 @@ A builder for nearest neighbor queries for LanceDB.
 - [metricType](Query.md#metrictype)
 - [nprobes](Query.md#nprobes)
 - [refineFactor](Query.md#refinefactor)
+- [select](Query.md#select)
 
 ## Constructors
 
@@ -60,27 +62,17 @@ A builder for nearest neighbor queries for LanceDB.
 
 #### Defined in
 
-[index.ts:241](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L241)
+[index.ts:362](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L362)
 
 ## Properties
 
-### \_columns
-
-• `Private` `Optional` `Readonly` **\_columns**: `string`[]
-
-#### Defined in
-
-[index.ts:236](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L236)
-
-___
-
 ### \_embeddings
 
 • `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
 
 #### Defined in
 
-[index.ts:239](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L239)
+[index.ts:360](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L360)
 
 ___
 
@@ -90,7 +82,7 @@ ___
 
 #### Defined in
 
-[index.ts:237](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L237)
+[index.ts:358](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L358)
 
 ___
 
@@ -100,7 +92,7 @@ ___
 
 #### Defined in
 
-[index.ts:233](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L233)
+[index.ts:354](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L354)
 
 ___
 
@@ -110,7 +102,7 @@ ___
 
 #### Defined in
 
-[index.ts:238](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L238)
+[index.ts:359](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L359)
 
 ___
 
@@ -120,7 +112,7 @@ ___
 
 #### Defined in
 
-[index.ts:235](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L235)
+[index.ts:356](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L356)
 
 ___
 
@@ -130,7 +122,7 @@ ___
 
 #### Defined in
 
-[index.ts:231](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L231)
+[index.ts:352](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L352)
 
 ___
 
@@ -140,7 +132,7 @@ ___
 
 #### Defined in
 
-[index.ts:232](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L232)
+[index.ts:353](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L353)
 
 ___
 
@@ -150,7 +142,17 @@ ___
 
 #### Defined in
 
-[index.ts:234](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L234)
+[index.ts:355](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L355)
+
+___
+
+### \_select
+
+• `Private` `Optional` **\_select**: `string`[]
+
+#### Defined in
+
+[index.ts:357](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L357)
 
 ___
 
@@ -160,7 +162,33 @@ ___
 
 #### Defined in
 
-[index.ts:230](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L230)
+[index.ts:351](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L351)
+
+___
+
+### where
+
+• **where**: (`value`: `string`) => [`Query`](Query.md)<`T`\>
+
+#### Type declaration
+
+▸ (`value`): [`Query`](Query.md)<`T`\>
+
+A filter statement to be applied to this query.
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `value` | `string` | A filter in the same format used by a sql WHERE clause. |
+
+##### Returns
+
+[`Query`](Query.md)<`T`\>
+
+#### Defined in
+
+[index.ts:410](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L410)
 
 ## Methods
 
@@ -182,7 +210,7 @@ Execute the query and return the results as an Array of Objects
 
 #### Defined in
 
-[index.ts:301](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L301)
+[index.ts:433](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L433)
 
 ___
 
@@ -204,7 +232,7 @@ A filter statement to be applied to this query.
 
 #### Defined in
 
-[index.ts:284](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L284)
+[index.ts:405](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L405)
 
 ___
 
@@ -226,7 +254,7 @@ Sets the number of results that will be returned
 
 #### Defined in
 
-[index.ts:257](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L257)
+[index.ts:378](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L378)
 
 ___
 
@@ -252,7 +280,7 @@ MetricType for the different options
 
 #### Defined in
 
-[index.ts:293](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L293)
+[index.ts:425](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L425)
 
 ___
 
@@ -274,7 +302,7 @@ The number of probes used. A higher number makes search more accurate but also s
 
 #### Defined in
 
-[index.ts:275](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L275)
+[index.ts:396](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L396)
 
 ___
 
@@ -296,4 +324,26 @@ Refine the results by reading extra elements and re-ranking them in memory.
 
 #### Defined in
 
-[index.ts:266](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L266)
+[index.ts:387](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L387)
+
+___
+
+### select
+
+▸ **select**(`value`): [`Query`](Query.md)<`T`\>
+
+Return only the specified columns.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `value` | `string`[] | Only select the specified columns. If not specified, all columns will be returned. |
+
+#### Returns
+
+[`Query`](Query.md)<`T`\>
+
+#### Defined in
+
+[index.ts:416](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L416)

docs/src/javascript/classes/Table.md

@@ -1,215 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / Table
-
-# Class: Table<T\>
-
-## Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-## Table of contents
-
-### Constructors
-
-- [constructor](Table.md#constructor)
-
-### Properties
-
-- [\_embeddings](Table.md#_embeddings)
-- [\_name](Table.md#_name)
-- [\_tbl](Table.md#_tbl)
-
-### Accessors
-
-- [name](Table.md#name)
-
-### Methods
-
-- [add](Table.md#add)
-- [create\_index](Table.md#create_index)
-- [overwrite](Table.md#overwrite)
-- [search](Table.md#search)
-
-## Constructors
-
-### constructor
-
-• **new Table**<`T`\>(`tbl`, `name`)
-
-#### Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `tbl` | `any` |
-| `name` | `string` |
-
-#### Defined in
-
-[index.ts:121](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L121)
-
-• **new Table**<`T`\>(`tbl`, `name`, `embeddings`)
-
-#### Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `tbl` | `any` |  |
-| `name` | `string` |  |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use when interacting with this table |
-
-#### Defined in
-
-[index.ts:127](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L127)
-
-## Properties
-
-### \_embeddings
-
-• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
-
-#### Defined in
-
-[index.ts:119](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L119)
-
-___
-
-### \_name
-
-• `Private` `Readonly` **\_name**: `string`
-
-#### Defined in
-
-[index.ts:118](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L118)
-
-___
-
-### \_tbl
-
-• `Private` `Readonly` **\_tbl**: `any`
-
-#### Defined in
-
-[index.ts:117](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L117)
-
-## Accessors
-
-### name
-
-• `get` **name**(): `string`
-
-#### Returns
-
-`string`
-
-#### Defined in
-
-[index.ts:134](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L134)
-
-## Methods
-
-### add
-
-▸ **add**(`data`): `Promise`<`number`\>
-
-Insert records into this Table.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
-
-#### Returns
-
-`Promise`<`number`\>
-
-The number of rows added to the table
-
-#### Defined in
-
-[index.ts:152](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L152)
-
-___
-
-### create\_index
-
-▸ **create_index**(`indexParams`): `Promise`<`any`\>
-
-Create an ANN index on this Table vector index.
-
-**`See`**
-
-VectorIndexParams.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `indexParams` | `IvfPQIndexConfig` | The parameters of this Index, |
-
-#### Returns
-
-`Promise`<`any`\>
-
-#### Defined in
-
-[index.ts:171](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L171)
-
-___
-
-### overwrite
-
-▸ **overwrite**(`data`): `Promise`<`number`\>
-
-Insert records into this Table, replacing its contents.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
-
-#### Returns
-
-`Promise`<`number`\>
-
-The number of rows added to the table
-
-#### Defined in
-
-[index.ts:162](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L162)
-
-___
-
-### search
-
-▸ **search**(`query`): [`Query`](Query.md)<`T`\>
-
-Creates a search query to find the nearest neighbors of the given search term
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `query` | `T` | The query search term |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:142](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L142)

docs/src/javascript/enums/MetricType.md

@@ -9,6 +9,7 @@ Distance metrics type.
 ### Enumeration Members
 
 - [Cosine](MetricType.md#cosine)
+- [Dot](MetricType.md#dot)
 - [L2](MetricType.md#l2)
 
 ## Enumeration Members
@@ -21,7 +22,19 @@ Cosine distance
 
 #### Defined in
 
-[index.ts:341](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L341)
+[index.ts:481](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L481)
+
+___
+
+### Dot
+
+• **Dot** = ``"dot"``
+
+Dot product
+
+#### Defined in
+
+[index.ts:486](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L486)
 
 ___
 
@@ -33,4 +46,4 @@ Euclidean distance
 
 #### Defined in
 
-[index.ts:336](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L336)
+[index.ts:476](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L476)

docs/src/javascript/enums/WriteMode.md

@@ -2,11 +2,14 @@
 
 # Enumeration: WriteMode
 
+Write mode for writing a table.
+
 ## Table of contents
 
 ### Enumeration Members
 
 - [Append](WriteMode.md#append)
+- [Create](WriteMode.md#create)
 - [Overwrite](WriteMode.md#overwrite)
 
 ## Enumeration Members
@@ -15,9 +18,23 @@
 
 • **Append** = ``"append"``
 
+Append new data to the table.
+
 #### Defined in
 
-[index.ts:326](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L326)
+[index.ts:466](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L466)
+
+___
+
+### Create
+
+• **Create** = ``"create"``
+
+Create a new [Table](../interfaces/Table.md).
+
+#### Defined in
+
+[index.ts:462](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L462)
 
 ___
 
@@ -25,6 +42,8 @@ ___
 
 • **Overwrite** = ``"overwrite"``
 
+Overwrite the existing [Table](../interfaces/Table.md) if presented.
+
 #### Defined in
 
-[index.ts:325](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L325)
+[index.ts:464](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L464)

docs/src/javascript/interfaces/Connection.md

@@ -0,0 +1,152 @@
+[vectordb](../README.md) / [Exports](../modules.md) / Connection
+
+# Interface: Connection
+
+A LanceDB Connection that allows you to open tables and create new ones.
+
+Connection could be local against filesystem or remote against a server.
+
+## Implemented by
+
+- [`LocalConnection`](../classes/LocalConnection.md)
+
+## Table of contents
+
+### Properties
+
+- [uri](Connection.md#uri)
+
+### Methods
+
+- [createTable](Connection.md#createtable)
+- [createTableArrow](Connection.md#createtablearrow)
+- [dropTable](Connection.md#droptable)
+- [openTable](Connection.md#opentable)
+- [tableNames](Connection.md#tablenames)
+
+## Properties
+
+### uri
+
+• **uri**: `string`
+
+#### Defined in
+
+[index.ts:45](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L45)
+
+## Methods
+
+### createTable
+
+▸ **createTable**<`T`\>(`name`, `data`, `mode?`, `embeddings?`): `Promise`<[`Table`](Table.md)<`T`\>\>
+
+Creates a new Table and initialize it with new data.
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
+| `mode?` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
+| `embeddings?` | [`EmbeddingFunction`](EmbeddingFunction.md)<`T`\> | An embedding function to use on this table |
+
+#### Returns
+
+`Promise`<[`Table`](Table.md)<`T`\>\>
+
+#### Defined in
+
+[index.ts:65](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L65)
+
+___
+
+### createTableArrow
+
+▸ **createTableArrow**(`name`, `table`): `Promise`<[`Table`](Table.md)<`number`[]\>\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `table` | `Table`<`any`\> |
+
+#### Returns
+
+`Promise`<[`Table`](Table.md)<`number`[]\>\>
+
+#### Defined in
+
+[index.ts:67](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L67)
+
+___
+
+### dropTable
+
+▸ **dropTable**(`name`): `Promise`<`void`\>
+
+Drop an existing table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table to drop. |
+
+#### Returns
+
+`Promise`<`void`\>
+
+#### Defined in
+
+[index.ts:73](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L73)
+
+___
+
+### openTable
+
+▸ **openTable**<`T`\>(`name`, `embeddings?`): `Promise`<[`Table`](Table.md)<`T`\>\>
+
+Open a table in the database.
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `embeddings?` | [`EmbeddingFunction`](EmbeddingFunction.md)<`T`\> | An embedding function to use on this table |
+
+#### Returns
+
+`Promise`<[`Table`](Table.md)<`T`\>\>
+
+#### Defined in
+
+[index.ts:55](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L55)
+
+___
+
+### tableNames
+
+▸ **tableNames**(): `Promise`<`string`[]\>
+
+#### Returns
+
+`Promise`<`string`[]\>
+
+#### Defined in
+
+[index.ts:47](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L47)

docs/src/javascript/interfaces/EmbeddingFunction.md

@@ -45,7 +45,7 @@ Creates a vector representation for the given values.
 
 #### Defined in
 
-[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/embedding_function.ts#L27)
+[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/embedding_function.ts#L27)
 
 ___
 
@@ -57,4 +57,4 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/embedding_function.ts#L22)
+[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/embedding_function.ts#L22)

docs/src/javascript/interfaces/Table.md

@@ -0,0 +1,195 @@
+[vectordb](../README.md) / [Exports](../modules.md) / Table
+
+# Interface: Table<T\>
+
+A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
+
+## Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+## Implemented by
+
+- [`LocalTable`](../classes/LocalTable.md)
+
+## Table of contents
+
+### Properties
+
+- [add](Table.md#add)
+- [countRows](Table.md#countrows)
+- [createIndex](Table.md#createindex)
+- [delete](Table.md#delete)
+- [name](Table.md#name)
+- [overwrite](Table.md#overwrite)
+- [search](Table.md#search)
+
+## Properties
+
+### add
+
+• **add**: (`data`: `Record`<`string`, `unknown`\>[]) => `Promise`<`number`\>
+
+#### Type declaration
+
+▸ (`data`): `Promise`<`number`\>
+
+Insert records into this Table.
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+
+##### Returns
+
+`Promise`<`number`\>
+
+The number of rows added to the table
+
+#### Defined in
+
+[index.ts:95](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L95)
+
+___
+
+### countRows
+
+• **countRows**: () => `Promise`<`number`\>
+
+#### Type declaration
+
+▸ (): `Promise`<`number`\>
+
+Returns the number of rows in this table.
+
+##### Returns
+
+`Promise`<`number`\>
+
+#### Defined in
+
+[index.ts:115](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L115)
+
+___
+
+### createIndex
+
+• **createIndex**: (`indexParams`: `IvfPQIndexConfig`) => `Promise`<`any`\>
+
+#### Type declaration
+
+▸ (`indexParams`): `Promise`<`any`\>
+
+Create an ANN index on this Table vector index.
+
+**`See`**
+
+VectorIndexParams.
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `indexParams` | `IvfPQIndexConfig` | The parameters of this Index, |
+
+##### Returns
+
+`Promise`<`any`\>
+
+#### Defined in
+
+[index.ts:110](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L110)
+
+___
+
+### delete
+
+• **delete**: (`filter`: `string`) => `Promise`<`void`\>
+
+#### Type declaration
+
+▸ (`filter`): `Promise`<`void`\>
+
+Delete rows from this table.
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `filter` | `string` | A filter in the same format used by a sql WHERE clause. |
+
+##### Returns
+
+`Promise`<`void`\>
+
+#### Defined in
+
+[index.ts:122](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L122)
+
+___
+
+### name
+
+• **name**: `string`
+
+#### Defined in
+
+[index.ts:81](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L81)
+
+___
+
+### overwrite
+
+• **overwrite**: (`data`: `Record`<`string`, `unknown`\>[]) => `Promise`<`number`\>
+
+#### Type declaration
+
+▸ (`data`): `Promise`<`number`\>
+
+Insert records into this Table, replacing its contents.
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+
+##### Returns
+
+`Promise`<`number`\>
+
+The number of rows added to the table
+
+#### Defined in
+
+[index.ts:103](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L103)
+
+___
+
+### search
+
+• **search**: (`query`: `T`) => [`Query`](../classes/Query.md)<`T`\>
+
+#### Type declaration
+
+▸ (`query`): [`Query`](../classes/Query.md)<`T`\>
+
+Creates a search query to find the nearest neighbors of the given search term
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `query` | `T` | The query search term |
+
+##### Returns
+
+[`Query`](../classes/Query.md)<`T`\>
+
+#### Defined in
+
+[index.ts:87](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L87)

docs/src/javascript/modules.md

@@ -11,14 +11,16 @@
 
 ### Classes
 
-- [Connection](classes/Connection.md)
+- [LocalConnection](classes/LocalConnection.md)
+- [LocalTable](classes/LocalTable.md)
 - [OpenAIEmbeddingFunction](classes/OpenAIEmbeddingFunction.md)
 - [Query](classes/Query.md)
-- [Table](classes/Table.md)
 
 ### Interfaces
 
+- [Connection](interfaces/Connection.md)
 - [EmbeddingFunction](interfaces/EmbeddingFunction.md)
+- [Table](interfaces/Table.md)
 
 ### Type Aliases
 
@@ -36,13 +38,13 @@
 
 #### Defined in
 
-[index.ts:224](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L224)
+[index.ts:345](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L345)
 
 ## Functions
 
 ### connect
 
-▸ **connect**(`uri`): `Promise`<[`Connection`](classes/Connection.md)\>
+▸ **connect**(`uri`): `Promise`<[`Connection`](interfaces/Connection.md)\>
 
 Connect to a LanceDB instance at the given URI
 
@@ -54,8 +56,8 @@ Connect to a LanceDB instance at the given URI
 
 #### Returns
 
-`Promise`<[`Connection`](classes/Connection.md)\>
+`Promise`<[`Connection`](interfaces/Connection.md)\>
 
 #### Defined in
 
-[index.ts:34](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L34)
+[index.ts:34](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L34)

docs/src/python/python.md

@@ -10,14 +10,16 @@ pip install lancedb
 
 ::: lancedb.connect
 
-::: lancedb.LanceDBConnection
+::: lancedb.db.DBConnection
 
 ## Table
 
-::: lancedb.table.LanceTable
+::: lancedb.table.Table
 
 ## Querying
 
+::: lancedb.query.Query
+
 ::: lancedb.query.LanceQueryBuilder
 
 ::: lancedb.query.LanceFtsQueryBuilder

docs/src/search.md

@@ -18,6 +18,7 @@ Currently, we support the following metrics:
 | ----------- | ------------------------------------ |
 | `L2`        | [Euclidean / L2 distance](https://en.wikipedia.org/wiki/Euclidean_distance) |
 | `Cosine`    | [Cosine Similarity](https://en.wikipedia.org/wiki/Cosine_similarity)|
+| `Dot`       | [Dot Production](https://en.wikipedia.org/wiki/Dot_product) |
 
 
 ## Search
@@ -28,16 +29,44 @@ Currently, we support the following metrics:
 If there is no [vector index is created](ann_indexes.md), LanceDB will just brute-force scan
 the vector column and compute the distance.
 
+<!-- Setup Code
+```python
+import lancedb
+import numpy as np
+uri = "data/sample-lancedb"
+db = lancedb.connect(uri)
+
+data = [{"vector": row, "item": f"item {i}"}
+     for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]
+
+db.create_table("my_vectors", data=data)
+```
+-->
+<!-- Setup Code
+```javascript
+const vectordb_setup = require('vectordb')
+const db_setup = await vectordb_setup.connect('data/sample-lancedb')
+
+let data = []
+for (let i = 0; i < 10_000; i++) {
+     data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
+}
+await db_setup.createTable('my_vectors', data)
+```
+-->
 === "Python"
 
+
     ```python
     import lancedb
+    import numpy as np
+
     db = lancedb.connect("data/sample-lancedb")
 
     tbl = db.open_table("my_vectors")
 
-    df = tbl.search(np.random.random((768)))
-        .limit(10)
+    df = tbl.search(np.random.random((1536))) \
+        .limit(10) \
         .to_df()
     ```
 
@@ -47,38 +76,41 @@ the vector column and compute the distance.
     const vectordb = require('vectordb')
     const db = await vectordb.connect('data/sample-lancedb')
 
-    tbl = db.open_table("my_vectors")
+    const tbl = await db.openTable("my_vectors")
 
-    const results = await tbl.search(Array(768))
+    const results_1 = await tbl.search(Array(1536).fill(1.2))
         .limit(20)
         .execute()
     ```
 
+
+<!-- Commenting out for now since metricType fails for JS on Ubuntu 22.04.
+
 By default, `l2` will be used as `Metric` type. You can customize the metric type
 as well.
+-->
 
+<!--
 === "Python"
-
-    ```python
-    df = tbl.search(np.random.random((768)))
-        .metric("cosine")
-        .limit(10)
+-->
+<!--    ```python
+    df = tbl.search(np.random.random((1536))) \
+        .metric("cosine") \
+        .limit(10) \
         .to_df()
     ```
-
+-->
+<!--
 === "JavaScript"
+-->
 
-    ```javascript
-    const vectordb = require('vectordb')
-    const db = await vectordb.connect('data/sample-lancedb')
-
-    tbl = db.open_table("my_vectors")
-
-    const results = await tbl.search(Array(768))
-        .metric("cosine")
+<!--   ```javascript
+    const results_2 = await tbl.search(Array(1536).fill(1.2))
+        .metricType("cosine")
         .limit(20)
         .execute()
     ```
+-->
 
 ### Search with Vector Index.
 

docs/src/sql.md

@@ -0,0 +1,120 @@
+# SQL filters
+
+LanceDB embraces the utilization of standard SQL expressions as predicates for hybrid
+filters. It can be used during hybrid vector search and deletion operations.
+
+Currently, Lance supports a growing list of expressions.
+
+* ``>``, ``>=``, ``<``, ``<=``, ``=``
+* ``AND``, ``OR``, ``NOT``
+* ``IS NULL``, ``IS NOT NULL``
+* ``IS TRUE``, ``IS NOT TRUE``, ``IS FALSE``, ``IS NOT FALSE``
+* ``IN``
+* ``LIKE``, ``NOT LIKE``
+* ``CAST``
+* ``regexp_match(column, pattern)``
+
+For example, the following filter string is acceptable:
+<!-- Setup Code
+```python 
+import lancedb
+import numpy as np
+uri = "data/sample-lancedb"
+db = lancedb.connect(uri)
+
+data = [{"vector": row, "item": f"item {i}"}
+     for i, row in enumerate(np.random.random((10_000, 2)).astype('int'))]
+
+tbl = db.create_table("my_vectors", data=data)
+```
+-->
+<!-- Setup Code
+```javascript 
+const vectordb = require('vectordb')
+const db = await vectordb.connect('data/sample-lancedb')
+
+let data = []
+for (let i = 0; i < 10_000; i++) {
+     data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
+}
+const tbl = await db.createTable('my_vectors', data)
+```
+-->
+=== "Python"
+
+    ```python
+    tbl.search([100, 102]) \
+       .where("""(
+        (label IN [10, 20])
+        AND
+        (note.email IS NOT NULL)
+        ) OR NOT note.created
+        """)
+
+    ```
+=== "Javascript"
+
+    ```javascript
+    tbl.search([100, 102])
+       .where(`(
+            (label IN [10, 20])
+            AND
+            (note.email IS NOT NULL)
+        ) OR NOT note.created
+       `)
+    ```
+
+
+If your column name contains special characters or is a [SQL Keyword](https://docs.rs/sqlparser/latest/sqlparser/keywords/index.html),
+you can use backtick (`` ` ``) to escape it. For nested fields, each segment of the
+path must be wrapped in backticks.
+
+=== "SQL"
+    ```sql
+    `CUBE` = 10 AND `column name with space` IS NOT NULL
+      AND `nested with space`.`inner with space` < 2
+    ```
+
+!!! warning
+    Field names containing periods (``.``) are not supported.
+
+Literals for dates, timestamps, and decimals can be written by writing the string
+value after the type name. For example
+
+=== "SQL"
+    ```sql
+    date_col = date '2021-01-01'
+    and timestamp_col = timestamp '2021-01-01 00:00:00'
+    and decimal_col = decimal(8,3) '1.000'
+    ```
+
+For timestamp columns, the precision can be specified as a number in the type
+parameter. Microsecond precision (6) is the default.
+
+| SQL              | Time unit    |
+|------------------|--------------|
+| ``timestamp(0)`` | Seconds      |
+| ``timestamp(3)`` | Milliseconds |
+| ``timestamp(6)`` | Microseconds |
+| ``timestamp(9)`` | Nanoseconds  |
+
+LanceDB internally stores data in [Apache Arrow](https://arrow.apache.org/) format.
+The mapping from SQL types to Arrow types is:
+
+| SQL type | Arrow type |
+|----------|------------|
+| ``boolean`` | ``Boolean`` |
+| ``tinyint`` / ``tinyint unsigned`` | ``Int8`` / ``UInt8`` |
+| ``smallint`` / ``smallint unsigned`` | ``Int16`` / ``UInt16`` |
+| ``int`` or ``integer`` / ``int unsigned`` or ``integer unsigned`` | ``Int32`` / ``UInt32`` |
+| ``bigint`` / ``bigint unsigned`` | ``Int64`` / ``UInt64`` |
+| ``float`` | ``Float32`` |
+| ``double`` | ``Float64`` |
+| ``decimal(precision, scale)`` | ``Decimal128`` |
+| ``date`` | ``Date32`` |
+| ``timestamp`` | ``Timestamp`` [^1] |
+| ``string`` | ``Utf8`` |
+| ``binary`` | ``Binary`` |
+
+[^1]: See precision mapping in previous table.
+

docs/test/md_testing.js

@@ -0,0 +1,51 @@
+const glob = require("glob");
+const fs = require("fs");
+const path = require("path");
+
+const excludedFiles = [
+  "../src/fts.md",
+  "../src/embedding.md",
+  "../src/examples/serverless_lancedb_with_s3_and_lambda.md",
+  "../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
+  "../src/examples/youtube_transcript_bot_with_nodejs.md",
+];
+const nodePrefix = "javascript";
+const nodeFile = ".js";
+const nodeFolder = "node";
+const globString = "../src/**/*.md";
+const asyncPrefix = "(async () => {\n";
+const asyncSuffix = "})();";
+
+function* yieldLines(lines, prefix, suffix) {
+  let inCodeBlock = false;
+  for (const line of lines) {
+    if (line.trim().startsWith(prefix + nodePrefix)) {
+      inCodeBlock = true;
+    } else if (inCodeBlock && line.trim().startsWith(suffix)) {
+      inCodeBlock = false;
+      yield "\n";
+    } else if (inCodeBlock) {
+      yield line;
+    }
+  }
+}
+
+const files = glob.sync(globString, { recursive: true });
+
+for (const file of files.filter((file) => !excludedFiles.includes(file))) {
+  const lines = [];
+  const data = fs.readFileSync(file, "utf-8");
+  const fileLines = data.split("\n");
+
+  for (const line of yieldLines(fileLines, "```", "```")) {
+    lines.push(line);
+  }
+
+  if (lines.length > 0) {
+    const fileName = path.basename(file, ".md");
+    const outPath = path.join(nodeFolder, fileName, `${fileName}${nodeFile}`);
+    console.log(outPath)
+    fs.mkdirSync(path.dirname(outPath), { recursive: true });
+    fs.writeFileSync(outPath, asyncPrefix + "\n" + lines.join("\n") + asyncSuffix);
+  }
+}

docs/test/md_testing.py

@@ -0,0 +1,41 @@
+import glob
+from typing import Iterator
+from pathlib import Path
+
+excluded_files = [
+    "../src/fts.md",
+    "../src/embedding.md",
+    "../src/examples/serverless_lancedb_with_s3_and_lambda.md",
+    "../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
+    "../src/examples/youtube_transcript_bot_with_nodejs.md"
+]
+
+python_prefix = "py"
+python_file = ".py"
+python_folder = "python"
+glob_string = "../src/**/*.md"
+
+def yield_lines(lines: Iterator[str], prefix: str, suffix: str):
+    in_code_block = False
+    # Python code has strict indentation
+    strip_length = 0
+    for line in lines:
+        if line.strip().startswith(prefix + python_prefix):
+            in_code_block = True
+            strip_length = len(line) - len(line.lstrip())
+        elif in_code_block and line.strip().startswith(suffix):
+            in_code_block = False
+            yield "\n"
+        elif in_code_block:
+            yield line[strip_length:]
+
+for file in filter(lambda file: file not in excluded_files, glob.glob(glob_string, recursive=True)):
+    with open(file, "r") as f:
+        lines = list(yield_lines(iter(f), "```", "```"))
+
+    if len(lines) > 0:
+        out_path = Path(python_folder) / Path(file).name.strip(".md") / (Path(file).name.strip(".md") + python_file)
+        print(out_path)
+        out_path.parent.mkdir(exist_ok=True, parents=True)
+        with open(out_path, "w") as out:
+            out.writelines(lines)

docs/test/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "lancedb-docs-test",
+  "version": "1.0.0",
+  "description": "",
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "fs": "^0.0.1-security",
+    "glob": "^10.2.7",
+    "path": "^0.12.7",
+    "vectordb": "https://gitpkg.now.sh/lancedb/lancedb/node?main"
+  }
+}

docs/test/requirements.txt

@@ -0,0 +1,5 @@
+lancedb @ git+https://github.com/lancedb/lancedb.git#egg=subdir&subdirectory=python
+numpy
+pandas
+pylance
+duckdb

node/.eslintrc.js

@@ -12,5 +12,6 @@ module.exports = {
     sourceType: 'module'
   },
   rules: {
+    "@typescript-eslint/method-signature-style": "off",
   }
 }

node/README.md

@@ -14,9 +14,11 @@ npm install vectordb
 
 ```javascript
 const lancedb = require('vectordb');
-const db = lancedb.connect('<PATH_TO_LANCEDB_DATASET>');
-const table = await db.openTable('my_table');
-const query = await table.search([0.1, 0.3]).setLimit(20).execute();
+const db = await lancedb.connect('data/sample-lancedb');
+const table = await db.createTable("my_table",
+      [{ id: 1, vector: [0.1, 1.0], item: "foo", price: 10.0 },
+      { id: 2, vector: [3.9, 0.5], item: "bar", price: 20.0 }])
+const results = await table.search([0.1, 0.3]).limit(20).execute();
 console.log(results);
 ```
 
@@ -24,12 +26,6 @@ The [examples](./examples) folder contains complete examples.
 
 ## Development
 
-The LanceDB javascript is built with npm:
-
-```bash
-npm run tsc
-```
-
 Run the tests with
 
 ```bash
@@ -46,4 +42,4 @@ To build documentation
 
 ```bash
 npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
-```
+```

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.1.8",
+  "version": "0.1.9",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.1.8",
+      "version": "0.1.9",
       "license": "Apache-2.0",
       "dependencies": {
         "@apache-arrow/ts": "^12.0.0",
@@ -14,6 +14,7 @@
       },
       "devDependencies": {
         "@types/chai": "^4.3.4",
+        "@types/chai-as-promised": "^7.1.5",
         "@types/mocha": "^10.0.1",
         "@types/node": "^18.16.2",
         "@types/sinon": "^10.0.15",
@@ -21,6 +22,7 @@
         "@typescript-eslint/eslint-plugin": "^5.59.1",
         "cargo-cp-artifact": "^0.1",
         "chai": "^4.3.7",
+        "chai-as-promised": "^7.1.1",
         "eslint": "^8.39.0",
         "eslint-config-standard-with-typescript": "^34.0.1",
         "eslint-plugin-import": "^2.26.0",
@@ -311,6 +313,15 @@
       "integrity": "sha512-KnRanxnpfpjUTqTCXslZSEdLfXExwgNxYPdiO2WGUj8+HDjFi8R3k5RVKPeSCzLjCcshCAtVO2QBbVuAV4kTnw==",
       "dev": true
     },
+    "node_modules/@types/chai-as-promised": {
+      "version": "7.1.5",
+      "resolved": "https://registry.npmjs.org/@types/chai-as-promised/-/chai-as-promised-7.1.5.tgz",
+      "integrity": "sha512-jStwss93SITGBwt/niYrkf2C+/1KTeZCZl1LaeezTlqppAKeoQC7jxyqYuP72sxBGKCIbw7oHgbYssIRzT5FCQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/chai": "*"
+      }
+    },
     "node_modules/@types/command-line-args": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz",
@@ -942,6 +953,18 @@
         "node": ">=4"
       }
     },
+    "node_modules/chai-as-promised": {
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz",
+      "integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==",
+      "dev": true,
+      "dependencies": {
+        "check-error": "^1.0.2"
+      },
+      "peerDependencies": {
+        "chai": ">= 2.1.2 < 5"
+      }
+    },
     "node_modules/chalk": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
@@ -4680,6 +4703,15 @@
       "integrity": "sha512-KnRanxnpfpjUTqTCXslZSEdLfXExwgNxYPdiO2WGUj8+HDjFi8R3k5RVKPeSCzLjCcshCAtVO2QBbVuAV4kTnw==",
       "dev": true
     },
+    "@types/chai-as-promised": {
+      "version": "7.1.5",
+      "resolved": "https://registry.npmjs.org/@types/chai-as-promised/-/chai-as-promised-7.1.5.tgz",
+      "integrity": "sha512-jStwss93SITGBwt/niYrkf2C+/1KTeZCZl1LaeezTlqppAKeoQC7jxyqYuP72sxBGKCIbw7oHgbYssIRzT5FCQ==",
+      "dev": true,
+      "requires": {
+        "@types/chai": "*"
+      }
+    },
     "@types/command-line-args": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz",
@@ -5137,6 +5169,15 @@
         "type-detect": "^4.0.5"
       }
     },
+    "chai-as-promised": {
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz",
+      "integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==",
+      "dev": true,
+      "requires": {
+        "check-error": "^1.0.2"
+      }
+    },
     "chalk": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.1.9",
+  "version": "0.1.10",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -8,7 +8,7 @@
     "tsc": "tsc -b",
     "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json-render-diagnostics",
     "build-release": "npm run build -- --release",
-    "test": "mocha -recursive dist/test",
+    "test": "npm run tsc; mocha -recursive dist/test",
     "lint": "eslint src --ext .js,.ts",
     "clean": "rm -rf node_modules *.node dist/"
   },
@@ -26,6 +26,7 @@
   "license": "Apache-2.0",
   "devDependencies": {
     "@types/chai": "^4.3.4",
+    "@types/chai-as-promised": "^7.1.5",
     "@types/mocha": "^10.0.1",
     "@types/node": "^18.16.2",
     "@types/sinon": "^10.0.15",
@@ -33,6 +34,7 @@
     "@typescript-eslint/eslint-plugin": "^5.59.1",
     "cargo-cp-artifact": "^0.1",
     "chai": "^4.3.7",
+    "chai-as-promised": "^7.1.1",
     "eslint": "^8.39.0",
     "eslint-config-standard-with-typescript": "^34.0.1",
     "eslint-plugin-import": "^2.26.0",

node/src/index.ts

@@ -33,13 +33,99 @@ export { OpenAIEmbeddingFunction } from './embedding/openai'
  */
 export async function connect (uri: string): Promise<Connection> {
   const db = await databaseNew(uri)
-  return new Connection(db, uri)
+  return new LocalConnection(db, uri)
+}
+
+/**
+ * A LanceDB Connection that allows you to open tables and create new ones.
+ *
+ * Connection could be local against filesystem or remote against a server.
+ */
+export interface Connection {
+  uri: string
+
+  tableNames(): Promise<string[]>
+
+  /**
+   * Open a table in the database.
+   *
+   * @param name The name of the table.
+   * @param embeddings An embedding function to use on this table
+   */
+  openTable<T>(name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
+
+  /**
+   * Creates a new Table and initialize it with new data.
+   *
+   * @param {string} name - The name of the table.
+   * @param data - Non-empty Array of Records to be inserted into the table
+   * @param {WriteMode} mode - The write mode to use when creating the table.
+   * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
+   */
+  createTable<T>(name: string, data: Array<Record<string, unknown>>, mode?: WriteMode, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
+
+  createTableArrow(name: string, table: ArrowTable): Promise<Table>
+
+  /**
+   * Drop an existing table.
+   * @param name The name of the table to drop.
+   */
+  dropTable(name: string): Promise<void>
+
+}
+
+/**
+ * A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
+ */
+export interface Table<T = number[]> {
+  name: string
+
+  /**
+   * Creates a search query to find the nearest neighbors of the given search term
+   * @param query The query search term
+   */
+  search: (query: T) => Query<T>
+
+  /**
+   * Insert records into this Table.
+   *
+   * @param data Records to be inserted into the Table
+   * @return The number of rows added to the table
+   */
+  add: (data: Array<Record<string, unknown>>) => Promise<number>
+
+  /**
+   * Insert records into this Table, replacing its contents.
+   *
+   * @param data Records to be inserted into the Table
+   * @return The number of rows added to the table
+   */
+  overwrite: (data: Array<Record<string, unknown>>) => Promise<number>
+
+  /**
+   * Create an ANN index on this Table vector index.
+   *
+   * @param indexParams The parameters of this Index, @see VectorIndexParams.
+   */
+  createIndex: (indexParams: VectorIndexParams) => Promise<any>
+
+  /**
+   * Returns the number of rows in this table.
+   */
+  countRows: () => Promise<number>
+
+  /**
+   * Delete rows from this table.
+   *
+   * @param filter  A filter in the same format used by a sql WHERE clause.
+   */
+  delete: (filter: string) => Promise<void>
 }
 
 /**
  * A connection to a LanceDB database.
  */
-export class Connection {
+export class LocalConnection implements Connection {
   private readonly _uri: string
   private readonly _db: any
 
@@ -75,9 +161,9 @@ export class Connection {
   async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
     const tbl = await databaseOpenTable.call(this._db, name)
     if (embeddings !== undefined) {
-      return new Table(tbl, name, embeddings)
+      return new LocalTable(tbl, name, embeddings)
     } else {
-      return new Table(tbl, name)
+      return new LocalTable(tbl, name)
     }
   }
 
@@ -86,23 +172,29 @@ export class Connection {
    *
    * @param name The name of the table.
    * @param data Non-empty Array of Records to be inserted into the Table
+   * @param mode The write mode to use when creating the table.
    */
+  async createTable (name: string, data: Array<Record<string, unknown>>, mode?: WriteMode): Promise<Table>
+  async createTable (name: string, data: Array<Record<string, unknown>>, mode: WriteMode): Promise<Table>
 
-  async createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table>
   /**
    * Creates a new Table and initialize it with new data.
    *
    * @param name The name of the table.
    * @param data Non-empty Array of Records to be inserted into the Table
+   * @param mode The write mode to use when creating the table.
    * @param embeddings An embedding function to use on this Table
    */
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
-    const tbl = await tableCreate.call(this._db, name, await fromRecordsToBuffer(data, embeddings))
+  async createTable<T> (name: string, data: Array<Record<string, unknown>>, mode: WriteMode, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
+  async createTable<T> (name: string, data: Array<Record<string, unknown>>, mode: WriteMode, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
+    if (mode === undefined) {
+      mode = WriteMode.Create
+    }
+    const tbl = await tableCreate.call(this._db, name, await fromRecordsToBuffer(data, embeddings), mode.toLowerCase())
     if (embeddings !== undefined) {
-      return new Table(tbl, name, embeddings)
+      return new LocalTable(tbl, name, embeddings)
     } else {
-      return new Table(tbl, name)
+      return new LocalTable(tbl, name)
     }
   }
 
@@ -121,7 +213,7 @@ export class Connection {
   }
 }
 
-export class Table<T = number[]> {
+export class LocalTable<T = number[]> implements Table<T> {
   private readonly _tbl: any
   private readonly _name: string
   private readonly _embeddings?: EmbeddingFunction<T>
@@ -180,13 +272,6 @@ export class Table<T = number[]> {
     return tableCreateVectorIndex.call(this._tbl, indexParams)
   }
 
-  /**
-   * @deprecated Use [Table.createIndex]
-   */
-  async create_index (indexParams: VectorIndexParams): Promise<any> {
-    return await this.createIndex(indexParams)
-  }
-
   /**
    * Returns the number of rows in this table.
    */
@@ -197,14 +282,16 @@ export class Table<T = number[]> {
   /**
    * Delete rows from this table.
    *
-   * @param filter The filter to be applied to this table.
+   * @param filter A filter in the same format used by a sql WHERE clause.
    */
   async delete (filter: string): Promise<void> {
     return tableDelete.call(this._tbl, filter)
   }
 }
 
-interface IvfPQIndexConfig {
+/// Config to build IVF_PQ index.
+///
+export interface IvfPQIndexConfig {
   /**
    * The column to be indexed
    */
@@ -249,6 +336,11 @@ interface IvfPQIndexConfig {
    */
   max_opq_iters?: number
 
+  /**
+   * Replace an existing index with the same name if it exists.
+   */
+  replace?: boolean
+
   type: 'ivf_pq'
 }
 
@@ -349,6 +441,7 @@ export class Query<T = number[]> {
 
     const buffer = await tableSearch.call(this._tbl, this)
     const data = tableFromIPC(buffer)
+
     return data.toArray().map((entry: Record<string, unknown>) => {
       const newObject: Record<string, unknown> = {}
       Object.keys(entry).forEach((key: string) => {
@@ -363,8 +456,15 @@ export class Query<T = number[]> {
   }
 }
 
+/**
+ * Write mode for writing a table.
+ */
 export enum WriteMode {
+  /** Create a new {@link Table}. */
+  Create = 'create',
+  /** Overwrite the existing {@link Table} if presented. */
   Overwrite = 'overwrite',
+  /** Append new data to the table. */
   Append = 'append'
 }
 
@@ -380,5 +480,10 @@ export enum MetricType {
   /**
    * Cosine distance
    */
-  Cosine = 'cosine'
+  Cosine = 'cosine',
+
+  /**
+   * Dot product
+   */
+  Dot = 'dot'
 }

node/src/test/test.ts

@@ -1,4 +1,4 @@
-// Copyright 2023 Lance Developers.
+// Copyright 2023 LanceDB Developers.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,11 +13,16 @@
 // limitations under the License.
 
 import { describe } from 'mocha'
-import { assert } from 'chai'
 import { track } from 'temp'
+import * as chai from 'chai'
+import * as chaiAsPromised from 'chai-as-promised'
 
 import * as lancedb from '../index'
-import { type EmbeddingFunction, MetricType, Query } from '../index'
+import { type EmbeddingFunction, MetricType, Query, WriteMode } from '../index'
+
+const expect = chai.expect
+const assert = chai.assert
+chai.use(chaiAsPromised)
 
 describe('LanceDB client', function () {
   describe('when creating a connection to lancedb', function () {
@@ -113,6 +118,31 @@ describe('LanceDB client', function () {
       assert.equal(await table.countRows(), 2)
     })
 
+    it('use overwrite flag to overwrite existing table', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const data = [
+        { id: 1, vector: [0.1, 0.2], price: 10 },
+        { id: 2, vector: [1.1, 1.2], price: 50 }
+      ]
+
+      const tableName = 'overwrite'
+      await con.createTable(tableName, data, WriteMode.Create)
+
+      const newData = [
+        { id: 1, vector: [0.1, 0.2], price: 10 },
+        { id: 2, vector: [1.1, 1.2], price: 50 },
+        { id: 3, vector: [1.1, 1.2], price: 50 }
+      ]
+
+      await expect(con.createTable(tableName, newData)).to.be.rejectedWith(Error, 'already exists')
+
+      const table = await con.createTable(tableName, newData, WriteMode.Overwrite)
+      assert.equal(table.name, tableName)
+      assert.equal(await table.countRows(), 3)
+    })
+
     it('appends records to an existing table ', async function () {
       const dir = await track().mkdir('lancejs')
       const con = await lancedb.connect(dir)
@@ -165,8 +195,25 @@ describe('LanceDB client', function () {
       const uri = await createTestDB(32, 300)
       const con = await lancedb.connect(uri)
       const table = await con.openTable('vectors')
-      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2 })
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
     }).timeout(10_000) // Timeout is high partially because GH macos runner is pretty slow
+
+    it('replace an existing index', async function () {
+      const uri = await createTestDB(16, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+
+      // Replace should fail if the index already exists
+      await expect(table.createIndex({
+        type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2, replace: false
+      })
+      ).to.be.rejectedWith('LanceError(Index)')
+
+      // Default replace = true
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+    }).timeout(50_000)
   })
 
   describe('when using a custom embedding function', function () {
@@ -196,7 +243,7 @@ describe('LanceDB client', function () {
         { price: 10, name: 'foo' },
         { price: 50, name: 'bar' }
       ]
-      const table = await con.createTable('vectors', data, embeddings)
+      const table = await con.createTable('vectors', data, WriteMode.Create, embeddings)
       const results = await table.search('foo').execute()
       assert.equal(results.length, 2)
     })

python/README.md

@@ -0,0 +1,85 @@
+# LanceDB
+
+A Python library for [LanceDB](https://github.com/lancedb/lancedb).
+
+## Installation
+
+```bash
+pip install lancedb
+```
+
+## Usage
+
+### Basic Example
+
+```python
+import lancedb
+db = lancedb.connect('<PATH_TO_LANCEDB_DATASET>')
+table = db.open_table('my_table')
+results = table.search([0.1, 0.3]).limit(20).to_df()
+print(results)
+```
+
+
+## Development
+
+Create a virtual environment and activate it:
+
+```bash
+python -m venv venv
+. ./venv/bin/activate
+```
+
+Install the necessary packages:
+
+```bash
+python -m pip install .
+```
+
+To run the unit tests:
+
+```bash
+pytest
+```
+
+To run linter and automatically fix all errors:
+
+```bash
+black .
+isort .
+```
+
+If any packages are missing, install them with:
+
+```bash
+pip install <PACKAGE_NAME>
+```
+
+
+___
+For **Windows** users, there may be errors when installing packages, so these commands may be helpful:
+
+Activate the virtual environment:
+```bash
+. .\venv\Scripts\activate
+```
+
+You may need to run the installs separately:
+```bash
+pip install -e .[tests]
+pip install -e .[dev]
+```
+
+
+`tantivy` requires `rust` to be installed, so install it with `conda`, as it doesn't support windows installation:
+```bash
+pip install wheel
+pip install cargo
+conda install rust
+pip install tantivy
+```
+
+To run the unit tests:
+```bash
+pytest
+```

python/lancedb/__init__.py

@@ -11,16 +11,24 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from .db import URI, LanceDBConnection
+from typing import Optional
+
+from .db import URI, DBConnection, LanceDBConnection
+from .remote.db import RemoteDBConnection
 
 
-def connect(uri: URI) -> LanceDBConnection:
-    """Connect to a LanceDB instance at the given URI
+def connect(
+    uri: URI, *, api_key: Optional[str] = None, region: str = "us-west-2"
+) -> DBConnection:
+    """Connect to a LanceDB database.
 
     Parameters
     ----------
     uri: str or Path
         The uri of the database.
+    api_token: str, optional
+        If presented, connect to LanceDB cloud.
+        Otherwise, connect to a database on file system or cloud storage.
 
     Examples
     --------
@@ -34,9 +42,17 @@ def connect(uri: URI) -> LanceDBConnection:
 
     >>> db = lancedb.connect("s3://my-bucket/lancedb")
 
+    Connect to LancdDB cloud:
+
+    >>> db = lancedb.connect("db://my_database", api_key="ldb_...")
+
     Returns
     -------
-    conn : LanceDBConnection
+    conn : DBConnection
         A connection to a LanceDB database.
     """
+    if isinstance(uri, str) and uri.startswith("db://"):
+        if api_key is None:
+            raise ValueError(f"api_key is required to connected LanceDB cloud: {uri}")
+        return RemoteDBConnection(uri, api_key, region)
     return LanceDBConnection(uri)

python/lancedb/common.py

@@ -23,3 +23,13 @@ URI = Union[str, Path]
 # TODO support generator
 DATA = Union[List[dict], dict, pd.DataFrame]
 VECTOR_COLUMN_NAME = "vector"
+
+
+class Credential(str):
+    """Credential field"""
+
+    def __repr__(self) -> str:
+        return "********"
+
+    def __str__(self) -> str:
+        return "********"

python/lancedb/conftest.py

@@ -1,10 +1,8 @@
-import builtins
 import os
 
 import pytest
 
 # import lancedb so we don't have to in every example
-import lancedb
 
 
 @pytest.fixture(autouse=True)

python/lancedb/db.py

@@ -15,151 +15,36 @@ from __future__ import annotations
 
 import functools
 import os
+from abc import ABC, abstractmethod
 from pathlib import Path
 
 import pyarrow as pa
 from pyarrow import fs
 
 from .common import DATA, URI
-from .table import LanceTable
+from .table import LanceTable, Table
 from .util import get_uri_location, get_uri_scheme
 
 
-class LanceDBConnection:
-    """
-    A connection to a LanceDB database.
-
-    Parameters
-    ----------
-    uri: str or Path
-        The root uri of the database.
-
-    Examples
-    --------
-    >>> import lancedb
-    >>> db = lancedb.connect("./.lancedb")
-    >>> db.create_table("my_table", data=[{"vector": [1.1, 1.2], "b": 2},
-    ...                                   {"vector": [0.5, 1.3], "b": 4}])
-    LanceTable(my_table)
-    >>> db.create_table("another_table", data=[{"vector": [0.4, 0.4], "b": 6}])
-    LanceTable(another_table)
-    >>> sorted(db.table_names())
-    ['another_table', 'my_table']
-    >>> len(db)
-    2
-    >>> db["my_table"]
-    LanceTable(my_table)
-    >>> "my_table" in db
-    True
-    >>> db.drop_table("my_table")
-    >>> db.drop_table("another_table")
-    """
-
-    def __init__(self, uri: URI):
-        if not isinstance(uri, Path):
-            scheme = get_uri_scheme(uri)
-        is_local = isinstance(uri, Path) or scheme == "file"
-        # managed lancedb remote uses schema like lancedb+[http|grpc|...]://
-        self._is_managed_remote = not is_local and scheme.startswith("lancedb")
-        if self._is_managed_remote:
-            if len(scheme.split("+")) != 2:
-                raise ValueError(
-                    f"Invalid LanceDB URI: {uri}, expected uri to have scheme like lancedb+<flavor>://..."
-                )
-        if is_local:
-            if isinstance(uri, str):
-                uri = Path(uri)
-            uri = uri.expanduser().absolute()
-            Path(uri).mkdir(parents=True, exist_ok=True)
-        self._uri = str(uri)
-
-        self._entered = False
-
-    @property
-    def uri(self) -> str:
-        return self._uri
-
-    @functools.cached_property
-    def is_managed_remote(self) -> bool:
-        return self._is_managed_remote
-
-    @functools.cached_property
-    def remote_flavor(self) -> str:
-        if not self.is_managed_remote:
-            raise ValueError(
-                "Not a managed remote LanceDB, there should be no server flavor"
-            )
-        return get_uri_scheme(self.uri).split("+")[1]
-
-    @functools.cached_property
-    def _client(self) -> "lancedb.remote.LanceDBClient":
-        if not self.is_managed_remote:
-            raise ValueError("Not a managed remote LanceDB, there should be no client")
-
-        # don't import unless we are really using remote
-        from lancedb.remote.client import RestfulLanceDBClient
-
-        if self.remote_flavor == "http":
-            return RestfulLanceDBClient(self._uri)
-
-        raise ValueError("Unsupported remote flavor: " + self.remote_flavor)
-
-    async def close(self):
-        if self._entered:
-            raise ValueError("Cannot re-enter the same LanceDBConnection twice")
-        self._entered = True
-        await self._client.close()
-
-    async def __aenter__(self) -> LanceDBConnection:
-        return self
-
-    async def __aexit__(self, exc_type, exc_value, traceback):
-        await self.close()
+class DBConnection(ABC):
+    """An active LanceDB connection interface."""
 
+    @abstractmethod
     def table_names(self) -> list[str]:
-        """Get the names of all tables in the database.
-
-        Returns
-        -------
-        list of str
-            A list of table names.
-        """
-        try:
-            filesystem, path = fs.FileSystem.from_uri(self.uri)
-        except pa.ArrowInvalid:
-            raise NotImplementedError("Unsupported scheme: " + self.uri)
-
-        try:
-            paths = filesystem.get_file_info(
-                fs.FileSelector(get_uri_location(self.uri))
-            )
-        except FileNotFoundError:
-            # It is ok if the file does not exist since it will be created
-            paths = []
-        tables = [
-            os.path.splitext(file_info.base_name)[0]
-            for file_info in paths
-            if file_info.extension == "lance"
-        ]
-        return tables
-
-    def __len__(self) -> int:
-        return len(self.table_names())
-
-    def __contains__(self, name: str) -> bool:
-        return name in self.table_names()
-
-    def __getitem__(self, name: str) -> LanceTable:
-        return self.open_table(name)
+        """List all table names in the database."""
+        pass
 
+    @abstractmethod
     def create_table(
         self,
         name: str,
         data: DATA = None,
         schema: pa.Schema = None,
         mode: str = "create",
-    ) -> LanceTable:
-        """Create a table in the database.
+        on_bad_vectors: str = "error",
+        fill_value: float = 0.0,
+    ) -> Table:
+        """Create a [Table][lancedb.table.Table] in the database.
 
         Parameters
         ----------
@@ -170,9 +55,14 @@ class LanceDBConnection:
         schema: pyarrow.Schema; optional
             The schema of the table.
         mode: str; default "create"
-            The mode to use when creating the table.
+            The mode to use when creating the table. Can be either "create" or "overwrite".
             By default, if the table already exists, an exception is raised.
             If you want to overwrite the table, use mode="overwrite".
+        on_bad_vectors: str, default "error"
+            What to do if any of the vectors are not the same size or contains NaNs.
+            One of "error", "drop", "fill".
+        fill_value: float
+            The value to use when filling vectors. Only used if on_bad_vectors="fill".
 
         Note
         ----
@@ -249,10 +139,235 @@ class LanceDBConnection:
         lat: [[45.5,40.1]]
         long: [[-122.7,-74.1]]
         """
+        raise NotImplementedError
+
+    def __getitem__(self, name: str) -> LanceTable:
+        return self.open_table(name)
+
+    def open_table(self, name: str) -> Table:
+        """Open a Lance Table in the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+
+        Returns
+        -------
+        A LanceTable object representing the table.
+        """
+        raise NotImplementedError
+
+    def drop_table(self, name: str):
+        """Drop a table from the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+        """
+        raise NotImplementedError
+
+
+class LanceDBConnection(DBConnection):
+    """
+    A connection to a LanceDB database.
+
+    Parameters
+    ----------
+    uri: str or Path
+        The root uri of the database.
+
+    Examples
+    --------
+    >>> import lancedb
+    >>> db = lancedb.connect("./.lancedb")
+    >>> db.create_table("my_table", data=[{"vector": [1.1, 1.2], "b": 2},
+    ...                                   {"vector": [0.5, 1.3], "b": 4}])
+    LanceTable(my_table)
+    >>> db.create_table("another_table", data=[{"vector": [0.4, 0.4], "b": 6}])
+    LanceTable(another_table)
+    >>> sorted(db.table_names())
+    ['another_table', 'my_table']
+    >>> len(db)
+    2
+    >>> db["my_table"]
+    LanceTable(my_table)
+    >>> "my_table" in db
+    True
+    >>> db.drop_table("my_table")
+    >>> db.drop_table("another_table")
+    """
+
+    def __init__(self, uri: URI):
+        if not isinstance(uri, Path):
+            scheme = get_uri_scheme(uri)
+        is_local = isinstance(uri, Path) or scheme == "file"
+        if is_local:
+            if isinstance(uri, str):
+                uri = Path(uri)
+            uri = uri.expanduser().absolute()
+            Path(uri).mkdir(parents=True, exist_ok=True)
+        self._uri = str(uri)
+
+        self._entered = False
+
+    @property
+    def uri(self) -> str:
+        return self._uri
+
+    def table_names(self) -> list[str]:
+        """Get the names of all tables in the database.
+
+        Returns
+        -------
+        list of str
+            A list of table names.
+        """
+        try:
+            filesystem, path = fs.FileSystem.from_uri(self.uri)
+        except pa.ArrowInvalid:
+            raise NotImplementedError("Unsupported scheme: " + self.uri)
+
+        try:
+            paths = filesystem.get_file_info(
+                fs.FileSelector(get_uri_location(self.uri))
+            )
+        except FileNotFoundError:
+            # It is ok if the file does not exist since it will be created
+            paths = []
+        tables = [
+            os.path.splitext(file_info.base_name)[0]
+            for file_info in paths
+            if file_info.extension == "lance"
+        ]
+        return tables
+
+    def __len__(self) -> int:
+        return len(self.table_names())
+
+    def __contains__(self, name: str) -> bool:
+        return name in self.table_names()
+
+    def create_table(
+        self,
+        name: str,
+        data: DATA = None,
+        schema: pa.Schema = None,
+        mode: str = "create",
+        on_bad_vectors: str = "error",
+        fill_value: float = 0.0,
+    ) -> LanceTable:
+        """Create a table in the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+        data: list, tuple, dict, pd.DataFrame; optional
+            The data to insert into the table.
+        schema: pyarrow.Schema; optional
+            The schema of the table.
+        mode: str; default "create"
+            The mode to use when creating the table. Can be either "create" or "overwrite".
+            By default, if the table already exists, an exception is raised.
+            If you want to overwrite the table, use mode="overwrite".
+        on_bad_vectors: str, default "error"
+            What to do if any of the vectors are not the same size or contains NaNs.
+            One of "error", "drop", "fill".
+        fill_value: float
+            The value to use when filling vectors. Only used if on_bad_vectors="fill".
+
+        Note
+        ----
+        The vector index won't be created by default.
+        To create the index, call the `create_index` method on the table.
+
+        Returns
+        -------
+        LanceTable
+            A reference to the newly created table.
+
+        Examples
+        --------
+
+        Can create with list of tuples or dictionaries:
+
+        >>> import lancedb
+        >>> db = lancedb.connect("./.lancedb")
+        >>> data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
+        ...         {"vector": [0.2, 1.8], "lat": 40.1, "long":  -74.1}]
+        >>> db.create_table("my_table", data)
+        LanceTable(my_table)
+        >>> db["my_table"].head()
+        pyarrow.Table
+        vector: fixed_size_list<item: float>[2]
+          child 0, item: float
+        lat: double
+        long: double
+        ----
+        vector: [[[1.1,1.2],[0.2,1.8]]]
+        lat: [[45.5,40.1]]
+        long: [[-122.7,-74.1]]
+
+        You can also pass a pandas DataFrame:
+
+        >>> import pandas as pd
+        >>> data = pd.DataFrame({
+        ...    "vector": [[1.1, 1.2], [0.2, 1.8]],
+        ...    "lat": [45.5, 40.1],
+        ...    "long": [-122.7, -74.1]
+        ... })
+        >>> db.create_table("table2", data)
+        LanceTable(table2)
+        >>> db["table2"].head()
+        pyarrow.Table
+        vector: fixed_size_list<item: float>[2]
+          child 0, item: float
+        lat: double
+        long: double
+        ----
+        vector: [[[1.1,1.2],[0.2,1.8]]]
+        lat: [[45.5,40.1]]
+        long: [[-122.7,-74.1]]
+
+        Data is converted to Arrow before being written to disk. For maximum
+        control over how data is saved, either provide the PyArrow schema to
+        convert to or else provide a PyArrow table directly.
+
+        >>> custom_schema = pa.schema([
+        ...   pa.field("vector", pa.list_(pa.float32(), 2)),
+        ...   pa.field("lat", pa.float32()),
+        ...   pa.field("long", pa.float32())
+        ... ])
+        >>> db.create_table("table3", data, schema = custom_schema)
+        LanceTable(table3)
+        >>> db["table3"].head()
+        pyarrow.Table
+        vector: fixed_size_list<item: float>[2]
+          child 0, item: float
+        lat: float
+        long: float
+        ----
+        vector: [[[1.1,1.2],[0.2,1.8]]]
+        lat: [[45.5,40.1]]
+        long: [[-122.7,-74.1]]
+        """
+        if mode.lower() not in ["create", "overwrite"]:
+            raise ValueError("mode must be either 'create' or 'overwrite'")
+
         if data is not None:
-            tbl = LanceTable.create(self, name, data, schema, mode=mode)
+            tbl = LanceTable.create(
+                self,
+                name,
+                data,
+                schema,
+                mode=mode,
+                on_bad_vectors=on_bad_vectors,
+                fill_value=fill_value,
+            )
         else:
-            tbl = LanceTable(self, name)
+            tbl = LanceTable.open(self, name)
         return tbl
 
     def open_table(self, name: str) -> LanceTable:
@@ -267,7 +382,7 @@ class LanceDBConnection:
         -------
         A LanceTable object representing the table.
         """
-        return LanceTable(self, name)
+        return LanceTable.open(self, name)
 
     def drop_table(self, name: str):
         """Drop a table from the database.

python/lancedb/query.py

@@ -10,18 +10,47 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
+
 from __future__ import annotations
 
-import asyncio
-from typing import Awaitable, Literal
+from typing import List, Literal, Optional, Union
 
 import numpy as np
 import pandas as pd
 import pyarrow as pa
+from pydantic import BaseModel
 
 from .common import VECTOR_COLUMN_NAME
 
 
+class Query(BaseModel):
+    """A Query"""
+
+    vector_column: str = VECTOR_COLUMN_NAME
+
+    # vector to search for
+    vector: List[float]
+
+    # sql filter to refine the query with
+    filter: Optional[str] = None
+
+    # top k results to return
+    k: int
+
+    # # metrics
+    metric: str = "L2"
+
+    # which columns to return in the results
+    columns: Optional[List[str]] = None
+
+    # optional query parameters for tuning the results,
+    # e.g. `{"nprobes": "10", "refine_factor": "10"}`
+    nprobes: int = 10
+
+    # Refine factor.
+    refine_factor: Optional[int] = None
+
+
 class LanceQueryBuilder:
     """
     A builder for nearest neighbor queries for LanceDB.
@@ -45,7 +74,12 @@ class LanceQueryBuilder:
     0  6  [0.4, 0.4]    0.0
     """
 
-    def __init__(self, table: "lancedb.table.LanceTable", query: np.ndarray):
+    def __init__(
+        self,
+        table: "lancedb.table.Table",
+        query: Union[np.ndarray, str],
+        vector_column: str = VECTOR_COLUMN_NAME,
+    ):
         self._metric = "L2"
         self._nprobes = 20
         self._refine_factor = None
@@ -54,6 +88,7 @@ class LanceQueryBuilder:
         self._limit = 10
         self._columns = None
         self._where = None
+        self._vector_column = vector_column
 
     def limit(self, limit: int) -> LanceQueryBuilder:
         """Set the maximum number of results to return.
@@ -175,52 +210,28 @@ class LanceQueryBuilder:
 
     def to_arrow(self) -> pa.Table:
         """
-        Execute the query and return the results as a arrow Table.
+        Execute the query and return the results as an
+        [Apache Arrow Table](https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table).
+
         In addition to the selected columns, LanceDB also returns a vector
         and also the "score" column which is the distance between the query
-        vector and the returned vector.
+        vector and the returned vectors.
         """
-        if self._table._conn.is_managed_remote:
-            try:
-                loop = asyncio.get_running_loop()
-            except RuntimeError:
-                loop = asyncio.get_event_loop()
-            result = self._table._conn._client.query(
-                self._table.name, self.to_remote_query()
-            )
-            return loop.run_until_complete(result).to_arrow()
-
-        ds = self._table.to_lance()
-        return ds.to_table(
-            columns=self._columns,
-            filter=self._where,
-            nearest={
-                "column": VECTOR_COLUMN_NAME,
-                "q": self._query,
-                "k": self._limit,
-                "metric": self._metric,
-                "nprobes": self._nprobes,
-                "refine_factor": self._refine_factor,
-            },
-        )
-
-    def to_remote_query(self) -> "VectorQuery":
-        # don't import unless we are connecting to remote
-        from lancedb.remote.client import VectorQuery
-
-        return VectorQuery(
-            vector=self._query.tolist(),
+        vector = self._query if isinstance(self._query, list) else self._query.tolist()
+        query = Query(
+            vector=vector,
             filter=self._where,
             k=self._limit,
-            _metric=self._metric,
+            metric=self._metric,
             columns=self._columns,
             nprobes=self._nprobes,
             refine_factor=self._refine_factor,
         )
+        return self._table._execute_query(query)
 
 
 class LanceFtsQueryBuilder(LanceQueryBuilder):
-    def to_df(self) -> pd.DataFrame:
+    def to_arrow(self) -> pd.Table:
         try:
             import tantivy
         except ImportError:
@@ -237,8 +248,9 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
         # get the scores and doc ids
         row_ids, scores = search_index(index, self._query, self._limit)
         if len(row_ids) == 0:
-            return pd.DataFrame()
+            empty_schema = pa.schema([pa.field("score", pa.float32())])
+            return pa.Table.from_pylist([], schema=empty_schema)
         scores = pa.array(scores)
         output_tbl = self._table.to_lance().take(row_ids, columns=self._columns)
         output_tbl = output_tbl.append_column("score", scores)
-        return output_tbl.to_pandas()
+        return output_tbl

python/lancedb/remote/__init__.py

@@ -15,7 +15,6 @@ import abc
 from typing import List, Optional
 
 import attr
-import pandas as pd
 import pyarrow as pa
 from pydantic import BaseModel
 

python/lancedb/remote/client.py

@@ -13,12 +13,13 @@
 
 
 import functools
-import urllib.parse
+from typing import Dict
 
 import aiohttp
 import attr
 import pyarrow as pa
 
+from lancedb.common import Credential
 from lancedb.remote import VectorQuery, VectorQueryResult
 from lancedb.remote.errors import LanceDBClientError
 
@@ -35,29 +36,32 @@ def _check_not_closed(f):
 
 @attr.define(slots=False)
 class RestfulLanceDBClient:
-    url: str
+    db_name: str
+    region: str
+    api_key: Credential
     closed: bool = attr.field(default=False, init=False)
 
     @functools.cached_property
     def session(self) -> aiohttp.ClientSession:
-        parsed = urllib.parse.urlparse(self.url)
-        scheme = parsed.scheme
-        if not scheme.startswith("lancedb"):
-            raise ValueError(
-                f"Invalid scheme: {scheme}, must be like lancedb+<flavor>://"
-            )
-        flavor = scheme.split("+")[1]
-        url = f"{flavor}://{parsed.hostname}:{parsed.port}"
+        url = f"https://{self.db_name}.{self.region}.api.lancedb.com"
         return aiohttp.ClientSession(url)
 
     async def close(self):
         await self.session.close()
         self.closed = True
 
+    @functools.cached_property
+    def headers(self) -> Dict[str, str]:
+        return {
+            "x-api-key": self.api_key,
+        }
+
     @_check_not_closed
     async def query(self, table_name: str, query: VectorQuery) -> VectorQueryResult:
         async with self.session.post(
-            f"/table/{table_name}/", json=query.dict(exclude_none=True)
+            f"/1/table/{table_name}/",
+            json=query.dict(exclude_none=True),
+            headers=self.headers,
         ) as resp:
             resp: aiohttp.ClientResponse = resp
             if 400 <= resp.status < 500:

python/lancedb/remote/db.py

@@ -0,0 +1,71 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from typing import List
+from urllib.parse import urlparse
+
+import pyarrow as pa
+
+from lancedb.common import DATA
+from lancedb.db import DBConnection
+from lancedb.table import Table
+
+from .client import RestfulLanceDBClient
+
+
+class RemoteDBConnection(DBConnection):
+    """A connection to a remote LanceDB database."""
+
+    def __init__(self, db_url: str, api_key: str, region: str):
+        """Connect to a remote LanceDB database."""
+        parsed = urlparse(db_url)
+        if parsed.scheme != "db":
+            raise ValueError(f"Invalid scheme: {parsed.scheme}, only accepts db://")
+        self.db_name = parsed.netloc
+        self.api_key = api_key
+        self._client = RestfulLanceDBClient(self.db_name, region, api_key)
+
+    def __repr__(self) -> str:
+        return f"RemoveConnect(name={self.db_name})"
+
+    def table_names(self) -> List[str]:
+        raise NotImplementedError
+
+    def open_table(self, name: str) -> Table:
+        """Open a Lance Table in the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+
+        Returns
+        -------
+        A LanceTable object representing the table.
+        """
+        from .table import RemoteTable
+
+        # TODO: check if table exists
+
+        return RemoteTable(self, name)
+
+    def create_table(
+        self,
+        name: str,
+        data: DATA = None,
+        schema: pa.Schema = None,
+        mode: str = "create",
+        on_bad_vectors: str = "error",
+        fill_value: float = 0.0,
+    ) -> Table:
+        raise NotImplementedError

python/lancedb/remote/table.py

@@ -0,0 +1,70 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import asyncio
+from typing import Union
+
+import pyarrow as pa
+
+from lancedb.common import DATA, VEC, VECTOR_COLUMN_NAME
+
+from ..query import LanceQueryBuilder, Query
+from ..table import Query, Table
+from .db import RemoteDBConnection
+
+
+class RemoteTable(Table):
+    def __init__(self, conn: RemoteDBConnection, name: str):
+        self._conn = conn
+        self._name = name
+
+    def __repr__(self) -> str:
+        return f"RemoteTable({self._conn.db_name}.{self.name})"
+
+    def schema(self) -> pa.Schema:
+        raise NotImplementedError
+
+    def to_arrow(self) -> pa.Table:
+        raise NotImplementedError
+
+    def create_index(
+        self,
+        metric="L2",
+        num_partitions=256,
+        num_sub_vectors=96,
+        vector_column_name: str = VECTOR_COLUMN_NAME,
+        replace: bool = True,
+    ):
+        raise NotImplementedError
+
+    def add(
+        self,
+        data: DATA,
+        mode: str = "append",
+        on_bad_vectors: str = "error",
+        fill_value: float = 0.0,
+    ) -> int:
+        raise NotImplementedError
+
+    def search(
+        self, query: Union[VEC, str], vector_column: str = VECTOR_COLUMN_NAME
+    ) -> LanceQueryBuilder:
+        return LanceQueryBuilder(self, query, vector_column)
+
+    def _execute_query(self, query: Query) -> pa.Table:
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            loop = asyncio.get_event_loop()
+        result = self._conn._client.query(self._name, query)
+        return loop.run_until_complete(result).to_arrow()

python/lancedb/table.py

@@ -14,6 +14,7 @@
 from __future__ import annotations
 
 import os
+from abc import ABC, abstractmethod
 from functools import cached_property
 from typing import List, Union
 
@@ -21,35 +22,41 @@ import lance
 import numpy as np
 import pandas as pd
 import pyarrow as pa
+import pyarrow.compute as pc
+import pyarrow.fs
 from lance import LanceDataset
 from lance.vector import vec_to_table
 
 from .common import DATA, VEC, VECTOR_COLUMN_NAME
-from .query import LanceFtsQueryBuilder, LanceQueryBuilder
+from .query import LanceFtsQueryBuilder, LanceQueryBuilder, Query
 
 
-def _sanitize_data(data, schema):
+def _sanitize_data(data, schema, on_bad_vectors, fill_value):
     if isinstance(data, list):
         data = pa.Table.from_pylist(data)
-        data = _sanitize_schema(data, schema=schema)
+        data = _sanitize_schema(
+            data, schema=schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+        )
     if isinstance(data, dict):
         data = vec_to_table(data)
     if isinstance(data, pd.DataFrame):
         data = pa.Table.from_pandas(data)
-        data = _sanitize_schema(data, schema=schema)
+        data = _sanitize_schema(
+            data, schema=schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+        )
     if not isinstance(data, pa.Table):
         raise TypeError(f"Unsupported data type: {type(data)}")
     return data
 
 
-class LanceTable:
+class Table(ABC):
     """
-    A table in a LanceDB database.
+    A [Table](Table) is a collection of Records in a LanceDB [Database](Database).
 
     Examples
     --------
 
-    Create using [LanceDBConnection.create_table][lancedb.LanceDBConnection.create_table]
+    Create using [DBConnection.create_table][lancedb.DBConnection.create_table]
     (more examples in that method's documentation).
 
     >>> import lancedb
@@ -64,12 +71,12 @@ class LanceTable:
     vector: [[[1.1,1.2]]]
     b: [[2]]
 
-    Can append new data with [LanceTable.add][lancedb.table.LanceTable.add].
+    Can append new data with [Table.add()][lancedb.table.Table.add].
 
     >>> table.add([{"vector": [0.5, 1.3], "b": 4}])
     2
 
-    Can query the table with [LanceTable.search][lancedb.table.LanceTable.search].
+    Can query the table with [Table.search][lancedb.table.Table.search].
 
     >>> table.search([0.4, 0.4]).select(["b"]).to_df()
        b      vector  score
@@ -77,8 +84,128 @@ class LanceTable:
     1  2  [1.1, 1.2]   1.13
 
     Search queries are much faster when an index is created. See
-    [LanceTable.create_index][lancedb.table.LanceTable.create_index].
+    [Table.create_index][lancedb.table.Table.create_index].
+    """
 
+    @abstractmethod
+    def schema(self) -> pa.Schema:
+        """Return the [Arrow Schema](https://arrow.apache.org/docs/python/api/datatypes.html#) of
+        this [Table](Table)
+
+        """
+        raise NotImplementedError
+
+    def to_pandas(self) -> pd.DataFrame:
+        """Return the table as a pandas DataFrame.
+
+        Returns
+        -------
+        pd.DataFrame
+        """
+        return self.to_arrow().to_pandas()
+
+    @abstractmethod
+    def to_arrow(self) -> pa.Table:
+        """Return the table as a pyarrow Table.
+
+        Returns
+        -------
+        pa.Table
+        """
+        raise NotImplementedError
+
+    def create_index(
+        self,
+        metric="L2",
+        num_partitions=256,
+        num_sub_vectors=96,
+        vector_column_name: str = VECTOR_COLUMN_NAME,
+        replace: bool = True,
+    ):
+        """Create an index on the table.
+
+        Parameters
+        ----------
+        metric: str, default "L2"
+            The distance metric to use when creating the index.
+            Valid values are "L2", "cosine", or "dot".
+            L2 is euclidean distance.
+        num_partitions: int
+            The number of IVF partitions to use when creating the index.
+            Default is 256.
+        num_sub_vectors: int
+            The number of PQ sub-vectors to use when creating the index.
+            Default is 96.
+        vector_column_name: str, default "vector"
+            The vector column name to create the index.
+        replace: bool, default True
+            If True, replace the existing index if it exists.
+            If False, raise an error if duplicate index exists.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def add(
+        self,
+        data: DATA,
+        mode: str = "append",
+        on_bad_vectors: str = "error",
+        fill_value: float = 0.0,
+    ) -> int:
+        """Add more data to the [Table](Table).
+
+        Parameters
+        ----------
+        data: list-of-dict, dict, pd.DataFrame
+            The data to insert into the table.
+        mode: str
+            The mode to use when writing the data. Valid values are
+            "append" and "overwrite".
+        on_bad_vectors: str, default "error"
+            What to do if any of the vectors are not the same size or contains NaNs.
+            One of "error", "drop", "fill".
+        fill_value: float, default 0.
+            The value to use when filling vectors. Only used if on_bad_vectors="fill".
+
+        Returns
+        -------
+        int
+            The number of vectors in the table.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def search(
+        self, query: Union[VEC, str], vector_column: str = VECTOR_COLUMN_NAME
+    ) -> LanceQueryBuilder:
+        """Create a search query to find the nearest neighbors
+        of the given query vector.
+
+        Parameters
+        ----------
+        query: list, np.ndarray
+            The query vector.
+        vector_column: str, default "vector"
+            The name of the vector column to search.
+
+        Returns
+        -------
+        LanceQueryBuilder
+            A query builder object representing the query.
+            Once executed, the query returns selected columns, the vector,
+            and also the "score" column which is the distance between the query
+            vector and the returned vector.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def _execute_query(self, query: Query) -> pa.Table:
+        pass
+
+
+class LanceTable(Table):
+    """
+    A table in a LanceDB database.
     """
 
     def __init__(
@@ -90,7 +217,8 @@ class LanceTable:
 
     def _reset_dataset(self):
         try:
-            del self.__dict__["_dataset"]
+            if "_dataset" in self.__dict__:
+                del self.__dict__["_dataset"]
         except AttributeError:
             pass
 
@@ -182,27 +310,22 @@ class LanceTable:
     def _dataset_uri(self) -> str:
         return os.path.join(self._conn.uri, f"{self.name}.lance")
 
-    def create_index(self, metric="L2", num_partitions=256, num_sub_vectors=96):
-        """Create an index on the table.
-
-        Parameters
-        ----------
-        metric: str, default "L2"
-            The distance metric to use when creating the index. Valid values are "L2" or "cosine".
-            L2 is euclidean distance.
-        num_partitions: int
-            The number of IVF partitions to use when creating the index.
-            Default is 256.
-        num_sub_vectors: int
-            The number of PQ sub-vectors to use when creating the index.
-            Default is 96.
-        """
+    def create_index(
+        self,
+        metric="L2",
+        num_partitions=256,
+        num_sub_vectors=96,
+        vector_column_name=VECTOR_COLUMN_NAME,
+        replace: bool = True,
+    ):
+        """Create an index on the table."""
         self._dataset.create_index(
-            column=VECTOR_COLUMN_NAME,
+            column=vector_column_name,
             index_type="IVF_PQ",
             metric=metric,
             num_partitions=num_partitions,
             num_sub_vectors=num_sub_vectors,
+            replace=replace,
         )
         self._reset_dataset()
 
@@ -235,7 +358,13 @@ class LanceTable:
         """Return the LanceDataset backing this table."""
         return self._dataset
 
-    def add(self, data: DATA, mode: str = "append") -> int:
+    def add(
+        self,
+        data: DATA,
+        mode: str = "append",
+        on_bad_vectors: str = "error",
+        fill_value: float = 0.0,
+    ) -> int:
         """Add data to the table.
 
         Parameters
@@ -245,18 +374,28 @@ class LanceTable:
         mode: str
             The mode to use when writing the data. Valid values are
             "append" and "overwrite".
+        on_bad_vectors: str, default "error"
+            What to do if any of the vectors are not the same size or contains NaNs.
+            One of "error", "drop", "fill".
+        fill_value: float, default 0.
+            The value to use when filling vectors. Only used if on_bad_vectors="fill".
 
         Returns
         -------
         int
             The number of vectors in the table.
         """
-        data = _sanitize_data(data, self.schema)
+        # TODO: manage table listing and metadata separately
+        data = _sanitize_data(
+            data, self.schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+        )
         lance.write_dataset(data, self._dataset_uri, mode=mode)
         self._reset_dataset()
         return len(self)
 
-    def search(self, query: Union[VEC, str]) -> LanceQueryBuilder:
+    def search(
+        self, query: Union[VEC, str], vector_column_name=VECTOR_COLUMN_NAME
+    ) -> LanceQueryBuilder:
         """Create a search query to find the nearest neighbors
         of the given query vector.
 
@@ -264,6 +403,8 @@ class LanceTable:
         ----------
         query: list, np.ndarray
             The query vector.
+        vector_column_name: str, default "vector"
+            The name of the vector column to search.
 
         Returns
         -------
@@ -275,7 +416,7 @@ class LanceTable:
         """
         if isinstance(query, str):
             # fts
-            return LanceFtsQueryBuilder(self, query)
+            return LanceFtsQueryBuilder(self, query, vector_column_name)
 
         if isinstance(query, list):
             query = np.array(query)
@@ -283,13 +424,75 @@ class LanceTable:
             query = query.astype(np.float32)
         else:
             raise TypeError(f"Unsupported query type: {type(query)}")
-        return LanceQueryBuilder(self, query)
+        return LanceQueryBuilder(self, query, vector_column_name)
 
     @classmethod
-    def create(cls, db, name, data, schema=None, mode="create"):
+    def create(
+        cls,
+        db,
+        name,
+        data=None,
+        schema=None,
+        mode="create",
+        on_bad_vectors: str = "error",
+        fill_value: float = 0.0,
+    ):
+        """
+        Create a new table.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> import pandas as pd
+        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
+        >>> db = lancedb.connect("./.lancedb")
+        >>> table = db.create_table("my_table", data)
+        >>> table.to_pandas()
+           x      vector
+        0  1  [1.0, 2.0]
+        1  2  [3.0, 4.0]
+        2  3  [5.0, 6.0]
+
+        Parameters
+        ----------
+        db: LanceDB
+            The LanceDB instance to create the table in.
+        name: str
+            The name of the table to create.
+        data: list-of-dict, dict, pd.DataFrame, default None
+            The data to insert into the table.
+            At least one of `data` or `schema` must be provided.
+        schema: dict, optional
+            The schema of the table. If not provided, the schema is inferred from the data.
+            At least one of `data` or `schema` must be provided.
+        mode: str, default "create"
+            The mode to use when writing the data. Valid values are
+            "create", "overwrite", and "append".
+        on_bad_vectors: str, default "error"
+            What to do if any of the vectors are not the same size or contains NaNs.
+            One of "error", "drop", "fill".
+        fill_value: float, default 0.
+            The value to use when filling vectors. Only used if on_bad_vectors="fill".
+        """
         tbl = LanceTable(db, name)
-        data = _sanitize_data(data, schema)
+        if data is not None:
+            data = _sanitize_data(
+                data, schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+            )
+        else:
+            if schema is None:
+                raise ValueError("Either data or schema must be provided")
+            data = pa.Table.from_pylist([], schema=schema)
         lance.write_dataset(data, tbl._dataset_uri, mode=mode)
+        return LanceTable(db, name)
+
+    @classmethod
+    def open(cls, db, name):
+        tbl = cls(db, name)
+        if not os.path.exists(tbl._dataset_uri):
+            raise FileNotFoundError(
+                f"Table {name} does not exist. Please first call db.create_table({name}, data)"
+            )
         return tbl
 
     def delete(self, where: str):
@@ -320,8 +523,28 @@ class LanceTable:
         """
         self._dataset.delete(where)
 
+    def _execute_query(self, query: Query) -> pa.Table:
+        ds = self.to_lance()
+        return ds.to_table(
+            columns=query.columns,
+            filter=query.filter,
+            nearest={
+                "column": query.vector_column,
+                "q": query.vector,
+                "k": query.k,
+                "metric": query.metric,
+                "nprobes": query.nprobes,
+                "refine_factor": query.refine_factor,
+            },
+        )
 
-def _sanitize_schema(data: pa.Table, schema: pa.Schema = None) -> pa.Table:
+
+def _sanitize_schema(
+    data: pa.Table,
+    schema: pa.Schema = None,
+    on_bad_vectors: str = "error",
+    fill_value: float = 0.0,
+) -> pa.Table:
     """Ensure that the table has the expected schema.
 
     Parameters
@@ -331,21 +554,41 @@ def _sanitize_schema(data: pa.Table, schema: pa.Schema = None) -> pa.Table:
     schema: pa.Schema; optional
         The expected schema. If not provided, this just converts the
         vector column to fixed_size_list(float32) if necessary.
+    on_bad_vectors: str, default "error"
+        What to do if any of the vectors are not the same size or contains NaNs.
+        One of "error", "drop", "fill".
+    fill_value: float, default 0.
+        The value to use when filling vectors. Only used if on_bad_vectors="fill".
     """
     if schema is not None:
         if data.schema == schema:
             return data
         # cast the columns to the expected types
         data = data.combine_chunks()
-        data = _sanitize_vector_column(data, vector_column_name=VECTOR_COLUMN_NAME)
+        data = _sanitize_vector_column(
+            data,
+            vector_column_name=VECTOR_COLUMN_NAME,
+            on_bad_vectors=on_bad_vectors,
+            fill_value=fill_value,
+        )
         return pa.Table.from_arrays(
             [data[name] for name in schema.names], schema=schema
         )
     # just check the vector column
-    return _sanitize_vector_column(data, vector_column_name=VECTOR_COLUMN_NAME)
+    return _sanitize_vector_column(
+        data,
+        vector_column_name=VECTOR_COLUMN_NAME,
+        on_bad_vectors=on_bad_vectors,
+        fill_value=fill_value,
+    )
 
 
-def _sanitize_vector_column(data: pa.Table, vector_column_name: str) -> pa.Table:
+def _sanitize_vector_column(
+    data: pa.Table,
+    vector_column_name: str,
+    on_bad_vectors: str = "error",
+    fill_value: float = 0.0,
+) -> pa.Table:
     """
     Ensure that the vector column exists and has type fixed_size_list(float32)
 
@@ -355,19 +598,103 @@ def _sanitize_vector_column(data: pa.Table, vector_column_name: str) -> pa.Table
         The table to sanitize.
     vector_column_name: str
         The name of the vector column.
+    on_bad_vectors: str, default "error"
+        What to do if any of the vectors are not the same size or contains NaNs.
+        One of "error", "drop", "fill".
+    fill_value: float, default 0.0
+        The value to use when filling vectors. Only used if on_bad_vectors="fill".
     """
     if vector_column_name not in data.column_names:
         raise ValueError(f"Missing vector column: {vector_column_name}")
+    # ChunkedArray is annoying to work with, so we combine chunks here
     vec_arr = data[vector_column_name].combine_chunks()
-    if pa.types.is_fixed_size_list(vec_arr.type):
-        return data
-    if not pa.types.is_list(vec_arr.type):
+    if pa.types.is_list(data[vector_column_name].type):
+        # if it's a variable size list array we make sure the dimensions are all the same
+        has_jagged_ndims = len(vec_arr.values) % len(data) != 0
+        if has_jagged_ndims:
+            data = _sanitize_jagged(
+                data, fill_value, on_bad_vectors, vec_arr, vector_column_name
+            )
+            vec_arr = data[vector_column_name].combine_chunks()
+    elif not pa.types.is_fixed_size_list(vec_arr.type):
         raise TypeError(f"Unsupported vector column type: {vec_arr.type}")
+
+    vec_arr = ensure_fixed_size_list_of_f32(vec_arr)
+    data = data.set_column(
+        data.column_names.index(vector_column_name), vector_column_name, vec_arr
+    )
+
+    has_nans = pc.any(pc.is_nan(vec_arr.values)).as_py()
+    if has_nans:
+        data = _sanitize_nans(
+            data, fill_value, on_bad_vectors, vec_arr, vector_column_name
+        )
+
+    return data
+
+
+def ensure_fixed_size_list_of_f32(vec_arr):
     values = vec_arr.values
     if not pa.types.is_float32(values.type):
         values = values.cast(pa.float32())
-    list_size = len(values) / len(data)
+    if pa.types.is_fixed_size_list(vec_arr.type):
+        list_size = vec_arr.type.list_size
+    else:
+        list_size = len(values) / len(vec_arr)
     vec_arr = pa.FixedSizeListArray.from_arrays(values, list_size)
-    return data.set_column(
-        data.column_names.index(vector_column_name), vector_column_name, vec_arr
-    )
+    return vec_arr
+
+
+def _sanitize_jagged(data, fill_value, on_bad_vectors, vec_arr, vector_column_name):
+    """Sanitize jagged vectors."""
+    if on_bad_vectors == "error":
+        raise ValueError(
+            f"Vector column {vector_column_name} has variable length vectors "
+            "Set on_bad_vectors='drop' to remove them, or "
+            "set on_bad_vectors='fill' and fill_value=<value> to replace them."
+        )
+
+    lst_lengths = pc.list_value_length(vec_arr)
+    ndims = pc.max(lst_lengths).as_py()
+    correct_ndims = pc.equal(lst_lengths, ndims)
+
+    if on_bad_vectors == "fill":
+        if fill_value is None:
+            raise ValueError(
+                "`fill_value` must not be None if `on_bad_vectors` is 'fill'"
+            )
+        fill_arr = pa.scalar([float(fill_value)] * ndims)
+        vec_arr = pc.if_else(correct_ndims, vec_arr, fill_arr)
+        data = data.set_column(
+            data.column_names.index(vector_column_name), vector_column_name, vec_arr
+        )
+    elif on_bad_vectors == "drop":
+        data = data.filter(correct_ndims)
+    return data
+
+
+def _sanitize_nans(data, fill_value, on_bad_vectors, vec_arr, vector_column_name):
+    """Sanitize NaNs in vectors"""
+    if on_bad_vectors == "error":
+        raise ValueError(
+            f"Vector column {vector_column_name} has NaNs. "
+            "Set on_bad_vectors='drop' to remove them, or "
+            "set on_bad_vectors='fill' and fill_value=<value> to replace them."
+        )
+    elif on_bad_vectors == "fill":
+        if fill_value is None:
+            raise ValueError(
+                "`fill_value` must not be None if `on_bad_vectors` is 'fill'"
+            )
+        fill_value = float(fill_value)
+        values = pc.if_else(pc.is_nan(vec_arr.values), fill_value, vec_arr.values)
+        ndims = len(vec_arr[0])
+        vec_arr = pa.FixedSizeListArray.from_arrays(values, ndims)
+        data = data.set_column(
+            data.column_names.index(vector_column_name), vector_column_name, vec_arr
+        )
+    elif on_bad_vectors == "drop":
+        is_value_nan = pc.is_nan(vec_arr.values).to_numpy(zero_copy_only=False)
+        is_full = np.any(~is_value_nan.reshape(-1, vec_arr.type.list_size), axis=1)
+        data = data.filter(is_full)
+    return data

python/lancedb/util.py

@@ -11,9 +11,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from urllib.parse import ParseResult, urlparse
-
-from pyarrow import fs
+from urllib.parse import urlparse
 
 
 def get_uri_scheme(uri: str) -> str:

python/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lancedb"
-version = "0.1.8"
+version = "0.1.10"
 dependencies = ["pylance~=0.5.0", "ratelimiter", "retry", "tqdm", "aiohttp", "pydantic", "attr"]
 description = "lancedb"
 authors = [

python/tests/test_db.py

@@ -11,6 +11,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import numpy as np
 import pandas as pd
 import pytest
 
@@ -120,3 +121,40 @@ def test_delete_table(tmp_path):
 
     db.create_table("test", data=data)
     assert db.table_names() == ["test"]
+
+
+def test_empty_or_nonexistent_table(tmp_path):
+    db = lancedb.connect(tmp_path)
+    with pytest.raises(Exception):
+        db.create_table("test_with_no_data")
+
+    with pytest.raises(Exception):
+        db.open_table("does_not_exist")
+
+
+def test_replace_index(tmp_path):
+    db = lancedb.connect(uri=tmp_path)
+    table = db.create_table(
+        "test",
+        [
+            {"vector": np.random.rand(128), "item": "foo", "price": float(i)}
+            for i in range(1000)
+        ],
+    )
+    table.create_index(
+        num_partitions=2,
+        num_sub_vectors=4,
+    )
+
+    with pytest.raises(Exception):
+        table.create_index(
+            num_partitions=2,
+            num_sub_vectors=4,
+            replace=False,
+        )
+
+    table.create_index(
+        num_partitions=2,
+        num_sub_vectors=4,
+        replace=True,
+    )

python/tests/test_query.py

@@ -11,25 +11,42 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import unittest.mock as mock
+
 import lance
 import numpy as np
-import pandas as pd
 import pandas.testing as tm
 import pyarrow as pa
 import pytest
 
 from lancedb.db import LanceDBConnection
-from lancedb.query import LanceQueryBuilder
+from lancedb.query import LanceQueryBuilder, Query
+from lancedb.table import LanceTable
 
 
 class MockTable:
     def __init__(self, tmp_path):
         self.uri = tmp_path
-        self._conn = LanceDBConnection("/tmp/lance/")
+        self._conn = LanceDBConnection(self.uri)
 
     def to_lance(self):
         return lance.dataset(self.uri)
 
+    def _execute_query(self, query):
+        ds = self.to_lance()
+        return ds.to_table(
+            columns=query.columns,
+            filter=query.filter,
+            nearest={
+                "column": query.vector_column,
+                "q": query.vector,
+                "k": query.k,
+                "metric": query.metric,
+                "nprobes": query.nprobes,
+                "refine_factor": query.refine_factor,
+            },
+        )
+
 
 @pytest.fixture
 def table(tmp_path) -> MockTable:
@@ -48,24 +65,30 @@ def table(tmp_path) -> MockTable:
 
 
 def test_query_builder(table):
-    df = LanceQueryBuilder(table, [0, 0]).limit(1).select(["id"]).to_df()
+    df = LanceQueryBuilder(table, [0, 0], "vector").limit(1).select(["id"]).to_df()
     assert df["id"].values[0] == 1
     assert all(df["vector"].values[0] == [1, 2])
 
 
 def test_query_builder_with_filter(table):
-    df = LanceQueryBuilder(table, [0, 0]).where("id = 2").to_df()
+    df = LanceQueryBuilder(table, [0, 0], "vector").where("id = 2").to_df()
     assert df["id"].values[0] == 2
     assert all(df["vector"].values[0] == [3, 4])
 
 
 def test_query_builder_with_metric(table):
     query = [4, 8]
-    df_default = LanceQueryBuilder(table, query).to_df()
-    df_l2 = LanceQueryBuilder(table, query).metric("L2").to_df()
+    vector_column_name = "vector"
+    df_default = LanceQueryBuilder(table, query, vector_column_name).to_df()
+    df_l2 = LanceQueryBuilder(table, query, vector_column_name).metric("L2").to_df()
     tm.assert_frame_equal(df_default, df_l2)
 
-    df_cosine = LanceQueryBuilder(table, query).metric("cosine").limit(1).to_df()
+    df_cosine = (
+        LanceQueryBuilder(table, query, vector_column_name)
+        .metric("cosine")
+        .limit(1)
+        .to_df()
+    )
     assert df_cosine.score[0] == pytest.approx(
         cosine_distance(query, df_cosine.vector[0]),
         abs=1e-6,
@@ -73,5 +96,32 @@ def test_query_builder_with_metric(table):
     assert 0 <= df_cosine.score[0] <= 1
 
 
+def test_query_builder_with_different_vector_column():
+    table = mock.MagicMock(spec=LanceTable)
+    query = [4, 8]
+    vector_column_name = "foo_vector"
+    builder = (
+        LanceQueryBuilder(table, query, vector_column_name)
+        .metric("cosine")
+        .where("b < 10")
+        .select(["b"])
+        .limit(2)
+    )
+    ds = mock.Mock()
+    table.to_lance.return_value = ds
+    builder.to_arrow()
+    table._execute_query.assert_called_once_with(
+        Query(
+            vector=query,
+            filter="b < 10",
+            k=2,
+            metric="cosine",
+            columns=["b"],
+            nprobes=20,
+            refine_factor=None,
+        )
+    )
+
+
 def cosine_distance(vec1, vec2):
     return 1 - np.dot(vec1, vec2) / (np.linalg.norm(vec1) * np.linalg.norm(vec2))

python/tests/test_remote_client.py

@@ -30,7 +30,7 @@ class MockLanceDBServer:
         table_name = request.match_info["table_name"]
         assert table_name == "test_table"
 
-        request_json = await request.json()
+        await request.json()
         # TODO: do some matching
 
         vecs = pd.Series([np.random.rand(128) for x in range(10)], name="vector")

python/tests/test_remote_db.py

@@ -13,7 +13,7 @@
 
 import pyarrow as pa
 
-from lancedb.db import LanceDBConnection
+import lancedb
 from lancedb.remote.client import VectorQuery, VectorQueryResult
 
 
@@ -28,7 +28,7 @@ class FakeLanceDBClient:
 
 
 def test_remote_db():
-    conn = LanceDBConnection("lancedb+http://client-will-be-injected")
+    conn = lancedb.connect("db://client-will-be-injected", api_key="fake")
     setattr(conn, "_client", FakeLanceDBClient())
 
     table = conn["test"]

python/tests/test_table.py

@@ -13,11 +13,15 @@
 
 import functools
 from pathlib import Path
+from unittest.mock import PropertyMock, patch
 
+import numpy as np
 import pandas as pd
 import pyarrow as pa
 import pytest
+from lance.vector import vec_to_table
 
+from lancedb.db import LanceDBConnection
 from lancedb.table import LanceTable
 
 
@@ -86,7 +90,31 @@ def test_create_table(db):
         assert expected == tbl
 
 
+def test_empty_table(db):
+    schema = pa.schema(
+        [
+            pa.field("vector", pa.list_(pa.float32(), 2)),
+            pa.field("item", pa.string()),
+            pa.field("price", pa.float32()),
+        ]
+    )
+    tbl = LanceTable.create(db, "test", schema=schema)
+    data = [
+        {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0},
+    ]
+    tbl.add(data=data)
+
+
 def test_add(db):
+    schema = pa.schema(
+        [
+            pa.field("vector", pa.list_(pa.float32(), 2)),
+            pa.field("item", pa.string()),
+            pa.field("price", pa.float64()),
+        ]
+    )
+
     table = LanceTable.create(
         db,
         "test",
@@ -95,7 +123,19 @@ def test_add(db):
             {"vector": [5.9, 26.5], "item": "bar", "price": 20.0},
         ],
     )
+    _add(table, schema)
 
+    table = LanceTable.create(db, "test2", schema=schema)
+    table.add(
+        data=[
+            {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+            {"vector": [5.9, 26.5], "item": "bar", "price": 20.0},
+        ],
+    )
+    _add(table, schema)
+
+
+def _add(table, schema):
     # table = LanceTable(db, "test")
     assert len(table) == 2
 
@@ -110,13 +150,7 @@ def test_add(db):
             pa.array(["foo", "bar", "new"]),
             pa.array([10.0, 20.0, 30.0]),
         ],
-        schema=pa.schema(
-            [
-                pa.field("vector", pa.list_(pa.float32(), 2)),
-                pa.field("item", pa.string()),
-                pa.field("price", pa.float64()),
-            ]
-        ),
+        schema=schema,
     )
     assert expected == table.to_arrow()
 
@@ -142,3 +176,83 @@ def test_versioning(db):
     table.checkout(1)
     assert table.version == 1
     assert len(table) == 2
+
+
+def test_create_index_method():
+    with patch.object(LanceTable, "_reset_dataset", return_value=None):
+        with patch.object(
+            LanceTable, "_dataset", new_callable=PropertyMock
+        ) as mock_dataset:
+            # Setup mock responses
+            mock_dataset.return_value.create_index.return_value = None
+
+            # Create a LanceTable object
+            connection = LanceDBConnection(uri="mock.uri")
+            table = LanceTable(connection, "test_table")
+
+            # Call the create_index method
+            table.create_index(
+                metric="L2",
+                num_partitions=256,
+                num_sub_vectors=96,
+                vector_column_name="vector",
+                replace=True,
+            )
+
+            # Check that the _dataset.create_index method was called
+            # with the right parameters
+            mock_dataset.return_value.create_index.assert_called_once_with(
+                column="vector",
+                index_type="IVF_PQ",
+                metric="L2",
+                num_partitions=256,
+                num_sub_vectors=96,
+                replace=True,
+            )
+
+
+def test_add_with_nans(db):
+    # by default we raise an error on bad input vectors
+    bad_data = [
+        {"vector": [np.nan], "item": "bar", "price": 20.0},
+        {"vector": [5], "item": "bar", "price": 20.0},
+        {"vector": [np.nan, np.nan], "item": "bar", "price": 20.0},
+        {"vector": [np.nan, 5.0], "item": "bar", "price": 20.0},
+    ]
+    for row in bad_data:
+        with pytest.raises(ValueError):
+            LanceTable.create(
+                db,
+                "error_test",
+                data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0}, row],
+            )
+
+    table = LanceTable.create(
+        db,
+        "drop_test",
+        data=[
+            {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+            {"vector": [np.nan], "item": "bar", "price": 20.0},
+            {"vector": [5], "item": "bar", "price": 20.0},
+            {"vector": [np.nan, np.nan], "item": "bar", "price": 20.0},
+        ],
+        on_bad_vectors="drop",
+    )
+    assert len(table) == 1
+
+    # We can fill bad input with some value
+    table = LanceTable.create(
+        db,
+        "fill_test",
+        data=[
+            {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+            {"vector": [np.nan], "item": "bar", "price": 20.0},
+            {"vector": [np.nan, np.nan], "item": "bar", "price": 20.0},
+        ],
+        on_bad_vectors="fill",
+        fill_value=0.0,
+    )
+    assert len(table) == 3
+    arrow_tbl = table.to_lance().to_table(filter="item == 'bar'")
+    v = arrow_tbl["vector"].to_pylist()[0]
+    assert np.allclose(v, np.array([0.0, 0.0]))

rust/ffi/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb-node"
-version = "0.1.9"
+version = "0.1.10"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
 edition = "2018"
@@ -10,12 +10,12 @@ exclude = ["index.node"]
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow-array = "40.0"
-arrow-ipc = "40.0"
-arrow-schema = "40.0"
+arrow-array = { workspace = true }
+arrow-ipc = { workspace = true }
+arrow-schema = { workspace = true }
 once_cell = "1"
 futures = "0.3"
-lance = "0.5.0"
+lance = { workspace = true }
 vectordb = { path = "../../vectordb" }
 tokio = { version = "1.23", features = ["rt-multi-thread"] }
 neon = {version = "0.10.1", default-features = false, features = ["channel-api", "napi-6", "promise-api", "task-api"] }

rust/ffi/node/src/index/vector.rs

@@ -122,6 +122,10 @@ fn get_index_params_builder(
                 .map_err(|t| t.to_string())?
                 .map(|s| pq_params.max_opq_iters = s.value(cx) as usize);
 
+            obj.get_opt::<JsBoolean, _, _>(cx, "replace")
+                .map_err(|t| t.to_string())?
+                .map(|s| index_builder.replace(s.value(cx)));
+
             Ok(index_builder)
         }
         t => Err(format!("{} is not a valid index type", t).to_string()),

rust/ffi/node/src/lib.rs

@@ -17,11 +17,10 @@ use std::convert::TryFrom;
 use std::ops::Deref;
 use std::sync::{Arc, Mutex};
 
-use arrow_array::{Float32Array, RecordBatchReader};
+use arrow_array::{Float32Array, RecordBatchIterator, RecordBatchReader};
 use arrow_ipc::writer::FileWriter;
 use futures::{TryFutureExt, TryStreamExt};
-use lance::arrow::RecordBatchBuffer;
-use lance::dataset::WriteMode;
+use lance::dataset::{WriteMode, WriteParams};
 use lance::index::vector::MetricType;
 use neon::prelude::*;
 use neon::types::buffer::TypedArray;
@@ -233,6 +232,17 @@ fn table_create(mut cx: FunctionContext) -> JsResult<JsPromise> {
     let table_name = cx.argument::<JsString>(0)?.value(&mut cx);
     let buffer = cx.argument::<JsBuffer>(1)?;
     let batches = arrow_buffer_to_record_batch(buffer.as_slice(&mut cx));
+    let schema = batches[0].schema();
+
+    // Write mode
+    let mode = match cx.argument::<JsString>(2)?.value(&mut cx).as_str() {
+        "overwrite" => WriteMode::Overwrite,
+        "append" => WriteMode::Append,
+        "create" => WriteMode::Create,
+        _ => return cx.throw_error("Table::create only supports 'overwrite' and 'create' modes"),
+    };
+    let mut params = WriteParams::default();
+    params.mode = mode;
 
     let rt = runtime(&mut cx)?;
     let channel = cx.channel();
@@ -241,8 +251,13 @@ fn table_create(mut cx: FunctionContext) -> JsResult<JsPromise> {
     let database = db.database.clone();
 
     rt.block_on(async move {
-        let batch_reader: Box<dyn RecordBatchReader> = Box::new(RecordBatchBuffer::new(batches));
-        let table_rst = database.create_table(&table_name, batch_reader).await;
+        let batch_reader: Box<dyn RecordBatchReader> = Box::new(RecordBatchIterator::new(
+            batches.into_iter().map(Ok),
+            schema,
+        ));
+        let table_rst = database
+            .create_table(&table_name, batch_reader, Some(params))
+            .await;
 
         deferred.settle_with(&channel, move |mut cx| {
             let table = Arc::new(Mutex::new(
@@ -265,6 +280,7 @@ fn table_add(mut cx: FunctionContext) -> JsResult<JsPromise> {
     let buffer = cx.argument::<JsBuffer>(0)?;
     let write_mode = cx.argument::<JsString>(1)?.value(&mut cx);
     let batches = arrow_buffer_to_record_batch(buffer.as_slice(&mut cx));
+    let schema = batches[0].schema();
 
     let rt = runtime(&mut cx)?;
     let channel = cx.channel();
@@ -274,7 +290,10 @@ fn table_add(mut cx: FunctionContext) -> JsResult<JsPromise> {
     let write_mode = write_mode_map.get(write_mode.as_str()).cloned();
 
     rt.block_on(async move {
-        let batch_reader: Box<dyn RecordBatchReader> = Box::new(RecordBatchBuffer::new(batches));
+        let batch_reader: Box<dyn RecordBatchReader> = Box::new(RecordBatchIterator::new(
+            batches.into_iter().map(Ok),
+            schema,
+        ));
         let add_result = table.lock().unwrap().add(batch_reader, write_mode).await;
 
         deferred.settle_with(&channel, move |mut cx| {

rust/vectordb/Cargo.toml

@@ -1,20 +1,19 @@
 [package]
 name = "vectordb"
-version = "0.1.9"
+version = "0.1.10"
 edition = "2021"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
 repository = "https://github.com/lancedb/lancedb"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies]
-arrow-array = "40.0"
-arrow-data = "40.0"
-arrow-schema = "40.0"
-object_store = "0.6.1"
+arrow-array = { workspace = true }
+arrow-data = { workspace = true }
+arrow-schema = { workspace = true }
+object_store = { workspace = true }
 snafu = "0.7.4"
-lance = "0.5.0"
+lance = { workspace = true }
 tokio = { version = "1.23", features = ["rt-multi-thread"] }
 
 [dev-dependencies]

rust/vectordb/src/database.rs

@@ -16,11 +16,12 @@ use std::fs::create_dir_all;
 use std::path::Path;
 
 use arrow_array::RecordBatchReader;
+use lance::dataset::WriteParams;
 use lance::io::object_store::ObjectStore;
 use snafu::prelude::*;
 
 use crate::error::{CreateDirSnafu, Result};
-use crate::table::Table;
+use crate::table::{OpenTableParams, Table};
 
 pub struct Database {
     object_store: ObjectStore,
@@ -90,12 +91,19 @@ impl Database {
         Ok(f)
     }
 
+    /// Create a new table in the database.
+    ///
+    /// # Arguments
+    /// * `name` - The name of the table.
+    /// * `batches` - The initial data to write to the table.
+    /// * `params` - Optional [`WriteParams`] to create the table.
     pub async fn create_table(
         &self,
         name: &str,
         batches: Box<dyn RecordBatchReader>,
+        params: Option<WriteParams>,
     ) -> Result<Table> {
-        Table::create(&self.uri, name, batches).await
+        Table::create(&self.uri, name, batches, params).await
     }
 
     /// Open a table in the database.
@@ -107,7 +115,25 @@ impl Database {
     ///
     /// * A [Table] object.
     pub async fn open_table(&self, name: &str) -> Result<Table> {
-        Table::open(&self.uri, name).await
+        self.open_table_with_params(name, OpenTableParams::default())
+            .await
+    }
+
+    /// Open a table in the database.
+    ///
+    /// # Arguments
+    /// * `name` - The name of the table.
+    /// * `params` - The parameters to open the table.
+    ///
+    /// # Returns
+    ///
+    /// * A [Table] object.
+    pub async fn open_table_with_params(
+        &self,
+        name: &str,
+        params: OpenTableParams,
+    ) -> Result<Table> {
+        Table::open_with_params(&self.uri, name, params).await
     }
 
     /// Drop a table in the database.

rust/vectordb/src/lib.rs

@@ -19,3 +19,6 @@ pub mod query;
 pub mod table;
 
 pub use database::Database;
+pub use table::Table;
+
+pub use lance::dataset::WriteMode;

rust/vectordb/src/query.rs

@@ -164,9 +164,8 @@ impl Query {
 mod tests {
     use std::sync::Arc;
 
-    use arrow_array::{Float32Array, RecordBatch, RecordBatchReader};
+    use arrow_array::{Float32Array, RecordBatch, RecordBatchIterator, RecordBatchReader};
     use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
-    use lance::arrow::RecordBatchBuffer;
     use lance::dataset::Dataset;
     use lance::index::vector::MetricType;
 
@@ -174,7 +173,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_setters_getters() {
-        let mut batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
+        let mut batches: Box<dyn RecordBatchReader> = make_test_batches();
         let ds = Dataset::write(&mut batches, "memory://foo", None)
             .await
             .unwrap();
@@ -203,7 +202,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_execute() {
-        let mut batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
+        let mut batches: Box<dyn RecordBatchReader> = make_test_batches();
         let ds = Dataset::write(&mut batches, "memory://foo", None)
             .await
             .unwrap();
@@ -214,7 +213,7 @@ mod tests {
         assert_eq!(result.is_ok(), true);
     }
 
-    fn make_test_batches() -> RecordBatchBuffer {
+    fn make_test_batches() -> Box<dyn RecordBatchReader> {
         let dim: usize = 128;
         let schema = Arc::new(ArrowSchema::new(vec![
             ArrowField::new("key", DataType::Int32, false),
@@ -228,7 +227,11 @@ mod tests {
             ),
             ArrowField::new("uri", DataType::Utf8, true),
         ]));
-
-        RecordBatchBuffer::new(vec![RecordBatch::new_empty(schema.clone())])
+        Box::new(RecordBatchIterator::new(
+            vec![RecordBatch::new_empty(schema.clone())]
+                .into_iter()
+                .map(Ok),
+            schema,
+        ))
     }
 }

rust/vectordb/src/table.rs

@@ -1,4 +1,4 @@
-// Copyright 2023 Lance Developers.
+// Copyright 2023 LanceDB Developers.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -16,12 +16,13 @@ use std::path::Path;
 use std::sync::Arc;
 
 use arrow_array::{Float32Array, RecordBatchReader};
-use lance::dataset::{Dataset, WriteMode, WriteParams};
+use lance::dataset::{Dataset, ReadParams, WriteParams};
 use lance::index::IndexType;
 use snafu::prelude::*;
 
 use crate::error::{Error, InvalidTableNameSnafu, Result};
 use crate::index::vector::VectorIndexBuilder;
+use crate::WriteMode;
 use crate::query::Query;
 
 pub const VECTOR_COLUMN_NAME: &str = "vector";
@@ -41,6 +42,11 @@ impl std::fmt::Display for Table {
     }
 }
 
+#[derive(Default)]
+pub struct OpenTableParams {
+    pub open_table_params: ReadParams,
+}
+
 impl Table {
     /// Opens an existing Table
     ///
@@ -53,6 +59,25 @@ impl Table {
     ///
     /// * A [Table] object.
     pub async fn open(base_uri: &str, name: &str) -> Result<Self> {
+        Self::open_with_params(base_uri, name, OpenTableParams::default()).await
+    }
+
+    /// Opens an existing Table
+    ///
+    /// # Arguments
+    ///
+    /// * `base_path` - The base path where the table is located
+    /// * `name` The Table name
+    /// * `params` The [OpenTableParams] to use when opening the table
+    ///
+    /// # Returns
+    ///
+    /// * A [Table] object.
+    pub async fn open_with_params(
+        base_uri: &str,
+        name: &str,
+        params: OpenTableParams,
+    ) -> Result<Self> {
         let path = Path::new(base_uri);
 
         let table_uri = path.join(format!("{}.{}", name, LANCE_FILE_EXTENSION));
@@ -61,14 +86,16 @@ impl Table {
             .to_str()
             .context(InvalidTableNameSnafu { name })?;
 
-        let dataset = Dataset::open(&uri).await.map_err(|e| match e {
-            lance::Error::DatasetNotFound { .. } => Error::TableNotFound {
-                name: name.to_string(),
-            },
-            e => Error::Lance {
-                message: e.to_string(),
-            },
-        })?;
+        let dataset = Dataset::open_with_params(uri, &params.open_table_params)
+            .await
+            .map_err(|e| match e {
+                lance::Error::DatasetNotFound { .. } => Error::TableNotFound {
+                    name: name.to_string(),
+                },
+                e => Error::Lance {
+                    message: e.to_string(),
+                },
+            })?;
         Ok(Table {
             name: name.to_string(),
             uri: uri.to_string(),
@@ -91,6 +118,7 @@ impl Table {
         base_uri: &str,
         name: &str,
         mut batches: Box<dyn RecordBatchReader>,
+        params: Option<WriteParams>,
     ) -> Result<Self> {
         let base_path = Path::new(base_uri);
         let table_uri = base_path.join(format!("{}.{}", name, LANCE_FILE_EXTENSION));
@@ -99,7 +127,7 @@ impl Table {
             .to_str()
             .context(InvalidTableNameSnafu { name })?
             .to_string();
-        let dataset = Dataset::write(&mut batches, &uri, Some(WriteParams::default()))
+        let dataset = Dataset::write(&mut batches, &uri, params)
             .await
             .map_err(|e| match e {
                 lance::Error::DatasetAlreadyExists { .. } => Error::TableAlreadyExists {
@@ -200,17 +228,19 @@ impl Table {
 
 #[cfg(test)]
 mod tests {
+    use std::sync::atomic::{AtomicBool, Ordering};
     use std::sync::Arc;
 
     use arrow_array::{
-        Array, FixedSizeListArray, Float32Array, Int32Array, RecordBatch, RecordBatchReader,
+        Array, FixedSizeListArray, Float32Array, Int32Array, RecordBatch, RecordBatchIterator,
+        RecordBatchReader,
     };
     use arrow_data::ArrayDataBuilder;
     use arrow_schema::{DataType, Field, Schema};
-    use lance::arrow::RecordBatchBuffer;
     use lance::dataset::{Dataset, WriteMode};
     use lance::index::vector::ivf::IvfBuildParams;
     use lance::index::vector::pq::PQBuildParams;
+    use lance::io::object_store::{ObjectStoreParams, WrappingObjectStore};
     use rand::Rng;
     use tempfile::tempdir;
 
@@ -223,7 +253,7 @@ mod tests {
         let dataset_path = tmp_dir.path().join("test.lance");
         let uri = tmp_dir.path().to_str().unwrap();
 
-        let mut batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
+        let mut batches: Box<dyn RecordBatchReader> = make_test_batches();
         Dataset::write(&mut batches, dataset_path.to_str().unwrap(), None)
             .await
             .unwrap();
@@ -254,12 +284,12 @@ mod tests {
         let tmp_dir = tempdir().unwrap();
         let uri = tmp_dir.path().to_str().unwrap();
 
-        let batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
+        let batches: Box<dyn RecordBatchReader> = make_test_batches();
         let _ = batches.schema().clone();
-        Table::create(&uri, "test", batches).await.unwrap();
+        Table::create(&uri, "test", batches, None).await.unwrap();
 
-        let batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
-        let result = Table::create(&uri, "test", batches).await;
+        let batches: Box<dyn RecordBatchReader> = make_test_batches();
+        let result = Table::create(&uri, "test", batches, None).await;
         assert!(matches!(
             result.unwrap_err(),
             Error::TableAlreadyExists { .. }
@@ -271,17 +301,21 @@ mod tests {
         let tmp_dir = tempdir().unwrap();
         let uri = tmp_dir.path().to_str().unwrap();
 
-        let batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
+        let batches: Box<dyn RecordBatchReader> = make_test_batches();
         let schema = batches.schema().clone();
-        let mut table = Table::create(&uri, "test", batches).await.unwrap();
+        let mut table = Table::create(&uri, "test", batches, None).await.unwrap();
         assert_eq!(table.count_rows().await.unwrap(), 10);
 
-        let new_batches: Box<dyn RecordBatchReader> =
-            Box::new(RecordBatchBuffer::new(vec![RecordBatch::try_new(
-                schema,
+        let new_batches: Box<dyn RecordBatchReader> = Box::new(RecordBatchIterator::new(
+            vec![RecordBatch::try_new(
+                schema.clone(),
                 vec![Arc::new(Int32Array::from_iter_values(100..110))],
             )
-            .unwrap()]));
+            .unwrap()]
+            .into_iter()
+            .map(Ok),
+            schema.clone(),
+        ));
 
         table.add(new_batches, None).await.unwrap();
         assert_eq!(table.count_rows().await.unwrap(), 20);
@@ -293,17 +327,21 @@ mod tests {
         let tmp_dir = tempdir().unwrap();
         let uri = tmp_dir.path().to_str().unwrap();
 
-        let batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
+        let batches: Box<dyn RecordBatchReader> = make_test_batches();
         let schema = batches.schema().clone();
-        let mut table = Table::create(uri, "test", batches).await.unwrap();
+        let mut table = Table::create(uri, "test", batches, None).await.unwrap();
         assert_eq!(table.count_rows().await.unwrap(), 10);
 
-        let new_batches: Box<dyn RecordBatchReader> =
-            Box::new(RecordBatchBuffer::new(vec![RecordBatch::try_new(
-                schema,
+        let new_batches: Box<dyn RecordBatchReader> = Box::new(RecordBatchIterator::new(
+            vec![RecordBatch::try_new(
+                schema.clone(),
                 vec![Arc::new(Int32Array::from_iter_values(100..110))],
             )
-            .unwrap()]));
+            .unwrap()]
+            .into_iter()
+            .map(Ok),
+            schema.clone(),
+        ));
 
         table
             .add(new_batches, Some(WriteMode::Overwrite))
@@ -319,7 +357,7 @@ mod tests {
         let dataset_path = tmp_dir.path().join("test.lance");
         let uri = tmp_dir.path().to_str().unwrap();
 
-        let mut batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
+        let mut batches: Box<dyn RecordBatchReader> = make_test_batches();
         Dataset::write(&mut batches, dataset_path.to_str().unwrap(), None)
             .await
             .unwrap();
@@ -331,13 +369,63 @@ mod tests {
         assert_eq!(vector, query.query_vector);
     }
 
-    fn make_test_batches() -> RecordBatchBuffer {
+    #[derive(Default)]
+    struct NoOpCacheWrapper {
+        called: AtomicBool,
+    }
+
+    impl NoOpCacheWrapper {
+        fn called(&self) -> bool {
+            self.called.load(Ordering::Relaxed)
+        }
+    }
+
+    impl WrappingObjectStore for NoOpCacheWrapper {
+        fn wrap(
+            &self,
+            original: Arc<dyn object_store::ObjectStore>,
+        ) -> Arc<dyn object_store::ObjectStore> {
+            self.called.store(true, Ordering::Relaxed);
+            return original;
+        }
+    }
+
+    #[tokio::test]
+    async fn test_open_table_options() {
+        let tmp_dir = tempdir().unwrap();
+        let dataset_path = tmp_dir.path().join("test.lance");
+        let uri = tmp_dir.path().to_str().unwrap();
+
+        let mut batches: Box<dyn RecordBatchReader> = make_test_batches();
+        Dataset::write(&mut batches, dataset_path.to_str().unwrap(), None)
+            .await
+            .unwrap();
+
+        let wrapper = Arc::new(NoOpCacheWrapper::default());
+
+        let mut object_store_params = ObjectStoreParams::default();
+        object_store_params.object_store_wrapper = Some(wrapper.clone());
+        let param = OpenTableParams {
+            open_table_params: ReadParams {
+                store_options: Some(object_store_params),
+                ..ReadParams::default()
+            },
+        };
+
+        assert!(!wrapper.called());
+        let _ = Table::open_with_params(uri, "test", param).await.unwrap();
+        assert!(wrapper.called());
+    }
+
+    fn make_test_batches() -> Box<dyn RecordBatchReader> {
         let schema = Arc::new(Schema::new(vec![Field::new("i", DataType::Int32, false)]));
-        RecordBatchBuffer::new(vec![RecordBatch::try_new(
-            schema.clone(),
-            vec![Arc::new(Int32Array::from_iter_values(0..10))],
-        )
-        .unwrap()])
+        Box::new(RecordBatchIterator::new(
+            vec![RecordBatch::try_new(
+                schema.clone(),
+                vec![Arc::new(Int32Array::from_iter_values(0..10))],
+            )],
+            schema,
+        ))
     }
 
     #[tokio::test]
@@ -370,14 +458,15 @@ mod tests {
         );
 
         let vectors = Arc::new(create_fixed_size_list(float_arr, dimension).unwrap());
-        let batches = RecordBatchBuffer::new(vec![RecordBatch::try_new(
-            schema.clone(),
-            vec![vectors.clone()],
-        )
-        .unwrap()]);
+        let batches = RecordBatchIterator::new(
+            vec![RecordBatch::try_new(schema.clone(), vec![vectors.clone()]).unwrap()]
+                .into_iter()
+                .map(Ok),
+            schema,
+        );
 
         let reader: Box<dyn RecordBatchReader + Send> = Box::new(batches);
-        let mut table = Table::create(uri, "test", reader).await.unwrap();
+        let mut table = Table::create(uri, "test", reader, None).await.unwrap();
 
         let mut i = IvfPQIndexBuilder::new();