WIP hastily moved types around

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.1
+current_version = 0.3.9
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True

.github/ISSUE_TEMPLATE/bug-node.yml

@@ -1,33 +0,0 @@
-name: Bug Report - Node / Typescript
-description: File a bug report
-title: "bug(node): "
-labels: [bug, typescript]
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Thanks for taking the time to fill out this bug report!
-  - type: input
-    id: version
-    attributes:
-      label: LanceDB version
-      description: What version of LanceDB are you using? `npm list | grep vectordb`.
-      placeholder: v0.3.2
-    validations:
-      required: false
-  - type: textarea
-    id: what-happened
-    attributes:
-      label: What happened?
-      description: Also tell us, what did you expect to happen?
-    validations:
-      required: true
-  - type: textarea
-    id: reproduction
-    attributes:
-      label: Are there known steps to reproduce?
-      description: |
-        Let us know how to reproduce the bug and we may be able to fix it more
-        quickly. This is not required, but it is helpful.
-    validations:
-      required: false

.github/ISSUE_TEMPLATE/bug-python.yml

@@ -1,33 +0,0 @@
-name: Bug Report - Python
-description: File a bug report
-title: "bug(python): "
-labels: [bug, python]
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Thanks for taking the time to fill out this bug report!
-  - type: input
-    id: version
-    attributes:
-      label: LanceDB version
-      description: What version of LanceDB are you using? `python -c "import lancedb; print(lancedb.__version__)"`.
-      placeholder: v0.3.2
-    validations:
-      required: false
-  - type: textarea
-    id: what-happened
-    attributes:
-      label: What happened?
-      description: Also tell us, what did you expect to happen?
-    validations:
-      required: true
-  - type: textarea
-    id: reproduction
-    attributes:
-      label: Are there known steps to reproduce?
-      description: |
-        Let us know how to reproduce the bug and we may be able to fix it more
-        quickly. This is not required, but it is helpful.
-    validations:
-      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +0,0 @@
-blank_issues_enabled: true
-contact_links:
-  - name: Discord Community Support
-    url: https://discord.com/invite/zMM32dvNtd
-    about: Please ask and answer questions here.

.github/ISSUE_TEMPLATE/documentation.yml

@@ -1,23 +0,0 @@
-name: 'Documentation improvement'
-description: Report an issue with the documentation.
-labels: [documentation]
-
-body:
-  - type: textarea
-    id: description
-    attributes:
-      label: Description
-      description: >
-        Describe the issue with the documentation and how it can be fixed or improved.
-    validations:
-      required: true
-
-  - type: input
-    id: link
-    attributes:
-      label: Link
-      description: >
-        Provide a link to the existing documentation, if applicable.
-      placeholder: ex. https://lancedb.github.io/lancedb/guides/tables/...
-    validations:
-      required: false

.github/ISSUE_TEMPLATE/feature.yml

@@ -1,31 +0,0 @@
-name: Feature suggestion
-description: Suggestion a new feature for LanceDB
-title: "Feature: "
-labels: [enhancement]
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Share a new idea for a feature or improvement. Be sure to search existing
-        issues first to avoid duplicates.
-  - type: dropdown
-    id: sdk
-    attributes:
-      label: SDK
-      description: Which SDK are you using? This helps us prioritize.
-      options:
-        - Python
-        - Node
-        - Rust
-      default: 0
-    validations:
-      required: false
-  - type: textarea
-    id: description
-    attributes:
-      label: Description
-      description: |
-        Describe the feature and why it would be useful. If applicable, consider
-        providing a code example of what it might be like to use the feature.
-    validations:
-      required: true

.github/workflows/npm-publish.yml

@@ -37,16 +37,8 @@ jobs:
           path: |
             node/vectordb-*.tgz
 
-  node-macos:
-    strategy:
-      matrix:
-        config:
-          - arch: x86_64-apple-darwin
-            runner: macos-13
-          - arch: aarch64-apple-darwin
-            # xlarge is implicitly arm64.
-            runner: macos-13-xlarge
-    runs-on: ${{ matrix.config.runner }}
+  node-macos-x86:
+    runs-on: macos-13
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
@@ -59,14 +51,35 @@ jobs:
           cd node
           npm ci
       - name: Build MacOS native node modules
-        run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }}
+        run: bash ci/build_macos_artifacts.sh x86_64-apple-darwin
       - name: Upload Darwin Artifacts
         uses: actions/upload-artifact@v3
         with:
           name: native-darwin
           path: |
             node/dist/lancedb-vectordb-darwin*.tgz
-         
+
+  node-macos-arm64:
+    runs-on: macos-13-xlarge
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install system dependencies
+        run: brew install protobuf
+      - name: Install npm dependencies
+        run: |
+          cd node
+          npm ci
+      - name: Build MacOS native node modules
+        run: bash ci/build_macos_artifacts.sh aarch64-apple-darwin
+      - name: Upload Darwin Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: native-darwin
+          path: |
+            node/dist/lancedb-vectordb-darwin*.tgz            
 
   node-linux:
     name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu

.github/workflows/python.yml

@@ -44,19 +44,12 @@ jobs:
       run: pytest -m "not slow" -x -v --durations=30 tests
     - name: doctest
       run: pytest --doctest-modules lancedb
-  platform:
-    name: "Platform: ${{ matrix.config.name }}"
+  mac:
     timeout-minutes: 30
     strategy:
       matrix:
-        config: 
-          - name: x86 Mac
-            runner: macos-13
-          - name: Arm Mac
-            runner: macos-13-xlarge
-          - name: x86 Windows
-            runner: windows-latest
-    runs-on: "${{ matrix.config.runner }}"
+        mac-runner: [ "macos-13", "macos-13-xlarge" ]
+    runs-on: "${{ matrix.mac-runner }}"
     defaults:
       run:
         shell: bash
@@ -98,7 +91,11 @@ jobs:
         pip install "pydantic<2"
         pip install -e .[tests]
         pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock
+        pip install pytest pytest-mock black isort
+    - name: Black
+      run: black --check --diff --no-color --quiet .
+    - name: isort
+      run: isort --check --diff --quiet .
     - name: Run tests
       run: pytest -m "not slow" -x -v --durations=30 tests
     - name: doctest

.github/workflows/rust.yml

@@ -24,29 +24,6 @@ env:
   RUST_BACKTRACE: "1"
 
 jobs:
-  lint:
-    timeout-minutes: 30
-    runs-on: ubuntu-22.04
-    defaults:
-      run:
-        shell: bash
-        working-directory: rust
-    steps:
-    - uses: actions/checkout@v3
-      with:
-          fetch-depth: 0
-          lfs: true
-    - uses: Swatinem/rust-cache@v2
-      with:
-        workspaces: rust
-    - name: Install dependencies
-      run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-    - name: Run format
-      run: cargo fmt --all -- --check
-    - name: Run clippy
-      run: cargo clippy --all --all-features -- -D warnings
   linux:
     timeout-minutes: 30
     runs-on: ubuntu-22.04

Cargo.toml

@@ -5,24 +5,24 @@ exclude = ["python"]
 resolver = "2"
 
 [workspace.dependencies]
-lance = { "version" = "=0.9.2", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.9.2" }
-lance-linalg = { "version" = "=0.9.2" }
-lance-testing = { "version" = "=0.9.2" }
+lance = { "version" = "=0.8.20", "features" = ["dynamodb"] }
+lance-index = { "version" = "=0.8.20" }
+lance-linalg = { "version" = "=0.8.20" }
+lance-testing = { "version" = "=0.8.20" }
 # Note that this one does not include pyarrow
-arrow = { version = "49.0.0", optional = false }
-arrow-array = "49.0"
-arrow-data = "49.0"
-arrow-ipc = "49.0"
-arrow-ord = "49.0"
-arrow-schema = "49.0"
-arrow-arith = "49.0"
-arrow-cast = "49.0"
+arrow = { version = "47.0.0", optional = false }
+arrow-array = "47.0"
+arrow-data = "47.0"
+arrow-ipc = "47.0"
+arrow-ord = "47.0"
+arrow-schema = "47.0"
+arrow-arith = "47.0"
+arrow-cast = "47.0"
 chrono = "0.4.23"
 half = { "version" = "=2.3.1", default-features = false, features = [
     "num-traits",
 ] }
 log = "0.4"
-object_store = "0.8.0"
+object_store = "0.7.1"
 snafu = "0.7.4"
 url = "2"

docs/requirements.txt

@@ -2,4 +2,3 @@ mkdocs==1.4.2
 mkdocs-jupyter==0.24.1
 mkdocs-material==9.1.3
 mkdocstrings[python]==0.20.0
-pydantic

docs/src/basic.md

@@ -64,26 +64,18 @@ We'll cover the basics of using LanceDB on your local machine in this section.
       tbl = db.create_table("table_from_df", data=df)
       ```
 
-      !!! warning
-
-            If the table already exists, LanceDB will raise an error by default.
-            If you want to overwrite the table, you can pass in `mode="overwrite"`
-            to the `createTable` function.
-
 === "Javascript"
       ```javascript
-      const tb = await db.createTable(
-        "myTable",
-        [{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-         {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+      const tb = await db.createTable("my_table",
+                        data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                              {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
       ```
-
-      !!! warning
-
-            If the table already exists, LanceDB will raise an error by default.
-            If you want to overwrite the table, you can pass in `"overwrite"`
-            to the `createTable` function like this: `await con.createTable(tableName, data, { writeMode: WriteMode.Overwrite })`
       
+!!! warning
+
+      If the table already exists, LanceDB will raise an error by default.
+      If you want to overwrite the table, you can pass in `mode="overwrite"`
+      to the `createTable` function.
 
 ??? info "Under the hood, LanceDB is converting the input data into an Apache Arrow table and persisting it to disk in [Lance format](https://www.github.com/lancedb/lance)."
 
@@ -116,7 +108,7 @@ Once created, you can open a table using the following code:
 
 === "Javascript"
       ```javascript
-      const tbl = await db.openTable("myTable");
+      const tbl = await db.openTable("my_table");
       ```
 
       If you forget the name of your table, you can always get a listing of all table names:
@@ -202,17 +194,10 @@ Use the `drop_table()` method on the database to remove a table.
       db.drop_table("my_table")
       ```
 
-      This permanently removes the table and is not recoverable, unlike deleting rows.
-      By default, if the table does not exist an exception is raised. To suppress this,
-      you can pass in `ignore_missing=True`.
+This permanently removes the table and is not recoverable, unlike deleting rows.
+By default, if the table does not exist an exception is raised. To suppress this,
+you can pass in `ignore_missing=True`.
 
-=== "JavaScript"
-      ```javascript
-      await db.dropTable('myTable')
-      ```
-
-      This permanently removes the table and is not recoverable, unlike deleting rows.
-      If the table does not exist an exception is raised. 
 
 ## What's next
 

docs/src/embeddings/default_embedding_functions.md

@@ -1,9 +1,9 @@
-There are various Embedding functions available out of the box with LanceDB. We're working on supporting other popular embedding APIs.
+There are various Embedding functions available out of the box with lancedb. We're working on supporting other popular embedding APIs.
 
 ## Text Embedding Functions
 Here are the text embedding functions registered by default.
-Embedding functions have an inbuilt rate limit handler wrapper for source and query embedding function calls that retry with exponential standoff. 
-Each `EmbeddingFunction` implementation automatically takes `max_retries` as an argument which has the default value of 7.
+Embedding functions have inbuilt rate limit handler wrapper for source and query embedding function calls that retry with exponential standoff. 
+Each `EmbeddingFunction` implementation automatically takes `max_retries` as an argument which has the deafult value of 7.
 
 ### Sentence Transformers
 Here are the parameters that you can set when registering a `sentence-transformers` object, and their default values:
@@ -69,15 +69,15 @@ print(actual.text)
 ```
 
 ### Instructor Embeddings
-Instructor is an instruction-finetuned text embedding model that can generate text embeddings tailored to any task (e.g. classification, retrieval, clustering, text evaluation, etc.) and domains (e.g. science, finance, etc.) by simply providing the task instruction, without any finetuning.
+Instructor is an instruction-finetuned text embedding model that can generate text embeddings tailored to any task (e.g., classification, retrieval, clustering, text evaluation, etc.) and domains (e.g., science, finance, etc.) by simply providing the task instruction, without any finetuning
 
 If you want to calculate customized embeddings for specific sentences, you may follow the unified template to write instructions:
 
                           Represent the `domain` `text_type` for `task_objective`:
 
-* `domain` is optional, and it specifies the domain of the text, e.g. science, finance, medicine, etc.
-* `text_type` is required, and it specifies the encoding unit, e.g. sentence, document, paragraph, etc.
-* `task_objective` is optional, and it specifies the objective of embedding, e.g. retrieve a document, classify the sentence, etc.
+* `domain` is optional, and it specifies the domain of the text, e.g., science, finance, medicine, etc.
+* `text_type` is required, and it specifies the encoding unit, e.g., sentence, document, paragraph, etc.
+* `task_objective` is optional, and it specifies the objective of embedding, e.g., retrieve a document, classify the sentence, etc.
 
 More information about the model can be found here - https://github.com/xlang-ai/instructor-embedding
 
@@ -119,10 +119,10 @@ tbl.add(texts)
 ```
 
 ## Multi-modal embedding functions
-Multi-modal embedding functions allow you to query your table using both images and text.
+Multi-modal embedding functions allow you query your table using both images and text.
 
 ### OpenClipEmbeddings
-We support CLIP model embeddings using the open source alternative, open-clip which supports various customizations. It is registered as `open-clip` and supports the following customizations:
+We support CLIP model embeddings using the open souce alternbative, open-clip which support various customizations. It is registered as `open-clip` and supports following customizations.
 
 
 | Parameter | Type | Default Value | Description |
@@ -205,4 +205,4 @@ print(actual.label)
 
 ```
 
-If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue.
+If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue.

docs/src/fts.md

@@ -29,9 +29,8 @@ uri = "data/sample-lancedb"
 db = lancedb.connect(uri)
 
 table = db.create_table("my_table",
-            data=[{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy", "meta": "foo"},
-                  {"vector": [5.9, 26.5], "text": "Sam was a loyal puppy", "meta": "bar"},
-                  {"vector": [15.9, 6.5], "text": "There are several kittens playing"}])
+            data=[{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
+                  {"vector": [5.9, 26.5], "text": "There are several kittens playing"}])
 
 ```
 
@@ -65,23 +64,10 @@ table.create_fts_index(["text1", "text2"])
 
 Note that the search API call does not change - you can search over all indexed columns at once.
 
-## Filtering
-
-Currently the LanceDB full text search feature supports *post-filtering*, meaning filters are
-applied on top of the full text search results. This can be invoked via the familiar
-`where` syntax:
-
-```python
-table.search("puppy").limit(10).where("meta='foo'").to_list()
-```
-
 ## Current limitations
 
 1. Currently we do not yet support incremental writes.
-   If you add data after fts index creation, it won't be reflected
-   in search results until you do a full reindex.
-
-2. We currently only support local filesystem paths for the fts index. 
-   This is a tantivy limitation. We've implemented an object store plugin
-   but there's no way in tantivy-py to specify to use it.
+If you add data after fts index creation, it won't be reflected
+in search results until you do a full reindex.
 
+2. We currently only support local filesystem paths for the fts index.

docs/src/guides/tables.md

@@ -1,7 +1,5 @@
  <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/tables_guide.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
-A Table is a collection of Records in a LanceDB Database. Tables in Lance have a schema that defines the columns and their types. These schemas can include nested columns and can evolve over time.
-
-This guide will show how to create tables, insert data into them, and update the data. You can follow along on colab!
+A Table is a collection of Records in a LanceDB Database. You can follow along on colab!
 
 ## Creating a LanceDB Table
 
@@ -118,84 +116,6 @@ This guide will show how to create tables, insert data into them, and update the
     table = db.create_table(table_name, schema=Content)
     ```
 
-    #### Nested schemas
-
-    Sometimes your data model may contain nested objects.
-    For example, you may want to store the document string
-    and the document soure name as a nested Document object:
-
-    ```python
-    class Document(BaseModel):
-        content: str
-        source: str
-    ```
-
-    This can be used as the type of a LanceDB table column:
-
-    ```python
-    class NestedSchema(LanceModel):
-        id: str
-        vector: Vector(1536)
-        document: Document
-
-    tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite")
-    ```
-
-    This creates a struct column called "document" that has two subfields 
-    called "content" and "source":
-
-    ```
-    In [28]: tbl.schema
-    Out[28]:
-    id: string not null
-    vector: fixed_size_list<item: float>[1536] not null
-        child 0, item: float
-    document: struct<content: string not null, source: string not null> not null
-        child 0, content: string not null
-        child 1, source: string not null    
-    ```
-
-    #### Validators
-
-    Note that neither pydantic nor pyarrow automatically validates that input data
-    is of the *correct* timezone, but this is easy to add as a custom field validator:
-
-    ```python
-    from datetime import datetime
-    from zoneinfo import ZoneInfo
-
-    from lancedb.pydantic import LanceModel
-    from pydantic import Field, field_validator, ValidationError, ValidationInfo
-
-    tzname = "America/New_York"
-    tz = ZoneInfo(tzname)
-
-    class TestModel(LanceModel):
-        dt_with_tz: datetime = Field(json_schema_extra={"tz": tzname})
-
-        @field_validator('dt_with_tz')
-        @classmethod
-        def tz_must_match(cls, dt: datetime) -> datetime:
-            assert dt.tzinfo == tz
-            return dt        
-
-    ok = TestModel(dt_with_tz=datetime.now(tz))
-
-    try:
-        TestModel(dt_with_tz=datetime.now(ZoneInfo("Asia/Shanghai")))
-        assert 0 == 1, "this should raise ValidationError"
-    except ValidationError:
-        print("A ValidationError was raised.")
-        pass
-    ```
-
-    When you run this code it should print "A ValidationError was raised."
-
-    #### Pydantic custom types
-
-    LanceDB does NOT yet support converting pydantic custom types. If this is something you need,
-    please file a feature request on the [LanceDB Github repo](https://github.com/lancedb/lancedb/issues/new).
-
     ### Using Iterators / Writing Large Datasets
 
     It is recommended to use itertators to add large datasets in batches when creating your table in one go. This does not create multiple versions of your dataset unlike manually adding batches using `table.add()`
@@ -231,7 +151,7 @@ This guide will show how to create tables, insert data into them, and update the
     You can also use iterators of other types like Pandas dataframe or Pylists directly in the above example.
 
     ## Creating Empty Table
-    You can create empty tables in python. Initialize it with schema and later ingest data into it.
+    You can also create empty tables in python. Initialize it with schema and later ingest data into it.
 
     ```python
     import lancedb
@@ -281,8 +201,8 @@ This guide will show how to create tables, insert data into them, and update the
     ```javascript
     data
     const tb = await db.createTable("my_table",
-      [{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-       {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+                  data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
     ```
 
     !!! info "Note"
@@ -441,28 +361,19 @@ Use the `delete()` method on tables to delete rows from a table. To choose which
     await tbl.countRows() // Returns 1
     ```
 
-## Updating a Table
+### Updating a Table [Experimental]
+EXPERIMENTAL: Update rows in the table (not threadsafe).
 
-This can be used to update zero to all rows depending on how many rows match the where clause. The update queries follow the form of a SQL UPDATE statement. The `where` parameter is a SQL filter that matches on the metadata columns. The `values` or `values_sql` parameters are used to provide the new values for the columns.
+This can be used to update zero to all rows depending on how many rows match the where clause.
 
-| Parameter   | Type | Description |
+| Parameter | Type | Description |
 |---|---|---|
 | `where` | `str` | The SQL where clause to use when updating rows. For example, `'x = 2'` or `'x IN (1, 2, 3)'`. The filter must not be empty, or it will error. |
 | `values` | `dict` | The values to update. The keys are the column names and the values are the values to set. |
-| `values_sql` | `dict` | The values to update. The keys are the column names and the values are the SQL expressions to set. For example, `{'x': 'x + 1'}` will increment the value of the `x` column by 1. |
 
-!!! info "SQL syntax"
-
-    See [SQL filters](sql.md) for more information on the supported SQL syntax.
-
-!!! warning "Warning"
-
-    Updating nested columns is not yet supported.
 
 === "Python"
 
-    API Reference: [lancedb.table.Table.update][]
-
     ```python
     import lancedb
     import pandas as pd
@@ -492,55 +403,6 @@ This can be used to update zero to all rows depending on how many rows match the
     2  2  [10.0, 10.0]
     ```
 
-=== "Javascript/Typescript"
-
-    API Reference: [vectordb.Table.update](../../javascript/interfaces/Table/#update)
-
-    ```javascript
-    const lancedb = require("vectordb");
-
-    const db = await lancedb.connect("./.lancedb");
-
-    const data = [
-      {x: 1, vector: [1, 2]},
-      {x: 2, vector: [3, 4]},
-      {x: 3, vector: [5, 6]},
-    ];
-    const tbl = await db.createTable("my_table", data)
-
-    await tbl.update({ where: "x = 2", values: {vector: [10, 10]} })
-    ```
-
-The `values` parameter is used to provide the new values for the columns as literal values. You can also use the `values_sql` / `valuesSql` parameter to provide SQL expressions for the new values. For example, you can use `values_sql="x + 1"` to increment the value of the `x` column by 1.
-
-=== "Python"
-
-    ```python
-    # Update the table where x = 2
-    table.update(valuesSql={"x": "x + 1"})
-
-    print(table.to_pandas())
-    ```
-
-    Output
-    ```shell
-        x  vector
-    0  2  [1.0, 2.0]
-    1  4  [5.0, 6.0]
-    2  3  [10.0, 10.0]
-    ```
-
-=== "Javascript/Typescript"
-
-    ```javascript
-    await tbl.update({ valuesSql: { x: "x + 1" } })
-    ```
-
-!!! info "Note"
-
-    When rows are updated, they are moved out of the index. The row will still show up in ANN queries, but the query will not be as fast as it would be if the row was in the index. If you update a large proportion of rows, consider rebuilding the index afterwards.
-
-
 ## What's Next?
 
 Learn how to Query your tables and create indices

docs/src/javascript/README.md

@@ -11,13 +11,8 @@ npm install vectordb
 ```
 
 This will download the appropriate native library for your platform. We currently
-support:
-
-* Linux (x86_64 and aarch64)
-* MacOS (Intel and ARM/M1/M2)
-* Windows (x86_64 only)
-
-We do not yet support musl-based Linux (such as Alpine Linux) or aarch64 Windows.
+support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
+yet support Windows or musl-based Linux (such as Alpine Linux).
 
 ## Usage
 

docs/src/javascript/classes/DefaultWriteOptions.md

@@ -1,41 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / DefaultWriteOptions
-
-# Class: DefaultWriteOptions
-
-Write options when creating a Table.
-
-## Implements
-
-- [`WriteOptions`](../interfaces/WriteOptions.md)
-
-## Table of contents
-
-### Constructors
-
-- [constructor](DefaultWriteOptions.md#constructor)
-
-### Properties
-
-- [writeMode](DefaultWriteOptions.md#writemode)
-
-## Constructors
-
-### constructor
-
-• **new DefaultWriteOptions**()
-
-## Properties
-
-### writeMode
-
-• **writeMode**: [`WriteMode`](../enums/WriteMode.md) = `WriteMode.Create`
-
-A [WriteMode](../enums/WriteMode.md) to use on this operation
-
-#### Implementation of
-
-[WriteOptions](../interfaces/WriteOptions.md).[writeMode](../interfaces/WriteOptions.md#writemode)
-
-#### Defined in
-
-[index.ts:778](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L778)

docs/src/javascript/classes/LocalConnection.md

@@ -26,7 +26,7 @@ A connection to a LanceDB database.
 ### Methods
 
 - [createTable](LocalConnection.md#createtable)
-- [createTableImpl](LocalConnection.md#createtableimpl)
+- [createTableArrow](LocalConnection.md#createtablearrow)
 - [dropTable](LocalConnection.md#droptable)
 - [openTable](LocalConnection.md#opentable)
 - [tableNames](LocalConnection.md#tablenames)
@@ -46,7 +46,7 @@ A connection to a LanceDB database.
 
 #### Defined in
 
-[index.ts:355](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L355)
+[index.ts:184](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L184)
 
 ## Properties
 
@@ -56,25 +56,17 @@ A connection to a LanceDB database.
 
 #### Defined in
 
-[index.ts:353](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L353)
+[index.ts:182](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L182)
 
 ___
 
 ### \_options
 
-• `Private` `Readonly` **\_options**: () => [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
-
-#### Type declaration
-
-▸ (): [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
-
-##### Returns
-
-[`ConnectionOptions`](../interfaces/ConnectionOptions.md)
+• `Private` `Readonly` **\_options**: [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
 
 #### Defined in
 
-[index.ts:352](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L352)
+[index.ts:181](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L181)
 
 ## Accessors
 
@@ -92,34 +84,27 @@ ___
 
 #### Defined in
 
-[index.ts:360](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L360)
+[index.ts:189](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L189)
 
 ## Methods
 
 ### createTable
 
-▸ **createTable**\<`T`\>(`name`, `data?`, `optsOrEmbedding?`, `opt?`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
+▸ **createTable**(`name`, `data`, `mode?`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
 
-Creates a new Table, optionally initializing it with new data.
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
+Creates a new Table and initialize it with new data.
 
 #### Parameters
 
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` \| [`CreateTableOptions`](../interfaces/CreateTableOptions.md)\<`T`\> |
-| `data?` | `Record`\<`string`, `unknown`\>[] |
-| `optsOrEmbedding?` | [`WriteOptions`](../interfaces/WriteOptions.md) \| [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> |
-| `opt?` | [`WriteOptions`](../interfaces/WriteOptions.md) |
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
+| `mode?` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
 
 #### Returns
 
-`Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
+`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
 
 #### Implementation of
 
@@ -127,94 +112,33 @@ Creates a new Table, optionally initializing it with new data.
 
 #### Defined in
 
-[index.ts:395](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L395)
+[index.ts:230](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L230)
 
-___
-
-### createTableImpl
-
-▸ `Private` **createTableImpl**\<`T`\>(`«destructured»`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
+▸ **createTable**(`name`, `data`, `mode`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
 
 #### Parameters
 
 | Name | Type |
 | :------ | :------ |
-| `«destructured»` | `Object` |
-| › `data?` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] |
-| › `embeddingFunction?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> |
-| › `name` | `string` |
-| › `schema?` | `Schema`\<`any`\> |
-| › `writeOptions?` | [`WriteOptions`](../interfaces/WriteOptions.md) |
+| `name` | `string` |
+| `data` | `Record`<`string`, `unknown`\>[] |
+| `mode` | [`WriteMode`](../enums/WriteMode.md) |
 
 #### Returns
 
-`Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
-
-#### Defined in
-
-[index.ts:413](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L413)
-
-___
-
-### dropTable
-
-▸ **dropTable**(`name`): `Promise`\<`void`\>
-
-Drop an existing table.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table to drop. |
-
-#### Returns
-
-`Promise`\<`void`\>
+`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
 
 #### Implementation of
 
-[Connection](../interfaces/Connection.md).[dropTable](../interfaces/Connection.md#droptable)
+Connection.createTable
 
 #### Defined in
 
-[index.ts:453](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L453)
+[index.ts:231](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L231)
 
-___
+▸ **createTable**<`T`\>(`name`, `data`, `mode`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
 
-### openTable
-
-▸ **openTable**(`name`): `Promise`\<[`Table`](../interfaces/Table.md)\<`number`[]\>\>
-
-Open a table in the database.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-
-#### Returns
-
-`Promise`\<[`Table`](../interfaces/Table.md)\<`number`[]\>\>
-
-#### Implementation of
-
-[Connection](../interfaces/Connection.md).[openTable](../interfaces/Connection.md#opentable)
-
-#### Defined in
-
-[index.ts:376](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L376)
-
-▸ **openTable**\<`T`\>(`name`, `embeddings`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
-
-Open a table in the database.
+Creates a new Table and initialize it with new data.
 
 #### Type parameters
 
@@ -227,21 +151,23 @@ Open a table in the database.
 | Name | Type | Description |
 | :------ | :------ | :------ |
 | `name` | `string` | The name of the table. |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> | An embedding function to use on this Table |
+| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
+| `mode` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
 
 #### Returns
 
-`Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
+`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
 
 #### Implementation of
 
-Connection.openTable
+Connection.createTable
 
 #### Defined in
 
-[index.ts:384](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L384)
+[index.ts:241](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L241)
 
-▸ **openTable**\<`T`\>(`name`, `embeddings?`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
+▸ **createTable**<`T`\>(`name`, `data`, `mode`, `embeddings?`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
 
 #### Type parameters
 
@@ -254,11 +180,119 @@ Connection.openTable
 | Name | Type |
 | :------ | :------ |
 | `name` | `string` |
-| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> |
+| `data` | `Record`<`string`, `unknown`\>[] |
+| `mode` | [`WriteMode`](../enums/WriteMode.md) |
+| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
 
 #### Returns
 
-`Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
+`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Implementation of
+
+Connection.createTable
+
+#### Defined in
+
+[index.ts:242](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L242)
+
+___
+
+### createTableArrow
+
+▸ **createTableArrow**(`name`, `table`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `table` | `Table`<`any`\> |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[createTableArrow](../interfaces/Connection.md#createtablearrow)
+
+#### Defined in
+
+[index.ts:266](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L266)
+
+___
+
+### dropTable
+
+▸ **dropTable**(`name`): `Promise`<`void`\>
+
+Drop an existing table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table to drop. |
+
+#### Returns
+
+`Promise`<`void`\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[dropTable](../interfaces/Connection.md#droptable)
+
+#### Defined in
+
+[index.ts:276](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L276)
+
+___
+
+### openTable
+
+▸ **openTable**(`name`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+Open a table in the database.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[openTable](../interfaces/Connection.md#opentable)
+
+#### Defined in
+
+[index.ts:205](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L205)
+
+▸ **openTable**<`T`\>(`name`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+Open a table in the database.
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
 
 #### Implementation of
 
@@ -266,19 +300,46 @@ Connection.openTable
 
 #### Defined in
 
-[index.ts:385](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L385)
+[index.ts:212](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L212)
+
+▸ **openTable**<`T`\>(`name`, `embeddings?`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Implementation of
+
+Connection.openTable
+
+#### Defined in
+
+[index.ts:213](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L213)
 
 ___
 
 ### tableNames
 
-▸ **tableNames**(): `Promise`\<`string`[]\>
+▸ **tableNames**(): `Promise`<`string`[]\>
 
 Get the names of all tables in the database.
 
 #### Returns
 
-`Promise`\<`string`[]\>
+`Promise`<`string`[]\>
 
 #### Implementation of
 
@@ -286,4 +347,4 @@ Get the names of all tables in the database.
 
 #### Defined in
 
-[index.ts:367](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L367)
+[index.ts:196](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L196)

docs/src/javascript/classes/LocalTable.md

@@ -1,6 +1,6 @@
 [vectordb](../README.md) / [Exports](../modules.md) / LocalTable
 
-# Class: LocalTable\<T\>
+# Class: LocalTable<T\>
 
 A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
 
@@ -12,7 +12,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 ## Implements
 
-- [`Table`](../interfaces/Table.md)\<`T`\>
+- [`Table`](../interfaces/Table.md)<`T`\>
 
 ## Table of contents
 
@@ -26,7 +26,6 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 - [\_name](LocalTable.md#_name)
 - [\_options](LocalTable.md#_options)
 - [\_tbl](LocalTable.md#_tbl)
-- [where](LocalTable.md#where)
 
 ### Accessors
 
@@ -35,23 +34,17 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 ### Methods
 
 - [add](LocalTable.md#add)
-- [cleanupOldVersions](LocalTable.md#cleanupoldversions)
-- [compactFiles](LocalTable.md#compactfiles)
 - [countRows](LocalTable.md#countrows)
 - [createIndex](LocalTable.md#createindex)
 - [delete](LocalTable.md#delete)
-- [filter](LocalTable.md#filter)
-- [indexStats](LocalTable.md#indexstats)
-- [listIndices](LocalTable.md#listindices)
 - [overwrite](LocalTable.md#overwrite)
 - [search](LocalTable.md#search)
-- [update](LocalTable.md#update)
 
 ## Constructors
 
 ### constructor
 
-• **new LocalTable**\<`T`\>(`tbl`, `name`, `options`)
+• **new LocalTable**<`T`\>(`tbl`, `name`, `options`)
 
 #### Type parameters
 
@@ -69,9 +62,9 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 #### Defined in
 
-[index.ts:464](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L464)
+[index.ts:287](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L287)
 
-• **new LocalTable**\<`T`\>(`tbl`, `name`, `options`, `embeddings`)
+• **new LocalTable**<`T`\>(`tbl`, `name`, `options`, `embeddings`)
 
 #### Type parameters
 
@@ -86,21 +79,21 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 | `tbl` | `any` |  |
 | `name` | `string` |  |
 | `options` | [`ConnectionOptions`](../interfaces/ConnectionOptions.md) |  |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> | An embedding function to use when interacting with this table |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use when interacting with this table |
 
 #### Defined in
 
-[index.ts:471](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L471)
+[index.ts:294](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L294)
 
 ## Properties
 
 ### \_embeddings
 
-• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\>
+• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
 
 #### Defined in
 
-[index.ts:461](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L461)
+[index.ts:284](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L284)
 
 ___
 
@@ -110,61 +103,27 @@ ___
 
 #### Defined in
 
-[index.ts:460](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L460)
+[index.ts:283](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L283)
 
 ___
 
 ### \_options
 
-• `Private` `Readonly` **\_options**: () => [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
-
-#### Type declaration
-
-▸ (): [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
-
-##### Returns
-
-[`ConnectionOptions`](../interfaces/ConnectionOptions.md)
+• `Private` `Readonly` **\_options**: [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
 
 #### Defined in
 
-[index.ts:462](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L462)
+[index.ts:285](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L285)
 
 ___
 
 ### \_tbl
 
-• `Private` **\_tbl**: `any`
+• `Private` `Readonly` **\_tbl**: `any`
 
 #### Defined in
 
-[index.ts:459](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L459)
-
-___
-
-### where
-
-• **where**: (`value`: `string`) => [`Query`](Query.md)\<`T`\>
-
-#### Type declaration
-
-▸ (`value`): [`Query`](Query.md)\<`T`\>
-
-Creates a filter query to find all rows matching the specified criteria
-
-##### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | `string` | The filter criteria (like SQL where clause syntax) |
-
-##### Returns
-
-[`Query`](Query.md)\<`T`\>
-
-#### Defined in
-
-[index.ts:499](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L499)
+[index.ts:282](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L282)
 
 ## Accessors
 
@@ -182,13 +141,13 @@ Creates a filter query to find all rows matching the specified criteria
 
 #### Defined in
 
-[index.ts:479](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L479)
+[index.ts:302](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L302)
 
 ## Methods
 
 ### add
 
-▸ **add**(`data`): `Promise`\<`number`\>
+▸ **add**(`data`): `Promise`<`number`\>
 
 Insert records into this Table.
 
@@ -196,11 +155,11 @@ Insert records into this Table.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
 
 #### Returns
 
-`Promise`\<`number`\>
+`Promise`<`number`\>
 
 The number of rows added to the table
 
@@ -210,69 +169,19 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:507](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L507)
-
-___
-
-### cleanupOldVersions
-
-▸ **cleanupOldVersions**(`olderThan?`, `deleteUnverified?`): `Promise`\<[`CleanupStats`](../interfaces/CleanupStats.md)\>
-
-Clean up old versions of the table, freeing disk space.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `olderThan?` | `number` | The minimum age in minutes of the versions to delete. If not provided, defaults to two weeks. |
-| `deleteUnverified?` | `boolean` | Because they may be part of an in-progress transaction, uncommitted files newer than 7 days old are not deleted by default. This means that failed transactions can leave around data that takes up disk space for up to 7 days. You can override this safety mechanism by setting this option to `true`, only if you promise there are no in progress writes while you run this operation. Failure to uphold this promise can lead to corrupted tables. |
-
-#### Returns
-
-`Promise`\<[`CleanupStats`](../interfaces/CleanupStats.md)\>
-
-#### Defined in
-
-[index.ts:596](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L596)
-
-___
-
-### compactFiles
-
-▸ **compactFiles**(`options?`): `Promise`\<[`CompactionMetrics`](../interfaces/CompactionMetrics.md)\>
-
-Run the compaction process on the table.
-
-This can be run after making several small appends to optimize the table
-for faster reads.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `options?` | [`CompactionOptions`](../interfaces/CompactionOptions.md) | Advanced options configuring compaction. In most cases, you can omit this arguments, as the default options are sensible for most tables. |
-
-#### Returns
-
-`Promise`\<[`CompactionMetrics`](../interfaces/CompactionMetrics.md)\>
-
-Metrics about the compaction operation.
-
-#### Defined in
-
-[index.ts:615](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L615)
+[index.ts:320](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L320)
 
 ___
 
 ### countRows
 
-▸ **countRows**(): `Promise`\<`number`\>
+▸ **countRows**(): `Promise`<`number`\>
 
 Returns the number of rows in this table.
 
 #### Returns
 
-`Promise`\<`number`\>
+`Promise`<`number`\>
 
 #### Implementation of
 
@@ -280,16 +189,20 @@ Returns the number of rows in this table.
 
 #### Defined in
 
-[index.ts:543](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L543)
+[index.ts:362](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L362)
 
 ___
 
 ### createIndex
 
-▸ **createIndex**(`indexParams`): `Promise`\<`any`\>
+▸ **createIndex**(`indexParams`): `Promise`<`any`\>
 
 Create an ANN index on this Table vector index.
 
+**`See`**
+
+VectorIndexParams.
+
 #### Parameters
 
 | Name | Type | Description |
@@ -298,11 +211,7 @@ Create an ANN index on this Table vector index.
 
 #### Returns
 
-`Promise`\<`any`\>
-
-**`See`**
-
-VectorIndexParams.
+`Promise`<`any`\>
 
 #### Implementation of
 
@@ -310,13 +219,13 @@ VectorIndexParams.
 
 #### Defined in
 
-[index.ts:536](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L536)
+[index.ts:355](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L355)
 
 ___
 
 ### delete
 
-▸ **delete**(`filter`): `Promise`\<`void`\>
+▸ **delete**(`filter`): `Promise`<`void`\>
 
 Delete rows from this table.
 
@@ -328,7 +237,7 @@ Delete rows from this table.
 
 #### Returns
 
-`Promise`\<`void`\>
+`Promise`<`void`\>
 
 #### Implementation of
 
@@ -336,81 +245,13 @@ Delete rows from this table.
 
 #### Defined in
 
-[index.ts:552](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L552)
-
-___
-
-### filter
-
-▸ **filter**(`value`): [`Query`](Query.md)\<`T`\>
-
-Creates a filter query to find all rows matching the specified criteria
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `value` | `string` | The filter criteria (like SQL where clause syntax) |
-
-#### Returns
-
-[`Query`](Query.md)\<`T`\>
-
-#### Defined in
-
-[index.ts:495](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L495)
-
-___
-
-### indexStats
-
-▸ **indexStats**(`indexUuid`): `Promise`\<[`IndexStats`](../interfaces/IndexStats.md)\>
-
-Get statistics about an index.
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `indexUuid` | `string` |
-
-#### Returns
-
-`Promise`\<[`IndexStats`](../interfaces/IndexStats.md)\>
-
-#### Implementation of
-
-[Table](../interfaces/Table.md).[indexStats](../interfaces/Table.md#indexstats)
-
-#### Defined in
-
-[index.ts:628](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L628)
-
-___
-
-### listIndices
-
-▸ **listIndices**(): `Promise`\<[`VectorIndex`](../interfaces/VectorIndex.md)[]\>
-
-List the indicies on this table.
-
-#### Returns
-
-`Promise`\<[`VectorIndex`](../interfaces/VectorIndex.md)[]\>
-
-#### Implementation of
-
-[Table](../interfaces/Table.md).[listIndices](../interfaces/Table.md#listindices)
-
-#### Defined in
-
-[index.ts:624](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L624)
+[index.ts:371](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L371)
 
 ___
 
 ### overwrite
 
-▸ **overwrite**(`data`): `Promise`\<`number`\>
+▸ **overwrite**(`data`): `Promise`<`number`\>
 
 Insert records into this Table, replacing its contents.
 
@@ -418,11 +259,11 @@ Insert records into this Table, replacing its contents.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
 
 #### Returns
 
-`Promise`\<`number`\>
+`Promise`<`number`\>
 
 The number of rows added to the table
 
@@ -432,13 +273,13 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:522](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L522)
+[index.ts:338](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L338)
 
 ___
 
 ### search
 
-▸ **search**(`query`): [`Query`](Query.md)\<`T`\>
+▸ **search**(`query`): [`Query`](Query.md)<`T`\>
 
 Creates a search query to find the nearest neighbors of the given search term
 
@@ -450,7 +291,7 @@ Creates a search query to find the nearest neighbors of the given search term
 
 #### Returns
 
-[`Query`](Query.md)\<`T`\>
+[`Query`](Query.md)<`T`\>
 
 #### Implementation of
 
@@ -458,30 +299,4 @@ Creates a search query to find the nearest neighbors of the given search term
 
 #### Defined in
 
-[index.ts:487](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L487)
-
-___
-
-### update
-
-▸ **update**(`args`): `Promise`\<`void`\>
-
-Update rows in this table.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `args` | [`UpdateArgs`](../interfaces/UpdateArgs.md) \| [`UpdateSqlArgs`](../interfaces/UpdateSqlArgs.md) | see [UpdateArgs](../interfaces/UpdateArgs.md) and [UpdateSqlArgs](../interfaces/UpdateSqlArgs.md) for more details |
-
-#### Returns
-
-`Promise`\<`void`\>
-
-#### Implementation of
-
-[Table](../interfaces/Table.md).[update](../interfaces/Table.md#update)
-
-#### Defined in
-
-[index.ts:563](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L563)
+[index.ts:310](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L310)

docs/src/javascript/classes/OpenAIEmbeddingFunction.md

@@ -6,7 +6,7 @@ An embedding function that automatically creates vector representation for a giv
 
 ## Implements
 
-- [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`string`\>
+- [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`string`\>
 
 ## Table of contents
 
@@ -40,7 +40,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L21)
+[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L21)
 
 ## Properties
 
@@ -50,7 +50,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L19)
+[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L19)
 
 ___
 
@@ -60,7 +60,7 @@ ___
 
 #### Defined in
 
-[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L18)
+[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L18)
 
 ___
 
@@ -76,13 +76,13 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L50)
+[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L50)
 
 ## Methods
 
 ### embed
 
-▸ **embed**(`data`): `Promise`\<`number`[][]\>
+▸ **embed**(`data`): `Promise`<`number`[][]\>
 
 Creates a vector representation for the given values.
 
@@ -94,7 +94,7 @@ Creates a vector representation for the given values.
 
 #### Returns
 
-`Promise`\<`number`[][]\>
+`Promise`<`number`[][]\>
 
 #### Implementation of
 
@@ -102,4 +102,4 @@ Creates a vector representation for the given values.
 
 #### Defined in
 
-[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L38)
+[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L38)

docs/src/javascript/classes/Query.md

@@ -1,6 +1,6 @@
 [vectordb](../README.md) / [Exports](../modules.md) / Query
 
-# Class: Query\<T\>
+# Class: Query<T\>
 
 A builder for nearest neighbor queries for LanceDB.
 
@@ -23,7 +23,6 @@ A builder for nearest neighbor queries for LanceDB.
 - [\_limit](Query.md#_limit)
 - [\_metricType](Query.md#_metrictype)
 - [\_nprobes](Query.md#_nprobes)
-- [\_prefilter](Query.md#_prefilter)
 - [\_query](Query.md#_query)
 - [\_queryVector](Query.md#_queryvector)
 - [\_refineFactor](Query.md#_refinefactor)
@@ -35,11 +34,9 @@ A builder for nearest neighbor queries for LanceDB.
 
 - [execute](Query.md#execute)
 - [filter](Query.md#filter)
-- [isElectron](Query.md#iselectron)
 - [limit](Query.md#limit)
 - [metricType](Query.md#metrictype)
 - [nprobes](Query.md#nprobes)
-- [prefilter](Query.md#prefilter)
 - [refineFactor](Query.md#refinefactor)
 - [select](Query.md#select)
 
@@ -47,7 +44,7 @@ A builder for nearest neighbor queries for LanceDB.
 
 ### constructor
 
-• **new Query**\<`T`\>(`query?`, `tbl?`, `embeddings?`)
+• **new Query**<`T`\>(`tbl`, `query`, `embeddings?`)
 
 #### Type parameters
 
@@ -59,23 +56,23 @@ A builder for nearest neighbor queries for LanceDB.
 
 | Name | Type |
 | :------ | :------ |
-| `query?` | `T` |
-| `tbl?` | `any` |
-| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> |
+| `tbl` | `any` |
+| `query` | `T` |
+| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
 
 #### Defined in
 
-[query.ts:38](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L38)
+[index.ts:448](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L448)
 
 ## Properties
 
 ### \_embeddings
 
-• `Protected` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\>
+• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
 
 #### Defined in
 
-[query.ts:36](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L36)
+[index.ts:446](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L446)
 
 ___
 
@@ -85,17 +82,17 @@ ___
 
 #### Defined in
 
-[query.ts:33](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L33)
+[index.ts:444](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L444)
 
 ___
 
 ### \_limit
 
-• `Private` `Optional` **\_limit**: `number`
+• `Private` **\_limit**: `number`
 
 #### Defined in
 
-[query.ts:29](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L29)
+[index.ts:440](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L440)
 
 ___
 
@@ -105,7 +102,7 @@ ___
 
 #### Defined in
 
-[query.ts:34](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L34)
+[index.ts:445](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L445)
 
 ___
 
@@ -115,27 +112,17 @@ ___
 
 #### Defined in
 
-[query.ts:31](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L31)
-
-___
-
-### \_prefilter
-
-• `Private` **\_prefilter**: `boolean`
-
-#### Defined in
-
-[query.ts:35](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L35)
+[index.ts:442](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L442)
 
 ___
 
 ### \_query
 
-• `Private` `Optional` `Readonly` **\_query**: `T`
+• `Private` `Readonly` **\_query**: `T`
 
 #### Defined in
 
-[query.ts:26](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L26)
+[index.ts:438](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L438)
 
 ___
 
@@ -145,7 +132,7 @@ ___
 
 #### Defined in
 
-[query.ts:28](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L28)
+[index.ts:439](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L439)
 
 ___
 
@@ -155,7 +142,7 @@ ___
 
 #### Defined in
 
-[query.ts:30](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L30)
+[index.ts:441](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L441)
 
 ___
 
@@ -165,27 +152,27 @@ ___
 
 #### Defined in
 
-[query.ts:32](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L32)
+[index.ts:443](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L443)
 
 ___
 
 ### \_tbl
 
-• `Private` `Optional` `Readonly` **\_tbl**: `any`
+• `Private` `Readonly` **\_tbl**: `any`
 
 #### Defined in
 
-[query.ts:27](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L27)
+[index.ts:437](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L437)
 
 ___
 
 ### where
 
-• **where**: (`value`: `string`) => [`Query`](Query.md)\<`T`\>
+• **where**: (`value`: `string`) => [`Query`](Query.md)<`T`\>
 
 #### Type declaration
 
-▸ (`value`): [`Query`](Query.md)\<`T`\>
+▸ (`value`): [`Query`](Query.md)<`T`\>
 
 A filter statement to be applied to this query.
 
@@ -197,17 +184,17 @@ A filter statement to be applied to this query.
 
 ##### Returns
 
-[`Query`](Query.md)\<`T`\>
+[`Query`](Query.md)<`T`\>
 
 #### Defined in
 
-[query.ts:87](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L87)
+[index.ts:496](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L496)
 
 ## Methods
 
 ### execute
 
-▸ **execute**\<`T`\>(): `Promise`\<`T`[]\>
+▸ **execute**<`T`\>(): `Promise`<`T`[]\>
 
 Execute the query and return the results as an Array of Objects
 
@@ -215,21 +202,21 @@ Execute the query and return the results as an Array of Objects
 
 | Name | Type |
 | :------ | :------ |
-| `T` | `Record`\<`string`, `unknown`\> |
+| `T` | `Record`<`string`, `unknown`\> |
 
 #### Returns
 
-`Promise`\<`T`[]\>
+`Promise`<`T`[]\>
 
 #### Defined in
 
-[query.ts:115](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L115)
+[index.ts:519](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L519)
 
 ___
 
 ### filter
 
-▸ **filter**(`value`): [`Query`](Query.md)\<`T`\>
+▸ **filter**(`value`): [`Query`](Query.md)<`T`\>
 
 A filter statement to be applied to this query.
 
@@ -241,31 +228,17 @@ A filter statement to be applied to this query.
 
 #### Returns
 
-[`Query`](Query.md)\<`T`\>
+[`Query`](Query.md)<`T`\>
 
 #### Defined in
 
-[query.ts:82](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L82)
-
-___
-
-### isElectron
-
-▸ `Private` **isElectron**(): `boolean`
-
-#### Returns
-
-`boolean`
-
-#### Defined in
-
-[query.ts:142](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L142)
+[index.ts:491](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L491)
 
 ___
 
 ### limit
 
-▸ **limit**(`value`): [`Query`](Query.md)\<`T`\>
+▸ **limit**(`value`): [`Query`](Query.md)<`T`\>
 
 Sets the number of results that will be returned
 
@@ -277,20 +250,24 @@ Sets the number of results that will be returned
 
 #### Returns
 
-[`Query`](Query.md)\<`T`\>
+[`Query`](Query.md)<`T`\>
 
 #### Defined in
 
-[query.ts:55](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L55)
+[index.ts:464](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L464)
 
 ___
 
 ### metricType
 
-▸ **metricType**(`value`): [`Query`](Query.md)\<`T`\>
+▸ **metricType**(`value`): [`Query`](Query.md)<`T`\>
 
 The MetricType used for this Query.
 
+**`See`**
+
+MetricType for the different options
+
 #### Parameters
 
 | Name | Type | Description |
@@ -299,21 +276,17 @@ The MetricType used for this Query.
 
 #### Returns
 
-[`Query`](Query.md)\<`T`\>
-
-**`See`**
-
-MetricType for the different options
+[`Query`](Query.md)<`T`\>
 
 #### Defined in
 
-[query.ts:102](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L102)
+[index.ts:511](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L511)
 
 ___
 
 ### nprobes
 
-▸ **nprobes**(`value`): [`Query`](Query.md)\<`T`\>
+▸ **nprobes**(`value`): [`Query`](Query.md)<`T`\>
 
 The number of probes used. A higher number makes search more accurate but also slower.
 
@@ -325,37 +298,17 @@ The number of probes used. A higher number makes search more accurate but also s
 
 #### Returns
 
-[`Query`](Query.md)\<`T`\>
+[`Query`](Query.md)<`T`\>
 
 #### Defined in
 
-[query.ts:73](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L73)
-
-___
-
-### prefilter
-
-▸ **prefilter**(`value`): [`Query`](Query.md)\<`T`\>
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `value` | `boolean` |
-
-#### Returns
-
-[`Query`](Query.md)\<`T`\>
-
-#### Defined in
-
-[query.ts:107](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L107)
+[index.ts:482](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L482)
 
 ___
 
 ### refineFactor
 
-▸ **refineFactor**(`value`): [`Query`](Query.md)\<`T`\>
+▸ **refineFactor**(`value`): [`Query`](Query.md)<`T`\>
 
 Refine the results by reading extra elements and re-ranking them in memory.
 
@@ -367,17 +320,17 @@ Refine the results by reading extra elements and re-ranking them in memory.
 
 #### Returns
 
-[`Query`](Query.md)\<`T`\>
+[`Query`](Query.md)<`T`\>
 
 #### Defined in
 
-[query.ts:64](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L64)
+[index.ts:473](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L473)
 
 ___
 
 ### select
 
-▸ **select**(`value`): [`Query`](Query.md)\<`T`\>
+▸ **select**(`value`): [`Query`](Query.md)<`T`\>
 
 Return only the specified columns.
 
@@ -389,8 +342,8 @@ Return only the specified columns.
 
 #### Returns
 
-[`Query`](Query.md)\<`T`\>
+[`Query`](Query.md)<`T`\>
 
 #### Defined in
 
-[query.ts:93](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L93)
+[index.ts:502](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L502)

docs/src/javascript/enums/MetricType.md

@@ -22,7 +22,7 @@ Cosine distance
 
 #### Defined in
 
-[index.ts:798](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L798)
+[index.ts:567](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L567)
 
 ___
 
@@ -34,7 +34,7 @@ Dot product
 
 #### Defined in
 
-[index.ts:803](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L803)
+[index.ts:572](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L572)
 
 ___
 
@@ -46,4 +46,4 @@ Euclidean distance
 
 #### Defined in
 
-[index.ts:793](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L793)
+[index.ts:562](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L562)

docs/src/javascript/enums/WriteMode.md

@@ -22,7 +22,7 @@ Append new data to the table.
 
 #### Defined in
 
-[index.ts:766](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L766)
+[index.ts:552](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L552)
 
 ___
 
@@ -34,7 +34,7 @@ Create a new [Table](../interfaces/Table.md).
 
 #### Defined in
 
-[index.ts:762](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L762)
+[index.ts:548](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L548)
 
 ___
 
@@ -46,4 +46,4 @@ Overwrite the existing [Table](../interfaces/Table.md) if presented.
 
 #### Defined in
 
-[index.ts:764](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L764)
+[index.ts:550](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L550)

docs/src/javascript/interfaces/AwsCredentials.md

@@ -18,7 +18,7 @@
 
 #### Defined in
 
-[index.ts:34](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L34)
+[index.ts:31](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L31)
 
 ___
 
@@ -28,7 +28,7 @@ ___
 
 #### Defined in
 
-[index.ts:36](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L36)
+[index.ts:33](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L33)
 
 ___
 
@@ -38,4 +38,4 @@ ___
 
 #### Defined in
 
-[index.ts:38](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L38)
+[index.ts:35](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L35)

docs/src/javascript/interfaces/CleanupStats.md

@@ -1,34 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / CleanupStats
-
-# Interface: CleanupStats
-
-## Table of contents
-
-### Properties
-
-- [bytesRemoved](CleanupStats.md#bytesremoved)
-- [oldVersions](CleanupStats.md#oldversions)
-
-## Properties
-
-### bytesRemoved
-
-• **bytesRemoved**: `number`
-
-The number of bytes removed from disk.
-
-#### Defined in
-
-[index.ts:637](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L637)
-
-___
-
-### oldVersions
-
-• **oldVersions**: `number`
-
-The number of old table versions removed.
-
-#### Defined in
-
-[index.ts:641](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L641)

docs/src/javascript/interfaces/CompactionMetrics.md

@@ -1,62 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / CompactionMetrics
-
-# Interface: CompactionMetrics
-
-## Table of contents
-
-### Properties
-
-- [filesAdded](CompactionMetrics.md#filesadded)
-- [filesRemoved](CompactionMetrics.md#filesremoved)
-- [fragmentsAdded](CompactionMetrics.md#fragmentsadded)
-- [fragmentsRemoved](CompactionMetrics.md#fragmentsremoved)
-
-## Properties
-
-### filesAdded
-
-• **filesAdded**: `number`
-
-The number of files added. This is typically equal to the number of
-fragments added.
-
-#### Defined in
-
-[index.ts:692](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L692)
-
-___
-
-### filesRemoved
-
-• **filesRemoved**: `number`
-
-The number of files that were removed. Each fragment may have more than one
-file.
-
-#### Defined in
-
-[index.ts:687](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L687)
-
-___
-
-### fragmentsAdded
-
-• **fragmentsAdded**: `number`
-
-The number of new fragments that were created.
-
-#### Defined in
-
-[index.ts:682](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L682)
-
-___
-
-### fragmentsRemoved
-
-• **fragmentsRemoved**: `number`
-
-The number of fragments that were removed.
-
-#### Defined in
-
-[index.ts:678](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L678)

docs/src/javascript/interfaces/CompactionOptions.md

@@ -1,80 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / CompactionOptions
-
-# Interface: CompactionOptions
-
-## Table of contents
-
-### Properties
-
-- [materializeDeletions](CompactionOptions.md#materializedeletions)
-- [materializeDeletionsThreshold](CompactionOptions.md#materializedeletionsthreshold)
-- [maxRowsPerGroup](CompactionOptions.md#maxrowspergroup)
-- [numThreads](CompactionOptions.md#numthreads)
-- [targetRowsPerFragment](CompactionOptions.md#targetrowsperfragment)
-
-## Properties
-
-### materializeDeletions
-
-• `Optional` **materializeDeletions**: `boolean`
-
-If true, fragments that have rows that are deleted may be compacted to
-remove the deleted rows. This can improve the performance of queries.
-Default is true.
-
-#### Defined in
-
-[index.ts:660](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L660)
-
-___
-
-### materializeDeletionsThreshold
-
-• `Optional` **materializeDeletionsThreshold**: `number`
-
-A number between 0 and 1, representing the proportion of rows that must be
-marked deleted before a fragment is a candidate for compaction to remove
-the deleted rows. Default is 10%.
-
-#### Defined in
-
-[index.ts:666](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L666)
-
-___
-
-### maxRowsPerGroup
-
-• `Optional` **maxRowsPerGroup**: `number`
-
-The maximum number of rows per group. Defaults to 1024.
-
-#### Defined in
-
-[index.ts:654](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L654)
-
-___
-
-### numThreads
-
-• `Optional` **numThreads**: `number`
-
-The number of threads to use for compaction. If not provided, defaults to
-the number of cores on the machine.
-
-#### Defined in
-
-[index.ts:671](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L671)
-
-___
-
-### targetRowsPerFragment
-
-• `Optional` **targetRowsPerFragment**: `number`
-
-The number of rows per fragment to target. Fragments that have fewer rows
-will be compacted into adjacent fragments to produce larger fragments.
-Defaults to 1024 * 1024.
-
-#### Defined in
-
-[index.ts:650](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L650)

docs/src/javascript/interfaces/Connection.md

@@ -19,6 +19,7 @@ Connection could be local against filesystem or remote against a server.
 ### Methods
 
 - [createTable](Connection.md#createtable)
+- [createTableArrow](Connection.md#createtablearrow)
 - [dropTable](Connection.md#droptable)
 - [openTable](Connection.md#opentable)
 - [tableNames](Connection.md#tablenames)
@@ -31,15 +32,15 @@ Connection could be local against filesystem or remote against a server.
 
 #### Defined in
 
-[index.ts:125](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L125)
+[index.ts:70](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L70)
 
 ## Methods
 
 ### createTable
 
-▸ **createTable**\<`T`\>(`«destructured»`): `Promise`\<[`Table`](Table.md)\<`T`\>\>
+▸ **createTable**<`T`\>(`name`, `data`, `mode?`, `embeddings?`): `Promise`<[`Table`](Table.md)<`T`\>\>
 
-Creates a new Table, optionally initializing it with new data.
+Creates a new Table and initialize it with new data.
 
 #### Type parameters
 
@@ -49,115 +50,47 @@ Creates a new Table, optionally initializing it with new data.
 
 #### Parameters
 
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
+| `mode?` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
+| `embeddings?` | [`EmbeddingFunction`](EmbeddingFunction.md)<`T`\> | An embedding function to use on this table |
+
+#### Returns
+
+`Promise`<[`Table`](Table.md)<`T`\>\>
+
+#### Defined in
+
+[index.ts:90](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L90)
+
+___
+
+### createTableArrow
+
+▸ **createTableArrow**(`name`, `table`): `Promise`<[`Table`](Table.md)<`number`[]\>\>
+
+#### Parameters
+
 | Name | Type |
 | :------ | :------ |
-| `«destructured»` | [`CreateTableOptions`](CreateTableOptions.md)\<`T`\> |
+| `name` | `string` |
+| `table` | `Table`<`any`\> |
 
 #### Returns
 
-`Promise`\<[`Table`](Table.md)\<`T`\>\>
+`Promise`<[`Table`](Table.md)<`number`[]\>\>
 
 #### Defined in
 
-[index.ts:146](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L146)
-
-▸ **createTable**(`name`, `data`): `Promise`\<[`Table`](Table.md)\<`number`[]\>\>
-
-Creates a new Table and initialize it with new data.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
-
-#### Returns
-
-`Promise`\<[`Table`](Table.md)\<`number`[]\>\>
-
-#### Defined in
-
-[index.ts:154](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L154)
-
-▸ **createTable**(`name`, `data`, `options`): `Promise`\<[`Table`](Table.md)\<`number`[]\>\>
-
-Creates a new Table and initialize it with new data.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
-| `options` | [`WriteOptions`](WriteOptions.md) | The write options to use when creating the table. |
-
-#### Returns
-
-`Promise`\<[`Table`](Table.md)\<`number`[]\>\>
-
-#### Defined in
-
-[index.ts:163](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L163)
-
-▸ **createTable**\<`T`\>(`name`, `data`, `embeddings`): `Promise`\<[`Table`](Table.md)\<`T`\>\>
-
-Creates a new Table and initialize it with new data.
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
-| `embeddings` | [`EmbeddingFunction`](EmbeddingFunction.md)\<`T`\> | An embedding function to use on this table |
-
-#### Returns
-
-`Promise`\<[`Table`](Table.md)\<`T`\>\>
-
-#### Defined in
-
-[index.ts:172](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L172)
-
-▸ **createTable**\<`T`\>(`name`, `data`, `embeddings`, `options`): `Promise`\<[`Table`](Table.md)\<`T`\>\>
-
-Creates a new Table and initialize it with new data.
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
-| `embeddings` | [`EmbeddingFunction`](EmbeddingFunction.md)\<`T`\> | An embedding function to use on this table |
-| `options` | [`WriteOptions`](WriteOptions.md) | The write options to use when creating the table. |
-
-#### Returns
-
-`Promise`\<[`Table`](Table.md)\<`T`\>\>
-
-#### Defined in
-
-[index.ts:181](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L181)
+[index.ts:92](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L92)
 
 ___
 
 ### dropTable
 
-▸ **dropTable**(`name`): `Promise`\<`void`\>
+▸ **dropTable**(`name`): `Promise`<`void`\>
 
 Drop an existing table.
 
@@ -169,17 +102,17 @@ Drop an existing table.
 
 #### Returns
 
-`Promise`\<`void`\>
+`Promise`<`void`\>
 
 #### Defined in
 
-[index.ts:187](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L187)
+[index.ts:98](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L98)
 
 ___
 
 ### openTable
 
-▸ **openTable**\<`T`\>(`name`, `embeddings?`): `Promise`\<[`Table`](Table.md)\<`T`\>\>
+▸ **openTable**<`T`\>(`name`, `embeddings?`): `Promise`<[`Table`](Table.md)<`T`\>\>
 
 Open a table in the database.
 
@@ -194,26 +127,26 @@ Open a table in the database.
 | Name | Type | Description |
 | :------ | :------ | :------ |
 | `name` | `string` | The name of the table. |
-| `embeddings?` | [`EmbeddingFunction`](EmbeddingFunction.md)\<`T`\> | An embedding function to use on this table |
+| `embeddings?` | [`EmbeddingFunction`](EmbeddingFunction.md)<`T`\> | An embedding function to use on this table |
 
 #### Returns
 
-`Promise`\<[`Table`](Table.md)\<`T`\>\>
+`Promise`<[`Table`](Table.md)<`T`\>\>
 
 #### Defined in
 
-[index.ts:135](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L135)
+[index.ts:80](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L80)
 
 ___
 
 ### tableNames
 
-▸ **tableNames**(): `Promise`\<`string`[]\>
+▸ **tableNames**(): `Promise`<`string`[]\>
 
 #### Returns
 
-`Promise`\<`string`[]\>
+`Promise`<`string`[]\>
 
 #### Defined in
 
-[index.ts:127](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L127)
+[index.ts:72](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L72)

docs/src/javascript/interfaces/ConnectionOptions.md

@@ -6,62 +6,18 @@
 
 ### Properties
 
-- [apiKey](ConnectionOptions.md#apikey)
 - [awsCredentials](ConnectionOptions.md#awscredentials)
-- [awsRegion](ConnectionOptions.md#awsregion)
-- [hostOverride](ConnectionOptions.md#hostoverride)
-- [region](ConnectionOptions.md#region)
 - [uri](ConnectionOptions.md#uri)
 
 ## Properties
 
-### apiKey
-
-• `Optional` **apiKey**: `string`
-
-#### Defined in
-
-[index.ts:49](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L49)
-
-___
-
 ### awsCredentials
 
 • `Optional` **awsCredentials**: [`AwsCredentials`](AwsCredentials.md)
 
 #### Defined in
 
-[index.ts:44](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L44)
-
-___
-
-### awsRegion
-
-• `Optional` **awsRegion**: `string`
-
-#### Defined in
-
-[index.ts:46](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L46)
-
-___
-
-### hostOverride
-
-• `Optional` **hostOverride**: `string`
-
-#### Defined in
-
-[index.ts:54](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L54)
-
-___
-
-### region
-
-• `Optional` **region**: `string`
-
-#### Defined in
-
-[index.ts:51](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L51)
+[index.ts:40](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L40)
 
 ___
 
@@ -71,4 +27,4 @@ ___
 
 #### Defined in
 
-[index.ts:42](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L42)
+[index.ts:39](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L39)

docs/src/javascript/interfaces/CreateTableOptions.md

@@ -1,69 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / CreateTableOptions
-
-# Interface: CreateTableOptions\<T\>
-
-## Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-## Table of contents
-
-### Properties
-
-- [data](CreateTableOptions.md#data)
-- [embeddingFunction](CreateTableOptions.md#embeddingfunction)
-- [name](CreateTableOptions.md#name)
-- [schema](CreateTableOptions.md#schema)
-- [writeOptions](CreateTableOptions.md#writeoptions)
-
-## Properties
-
-### data
-
-• `Optional` **data**: `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[]
-
-#### Defined in
-
-[index.ts:79](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L79)
-
-___
-
-### embeddingFunction
-
-• `Optional` **embeddingFunction**: [`EmbeddingFunction`](EmbeddingFunction.md)\<`T`\>
-
-#### Defined in
-
-[index.ts:85](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L85)
-
-___
-
-### name
-
-• **name**: `string`
-
-#### Defined in
-
-[index.ts:76](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L76)
-
-___
-
-### schema
-
-• `Optional` **schema**: `Schema`\<`any`\>
-
-#### Defined in
-
-[index.ts:82](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L82)
-
-___
-
-### writeOptions
-
-• `Optional` **writeOptions**: [`WriteOptions`](WriteOptions.md)
-
-#### Defined in
-
-[index.ts:88](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L88)

docs/src/javascript/interfaces/EmbeddingFunction.md

@@ -1,6 +1,6 @@
 [vectordb](../README.md) / [Exports](../modules.md) / EmbeddingFunction
 
-# Interface: EmbeddingFunction\<T\>
+# Interface: EmbeddingFunction<T\>
 
 An embedding function that automatically creates vector representation for a given column.
 
@@ -25,11 +25,11 @@ An embedding function that automatically creates vector representation for a giv
 
 ### embed
 
-• **embed**: (`data`: `T`[]) => `Promise`\<`number`[][]\>
+• **embed**: (`data`: `T`[]) => `Promise`<`number`[][]\>
 
 #### Type declaration
 
-▸ (`data`): `Promise`\<`number`[][]\>
+▸ (`data`): `Promise`<`number`[][]\>
 
 Creates a vector representation for the given values.
 
@@ -41,11 +41,11 @@ Creates a vector representation for the given values.
 
 ##### Returns
 
-`Promise`\<`number`[][]\>
+`Promise`<`number`[][]\>
 
 #### Defined in
 
-[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/embedding_function.ts#L27)
+[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/embedding_function.ts#L27)
 
 ___
 
@@ -57,4 +57,4 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/embedding_function.ts#L22)
+[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/embedding_function.ts#L22)

docs/src/javascript/interfaces/IndexStats.md

@@ -1,30 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / IndexStats
-
-# Interface: IndexStats
-
-## Table of contents
-
-### Properties
-
-- [numIndexedRows](IndexStats.md#numindexedrows)
-- [numUnindexedRows](IndexStats.md#numunindexedrows)
-
-## Properties
-
-### numIndexedRows
-
-• **numIndexedRows**: ``null`` \| `number`
-
-#### Defined in
-
-[index.ts:344](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L344)
-
-___
-
-### numUnindexedRows
-
-• **numUnindexedRows**: ``null`` \| `number`
-
-#### Defined in
-
-[index.ts:345](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L345)

docs/src/javascript/interfaces/IvfPQIndexConfig.md

@@ -7,7 +7,6 @@
 ### Properties
 
 - [column](IvfPQIndexConfig.md#column)
-- [index\_cache\_size](IvfPQIndexConfig.md#index_cache_size)
 - [index\_name](IvfPQIndexConfig.md#index_name)
 - [max\_iters](IvfPQIndexConfig.md#max_iters)
 - [max\_opq\_iters](IvfPQIndexConfig.md#max_opq_iters)
@@ -29,19 +28,7 @@ The column to be indexed
 
 #### Defined in
 
-[index.ts:701](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L701)
-
-___
-
-### index\_cache\_size
-
-• `Optional` **index\_cache\_size**: `number`
-
-Cache size of the index
-
-#### Defined in
-
-[index.ts:750](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L750)
+[index.ts:382](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L382)
 
 ___
 
@@ -53,7 +40,7 @@ A unique name for the index
 
 #### Defined in
 
-[index.ts:706](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L706)
+[index.ts:387](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L387)
 
 ___
 
@@ -65,7 +52,7 @@ The max number of iterations for kmeans training.
 
 #### Defined in
 
-[index.ts:721](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L721)
+[index.ts:402](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L402)
 
 ___
 
@@ -77,7 +64,7 @@ Max number of iterations to train OPQ, if `use_opq` is true.
 
 #### Defined in
 
-[index.ts:740](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L740)
+[index.ts:421](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L421)
 
 ___
 
@@ -89,7 +76,7 @@ Metric type, L2 or Cosine
 
 #### Defined in
 
-[index.ts:711](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L711)
+[index.ts:392](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L392)
 
 ___
 
@@ -101,7 +88,7 @@ The number of bits to present one PQ centroid.
 
 #### Defined in
 
-[index.ts:735](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L735)
+[index.ts:416](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L416)
 
 ___
 
@@ -113,7 +100,7 @@ The number of partitions this index
 
 #### Defined in
 
-[index.ts:716](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L716)
+[index.ts:397](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L397)
 
 ___
 
@@ -125,7 +112,7 @@ Number of subvectors to build PQ code
 
 #### Defined in
 
-[index.ts:731](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L731)
+[index.ts:412](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L412)
 
 ___
 
@@ -137,7 +124,7 @@ Replace an existing index with the same name if it exists.
 
 #### Defined in
 
-[index.ts:745](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L745)
+[index.ts:426](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L426)
 
 ___
 
@@ -147,7 +134,7 @@ ___
 
 #### Defined in
 
-[index.ts:752](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L752)
+[index.ts:428](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L428)
 
 ___
 
@@ -159,4 +146,4 @@ Train as optimized product quantization.
 
 #### Defined in
 
-[index.ts:726](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L726)
+[index.ts:407](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L407)

docs/src/javascript/interfaces/Table.md

@@ -1,6 +1,6 @@
 [vectordb](../README.md) / [Exports](../modules.md) / Table
 
-# Interface: Table\<T\>
+# Interface: Table<T\>
 
 A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
 
@@ -22,22 +22,19 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 - [countRows](Table.md#countrows)
 - [createIndex](Table.md#createindex)
 - [delete](Table.md#delete)
-- [indexStats](Table.md#indexstats)
-- [listIndices](Table.md#listindices)
 - [name](Table.md#name)
 - [overwrite](Table.md#overwrite)
 - [search](Table.md#search)
-- [update](Table.md#update)
 
 ## Properties
 
 ### add
 
-• **add**: (`data`: `Record`\<`string`, `unknown`\>[]) => `Promise`\<`number`\>
+• **add**: (`data`: `Record`<`string`, `unknown`\>[]) => `Promise`<`number`\>
 
 #### Type declaration
 
-▸ (`data`): `Promise`\<`number`\>
+▸ (`data`): `Promise`<`number`\>
 
 Insert records into this Table.
 
@@ -45,50 +42,54 @@ Insert records into this Table.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
 
 ##### Returns
 
-`Promise`\<`number`\>
+`Promise`<`number`\>
 
 The number of rows added to the table
 
 #### Defined in
 
-[index.ts:209](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L209)
+[index.ts:120](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L120)
 
 ___
 
 ### countRows
 
-• **countRows**: () => `Promise`\<`number`\>
+• **countRows**: () => `Promise`<`number`\>
 
 #### Type declaration
 
-▸ (): `Promise`\<`number`\>
+▸ (): `Promise`<`number`\>
 
 Returns the number of rows in this table.
 
 ##### Returns
 
-`Promise`\<`number`\>
+`Promise`<`number`\>
 
 #### Defined in
 
-[index.ts:229](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L229)
+[index.ts:140](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L140)
 
 ___
 
 ### createIndex
 
-• **createIndex**: (`indexParams`: [`IvfPQIndexConfig`](IvfPQIndexConfig.md)) => `Promise`\<`any`\>
+• **createIndex**: (`indexParams`: [`IvfPQIndexConfig`](IvfPQIndexConfig.md)) => `Promise`<`any`\>
 
 #### Type declaration
 
-▸ (`indexParams`): `Promise`\<`any`\>
+▸ (`indexParams`): `Promise`<`any`\>
 
 Create an ANN index on this Table vector index.
 
+**`See`**
+
+VectorIndexParams.
+
 ##### Parameters
 
 | Name | Type | Description |
@@ -97,41 +98,27 @@ Create an ANN index on this Table vector index.
 
 ##### Returns
 
-`Promise`\<`any`\>
-
-**`See`**
-
-VectorIndexParams.
+`Promise`<`any`\>
 
 #### Defined in
 
-[index.ts:224](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L224)
+[index.ts:135](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L135)
 
 ___
 
 ### delete
 
-• **delete**: (`filter`: `string`) => `Promise`\<`void`\>
+• **delete**: (`filter`: `string`) => `Promise`<`void`\>
 
 #### Type declaration
 
-▸ (`filter`): `Promise`\<`void`\>
+▸ (`filter`): `Promise`<`void`\>
 
 Delete rows from this table.
 
 This can be used to delete a single row, many rows, all rows, or
 sometimes no rows (if your predicate matches nothing).
 
-##### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `filter` | `string` | A filter in the same format used by a sql WHERE clause. The filter must not be empty. |
-
-##### Returns
-
-`Promise`\<`void`\>
-
 **`Examples`**
 
 ```ts
@@ -155,55 +142,19 @@ await tbl.delete(`id IN (${to_remove.join(",")})`)
 await tbl.countRows() // Returns 1
 ```
 
-#### Defined in
-
-[index.ts:263](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L263)
-
-___
-
-### indexStats
-
-• **indexStats**: (`indexUuid`: `string`) => `Promise`\<[`IndexStats`](IndexStats.md)\>
-
-#### Type declaration
-
-▸ (`indexUuid`): `Promise`\<[`IndexStats`](IndexStats.md)\>
-
-Get statistics about an index.
-
 ##### Parameters
 
-| Name | Type |
-| :------ | :------ |
-| `indexUuid` | `string` |
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `filter` | `string` | A filter in the same format used by a sql WHERE clause. The filter must not be empty. |
 
 ##### Returns
 
-`Promise`\<[`IndexStats`](IndexStats.md)\>
+`Promise`<`void`\>
 
 #### Defined in
 
-[index.ts:306](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L306)
-
-___
-
-### listIndices
-
-• **listIndices**: () => `Promise`\<[`VectorIndex`](VectorIndex.md)[]\>
-
-#### Type declaration
-
-▸ (): `Promise`\<[`VectorIndex`](VectorIndex.md)[]\>
-
-List the indicies on this table.
-
-##### Returns
-
-`Promise`\<[`VectorIndex`](VectorIndex.md)[]\>
-
-#### Defined in
-
-[index.ts:301](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L301)
+[index.ts:174](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L174)
 
 ___
 
@@ -213,17 +164,17 @@ ___
 
 #### Defined in
 
-[index.ts:195](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L195)
+[index.ts:106](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L106)
 
 ___
 
 ### overwrite
 
-• **overwrite**: (`data`: `Record`\<`string`, `unknown`\>[]) => `Promise`\<`number`\>
+• **overwrite**: (`data`: `Record`<`string`, `unknown`\>[]) => `Promise`<`number`\>
 
 #### Type declaration
 
-▸ (`data`): `Promise`\<`number`\>
+▸ (`data`): `Promise`<`number`\>
 
 Insert records into this Table, replacing its contents.
 
@@ -231,27 +182,27 @@ Insert records into this Table, replacing its contents.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
 
 ##### Returns
 
-`Promise`\<`number`\>
+`Promise`<`number`\>
 
 The number of rows added to the table
 
 #### Defined in
 
-[index.ts:217](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L217)
+[index.ts:128](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L128)
 
 ___
 
 ### search
 
-• **search**: (`query`: `T`) => [`Query`](../classes/Query.md)\<`T`\>
+• **search**: (`query`: `T`) => [`Query`](../classes/Query.md)<`T`\>
 
 #### Type declaration
 
-▸ (`query`): [`Query`](../classes/Query.md)\<`T`\>
+▸ (`query`): [`Query`](../classes/Query.md)<`T`\>
 
 Creates a search query to find the nearest neighbors of the given search term
 
@@ -263,59 +214,8 @@ Creates a search query to find the nearest neighbors of the given search term
 
 ##### Returns
 
-[`Query`](../classes/Query.md)\<`T`\>
+[`Query`](../classes/Query.md)<`T`\>
 
 #### Defined in
 
-[index.ts:201](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L201)
-
-___
-
-### update
-
-• **update**: (`args`: [`UpdateArgs`](UpdateArgs.md) \| [`UpdateSqlArgs`](UpdateSqlArgs.md)) => `Promise`\<`void`\>
-
-#### Type declaration
-
-▸ (`args`): `Promise`\<`void`\>
-
-Update rows in this table.
-
-This can be used to update a single row, many rows, all rows, or
-sometimes no rows (if your predicate matches nothing).
-
-##### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `args` | [`UpdateArgs`](UpdateArgs.md) \| [`UpdateSqlArgs`](UpdateSqlArgs.md) | see [UpdateArgs](UpdateArgs.md) and [UpdateSqlArgs](UpdateSqlArgs.md) for more details |
-
-##### Returns
-
-`Promise`\<`void`\>
-
-**`Examples`**
-
-```ts
-const con = await lancedb.connect("./.lancedb")
-const data = [
-   {id: 1, vector: [3, 3], name: 'Ye'},
-   {id: 2, vector: [4, 4], name: 'Mike'},
-];
-const tbl = await con.createTable("my_table", data)
-
-await tbl.update({
-  filter: "id = 2",
-  updates: { vector: [2, 2], name: "Michael" },
-})
-
-let results = await tbl.search([1, 1]).execute();
-// Returns [
-//   {id: 2, vector: [2, 2], name: 'Michael'}
-//   {id: 1, vector: [3, 3], name: 'Ye'}
-// ]
-```
-
-#### Defined in
-
-[index.ts:296](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L296)
+[index.ts:112](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L112)

docs/src/javascript/interfaces/UpdateArgs.md

@@ -1,36 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / UpdateArgs
-
-# Interface: UpdateArgs
-
-## Table of contents
-
-### Properties
-
-- [values](UpdateArgs.md#values)
-- [where](UpdateArgs.md#where)
-
-## Properties
-
-### values
-
-• **values**: `Record`\<`string`, `Literal`\>
-
-A key-value map of updates. The keys are the column names, and the values are the
-new values to set
-
-#### Defined in
-
-[index.ts:320](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L320)
-
-___
-
-### where
-
-• `Optional` **where**: `string`
-
-A filter in the same format used by a sql WHERE clause. The filter may be empty,
-in which case all rows will be updated.
-
-#### Defined in
-
-[index.ts:314](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L314)

docs/src/javascript/interfaces/UpdateSqlArgs.md

@@ -1,36 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / UpdateSqlArgs
-
-# Interface: UpdateSqlArgs
-
-## Table of contents
-
-### Properties
-
-- [valuesSql](UpdateSqlArgs.md#valuessql)
-- [where](UpdateSqlArgs.md#where)
-
-## Properties
-
-### valuesSql
-
-• **valuesSql**: `Record`\<`string`, `string`\>
-
-A key-value map of updates. The keys are the column names, and the values are the
-new values to set as SQL expressions.
-
-#### Defined in
-
-[index.ts:334](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L334)
-
-___
-
-### where
-
-• `Optional` **where**: `string`
-
-A filter in the same format used by a sql WHERE clause. The filter may be empty,
-in which case all rows will be updated.
-
-#### Defined in
-
-[index.ts:328](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L328)

docs/src/javascript/interfaces/VectorIndex.md

@@ -1,41 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / VectorIndex
-
-# Interface: VectorIndex
-
-## Table of contents
-
-### Properties
-
-- [columns](VectorIndex.md#columns)
-- [name](VectorIndex.md#name)
-- [uuid](VectorIndex.md#uuid)
-
-## Properties
-
-### columns
-
-• **columns**: `string`[]
-
-#### Defined in
-
-[index.ts:338](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L338)
-
-___
-
-### name
-
-• **name**: `string`
-
-#### Defined in
-
-[index.ts:339](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L339)
-
-___
-
-### uuid
-
-• **uuid**: `string`
-
-#### Defined in
-
-[index.ts:340](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L340)

docs/src/javascript/interfaces/WriteOptions.md

@@ -1,27 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / WriteOptions
-
-# Interface: WriteOptions
-
-Write options when creating a Table.
-
-## Implemented by
-
-- [`DefaultWriteOptions`](../classes/DefaultWriteOptions.md)
-
-## Table of contents
-
-### Properties
-
-- [writeMode](WriteOptions.md#writemode)
-
-## Properties
-
-### writeMode
-
-• `Optional` **writeMode**: [`WriteMode`](../enums/WriteMode.md)
-
-A [WriteMode](../enums/WriteMode.md) to use on this operation
-
-#### Defined in
-
-[index.ts:774](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L774)

docs/src/javascript/modules.md

@@ -11,7 +11,6 @@
 
 ### Classes
 
-- [DefaultWriteOptions](classes/DefaultWriteOptions.md)
 - [LocalConnection](classes/LocalConnection.md)
 - [LocalTable](classes/LocalTable.md)
 - [OpenAIEmbeddingFunction](classes/OpenAIEmbeddingFunction.md)
@@ -20,20 +19,11 @@
 ### Interfaces
 
 - [AwsCredentials](interfaces/AwsCredentials.md)
-- [CleanupStats](interfaces/CleanupStats.md)
-- [CompactionMetrics](interfaces/CompactionMetrics.md)
-- [CompactionOptions](interfaces/CompactionOptions.md)
 - [Connection](interfaces/Connection.md)
 - [ConnectionOptions](interfaces/ConnectionOptions.md)
-- [CreateTableOptions](interfaces/CreateTableOptions.md)
 - [EmbeddingFunction](interfaces/EmbeddingFunction.md)
-- [IndexStats](interfaces/IndexStats.md)
 - [IvfPQIndexConfig](interfaces/IvfPQIndexConfig.md)
 - [Table](interfaces/Table.md)
-- [UpdateArgs](interfaces/UpdateArgs.md)
-- [UpdateSqlArgs](interfaces/UpdateSqlArgs.md)
-- [VectorIndex](interfaces/VectorIndex.md)
-- [WriteOptions](interfaces/WriteOptions.md)
 
 ### Type Aliases
 
@@ -42,7 +32,6 @@
 ### Functions
 
 - [connect](modules.md#connect)
-- [isWriteOptions](modules.md#iswriteoptions)
 
 ## Type Aliases
 
@@ -52,13 +41,13 @@
 
 #### Defined in
 
-[index.ts:755](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L755)
+[index.ts:431](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L431)
 
 ## Functions
 
 ### connect
 
-▸ **connect**(`uri`): `Promise`\<[`Connection`](interfaces/Connection.md)\>
+▸ **connect**(`uri`): `Promise`<[`Connection`](interfaces/Connection.md)\>
 
 Connect to a LanceDB instance at the given URI
 
@@ -70,44 +59,24 @@ Connect to a LanceDB instance at the given URI
 
 #### Returns
 
-`Promise`\<[`Connection`](interfaces/Connection.md)\>
+`Promise`<[`Connection`](interfaces/Connection.md)\>
 
 #### Defined in
 
-[index.ts:95](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L95)
+[index.ts:47](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L47)
 
-▸ **connect**(`opts`): `Promise`\<[`Connection`](interfaces/Connection.md)\>
+▸ **connect**(`opts`): `Promise`<[`Connection`](interfaces/Connection.md)\>
 
 #### Parameters
 
 | Name | Type |
 | :------ | :------ |
-| `opts` | `Partial`\<[`ConnectionOptions`](interfaces/ConnectionOptions.md)\> |
+| `opts` | `Partial`<[`ConnectionOptions`](interfaces/ConnectionOptions.md)\> |
 
 #### Returns
 
-`Promise`\<[`Connection`](interfaces/Connection.md)\>
+`Promise`<[`Connection`](interfaces/Connection.md)\>
 
 #### Defined in
 
-[index.ts:96](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L96)
-
-___
-
-### isWriteOptions
-
-▸ **isWriteOptions**(`value`): value is WriteOptions
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `value` | `any` |
-
-#### Returns
-
-value is WriteOptions
-
-#### Defined in
-
-[index.ts:781](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L781)
+[index.ts:48](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L48)

docs/src/python/pydantic.md

@@ -7,7 +7,7 @@ LanceDB integrates with Pydantic for schema inference, data ingestion, and query
 
 LanceDB supports to create Apache Arrow Schema from a
 [Pydantic BaseModel](https://docs.pydantic.dev/latest/api/main/#pydantic.main.BaseModel)
-via [pydantic_to_schema()](python.md#lancedb.pydantic.pydantic_to_schema) method.
+via [pydantic_to_schema()](python.md##lancedb.pydantic.pydantic_to_schema) method.
 
 ::: lancedb.pydantic.pydantic_to_schema
 

docs/src/search.md

@@ -118,101 +118,4 @@ However, fast vector search using indices often entails making a trade-off with
 This is why it is often called **Approximate Nearest Neighbors (ANN)** search, while the Flat Search (KNN)
 always returns 100% recall.
 
-See [ANN Index](ann_indexes.md) for more details.
-
-
-### Output formats
-
-LanceDB returns results in many different formats commonly used in python.
-Let's create a LanceDB table with a nested schema:
-
-```python
-from datetime import datetime
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-import numpy as np
-from pydantic import BaseModel
-uri = "data/sample-lancedb-nested"
-
-class Metadata(BaseModel):
-    source: str
-    timestamp: datetime
-
-class Document(BaseModel):
-    content: str
-    meta: Metadata
-
-class LanceSchema(LanceModel):
-    id: str
-    vector: Vector(1536)
-    payload: Document
-
-# Let's add 100 sample rows to our dataset
-data = [LanceSchema(
-    id=f"id{i}",
-    vector=np.random.randn(1536),
-    payload=Document(
-        content=f"document{i}", meta=Metadata(source=f"source{i%10}", timestamp=datetime.now())
-    ),
-) for i in range(100)]
-
-tbl = db.create_table("documents", data=data)
-```
-
-#### As a pyarrow table
-
-Using `to_arrow()` we can get the results back as a pyarrow Table.
-This result table has the same columns as the LanceDB table, with 
-the addition of an `_distance` column for vector search or a `score`
-column for full text search.
-
-```python
-tbl.search(np.random.randn(1536)).to_arrow()
-```
-
-#### As a pandas dataframe
-
-You can also get the results as a pandas dataframe.
-
-```python
-tbl.search(np.random.randn(1536)).to_pandas()
-```
-
-While other formats like Arrow/Pydantic/Python dicts have a natural 
-way to handle nested schemas, pandas can only store nested data as a 
-python dict column, which makes it difficult to support nested references.
-So for convenience, you can also tell LanceDB to flatten a nested schema 
-when creating the pandas dataframe. 
-
-```python
-tbl.search(np.random.randn(1536)).to_pandas(flatten=True)
-```
-
-If your table has a deeply nested struct, you can control how many levels
-of nesting to flatten by passing in a positive integer.
-
-```python
-tbl.search(np.random.randn(1536)).to_pandas(flatten=1)
-```
-
-
-#### As a list of python dicts
-
-You can of course return results as a list of python dicts.
-
-```python
-tbl.search(np.random.randn(1536)).to_list()
-```
-
-#### As a list of pydantic models
-
-We can add data using pydantic models, and we can certainly
-retrieve results as pydantic models
-
-```python
-tbl.search(np.random.randn(1536)).to_pydantic(LanceSchema)
-```
-
-Note that in this case the extra `_distance` field is discarded since
-it's not part of the LanceSchema.
-
+See [ANN Index](ann_indexes.md) for more details.

docs/src/sql.md

@@ -1,7 +1,7 @@
 # SQL filters
 
 LanceDB embraces the utilization of standard SQL expressions as predicates for hybrid
-filters. It can be used during hybrid vector search, update, and deletion operations.
+filters. It can be used during hybrid vector search and deletion operations.
 
 Currently, Lance supports a growing list of expressions.
 
@@ -22,7 +22,7 @@ import numpy as np
 uri = "data/sample-lancedb"
 db = lancedb.connect(uri)
 
-data = [{"vector": row, "item": f"item {i}", "id": i}
+data = [{"vector": row, "item": f"item {i}"}
      for i, row in enumerate(np.random.random((10_000, 2)).astype('int'))]
 
 tbl = db.create_table("my_vectors", data=data)
@@ -35,25 +35,33 @@ const db = await vectordb.connect('data/sample-lancedb')
 
 let data = []
 for (let i = 0; i < 10_000; i++) {
-     data.push({vector: Array(1536).fill(i), id: i, item: `item ${i}`, strId: `${i}`})
+     data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
 }
-const tbl = await db.createTable('myVectors', data)
+const tbl = await db.createTable('my_vectors', data)
 ```
 -->
 === "Python"
 
     ```python
     tbl.search([100, 102]) \
-       .where("(item IN ('item 0', 'item 2')) AND (id > 10)") \
-       .to_arrow()
-    ```
+       .where("""(
+        (label IN [10, 20])
+        AND
+        (note.email IS NOT NULL)
+        ) OR NOT note.created
+        """)
 
+    ```
 === "Javascript"
 
     ```javascript
-    await tbl.search(Array(1536).fill(0))
-       .where("(item IN ('item 0', 'item 2')) AND (id > 10)")
-       .execute()
+    tbl.search([100, 102])
+       .where(`(
+            (label IN [10, 20])
+            AND
+            (note.email IS NOT NULL)
+        ) OR NOT note.created
+       `)
     ```
 
 
@@ -110,22 +118,3 @@ The mapping from SQL types to Arrow types is:
 
 [^1]: See precision mapping in previous table.
 
-
-## Filtering without Vector Search
-
-You can also filter your data without search.
-
-=== "Python"
-      ```python
-      tbl.search().where("id=10").limit(10).to_arrow()
-      ```
-
-=== "JavaScript"
-      ```javascript
-      await tbl.where('id=10').limit(10).execute()
-      ```
-
-!!! warning
-    If your table is large, this could potentially return a very large
-    amount of data. Please be sure to use a `limit` clause unless
-    you're sure you want to return the whole result set.

node/README.md

@@ -9,13 +9,8 @@ npm install vectordb
 ```
 
 This will download the appropriate native library for your platform. We currently
-support:
-
-* Linux (x86_64 and aarch64)
-* MacOS (Intel and ARM/M1/M2)
-* Windows (x86_64 only)
-
-We do not yet support musl-based Linux (such as Alpine Linux) or aarch64 Windows.
+support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
+yet support musl-based Linux (such as Alpine Linux).
 
 ## Usage
 

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.4.1",
+  "version": "0.3.9",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.4.1",
+      "version": "0.3.9",
       "cpu": [
         "x64",
         "arm64"
@@ -53,11 +53,11 @@
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.4.1",
-        "@lancedb/vectordb-darwin-x64": "0.4.1",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.4.1",
-        "@lancedb/vectordb-linux-x64-gnu": "0.4.1",
-        "@lancedb/vectordb-win32-x64-msvc": "0.4.1"
+        "@lancedb/vectordb-darwin-arm64": "0.3.9",
+        "@lancedb/vectordb-darwin-x64": "0.3.9",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.3.9",
+        "@lancedb/vectordb-linux-x64-gnu": "0.3.9",
+        "@lancedb/vectordb-win32-x64-msvc": "0.3.9"
       }
     },
     "node_modules/@apache-arrow/ts": {
@@ -317,9 +317,9 @@
       }
     },
     "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.1.tgz",
-      "integrity": "sha512-ul/Hvv5RX2RThpKSuiUjJRVrmXuBPvpU+HrLjcBmu4dzpuWN4+IeHIUM6xe79gLxOKlwkscVweTOuZnmMfsZeg==",
+      "version": "0.3.9",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.3.9.tgz",
+      "integrity": "sha512-irtAdfSRQDcfnMnB8T7D0atLFfu1MMZZ1JaxMKu24DDZ8e4IMYKUplxwvWni3241yA9yDE/pliRZCNQbQCEfrg==",
       "cpu": [
         "arm64"
       ],
@@ -329,9 +329,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.4.1.tgz",
-      "integrity": "sha512-sJtF2Cv6T9RhUpdeHNkryiJwPuW9QPQ3aMs5fID1hMCJA2U3BX27t/WlkiPT2+kTLeUcwF1JvAOgsfvZkfvI8w==",
+      "version": "0.3.9",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.3.9.tgz",
+      "integrity": "sha512-4xXQoPheyIl1P5kRoKmZtaAHFrYdL9pw5yq+r6ewIx0TCemN4LSvzSUTqM5nZl3QPU8FeL0CGD8Gt2gMU0HQ2A==",
       "cpu": [
         "x64"
       ],
@@ -341,9 +341,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.4.1.tgz",
-      "integrity": "sha512-tNnziT0BRjPsznKI4GgWROFdCOsCGx0inFu0z+WV1UomwXKcMWGslpWBqKE8IUiCq14duPVx/ie7Wwcf51IeJQ==",
+      "version": "0.3.9",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.3.9.tgz",
+      "integrity": "sha512-WIxCZKnLeSlz0PGURtKSX6hJ4CYE2o5P+IFmmuWOWB1uNapQu6zOpea6rNxcRFHUA0IJdO02lVxVfn2hDX4SMg==",
       "cpu": [
         "arm64"
       ],
@@ -353,9 +353,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.4.1.tgz",
-      "integrity": "sha512-PAcF2p1FUsC0AD+qkLfgE5+ZlQwlHe9eTP9dSsX43V/NGPDQ9+gBzaBTEDbvyHj1wl2Wft2NwOqB1HAFhilSDg==",
+      "version": "0.3.9",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.3.9.tgz",
+      "integrity": "sha512-bQbcV9adKzYbJLNzDjk9OYsMnT2IjmieLfb4IQ1hj5IUoWfbg80Bd0+gZUnrmrhG6fe56TIriFZYQR9i7TSE9Q==",
       "cpu": [
         "x64"
       ],
@@ -365,9 +365,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.4.1.tgz",
-      "integrity": "sha512-8mvThCppI/AfSPby6Y3t6xpCfbo8IY6JH5exO8fDGTwBFHOqgwR4Izb2K7FgXxkwUYcN4EfGSsk/6B1GpwMudg==",
+      "version": "0.3.9",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.3.9.tgz",
+      "integrity": "sha512-7EXI7P1QvAfgJNPWWBMDOkoJ696gSBAClcyEJNYg0JV21jVFZRwJVI3bZXflesWduFi/mTuzPkFFA68us1u19A==",
       "cpu": [
         "x64"
       ],
@@ -4869,33 +4869,33 @@
       }
     },
     "@lancedb/vectordb-darwin-arm64": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.1.tgz",
-      "integrity": "sha512-ul/Hvv5RX2RThpKSuiUjJRVrmXuBPvpU+HrLjcBmu4dzpuWN4+IeHIUM6xe79gLxOKlwkscVweTOuZnmMfsZeg==",
+      "version": "0.3.9",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.3.9.tgz",
+      "integrity": "sha512-irtAdfSRQDcfnMnB8T7D0atLFfu1MMZZ1JaxMKu24DDZ8e4IMYKUplxwvWni3241yA9yDE/pliRZCNQbQCEfrg==",
       "optional": true
     },
     "@lancedb/vectordb-darwin-x64": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.4.1.tgz",
-      "integrity": "sha512-sJtF2Cv6T9RhUpdeHNkryiJwPuW9QPQ3aMs5fID1hMCJA2U3BX27t/WlkiPT2+kTLeUcwF1JvAOgsfvZkfvI8w==",
+      "version": "0.3.9",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.3.9.tgz",
+      "integrity": "sha512-4xXQoPheyIl1P5kRoKmZtaAHFrYdL9pw5yq+r6ewIx0TCemN4LSvzSUTqM5nZl3QPU8FeL0CGD8Gt2gMU0HQ2A==",
       "optional": true
     },
     "@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.4.1.tgz",
-      "integrity": "sha512-tNnziT0BRjPsznKI4GgWROFdCOsCGx0inFu0z+WV1UomwXKcMWGslpWBqKE8IUiCq14duPVx/ie7Wwcf51IeJQ==",
+      "version": "0.3.9",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.3.9.tgz",
+      "integrity": "sha512-WIxCZKnLeSlz0PGURtKSX6hJ4CYE2o5P+IFmmuWOWB1uNapQu6zOpea6rNxcRFHUA0IJdO02lVxVfn2hDX4SMg==",
       "optional": true
     },
     "@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.4.1.tgz",
-      "integrity": "sha512-PAcF2p1FUsC0AD+qkLfgE5+ZlQwlHe9eTP9dSsX43V/NGPDQ9+gBzaBTEDbvyHj1wl2Wft2NwOqB1HAFhilSDg==",
+      "version": "0.3.9",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.3.9.tgz",
+      "integrity": "sha512-bQbcV9adKzYbJLNzDjk9OYsMnT2IjmieLfb4IQ1hj5IUoWfbg80Bd0+gZUnrmrhG6fe56TIriFZYQR9i7TSE9Q==",
       "optional": true
     },
     "@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.4.1.tgz",
-      "integrity": "sha512-8mvThCppI/AfSPby6Y3t6xpCfbo8IY6JH5exO8fDGTwBFHOqgwR4Izb2K7FgXxkwUYcN4EfGSsk/6B1GpwMudg==",
+      "version": "0.3.9",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.3.9.tgz",
+      "integrity": "sha512-7EXI7P1QvAfgJNPWWBMDOkoJ696gSBAClcyEJNYg0JV21jVFZRwJVI3bZXflesWduFi/mTuzPkFFA68us1u19A==",
       "optional": true
     },
     "@neon-rs/cli": {

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.4.1",
+  "version": "0.3.9",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -81,10 +81,10 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.4.1",
-    "@lancedb/vectordb-darwin-x64": "0.4.1",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.4.1",
-    "@lancedb/vectordb-linux-x64-gnu": "0.4.1",
-    "@lancedb/vectordb-win32-x64-msvc": "0.4.1"
+    "@lancedb/vectordb-darwin-arm64": "0.3.9",
+    "@lancedb/vectordb-darwin-x64": "0.3.9",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.3.9",
+    "@lancedb/vectordb-linux-x64-gnu": "0.3.9",
+    "@lancedb/vectordb-win32-x64-msvc": "0.3.9"
   }
 }

node/src/index.ts

@@ -21,39 +21,23 @@ import type { EmbeddingFunction } from './embedding/embedding_function'
 import { RemoteConnection } from './remote'
 import { Query } from './query'
 import { isEmbeddingFunction } from './embedding/embedding_function'
-import { type Literal, toSQL } from './util'
+import {
+  type Connection, type CreateTableOptions, type Table,
+  type VectorIndexParams, type UpdateArgs, type UpdateSqlArgs,
+  type VectorIndex, type IndexStats,
+  type ConnectionOptions, WriteMode, type WriteOptions
+} from './types'
+import { toSQL } from './util'
+
+export { type WriteMode }
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateScalarIndex, tableCreateVectorIndex, tableCountRows, tableDelete, tableUpdate, tableCleanupOldVersions, tableCompactFiles, tableListIndices, tableIndexStats } = require('../native.js')
+const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete, tableUpdate, tableCleanupOldVersions, tableCompactFiles, tableListIndices, tableIndexStats } = require('../native.js')
 
 export { Query }
 export type { EmbeddingFunction }
 export { OpenAIEmbeddingFunction } from './embedding/openai'
 
-export interface AwsCredentials {
-  accessKeyId: string
-
-  secretKey: string
-
-  sessionToken?: string
-}
-
-export interface ConnectionOptions {
-  uri: string
-
-  awsCredentials?: AwsCredentials
-
-  awsRegion?: string
-
-  // API key for the remote connections
-  apiKey?: string
-  // Region to connect
-  region?: string
-
-  // override the host for the remote connections
-  hostOverride?: string
-}
-
 function getAwsArgs (opts: ConnectionOptions): any[] {
   const callArgs = []
   const awsCredentials = opts.awsCredentials
@@ -71,23 +55,6 @@ function getAwsArgs (opts: ConnectionOptions): any[] {
   return callArgs
 }
 
-export interface CreateTableOptions<T> {
-  // Name of Table
-  name: string
-
-  // Data to insert into the Table
-  data?: Array<Record<string, unknown>> | ArrowTable | undefined
-
-  // Optional Arrow Schema for this table
-  schema?: Schema | undefined
-
-  // Optional embedding function used to create embeddings
-  embeddingFunction?: EmbeddingFunction<T> | undefined
-
-  // WriteOptions for this operation
-  writeOptions?: WriteOptions | undefined
-}
-
 /**
  * Connect to a LanceDB instance at the given URI
  * @param uri The uri of the database.
@@ -116,285 +83,6 @@ export async function connect (arg: string | Partial<ConnectionOptions>): Promis
   return new LocalConnection(db, opts)
 }
 
-/**
- * A LanceDB Connection that allows you to open tables and create new ones.
- *
- * Connection could be local against filesystem or remote against a server.
- */
-export interface Connection {
-  uri: string
-
-  tableNames(): Promise<string[]>
-
-  /**
-   * Open a table in the database.
-   *
-   * @param name The name of the table.
-   * @param embeddings An embedding function to use on this table
-   */
-  openTable<T>(name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
-
-  /**
-   * Creates a new Table, optionally initializing it with new data.
-   *
-   * @param {string} name - The name of the table.
-   * @param data - Array of Records to be inserted into the table
-   * @param schema - An Arrow Schema that describe this table columns
-   * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
-   * @param {WriteOptions} writeOptions - The write options to use when creating the table.
-   */
-  createTable<T> ({ name, data, schema, embeddingFunction, writeOptions }: CreateTableOptions<T>): Promise<Table<T>>
-
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param {string} name - The name of the table.
-   * @param data - Non-empty Array of Records to be inserted into the table
-   */
-  createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table>
-
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param {string} name - The name of the table.
-   * @param data - Non-empty Array of Records to be inserted into the table
-   * @param {WriteOptions} options - The write options to use when creating the table.
-   */
-  createTable (name: string, data: Array<Record<string, unknown>>, options: WriteOptions): Promise<Table>
-
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param {string} name - The name of the table.
-   * @param data - Non-empty Array of Records to be inserted into the table
-   * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
-   */
-  createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param {string} name - The name of the table.
-   * @param data - Non-empty Array of Records to be inserted into the table
-   * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
-   * @param {WriteOptions} options - The write options to use when creating the table.
-   */
-  createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>, options: WriteOptions): Promise<Table<T>>
-
-  /**
-   * Drop an existing table.
-   * @param name The name of the table to drop.
-   */
-  dropTable(name: string): Promise<void>
-
-}
-
-/**
- * A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
- */
-export interface Table<T = number[]> {
-  name: string
-
-  /**
-   * Creates a search query to find the nearest neighbors of the given search term
-   * @param query The query search term
-   */
-  search: (query: T) => Query<T>
-
-  /**
-   * Insert records into this Table.
-   *
-   * @param data Records to be inserted into the Table
-   * @return The number of rows added to the table
-   */
-  add: (data: Array<Record<string, unknown>>) => Promise<number>
-
-  /**
-   * Insert records into this Table, replacing its contents.
-   *
-   * @param data Records to be inserted into the Table
-   * @return The number of rows added to the table
-   */
-  overwrite: (data: Array<Record<string, unknown>>) => Promise<number>
-
-  /**
-   * Create an ANN index on this Table vector index.
-   *
-   * @param indexParams The parameters of this Index, @see VectorIndexParams.
-   */
-  createIndex: (indexParams: VectorIndexParams) => Promise<any>
-
-  /**
-   * Create a scalar index on this Table for the given column
-   *
-   * @param column The column to index
-   * @param replace If false, fail if an index already exists on the column
-   *
-   * Scalar indices, like vector indices, can be used to speed up scans.  A scalar
-   * index can speed up scans that contain filter expressions on the indexed column.
-   * For example, the following scan will be faster if the column `my_col` has
-   * a scalar index:
-   *
-   * ```ts
-   * const con = await lancedb.connect('./.lancedb');
-   * const table = await con.openTable('images');
-   * const results = await table.where('my_col = 7').execute();
-   * ```
-   *
-   * Scalar indices can also speed up scans containing a vector search and a
-   * prefilter:
-   *
-   * ```ts
-   * const con = await lancedb.connect('././lancedb');
-   * const table = await con.openTable('images');
-   * const results = await table.search([1.0, 2.0]).where('my_col != 7').prefilter(true);
-   * ```
-   *
-   * Scalar indices can only speed up scans for basic filters using
-   * equality, comparison, range (e.g. `my_col BETWEEN 0 AND 100`), and set
-   * membership (e.g. `my_col IN (0, 1, 2)`)
-   *
-   * Scalar indices can be used if the filter contains multiple indexed columns and
-   * the filter criteria are AND'd or OR'd together
-   * (e.g. `my_col < 0 AND other_col> 100`)
-   *
-   * Scalar indices may be used if the filter contains non-indexed columns but,
-   * depending on the structure of the filter, they may not be usable.  For example,
-   * if the column `not_indexed` does not have a scalar index then the filter
-   * `my_col = 0 OR not_indexed = 1` will not be able to use any scalar index on
-   * `my_col`.
-   *
-   * @examples
-   *
-   * ```ts
-   * const con = await lancedb.connect('././lancedb')
-   * const table = await con.openTable('images')
-   * await table.createScalarIndex('my_col')
-   * ```
-   */
-  createScalarIndex: (column: string, replace: boolean) => Promise<void>
-
-  /**
-   * Returns the number of rows in this table.
-   */
-  countRows: () => Promise<number>
-
-  /**
-   * Delete rows from this table.
-   *
-   * This can be used to delete a single row, many rows, all rows, or
-   * sometimes no rows (if your predicate matches nothing).
-   *
-   * @param filter  A filter in the same format used by a sql WHERE clause. The
-   *                filter must not be empty.
-   *
-   * @examples
-   *
-   * ```ts
-   * const con = await lancedb.connect("./.lancedb")
-   * const data = [
-   *    {id: 1, vector: [1, 2]},
-   *    {id: 2, vector: [3, 4]},
-   *    {id: 3, vector: [5, 6]},
-   * ];
-   * const tbl = await con.createTable("my_table", data)
-   * await tbl.delete("id = 2")
-   * await tbl.countRows() // Returns 2
-   * ```
-   *
-   * If you have a list of values to delete, you can combine them into a
-   * stringified list and use the `IN` operator:
-   *
-   * ```ts
-   * const to_remove = [1, 5];
-   * await tbl.delete(`id IN (${to_remove.join(",")})`)
-   * await tbl.countRows() // Returns 1
-   * ```
-   */
-  delete: (filter: string) => Promise<void>
-
-  /**
-   * Update rows in this table.
-   *
-   * This can be used to update a single row, many rows, all rows, or
-   * sometimes no rows (if your predicate matches nothing).
-   *
-   * @param args see {@link UpdateArgs} and {@link UpdateSqlArgs} for more details
-   *
-   * @examples
-   *
-   * ```ts
-   * const con = await lancedb.connect("./.lancedb")
-   * const data = [
-   *    {id: 1, vector: [3, 3], name: 'Ye'},
-   *    {id: 2, vector: [4, 4], name: 'Mike'},
-   * ];
-   * const tbl = await con.createTable("my_table", data)
-   *
-   * await tbl.update({
-   *   where: "id = 2",
-   *   values: { vector: [2, 2], name: "Michael" },
-   * })
-   *
-   * let results = await tbl.search([1, 1]).execute();
-   * // Returns [
-   * //   {id: 2, vector: [2, 2], name: 'Michael'}
-   * //   {id: 1, vector: [3, 3], name: 'Ye'}
-   * // ]
-   * ```
-   *
-   */
-  update: (args: UpdateArgs | UpdateSqlArgs) => Promise<void>
-
-  /**
-   * List the indicies on this table.
-   */
-  listIndices: () => Promise<VectorIndex[]>
-
-  /**
-   * Get statistics about an index.
-   */
-  indexStats: (indexUuid: string) => Promise<IndexStats>
-}
-
-export interface UpdateArgs {
-  /**
-   * A filter in the same format used by a sql WHERE clause. The filter may be empty,
-   * in which case all rows will be updated.
-   */
-  where?: string
-
-  /**
-   * A key-value map of updates. The keys are the column names, and the values are the
-   * new values to set
-   */
-  values: Record<string, Literal>
-}
-
-export interface UpdateSqlArgs {
-  /**
-   * A filter in the same format used by a sql WHERE clause. The filter may be empty,
-   * in which case all rows will be updated.
-   */
-  where?: string
-
-  /**
-   * A key-value map of updates. The keys are the column names, and the values are the
-   * new values to set as SQL expressions.
-   */
-  valuesSql: Record<string, string>
-}
-
-export interface VectorIndex {
-  columns: string[]
-  name: string
-  uuid: string
-}
-
-export interface IndexStats {
-  numIndexedRows: number | null
-  numUnindexedRows: number | null
-}
-
 /**
  * A connection to a LanceDB database.
  */
@@ -587,10 +275,6 @@ export class LocalTable<T = number[]> implements Table<T> {
     return tableCreateVectorIndex.call(this._tbl, indexParams).then((newTable: any) => { this._tbl = newTable })
   }
 
-  async createScalarIndex (column: string, replace: boolean): Promise<void> {
-    return tableCreateScalarIndex.call(this._tbl, column, replace)
-  }
-
   /**
    * Returns the number of rows in this table.
    */
@@ -746,88 +430,6 @@ export interface CompactionMetrics {
   filesAdded: number
 }
 
-/// Config to build IVF_PQ index.
-///
-export interface IvfPQIndexConfig {
-  /**
-   * The column to be indexed
-   */
-  column?: string
-
-  /**
-   * A unique name for the index
-   */
-  index_name?: string
-
-  /**
-   * Metric type, L2 or Cosine
-   */
-  metric_type?: MetricType
-
-  /**
-   * The number of partitions this index
-   */
-  num_partitions?: number
-
-  /**
-   * The max number of iterations for kmeans training.
-   */
-  max_iters?: number
-
-  /**
-   * Train as optimized product quantization.
-   */
-  use_opq?: boolean
-
-  /**
-   * Number of subvectors to build PQ code
-   */
-  num_sub_vectors?: number
-  /**
-   * The number of bits to present one PQ centroid.
-   */
-  num_bits?: number
-
-  /**
-   * Max number of iterations to train OPQ, if `use_opq` is true.
-   */
-  max_opq_iters?: number
-
-  /**
-   * Replace an existing index with the same name if it exists.
-   */
-  replace?: boolean
-
-  /**
-   * Cache size of the index
-   */
-  index_cache_size?: number
-
-  type: 'ivf_pq'
-}
-
-export type VectorIndexParams = IvfPQIndexConfig
-
-/**
- * Write mode for writing a table.
- */
-export enum WriteMode {
-  /** Create a new {@link Table}. */
-  Create = 'create',
-  /** Overwrite the existing {@link Table} if presented. */
-  Overwrite = 'overwrite',
-  /** Append new data to the table. */
-  Append = 'append'
-}
-
-/**
- * Write options when creating a Table.
- */
-export interface WriteOptions {
-  /** A {@link WriteMode} to use on this operation */
-  writeMode?: WriteMode
-}
-
 export class DefaultWriteOptions implements WriteOptions {
   writeMode = WriteMode.Create
 }
@@ -836,23 +438,3 @@ export function isWriteOptions (value: any): value is WriteOptions {
   return Object.keys(value).length === 1 &&
       (value.writeMode === undefined || typeof value.writeMode === 'string')
 }
-
-/**
- * Distance metrics type.
- */
-export enum MetricType {
-  /**
-   * Euclidean distance
-   */
-  L2 = 'l2',
-
-  /**
-   * Cosine distance
-   */
-  Cosine = 'cosine',
-
-  /**
-   * Dot product
-   */
-  Dot = 'dot'
-}

node/src/integration_test/test.ts

@@ -1,180 +0,0 @@
-// Copyright 2023 LanceDB Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { describe } from 'mocha'
-import * as chai from 'chai'
-import * as chaiAsPromised from 'chai-as-promised'
-import { v4 as uuidv4 } from 'uuid'
-
-import * as lancedb from '../index'
-import { tmpdir } from 'os'
-import * as fs from 'fs'
-import * as path from 'path'
-
-const assert = chai.assert
-chai.use(chaiAsPromised)
-
-describe('LanceDB AWS Integration test', function () {
-  it('s3+ddb schema is processed correctly', async function () {
-    this.timeout(15000)
-
-    // WARNING: specifying engine is NOT a publicly supported feature in lancedb yet
-    // THE API WILL CHANGE
-    const conn = await lancedb.connect('s3://lancedb-integtest?engine=ddb&ddbTableName=lancedb-integtest')
-    const data = [{ vector: Array(128).fill(1.0) }]
-
-    const tableName = uuidv4()
-    let table = await conn.createTable(tableName, data, { writeMode: lancedb.WriteMode.Overwrite })
-
-    const futs = [table.add(data), table.add(data), table.add(data), table.add(data), table.add(data)]
-    await Promise.allSettled(futs)
-
-    table = await conn.openTable(tableName)
-    assert.equal(await table.countRows(), 6)
-  })
-})
-
-describe('LanceDB Mirrored Store Integration test', function () {
-  it('s3://...?mirroredStore=... param is processed correctly', async function () {
-    this.timeout(600000)
-
-    const dir = tmpdir()
-    console.log(dir)
-    const conn = await lancedb.connect(`s3://lancedb-integtest?mirroredStore=${dir}`)
-    const data = Array(200).fill({ vector: Array(128).fill(1.0), id: 0 })
-    data.push(...Array(200).fill({ vector: Array(128).fill(1.0), id: 1 }))
-    data.push(...Array(200).fill({ vector: Array(128).fill(1.0), id: 2 }))
-    data.push(...Array(200).fill({ vector: Array(128).fill(1.0), id: 3 }))
-
-    const tableName = uuidv4()
-
-    // try create table and check if it's mirrored
-    const t = await conn.createTable(tableName, data, { writeMode: lancedb.WriteMode.Overwrite })
-
-    const mirroredPath = path.join(dir, `${tableName}.lance`)
-    fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
-      if (err != null) throw err
-      // there should be three dirs
-      assert.equal(files.length, 3)
-      assert.isTrue(files[0].isDirectory())
-      assert.isTrue(files[1].isDirectory())
-
-      fs.readdir(path.join(mirroredPath, '_transactions'), { withFileTypes: true }, (err, files) => {
-        if (err != null) throw err
-        assert.equal(files.length, 1)
-        assert.isTrue(files[0].name.endsWith('.txn'))
-      })
-
-      fs.readdir(path.join(mirroredPath, '_versions'), { withFileTypes: true }, (err, files) => {
-        if (err != null) throw err
-        assert.equal(files.length, 1)
-        assert.isTrue(files[0].name.endsWith('.manifest'))
-      })
-
-      fs.readdir(path.join(mirroredPath, 'data'), { withFileTypes: true }, (err, files) => {
-        if (err != null) throw err
-        assert.equal(files.length, 1)
-        assert.isTrue(files[0].name.endsWith('.lance'))
-      })
-    })
-
-    // try create index and check if it's mirrored
-    await t.createIndex({ column: 'vector', type: 'ivf_pq' })
-
-    fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
-      if (err != null) throw err
-      // there should be four dirs
-      assert.equal(files.length, 4)
-      assert.isTrue(files[0].isDirectory())
-      assert.isTrue(files[1].isDirectory())
-      assert.isTrue(files[2].isDirectory())
-
-      // Two TXs now
-      fs.readdir(path.join(mirroredPath, '_transactions'), { withFileTypes: true }, (err, files) => {
-        if (err != null) throw err
-        assert.equal(files.length, 2)
-        assert.isTrue(files[0].name.endsWith('.txn'))
-        assert.isTrue(files[1].name.endsWith('.txn'))
-      })
-
-      fs.readdir(path.join(mirroredPath, 'data'), { withFileTypes: true }, (err, files) => {
-        if (err != null) throw err
-        assert.equal(files.length, 1)
-        assert.isTrue(files[0].name.endsWith('.lance'))
-      })
-
-      fs.readdir(path.join(mirroredPath, '_indices'), { withFileTypes: true }, (err, files) => {
-        if (err != null) throw err
-        assert.equal(files.length, 1)
-        assert.isTrue(files[0].isDirectory())
-
-        fs.readdir(path.join(mirroredPath, '_indices', files[0].name), { withFileTypes: true }, (err, files) => {
-          if (err != null) throw err
-
-          assert.equal(files.length, 1)
-          assert.isTrue(files[0].isFile())
-          assert.isTrue(files[0].name.endsWith('.idx'))
-        })
-      })
-    })
-
-    // try delete and check if it's mirrored
-    await t.delete('id = 0')
-
-    fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
-      if (err != null) throw err
-      // there should be five dirs
-      assert.equal(files.length, 5)
-      assert.isTrue(files[0].isDirectory())
-      assert.isTrue(files[1].isDirectory())
-      assert.isTrue(files[2].isDirectory())
-      assert.isTrue(files[3].isDirectory())
-      assert.isTrue(files[4].isDirectory())
-
-      // Three TXs now
-      fs.readdir(path.join(mirroredPath, '_transactions'), { withFileTypes: true }, (err, files) => {
-        if (err != null) throw err
-        assert.equal(files.length, 3)
-        assert.isTrue(files[0].name.endsWith('.txn'))
-        assert.isTrue(files[1].name.endsWith('.txn'))
-      })
-
-      fs.readdir(path.join(mirroredPath, 'data'), { withFileTypes: true }, (err, files) => {
-        if (err != null) throw err
-        assert.equal(files.length, 1)
-        assert.isTrue(files[0].name.endsWith('.lance'))
-      })
-
-      fs.readdir(path.join(mirroredPath, '_indices'), { withFileTypes: true }, (err, files) => {
-        if (err != null) throw err
-        assert.equal(files.length, 1)
-        assert.isTrue(files[0].isDirectory())
-
-        fs.readdir(path.join(mirroredPath, '_indices', files[0].name), { withFileTypes: true }, (err, files) => {
-          if (err != null) throw err
-
-          assert.equal(files.length, 1)
-          assert.isTrue(files[0].isFile())
-          assert.isTrue(files[0].name.endsWith('.idx'))
-        })
-      })
-
-      fs.readdir(path.join(mirroredPath, '_deletions'), { withFileTypes: true }, (err, files) => {
-        if (err != null) throw err
-        assert.equal(files.length, 1)
-        assert.isTrue(files[0].name.endsWith('.arrow'))
-      })
-    })
-  })
-})

node/src/query.ts

@@ -14,10 +14,16 @@
 
 import { Vector, tableFromIPC } from 'apache-arrow'
 import { type EmbeddingFunction } from './embedding/embedding_function'
-import { type MetricType } from '.'
+import { type MetricType } from './types'
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const { tableSearch } = require('../native.js')
+// const { tableSearch } = require('../native.js')
+
+const tableSearch = async function (args: any, arg2: any): Promise<any> {
+  return await new Promise((resolve, reject) => {
+    resolve('')
+  })
+}
 
 /**
  * A builder for nearest neighbor queries for LanceDB.

node/src/remote/index.ts

@@ -13,19 +13,21 @@
 // limitations under the License.
 
 import {
-  type EmbeddingFunction, type Table, type VectorIndexParams, type Connection,
-  type ConnectionOptions, type CreateTableOptions, type VectorIndex,
-  type WriteOptions,
+  type Table, type VectorIndexParams,
+  type VectorIndex,
   type IndexStats,
-  type UpdateArgs, type UpdateSqlArgs
-} from '../index'
+  type UpdateArgs, type UpdateSqlArgs,
+  type Connection,
+  type ConnectionOptions, type CreateTableOptions,
+  type WriteOptions
+} from '../types'
+import { type EmbeddingFunction } from '../embedding/embedding_function'
 import { Query } from '../query'
 
 import { Vector, Table as ArrowTable } from 'apache-arrow'
 import { HttpLancedbClient } from './client'
 import { isEmbeddingFunction } from '../embedding/embedding_function'
 import { createEmptyTable, fromRecordsToStreamBuffer, fromTableToStreamBuffer } from '../arrow'
-import { toSQL } from '../util'
 
 /**
  * Remote connection.
@@ -57,8 +59,8 @@ export class RemoteConnection implements Connection {
     return 'db://' + this._client.uri
   }
 
-  async tableNames (pageToken: string = '', limit: number = 10): Promise<string[]> {
-    const response = await this._client.get('/v1/table/', { limit, page_token: pageToken })
+  async tableNames (): Promise<string[]> {
+    const response = await this._client.get('/v1/table/')
     return response.data.tables
   }
 
@@ -195,17 +197,6 @@ export class RemoteTable<T = number[]> implements Table<T> {
     return this._name
   }
 
-  get schema (): Promise<any> {
-    return this._client.post(`/v1/table/${this._name}/describe/`).then(res => {
-      if (res.status !== 200) {
-        throw new Error(`Server Error, status: ${res.status}, ` +
-        // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-          `message: ${res.statusText}: ${res.data}`)
-      }
-      return res.data?.schema
-    })
-  }
-
   search (query: T): Query<T> {
     return new RemoteQuery(query, this._client, this._name)//, this._embeddings_new)
   }
@@ -246,44 +237,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
     return data.length
   }
 
-  async createIndex (indexParams: VectorIndexParams): Promise<void> {
-    const unsupportedParams = [
-      'index_name',
-      'num_partitions',
-      'max_iters',
-      'use_opq',
-      'num_sub_vectors',
-      'num_bits',
-      'max_opq_iters',
-      'replace'
-    ]
-    for (const param of unsupportedParams) {
-      // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
-      if (indexParams[param as keyof VectorIndexParams]) {
-        throw new Error(`${param} is not supported for remote connections`)
-      }
-    }
-
-    const column = indexParams.column ?? 'vector'
-    const indexType = 'vector' // only vector index is supported for remote connections
-    const metricType = indexParams.metric_type ?? 'L2'
-    const indexCacheSize = indexParams.index_cache_size ?? null
-
-    const data = {
-      column,
-      index_type: indexType,
-      metric_type: metricType,
-      index_cache_size: indexCacheSize
-    }
-    const res = await this._client.post(`/v1/table/${this._name}/create_index/`, data)
-    if (res.status !== 200) {
-      throw new Error(`Server Error, status: ${res.status}, ` +
-      // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-        `message: ${res.statusText}: ${res.data}`)
-    }
-  }
-
-  async createScalarIndex (column: string, replace: boolean): Promise<void> {
+  async createIndex (indexParams: VectorIndexParams): Promise<any> {
     throw new Error('Not implemented')
   }
 
@@ -297,23 +251,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
   }
 
   async update (args: UpdateArgs | UpdateSqlArgs): Promise<void> {
-    let filter: string | null
-    let updates: Record<string, string>
-
-    if ('valuesSql' in args) {
-      filter = args.where ?? null
-      updates = args.valuesSql
-    } else {
-      filter = args.where ?? null
-      updates = {}
-      for (const [key, value] of Object.entries(args.values)) {
-        updates[key] = toSQL(value)
-      }
-    }
-    await this._client.post(`/v1/table/${this._name}/update/`, {
-      predicate: filter,
-      updates: Object.entries(updates).map(([key, value]) => [key, value])
-    })
+    throw new Error('Not implemented')
   }
 
   async listIndices (): Promise<VectorIndex[]> {

node/src/test/embedding/openai.ts

@@ -1,57 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { describe } from 'mocha'
-import { assert } from 'chai'
-
-import { OpenAIEmbeddingFunction } from '../../embedding/openai'
-import { isEmbeddingFunction } from '../../embedding/embedding_function'
-
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const { OpenAIApi } = require('openai')
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const { stub } = require('sinon')
-
-describe('OpenAPIEmbeddings', function () {
-  const stubValue = {
-    data: {
-      data: [
-        {
-          embedding: Array(1536).fill(1.0)
-        },
-        {
-          embedding: Array(1536).fill(2.0)
-        }
-      ]
-    }
-  }
-
-  describe('#embed', function () {
-    it('should create vector embeddings', async function () {
-      const openAIStub = stub(OpenAIApi.prototype, 'createEmbedding').returns(stubValue)
-      const f = new OpenAIEmbeddingFunction('text', 'sk-key')
-      const vectors = await f.embed(['abc', 'def'])
-      assert.isTrue(openAIStub.calledOnce)
-      assert.equal(vectors.length, 2)
-      assert.deepEqual(vectors[0], stubValue.data.data[0].embedding)
-      assert.deepEqual(vectors[1], stubValue.data.data[1].embedding)
-    })
-  })
-
-  describe('isEmbeddingFunction', function () {
-    it('should match the isEmbeddingFunction guard', function () {
-      assert.isTrue(isEmbeddingFunction(new OpenAIEmbeddingFunction('text', 'sk-key')))
-    })
-  })
-})

node/src/test/io.ts

@@ -1,76 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// IO tests
-
-import { describe } from 'mocha'
-import { assert } from 'chai'
-
-import * as lancedb from '../index'
-import { type ConnectionOptions } from '../index'
-
-describe('LanceDB S3 client', function () {
-  if (process.env.TEST_S3_BASE_URL != null) {
-    const baseUri = process.env.TEST_S3_BASE_URL
-    it('should have a valid url', async function () {
-      const opts = { uri: `${baseUri}/valid_url` }
-      const table = await createTestDB(opts, 2, 20)
-      const con = await lancedb.connect(opts)
-      assert.equal(con.uri, opts.uri)
-
-      const results = await table.search([0.1, 0.3]).limit(5).execute()
-      assert.equal(results.length, 5)
-    }).timeout(10_000)
-  } else {
-    describe.skip('Skip S3 test', function () {})
-  }
-
-  if (process.env.TEST_S3_BASE_URL != null && process.env.TEST_AWS_ACCESS_KEY_ID != null && process.env.TEST_AWS_SECRET_ACCESS_KEY != null) {
-    const baseUri = process.env.TEST_S3_BASE_URL
-    it('use custom credentials', async function () {
-      const opts: ConnectionOptions = {
-        uri: `${baseUri}/custom_credentials`,
-        awsCredentials: {
-          accessKeyId: process.env.TEST_AWS_ACCESS_KEY_ID as string,
-          secretKey: process.env.TEST_AWS_SECRET_ACCESS_KEY as string
-        }
-      }
-      const table = await createTestDB(opts, 2, 20)
-      console.log(table)
-      const con = await lancedb.connect(opts)
-      console.log(con)
-      assert.equal(con.uri, opts.uri)
-
-      const results = await table.search([0.1, 0.3]).limit(5).execute()
-      assert.equal(results.length, 5)
-    }).timeout(10_000)
-  } else {
-    describe.skip('Skip S3 test', function () {})
-  }
-})
-
-async function createTestDB (opts: ConnectionOptions, numDimensions: number = 2, numRows: number = 2): Promise<lancedb.Table> {
-  const con = await lancedb.connect(opts)
-
-  const data = []
-  for (let i = 0; i < numRows; i++) {
-    const vector = []
-    for (let j = 0; j < numDimensions; j++) {
-      vector.push(i + (j * 0.1))
-    }
-    data.push({ id: i + 1, name: `name_${i}`, price: i + 10, is_active: (i % 2 === 0), vector })
-  }
-
-  return await con.createTable('vectors_2', data)
-}

node/src/test/test.ts

@@ -1,627 +0,0 @@
-// Copyright 2023 LanceDB Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { describe } from 'mocha'
-import { track } from 'temp'
-import * as chai from 'chai'
-import * as chaiAsPromised from 'chai-as-promised'
-
-import * as lancedb from '../index'
-import { type AwsCredentials, type EmbeddingFunction, MetricType, Query, WriteMode, DefaultWriteOptions, isWriteOptions, type LocalTable } from '../index'
-import { FixedSizeList, Field, Int32, makeVector, Schema, Utf8, Table as ArrowTable, vectorFromArray, Float32 } from 'apache-arrow'
-
-const expect = chai.expect
-const assert = chai.assert
-chai.use(chaiAsPromised)
-
-describe('LanceDB client', function () {
-  describe('when creating a connection to lancedb', function () {
-    it('should have a valid url', async function () {
-      const uri = await createTestDB()
-      const con = await lancedb.connect(uri)
-      assert.equal(con.uri, uri)
-    })
-
-    it('should accept an options object', async function () {
-      const uri = await createTestDB()
-      const con = await lancedb.connect({ uri })
-      assert.equal(con.uri, uri)
-    })
-
-    it('should accept custom aws credentials', async function () {
-      const uri = await createTestDB()
-      const awsCredentials: AwsCredentials = {
-        accessKeyId: '',
-        secretKey: ''
-      }
-      const con = await lancedb.connect({ uri, awsCredentials })
-      assert.equal(con.uri, uri)
-    })
-
-    it('should return the existing table names', async function () {
-      const uri = await createTestDB()
-      const con = await lancedb.connect(uri)
-      assert.deepEqual(await con.tableNames(), ['vectors'])
-    })
-  })
-
-  describe('when querying an existing dataset', function () {
-    it('should open a table', async function () {
-      const uri = await createTestDB()
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-      assert.equal(table.name, 'vectors')
-    })
-
-    it('execute a query', async function () {
-      const uri = await createTestDB()
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-      const results = await table.search([0.1, 0.3]).execute()
-
-      assert.equal(results.length, 2)
-      assert.equal(results[0].price, 10)
-      const vector = results[0].vector as Float32Array
-      assert.approximately(vector[0], 0.0, 0.2)
-      assert.approximately(vector[0], 0.1, 0.3)
-    })
-
-    it('limits # of results', async function () {
-      const uri = await createTestDB(2, 100)
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-      let results = await table.search([0.1, 0.3]).limit(1).execute()
-      assert.equal(results.length, 1)
-      assert.equal(results[0].id, 1)
-
-      // there is a default limit if unspecified
-      results = await table.search([0.1, 0.3]).execute()
-      assert.equal(results.length, 10)
-    })
-
-    it('uses a filter / where clause without vector search', async function () {
-      // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
-      const assertResults = (results: Array<Record<string, unknown>>) => {
-        assert.equal(results.length, 50)
-      }
-
-      const uri = await createTestDB(2, 100)
-      const con = await lancedb.connect(uri)
-      const table = (await con.openTable('vectors')) as LocalTable
-      let results = await table.filter('id % 2 = 0').execute()
-      assertResults(results)
-      results = await table.where('id % 2 = 0').execute()
-      assertResults(results)
-    })
-
-    it('uses a filter / where clause', async function () {
-      // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
-      const assertResults = (results: Array<Record<string, unknown>>) => {
-        assert.equal(results.length, 1)
-        assert.equal(results[0].id, 2)
-      }
-
-      const uri = await createTestDB()
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-      let results = await table.search([0.1, 0.1]).filter('id == 2').execute()
-      assertResults(results)
-      results = await table.search([0.1, 0.1]).where('id == 2').execute()
-      assertResults(results)
-    })
-
-    it('should correctly process prefilter/postfilter', async function () {
-      const uri = await createTestDB(16, 300)
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
-      // post filter should return less than the limit
-      let results = await table.search(new Array(16).fill(0.1)).limit(10).filter('id >= 10').prefilter(false).execute()
-      assert.isTrue(results.length < 10)
-
-      // pre filter should return exactly the limit
-      results = await table.search(new Array(16).fill(0.1)).limit(10).filter('id >= 10').prefilter(true).execute()
-      assert.isTrue(results.length === 10)
-    })
-
-    it('should allow creation and use of scalar indices', async function () {
-      const uri = await createTestDB(16, 300)
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-      await table.createScalarIndex('id', true)
-
-      // Prefiltering should still work the same
-      const results = await table.search(new Array(16).fill(0.1)).limit(10).filter('id >= 10').prefilter(true).execute()
-      assert.isTrue(results.length === 10)
-    })
-
-    it('select only a subset of columns', async function () {
-      const uri = await createTestDB()
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-      const results = await table.search([0.1, 0.1]).select(['is_active']).execute()
-      assert.equal(results.length, 2)
-      // vector and _distance are always returned
-      assert.isDefined(results[0].vector)
-      assert.isDefined(results[0]._distance)
-      assert.isDefined(results[0].is_active)
-
-      assert.isUndefined(results[0].id)
-      assert.isUndefined(results[0].name)
-      assert.isUndefined(results[0].price)
-    })
-  })
-
-  describe('when creating a new dataset', function () {
-    it('create an empty table', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-
-      const schema = new Schema(
-        [new Field('id', new Int32()), new Field('name', new Utf8())]
-      )
-      const table = await con.createTable({ name: 'vectors', schema })
-      assert.equal(table.name, 'vectors')
-      assert.deepEqual(await con.tableNames(), ['vectors'])
-    })
-
-    it('create a table with a empty data array', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-
-      const schema = new Schema(
-        [new Field('id', new Int32()), new Field('name', new Utf8())]
-      )
-      const table = await con.createTable({ name: 'vectors', schema, data: [] })
-      assert.equal(table.name, 'vectors')
-      assert.deepEqual(await con.tableNames(), ['vectors'])
-    })
-
-    it('create a table from an Arrow Table', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-
-      const i32s = new Int32Array(new Array<number>(10))
-      const i32 = makeVector(i32s)
-
-      const data = new ArrowTable({ vector: i32 })
-
-      const table = await con.createTable({ name: 'vectors', data })
-      assert.equal(table.name, 'vectors')
-      assert.equal(await table.countRows(), 10)
-      assert.deepEqual(await con.tableNames(), ['vectors'])
-    })
-
-    it('creates a new table from javascript objects', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-
-      const data = [
-        { id: 1, vector: [0.1, 0.2], price: 10 },
-        { id: 2, vector: [1.1, 1.2], price: 50 }
-      ]
-
-      const tableName = `vectors_${Math.floor(Math.random() * 100)}`
-      const table = await con.createTable(tableName, data)
-      assert.equal(table.name, tableName)
-      assert.equal(await table.countRows(), 2)
-    })
-
-    it('fails to create a new table when the vector column is missing', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-
-      const data = [
-        { id: 1, price: 10 }
-      ]
-
-      const create = con.createTable('missing_vector', data)
-      await expect(create).to.be.rejectedWith(Error, 'column \'vector\' is missing')
-    })
-
-    it('use overwrite flag to overwrite existing table', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-
-      const data = [
-        { id: 1, vector: [0.1, 0.2], price: 10 },
-        { id: 2, vector: [1.1, 1.2], price: 50 }
-      ]
-
-      const tableName = 'overwrite'
-      await con.createTable(tableName, data, { writeMode: WriteMode.Create })
-
-      const newData = [
-        { id: 1, vector: [0.1, 0.2], price: 10 },
-        { id: 2, vector: [1.1, 1.2], price: 50 },
-        { id: 3, vector: [1.1, 1.2], price: 50 }
-      ]
-
-      await expect(con.createTable(tableName, newData)).to.be.rejectedWith(Error, 'already exists')
-
-      const table = await con.createTable(tableName, newData, { writeMode: WriteMode.Overwrite })
-      assert.equal(table.name, tableName)
-      assert.equal(await table.countRows(), 3)
-    })
-
-    it('appends records to an existing table ', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-
-      const data = [
-        { id: 1, vector: [0.1, 0.2], price: 10, name: 'a' },
-        { id: 2, vector: [1.1, 1.2], price: 50, name: 'b' }
-      ]
-
-      const table = await con.createTable('vectors', data)
-      assert.equal(await table.countRows(), 2)
-
-      const dataAdd = [
-        { id: 3, vector: [2.1, 2.2], price: 10, name: 'c' },
-        { id: 4, vector: [3.1, 3.2], price: 50, name: 'd' }
-      ]
-      await table.add(dataAdd)
-      assert.equal(await table.countRows(), 4)
-    })
-
-    it('overwrite all records in a table', async function () {
-      const uri = await createTestDB()
-      const con = await lancedb.connect(uri)
-
-      const table = await con.openTable('vectors')
-      assert.equal(await table.countRows(), 2)
-
-      const dataOver = [
-        { vector: [2.1, 2.2], price: 10, name: 'foo' },
-        { vector: [3.1, 3.2], price: 50, name: 'bar' }
-      ]
-      await table.overwrite(dataOver)
-      assert.equal(await table.countRows(), 2)
-    })
-
-    it('can update records in the table', async function () {
-      const uri = await createTestDB()
-      const con = await lancedb.connect(uri)
-
-      const table = await con.openTable('vectors')
-      assert.equal(await table.countRows(), 2)
-
-      await table.update({ where: 'price = 10', valuesSql: { price: '100' } })
-      const results = await table.search([0.1, 0.2]).execute()
-      assert.equal(results[0].price, 100)
-      assert.equal(results[1].price, 11)
-    })
-
-    it('can update the records using a literal value', async function () {
-      const uri = await createTestDB()
-      const con = await lancedb.connect(uri)
-
-      const table = await con.openTable('vectors')
-      assert.equal(await table.countRows(), 2)
-
-      await table.update({ where: 'price = 10', values: { price: 100 } })
-      const results = await table.search([0.1, 0.2]).execute()
-      assert.equal(results[0].price, 100)
-      assert.equal(results[1].price, 11)
-    })
-
-    it('can update every record in the table', async function () {
-      const uri = await createTestDB()
-      const con = await lancedb.connect(uri)
-
-      const table = await con.openTable('vectors')
-      assert.equal(await table.countRows(), 2)
-
-      await table.update({ valuesSql: { price: '100' } })
-      const results = await table.search([0.1, 0.2]).execute()
-
-      assert.equal(results[0].price, 100)
-      assert.equal(results[1].price, 100)
-    })
-
-    it('can delete records from a table', async function () {
-      const uri = await createTestDB()
-      const con = await lancedb.connect(uri)
-
-      const table = await con.openTable('vectors')
-      assert.equal(await table.countRows(), 2)
-
-      await table.delete('price = 10')
-      assert.equal(await table.countRows(), 1)
-    })
-  })
-
-  describe('when searching an empty dataset', function () {
-    it('should not fail', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-
-      const schema = new Schema(
-        [new Field('vector', new FixedSizeList(128, new Field('float32', new Float32())))]
-      )
-      const table = await con.createTable({ name: 'vectors', schema })
-      const result = await table.search(Array(128).fill(0.1)).execute()
-      assert.isEmpty(result)
-    })
-  })
-
-  describe('when searching an empty-after-delete dataset', function () {
-    it('should not fail', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-
-      const schema = new Schema(
-        [new Field('vector', new FixedSizeList(128, new Field('float32', new Float32())))]
-      )
-      const table = await con.createTable({ name: 'vectors', schema })
-      await table.add([{ vector: Array(128).fill(0.1) }])
-      // https://github.com/lancedb/lance/issues/1635
-      await table.delete('true')
-      const result = await table.search(Array(128).fill(0.1)).execute()
-      assert.isEmpty(result)
-    })
-  })
-
-  describe('when creating a vector index', function () {
-    it('overwrite all records in a table', async function () {
-      const uri = await createTestDB(32, 300)
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
-    }).timeout(10_000) // Timeout is high partially because GH macos runner is pretty slow
-
-    it('replace an existing index', async function () {
-      const uri = await createTestDB(16, 300)
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-
-      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
-
-      // Replace should fail if the index already exists
-      await expect(table.createIndex({
-        type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2, replace: false
-      })
-      ).to.be.rejectedWith('LanceError(Index)')
-
-      // Default replace = true
-      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
-    }).timeout(50_000)
-
-    it('it should fail when the column is not a vector', async function () {
-      const uri = await createTestDB(32, 300)
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-      const createIndex = table.createIndex({ type: 'ivf_pq', column: 'name', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
-      await expect(createIndex).to.be.rejectedWith(/VectorIndex requires the column data type to be fixed size list of float32s/)
-    })
-
-    it('it should fail when the column is not a vector', async function () {
-      const uri = await createTestDB(32, 300)
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-      const createIndex = table.createIndex({ type: 'ivf_pq', column: 'name', num_partitions: -1, max_iters: 2, num_sub_vectors: 2 })
-      await expect(createIndex).to.be.rejectedWith('num_partitions: must be > 0')
-    })
-
-    it('should be able to list index and stats', async function () {
-      const uri = await createTestDB(32, 300)
-      const con = await lancedb.connect(uri)
-      const table = await con.openTable('vectors')
-      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
-
-      const indices = await table.listIndices()
-      expect(indices).to.have.lengthOf(1)
-      expect(indices[0].name).to.equal('vector_idx')
-      expect(indices[0].uuid).to.not.be.equal(undefined)
-      expect(indices[0].columns).to.have.lengthOf(1)
-      expect(indices[0].columns[0]).to.equal('vector')
-
-      const stats = await table.indexStats(indices[0].uuid)
-      expect(stats.numIndexedRows).to.equal(300)
-      expect(stats.numUnindexedRows).to.equal(0)
-    }).timeout(50_000)
-  })
-
-  describe('when using a custom embedding function', function () {
-    class TextEmbedding implements EmbeddingFunction<string> {
-      sourceColumn: string
-
-      constructor (targetColumn: string) {
-        this.sourceColumn = targetColumn
-      }
-
-      _embedding_map = new Map<string, number[]>([
-        ['foo', [2.1, 2.2]],
-        ['bar', [3.1, 3.2]]
-      ])
-
-      async embed (data: string[]): Promise<number[][]> {
-        return data.map(datum => this._embedding_map.get(datum) ?? [0.0, 0.0])
-      }
-    }
-
-    it('should encode the original data into embeddings', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-      const embeddings = new TextEmbedding('name')
-
-      const data = [
-        { price: 10, name: 'foo' },
-        { price: 50, name: 'bar' }
-      ]
-      const table = await con.createTable('vectors', data, embeddings, { writeMode: WriteMode.Create })
-      const results = await table.search('foo').execute()
-      assert.equal(results.length, 2)
-    })
-
-    it('should create embeddings for Arrow Table', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-      const embeddingFunction = new TextEmbedding('name')
-
-      const names = vectorFromArray(['foo', 'bar'], new Utf8())
-      const data = new ArrowTable({ name: names })
-
-      const table = await con.createTable({ name: 'vectors', data, embeddingFunction })
-      assert.equal(table.name, 'vectors')
-      const results = await table.search('foo').execute()
-      assert.equal(results.length, 2)
-    })
-  })
-})
-
-describe('Remote LanceDB client', function () {
-  describe('when the server is not reachable', function () {
-    it('produces a network error', async function () {
-      const con = await lancedb.connect({
-        uri: 'db://test-1234',
-        region: 'asdfasfasfdf',
-        apiKey: 'some-api-key'
-      })
-
-      // GET
-      try {
-        await con.tableNames()
-      } catch (err) {
-        expect(err).to.have.property('message', 'Network Error: getaddrinfo ENOTFOUND test-1234.asdfasfasfdf.api.lancedb.com')
-      }
-
-      // POST
-      try {
-        await con.createTable({ name: 'vectors', schema: new Schema([]) })
-      } catch (err) {
-        expect(err).to.have.property('message', 'Network Error: getaddrinfo ENOTFOUND test-1234.asdfasfasfdf.api.lancedb.com')
-      }
-
-      // Search
-      const table = await con.openTable('vectors')
-      try {
-        await table.search([0.1, 0.3]).execute()
-      } catch (err) {
-        expect(err).to.have.property('message', 'Network Error: getaddrinfo ENOTFOUND test-1234.asdfasfasfdf.api.lancedb.com')
-      }
-    })
-  })
-})
-
-describe('Query object', function () {
-  it('sets custom parameters', async function () {
-    const query = new Query([0.1, 0.3])
-      .limit(1)
-      .metricType(MetricType.Cosine)
-      .refineFactor(100)
-      .select(['a', 'b'])
-      .nprobes(20) as Record<string, any>
-    assert.equal(query._limit, 1)
-    assert.equal(query._metricType, MetricType.Cosine)
-    assert.equal(query._refineFactor, 100)
-    assert.equal(query._nprobes, 20)
-    assert.deepEqual(query._select, ['a', 'b'])
-  })
-})
-
-async function createTestDB (numDimensions: number = 2, numRows: number = 2): Promise<string> {
-  const dir = await track().mkdir('lancejs')
-  const con = await lancedb.connect(dir)
-
-  const data = []
-  for (let i = 0; i < numRows; i++) {
-    const vector = []
-    for (let j = 0; j < numDimensions; j++) {
-      vector.push(i + (j * 0.1))
-    }
-    data.push({ id: i + 1, name: `name_${i}`, price: i + 10, is_active: (i % 2 === 0), vector })
-  }
-
-  await con.createTable('vectors', data)
-  return dir
-}
-
-describe('Drop table', function () {
-  it('drop a table', async function () {
-    const dir = await track().mkdir('lancejs')
-    const con = await lancedb.connect(dir)
-
-    const data = [
-      { price: 10, name: 'foo', vector: [1, 2, 3] },
-      { price: 50, name: 'bar', vector: [4, 5, 6] }
-    ]
-    await con.createTable('t1', data)
-    await con.createTable('t2', data)
-
-    assert.deepEqual(await con.tableNames(), ['t1', 't2'])
-
-    await con.dropTable('t1')
-    assert.deepEqual(await con.tableNames(), ['t2'])
-  })
-})
-
-describe('WriteOptions', function () {
-  context('#isWriteOptions', function () {
-    it('should not match empty object', function () {
-      assert.equal(isWriteOptions({}), false)
-    })
-    it('should match write options', function () {
-      assert.equal(isWriteOptions({ writeMode: WriteMode.Create }), true)
-    })
-    it('should match undefined write mode', function () {
-      assert.equal(isWriteOptions({ writeMode: undefined }), true)
-    })
-    it('should match default write options', function () {
-      assert.equal(isWriteOptions(new DefaultWriteOptions()), true)
-    })
-  })
-})
-
-describe('Compact and cleanup', function () {
-  it('can cleanup after compaction', async function () {
-    const dir = await track().mkdir('lancejs')
-    const con = await lancedb.connect(dir)
-
-    const data = [
-      { price: 10, name: 'foo', vector: [1, 2, 3] },
-      { price: 50, name: 'bar', vector: [4, 5, 6] }
-    ]
-    const table = await con.createTable('t1', data) as LocalTable
-
-    const newData = [
-      { price: 30, name: 'baz', vector: [7, 8, 9] }
-    ]
-    await table.add(newData)
-
-    const compactionMetrics = await table.compactFiles({
-      numThreads: 2
-    })
-    assert.equal(compactionMetrics.fragmentsRemoved, 2)
-    assert.equal(compactionMetrics.fragmentsAdded, 1)
-    assert.equal(await table.countRows(), 3)
-
-    await table.cleanupOldVersions()
-    assert.equal(await table.countRows(), 3)
-
-    // should have no effect, but this validates the arguments are parsed.
-    await table.compactFiles({
-      targetRowsPerFragment: 102410,
-      maxRowsPerGroup: 1024,
-      materializeDeletions: true,
-      materializeDeletionsThreshold: 0.5,
-      numThreads: 2
-    })
-
-    const cleanupMetrics = await table.cleanupOldVersions(0, true)
-    assert.isAtLeast(cleanupMetrics.bytesRemoved, 1)
-    assert.isAtLeast(cleanupMetrics.oldVersions, 1)
-    assert.equal(await table.countRows(), 3)
-  })
-})

node/src/test/util.ts

@@ -1,45 +0,0 @@
-// Copyright 2023 LanceDB Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { toSQL } from '../util'
-import * as chai from 'chai'
-
-const expect = chai.expect
-
-describe('toSQL', function () {
-  it('should turn string to SQL expression', function () {
-    expect(toSQL('foo')).to.equal("'foo'")
-  })
-
-  it('should turn number to SQL expression', function () {
-    expect(toSQL(123)).to.equal('123')
-  })
-
-  it('should turn boolean to SQL expression', function () {
-    expect(toSQL(true)).to.equal('TRUE')
-  })
-
-  it('should turn null to SQL expression', function () {
-    expect(toSQL(null)).to.equal('NULL')
-  })
-
-  it('should turn Date to SQL expression', function () {
-    const date = new Date('05 October 2011 14:48 UTC')
-    expect(toSQL(date)).to.equal("'2011-10-05T14:48:00.000Z'")
-  })
-
-  it('should turn array to SQL expression', function () {
-    expect(toSQL(['foo', 'bar', true, 1])).to.equal("['foo', 'bar', TRUE, 1]")
-  })
-})

node/src/types.ts

@@ -0,0 +1,375 @@
+
+import {
+  type Schema,
+  type Table as ArrowTable
+} from 'apache-arrow'
+
+import { type Literal } from './util'
+import type { EmbeddingFunction } from './embedding/embedding_function'
+import { type Query } from './query'
+
+export interface AwsCredentials {
+  accessKeyId: string
+
+  secretKey: string
+
+  sessionToken?: string
+}
+
+/**
+ * Write options when creating a Table.
+ */
+export interface WriteOptions {
+  /** A {@link WriteMode} to use on this operation */
+  writeMode?: WriteMode
+}
+
+/**
+ * Write mode for writing a table.
+ */
+export enum WriteMode {
+  /** Create a new {@link Table}. */
+  Create = 'create',
+  /** Overwrite the existing {@link Table} if presented. */
+  Overwrite = 'overwrite',
+  /** Append new data to the table. */
+  Append = 'append'
+}
+
+/**
+ * A LanceDB Connection that allows you to open tables and create new ones.
+ *
+ * Connection could be local against filesystem or remote against a server.
+ */
+export interface Connection {
+  uri: string
+
+  tableNames(): Promise<string[]>
+
+  /**
+    * Open a table in the database.
+    *
+    * @param name The name of the table.
+    * @param embeddings An embedding function to use on this table
+    */
+  openTable<T>(name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
+
+  /**
+    * Creates a new Table, optionally initializing it with new data.
+    *
+    * @param {string} name - The name of the table.
+    * @param data - Array of Records to be inserted into the table
+    * @param schema - An Arrow Schema that describe this table columns
+    * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
+    * @param {WriteOptions} writeOptions - The write options to use when creating the table.
+    */
+  createTable<T> ({ name, data, schema, embeddingFunction, writeOptions }: CreateTableOptions<T>): Promise<Table<T>>
+
+  /**
+    * Creates a new Table and initialize it with new data.
+    *
+    * @param {string} name - The name of the table.
+    * @param data - Non-empty Array of Records to be inserted into the table
+    */
+  createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table>
+
+  /**
+    * Creates a new Table and initialize it with new data.
+    *
+    * @param {string} name - The name of the table.
+    * @param data - Non-empty Array of Records to be inserted into the table
+    * @param {WriteOptions} options - The write options to use when creating the table.
+    */
+  createTable (name: string, data: Array<Record<string, unknown>>, options: WriteOptions): Promise<Table>
+
+  /**
+    * Creates a new Table and initialize it with new data.
+    *
+    * @param {string} name - The name of the table.
+    * @param data - Non-empty Array of Records to be inserted into the table
+    * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
+    */
+  createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
+  /**
+    * Creates a new Table and initialize it with new data.
+    *
+    * @param {string} name - The name of the table.
+    * @param data - Non-empty Array of Records to be inserted into the table
+    * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
+    * @param {WriteOptions} options - The write options to use when creating the table.
+    */
+  createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>, options: WriteOptions): Promise<Table<T>>
+
+  /**
+    * Drop an existing table.
+    * @param name The name of the table to drop.
+    */
+  dropTable(name: string): Promise<void>
+
+}
+
+export interface CreateTableOptions<T> {
+  // Name of Table
+  name: string
+
+  // Data to insert into the Table
+  data?: Array<Record<string, unknown>> | ArrowTable | undefined
+
+  // Optional Arrow Schema for this table
+  schema?: Schema | undefined
+
+  // Optional embedding function used to create embeddings
+  embeddingFunction?: EmbeddingFunction<T> | undefined
+
+  // WriteOptions for this operation
+  writeOptions?: WriteOptions | undefined
+}
+
+export interface ConnectionOptions {
+  uri: string
+
+  awsCredentials?: AwsCredentials
+
+  awsRegion?: string
+
+  // API key for the remote connections
+  apiKey?: string
+  // Region to connect
+  region?: string
+
+  // override the host for the remote connections
+  hostOverride?: string
+}
+
+/**
+ * Distance metrics type.
+ */
+export enum MetricType {
+  /**
+    * Euclidean distance
+    */
+  L2 = 'l2',
+
+  /**
+    * Cosine distance
+    */
+  Cosine = 'cosine',
+
+  /**
+    * Dot product
+    */
+  Dot = 'dot'
+}
+
+/// Config to build IVF_PQ index.
+///
+export interface IvfPQIndexConfig {
+  /**
+    * The column to be indexed
+    */
+  column?: string
+
+  /**
+    * A unique name for the index
+    */
+  index_name?: string
+
+  /**
+    * Metric type, L2 or Cosine
+    */
+  metric_type?: MetricType
+
+  /**
+    * The number of partitions this index
+    */
+  num_partitions?: number
+
+  /**
+    * The max number of iterations for kmeans training.
+    */
+  max_iters?: number
+
+  /**
+    * Train as optimized product quantization.
+    */
+  use_opq?: boolean
+
+  /**
+    * Number of subvectors to build PQ code
+    */
+  num_sub_vectors?: number
+  /**
+    * The number of bits to present one PQ centroid.
+    */
+  num_bits?: number
+
+  /**
+    * Max number of iterations to train OPQ, if `use_opq` is true.
+    */
+  max_opq_iters?: number
+
+  /**
+    * Replace an existing index with the same name if it exists.
+    */
+  replace?: boolean
+
+  type: 'ivf_pq'
+}
+
+export type VectorIndexParams = IvfPQIndexConfig
+
+/**
+ * A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
+ */
+export interface Table<T = number[]> {
+  name: string
+
+  /**
+     * Creates a search query to find the nearest neighbors of the given search term
+     * @param query The query search term
+     */
+  search: (query: T) => Query<T>
+
+  /**
+     * Insert records into this Table.
+     *
+     * @param data Records to be inserted into the Table
+     * @return The number of rows added to the table
+     */
+  add: (data: Array<Record<string, unknown>>) => Promise<number>
+
+  /**
+     * Insert records into this Table, replacing its contents.
+     *
+     * @param data Records to be inserted into the Table
+     * @return The number of rows added to the table
+     */
+  overwrite: (data: Array<Record<string, unknown>>) => Promise<number>
+
+  /**
+     * Create an ANN index on this Table vector index.
+     *
+     * @param indexParams The parameters of this Index, @see VectorIndexParams.
+     */
+  createIndex: (indexParams: VectorIndexParams) => Promise<any>
+
+  /**
+     * Returns the number of rows in this table.
+     */
+  countRows: () => Promise<number>
+
+  /**
+     * Delete rows from this table.
+     *
+     * This can be used to delete a single row, many rows, all rows, or
+     * sometimes no rows (if your predicate matches nothing).
+     *
+     * @param filter  A filter in the same format used by a sql WHERE clause. The
+     *                filter must not be empty.
+     *
+     * @examples
+     *
+     * ```ts
+     * const con = await lancedb.connect("./.lancedb")
+     * const data = [
+     *    {id: 1, vector: [1, 2]},
+     *    {id: 2, vector: [3, 4]},
+     *    {id: 3, vector: [5, 6]},
+     * ];
+     * const tbl = await con.createTable("my_table", data)
+     * await tbl.delete("id = 2")
+     * await tbl.countRows() // Returns 2
+     * ```
+     *
+     * If you have a list of values to delete, you can combine them into a
+     * stringified list and use the `IN` operator:
+     *
+     * ```ts
+     * const to_remove = [1, 5];
+     * await tbl.delete(`id IN (${to_remove.join(",")})`)
+     * await tbl.countRows() // Returns 1
+     * ```
+     */
+  delete: (filter: string) => Promise<void>
+
+  /**
+     * Update rows in this table.
+     *
+     * This can be used to update a single row, many rows, all rows, or
+     * sometimes no rows (if your predicate matches nothing).
+     *
+     * @param args see {@link UpdateArgs} and {@link UpdateSqlArgs} for more details
+     *
+     * @examples
+     *
+     * ```ts
+     * const con = await lancedb.connect("./.lancedb")
+     * const data = [
+     *    {id: 1, vector: [3, 3], name: 'Ye'},
+     *    {id: 2, vector: [4, 4], name: 'Mike'},
+     * ];
+     * const tbl = await con.createTable("my_table", data)
+     *
+     * await tbl.update({
+     *   filter: "id = 2",
+     *   updates: { vector: [2, 2], name: "Michael" },
+     * })
+     *
+     * let results = await tbl.search([1, 1]).execute();
+     * // Returns [
+     * //   {id: 2, vector: [2, 2], name: 'Michael'}
+     * //   {id: 1, vector: [3, 3], name: 'Ye'}
+     * // ]
+     * ```
+     *
+     */
+  update: (args: UpdateArgs | UpdateSqlArgs) => Promise<void>
+
+  /**
+     * List the indicies on this table.
+     */
+  listIndices: () => Promise<VectorIndex[]>
+
+  /**
+     * Get statistics about an index.
+     */
+  indexStats: (indexUuid: string) => Promise<IndexStats>
+}
+export interface UpdateArgs {
+  /**
+     * A filter in the same format used by a sql WHERE clause. The filter may be empty,
+     * in which case all rows will be updated.
+     */
+  where?: string
+
+  /**
+     * A key-value map of updates. The keys are the column names, and the values are the
+     * new values to set
+     */
+  values: Record<string, Literal>
+}
+
+export interface UpdateSqlArgs {
+  /**
+     * A filter in the same format used by a sql WHERE clause. The filter may be empty,
+     * in which case all rows will be updated.
+     */
+  where?: string
+
+  /**
+     * A key-value map of updates. The keys are the column names, and the values are the
+     * new values to set as SQL expressions.
+     */
+  valuesSql: Record<string, string>
+}
+
+export interface VectorIndex {
+  columns: string[]
+  name: string
+  uuid: string
+}
+
+export interface IndexStats {
+  numIndexedRows: number | null
+  numUnindexedRows: number | null
+}

python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.3
+current_version = 0.3.4
 commit = True
 message = [python] Bump version: {current_version} → {new_version}
 tag = True

python/lancedb/db.py

@@ -23,7 +23,7 @@ from overrides import EnforceOverrides, override
 from pyarrow import fs
 
 from .table import LanceTable, Table
-from .util import fs_from_uri, get_uri_location, get_uri_scheme, join_uri
+from .util import fs_from_uri, get_uri_location, get_uri_scheme
 
 if TYPE_CHECKING:
     from .common import DATA, URI
@@ -288,13 +288,14 @@ class LanceDBConnection(DBConnection):
             A list of table names.
         """
         try:
-            filesystem = fs_from_uri(self.uri)[0]
+            filesystem, path = fs_from_uri(self.uri)
         except pa.ArrowInvalid:
             raise NotImplementedError("Unsupported scheme: " + self.uri)
 
         try:
-            loc = get_uri_location(self.uri)
-            paths = filesystem.get_file_info(fs.FileSelector(loc))
+            paths = filesystem.get_file_info(
+                fs.FileSelector(get_uri_location(self.uri))
+            )
         except FileNotFoundError:
             # It is ok if the file does not exist since it will be created
             paths = []
@@ -372,7 +373,7 @@ class LanceDBConnection(DBConnection):
         """
         try:
             filesystem, path = fs_from_uri(self.uri)
-            table_path = join_uri(path, name + ".lance")
+            table_path = os.path.join(path, name + ".lance")
             filesystem.delete_dir(table_path)
         except FileNotFoundError:
             if not ignore_missing:

python/lancedb/embeddings/openai.py

@@ -10,7 +10,6 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-from functools import cached_property
 from typing import List, Union
 
 import numpy as np
@@ -45,10 +44,6 @@ class OpenAIEmbeddings(TextEmbeddingFunction):
             The texts to embed
         """
         # TODO retry, rate limit, token limit
-        rs = self._openai_client.embeddings.create(input=texts, model=self.name)
-        return [v.embedding for v in rs.data]
-
-    @cached_property
-    def _openai_client(self):
         openai = self.safe_import("openai")
-        return openai.OpenAI()
+        rs = openai.Embedding.create(input=texts, model=self.name)["data"]
+        return [v["embedding"] for v in rs]

python/lancedb/embeddings/utils.py

@@ -249,7 +249,7 @@ def retry_with_exponential_backoff(
 
                 if num_retries > max_retries:
                     raise Exception(
-                        f"Maximum number of retries ({max_retries}) exceeded.", e
+                        f"Maximum number of retries ({max_retries}) exceeded."
                     )
 
                 delay *= exponential_base * (1 + jitter * random.random())

python/lancedb/fts.py

@@ -75,14 +75,8 @@ def populate_index(index: tantivy.Index, table: LanceTable, fields: List[str]) -
         The number of rows indexed
     """
     # first check the fields exist and are string or large string type
-    nested = []
     for name in fields:
-        try:
-            f = table.schema.field(name)  # raises KeyError if not found
-        except KeyError:
-            f = resolve_path(table.schema, name)
-            nested.append(name)
-
+        f = table.schema.field(name)  # raises KeyError if not found
         if not pa.types.is_string(f.type) and not pa.types.is_large_string(f.type):
             raise TypeError(f"Field {name} is not a string type")
 
@@ -91,16 +85,7 @@ def populate_index(index: tantivy.Index, table: LanceTable, fields: List[str]) -
     # write data into index
     dataset = table.to_lance()
     row_id = 0
-
-    max_nested_level = 0
-    if len(nested) > 0:
-        max_nested_level = max([len(name.split(".")) for name in nested])
-
     for b in dataset.to_batches(columns=fields):
-        if max_nested_level > 0:
-            b = pa.Table.from_batches([b])
-            for _ in range(max_nested_level - 1):
-                b = b.flatten()
         for i in range(b.num_rows):
             doc = tantivy.Document()
             doc.add_integer("doc_id", row_id)
@@ -113,30 +98,6 @@ def populate_index(index: tantivy.Index, table: LanceTable, fields: List[str]) -
     return row_id
 
 
-def resolve_path(schema, field_name: str) -> pa.Field:
-    """
-    Resolve a nested field path to a list of field names
-
-    Parameters
-    ----------
-    field_name : str
-        The field name to resolve
-
-    Returns
-    -------
-    List[str]
-        The resolved path
-    """
-    path = field_name.split(".")
-    field = schema.field(path.pop(0))
-    for segment in path:
-        if pa.types.is_struct(field.type):
-            field = field.type.field(segment)
-        else:
-            raise KeyError(f"field {field_name} not found in schema {schema}")
-    return field
-
-
 def search_index(
     index: tantivy.Index, query: str, limit: int = 10
 ) -> Tuple[Tuple[int], Tuple[float]]:

python/lancedb/pydantic.py

@@ -26,7 +26,6 @@ import numpy as np
 import pyarrow as pa
 import pydantic
 import semver
-from pydantic.fields import FieldInfo
 
 from .embeddings import EmbeddingFunctionRegistry
 
@@ -143,8 +142,8 @@ def Vector(
     return FixedSizeList
 
 
-def _py_type_to_arrow_type(py_type: Type[Any], field: FieldInfo) -> pa.DataType:
-    """Convert a field with native Python type to Arrow data type.
+def _py_type_to_arrow_type(py_type: Type[Any]) -> pa.DataType:
+    """Convert Python Type to Arrow DataType.
 
     Raises
     ------
@@ -164,13 +163,9 @@ def _py_type_to_arrow_type(py_type: Type[Any], field: FieldInfo) -> pa.DataType:
     elif py_type == date:
         return pa.date32()
     elif py_type == datetime:
-        tz = get_extras(field, "tz")
-        return pa.timestamp("us", tz=tz)
-    elif getattr(py_type, "__origin__", None) in (list, tuple):
-        child = py_type.__args__[0]
-        return pa.list_(_py_type_to_arrow_type(child, field))
+        return pa.timestamp("us")
     raise TypeError(
-        f"Converting Pydantic type to Arrow Type: unsupported type {py_type}."
+        f"Converting Pydantic type to Arrow Type: unsupported type {py_type}"
     )
 
 
@@ -199,10 +194,10 @@ def _pydantic_to_arrow_type(field: pydantic.fields.FieldInfo) -> pa.DataType:
         args = field.annotation.__args__
         if origin == list:
             child = args[0]
-            return pa.list_(_py_type_to_arrow_type(child, field))
+            return pa.list_(_py_type_to_arrow_type(child))
         elif origin == Union:
             if len(args) == 2 and args[1] == type(None):
-                return _py_type_to_arrow_type(args[0], field)
+                return _py_type_to_arrow_type(args[0])
     elif inspect.isclass(field.annotation):
         if issubclass(field.annotation, pydantic.BaseModel):
             # Struct
@@ -210,7 +205,7 @@ def _pydantic_to_arrow_type(field: pydantic.fields.FieldInfo) -> pa.DataType:
             return pa.struct(fields)
         elif issubclass(field.annotation, FixedSizeListMixin):
             return pa.list_(field.annotation.value_arrow_type(), field.annotation.dim())
-    return _py_type_to_arrow_type(field.annotation, field)
+    return _py_type_to_arrow_type(field.annotation)
 
 
 def is_nullable(field: pydantic.fields.FieldInfo) -> bool:
@@ -353,20 +348,3 @@ def get_extras(field_info: pydantic.fields.FieldInfo, key: str) -> Any:
     if PYDANTIC_VERSION.major >= 2:
         return (field_info.json_schema_extra or {}).get(key)
     return (field_info.field_info.extra or {}).get("json_schema_extra", {}).get(key)
-
-
-if PYDANTIC_VERSION.major < 2:
-
-    def model_to_dict(model: pydantic.BaseModel) -> Dict[str, Any]:
-        """
-        Convert a Pydantic model to a dictionary.
-        """
-        return model.dict()
-
-else:
-
-    def model_to_dict(model: pydantic.BaseModel) -> Dict[str, Any]:
-        """
-        Convert a Pydantic model to a dictionary.
-        """
-        return model.model_dump()

python/lancedb/query.py

@@ -70,7 +70,7 @@ class Query(pydantic.BaseModel):
     vector_column: str = VECTOR_COLUMN_NAME
 
     # vector to search for
-    vector: Union[List[float], List[List[float]]]
+    vector: List[float]
 
     # sql filter to refine the query with
     filter: Optional[str] = None
@@ -185,40 +185,14 @@ class LanceQueryBuilder(ABC):
         """
         return self.to_pandas()
 
-    def to_pandas(self, flatten: Optional[Union[int, bool]] = None) -> "pd.DataFrame":
+    def to_pandas(self) -> "pd.DataFrame":
         """
         Execute the query and return the results as a pandas DataFrame.
         In addition to the selected columns, LanceDB also returns a vector
         and also the "_distance" column which is the distance between the query
         vector and the returned vector.
-
-        Parameters
-        ----------
-        flatten: Optional[Union[int, bool]]
-            If flatten is True, flatten all nested columns.
-            If flatten is an integer, flatten the nested columns up to the
-            specified depth.
-            If unspecified, do not flatten the nested columns.
         """
-        tbl = self.to_arrow()
-        if flatten is True:
-            while True:
-                tbl = tbl.flatten()
-                has_struct = False
-                # loop through all columns to check if there is any struct column
-                if any(pa.types.is_struct(col.type) for col in tbl.schema):
-                    continue
-                else:
-                    break
-        elif isinstance(flatten, int):
-            if flatten <= 0:
-                raise ValueError(
-                    "Please specify a positive integer for flatten or the boolean value `True`"
-                )
-            while flatten > 0:
-                tbl = tbl.flatten()
-                flatten -= 1
-        return tbl.to_pandas()
+        return self.to_arrow().to_pandas()
 
     @abstractmethod
     def to_arrow(self) -> pa.Table:
@@ -421,8 +395,6 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
         vector and the returned vectors.
         """
         vector = self._query if isinstance(self._query, list) else self._query.tolist()
-        if isinstance(vector[0], np.ndarray):
-            vector = [v.tolist() for v in vector]
         query = Query(
             vector=vector,
             filter=self._where,
@@ -490,27 +462,6 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
         scores = pa.array(scores)
         output_tbl = self._table.to_lance().take(row_ids, columns=self._columns)
         output_tbl = output_tbl.append_column("score", scores)
-
-        if self._where is not None:
-            try:
-                # TODO would be great to have Substrait generate pyarrow compute expressions
-                # or conversely have pyarrow support SQL expressions using Substrait
-                import duckdb
-
-                output_tbl = (
-                    duckdb.sql(f"SELECT * FROM output_tbl")
-                    .filter(self._where)
-                    .to_arrow_table()
-                )
-            except ImportError:
-                import lance
-                import tempfile
-
-                # TODO Use "memory://" instead once that's supported
-                with tempfile.TemporaryDirectory() as tmp:
-                    ds = lance.write_dataset(output_tbl, tmp)
-                    output_tbl = ds.to_table(filter=self._where)
-
         return output_tbl
 
 

python/lancedb/remote/__init__.py

@@ -18,8 +18,6 @@ import attrs
 import pyarrow as pa
 from pydantic import BaseModel
 
-from lancedb.common import VECTOR_COLUMN_NAME
-
 __all__ = ["LanceDBClient", "VectorQuery", "VectorQueryResult"]
 
 
@@ -45,8 +43,6 @@ class VectorQuery(BaseModel):
 
     refine_factor: Optional[int] = None
 
-    vector_column: str = VECTOR_COLUMN_NAME
-
 
 @attrs.define
 class VectorQueryResult:

python/lancedb/remote/table.py

@@ -11,10 +11,9 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-import asyncio
 import uuid
 from functools import cached_property
-from typing import Dict, Optional, Union
+from typing import Optional, Union
 
 import pyarrow as pa
 from lance import json_to_schema
@@ -23,7 +22,6 @@ from lancedb.common import DATA, VEC, VECTOR_COLUMN_NAME
 
 from ..query import LanceVectorQueryBuilder
 from ..table import Query, Table, _sanitize_data
-from ..util import value_to_sql
 from .arrow import to_ipc_binary
 from .client import ARROW_STREAM_CONTENT_TYPE
 from .db import RemoteDBConnection
@@ -65,12 +63,6 @@ class RemoteTable(Table):
         """to_pandas() is not supported on the LanceDB cloud"""
         return NotImplementedError("to_pandas() is not supported on the LanceDB cloud")
 
-    def create_scalar_index(self, *args, **kwargs):
-        """Creates a scalar index"""
-        return NotImplementedError(
-            "create_scalar_index() is not supported on the LanceDB cloud"
-        )
-
     def create_index(
         self,
         metric="L2",
@@ -228,24 +220,8 @@ class RemoteTable(Table):
         return LanceVectorQueryBuilder(self, query, vector_column_name)
 
     def _execute_query(self, query: Query) -> pa.Table:
-        if (
-            query.vector is not None
-            and len(query.vector) > 0
-            and not isinstance(query.vector[0], float)
-        ):
-            futures = []
-            for v in query.vector:
-                v = list(v)
-                q = query.copy()
-                q.vector = v
-                futures.append(self._conn._client.query(self._name, q))
-            result = self._conn._loop.run_until_complete(asyncio.gather(*futures))
-            return pa.concat_tables(
-                [add_index(r.to_arrow(), i) for i, r in enumerate(result)]
-            )
-        else:
-            result = self._conn._client.query(self._name, query)
-            return self._conn._loop.run_until_complete(result).to_arrow()
+        result = self._conn._client.query(self._name, query)
+        return self._conn._loop.run_until_complete(result).to_arrow()
 
     def delete(self, predicate: str):
         """Delete rows from the table.
@@ -297,73 +273,3 @@ class RemoteTable(Table):
         self._conn._loop.run_until_complete(
             self._conn._client.post(f"/v1/table/{self._name}/delete/", data=payload)
         )
-
-    def update(
-        self,
-        where: Optional[str] = None,
-        values: Optional[dict] = None,
-        *,
-        values_sql: Optional[Dict[str, str]] = None,
-    ):
-        """
-        This can be used to update zero to all rows depending on how many
-        rows match the where clause.
-
-        Parameters
-        ----------
-        where: str, optional
-            The SQL where clause to use when updating rows. For example, 'x = 2'
-            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
-        values: dict, optional
-            The values to update. The keys are the column names and the values
-            are the values to set.
-        values_sql: dict, optional
-            The values to update, expressed as SQL expression strings. These can
-            reference existing columns. For example, {"x": "x + 1"} will increment
-            the x column by 1.
-
-        Examples
-        --------
-        >>> import lancedb
-        >>> data = [
-        ...    {"x": 1, "vector": [1, 2]},
-        ...    {"x": 2, "vector": [3, 4]},
-        ...    {"x": 3, "vector": [5, 6]}
-        ... ]
-        >>> db = lancedb.connect("db://...", api_key="...", region="...") # doctest: +SKIP
-        >>> table = db.create_table("my_table", data) # doctest: +SKIP
-        >>> table.to_pandas() # doctest: +SKIP
-           x      vector # doctest: +SKIP
-        0  1  [1.0, 2.0] # doctest: +SKIP
-        1  2  [3.0, 4.0] # doctest: +SKIP
-        2  3  [5.0, 6.0] # doctest: +SKIP
-        >>> table.update(where="x = 2", values={"vector": [10, 10]}) # doctest: +SKIP
-        >>> table.to_pandas() # doctest: +SKIP
-           x        vector # doctest: +SKIP
-        0  1    [1.0, 2.0] # doctest: +SKIP
-        1  3    [5.0, 6.0] # doctest: +SKIP
-        2  2  [10.0, 10.0] # doctest: +SKIP
-
-        """
-        if values is not None and values_sql is not None:
-            raise ValueError("Only one of values or values_sql can be provided")
-        if values is None and values_sql is None:
-            raise ValueError("Either values or values_sql must be provided")
-
-        if values is not None:
-            updates = [[k, value_to_sql(v)] for k, v in values.items()]
-        else:
-            updates = [[k, v] for k, v in values_sql.items()]
-
-        payload = {"predicate": where, "updates": updates}
-        self._conn._loop.run_until_complete(
-            self._conn._client.post(f"/v1/table/{self._name}/update/", data=payload)
-        )
-
-
-def add_index(tbl: pa.Table, i: int) -> pa.Table:
-    return tbl.add_column(
-        0,
-        pa.field("query_index", pa.uint32()),
-        pa.array([i] * len(tbl), pa.uint32()),
-    )

python/lancedb/table.py

@@ -17,21 +17,20 @@ import inspect
 import os
 from abc import ABC, abstractmethod
 from functools import cached_property
-from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union
 
 import lance
 import numpy as np
 import pyarrow as pa
 import pyarrow.compute as pc
-import pyarrow.fs as pa_fs
 from lance import LanceDataset
 from lance.vector import vec_to_table
 
 from .common import DATA, VEC, VECTOR_COLUMN_NAME
 from .embeddings import EmbeddingFunctionConfig, EmbeddingFunctionRegistry
-from .pydantic import LanceModel, model_to_dict
+from .pydantic import LanceModel
 from .query import LanceQueryBuilder, Query
-from .util import fs_from_uri, safe_import_pandas, value_to_sql, join_uri
+from .util import fs_from_uri, safe_import_pandas
 from .utils.events import register_event
 
 if TYPE_CHECKING:
@@ -54,10 +53,8 @@ def _sanitize_data(
         # convert to list of dict if data is a bunch of LanceModels
         if isinstance(data[0], LanceModel):
             schema = data[0].__class__.to_arrow_schema()
-            data = [model_to_dict(d) for d in data]
-            data = pa.Table.from_pylist(data, schema=schema)
-        else:
-            data = pa.Table.from_pylist(data)
+            data = [dict(d) for d in data]
+        data = pa.Table.from_pylist(data)
     elif isinstance(data, dict):
         data = vec_to_table(data)
     elif pd is not None and isinstance(data, pd.DataFrame):
@@ -221,77 +218,6 @@ class Table(ABC):
         """
         raise NotImplementedError
 
-    @abstractmethod
-    def create_scalar_index(
-        self,
-        column: str,
-        *,
-        replace: bool = True,
-    ):
-        """Create a scalar index on a column.
-
-        Scalar indices, like vector indices, can be used to speed up scans.  A scalar
-        index can speed up scans that contain filter expressions on the indexed column.
-        For example, the following scan will be faster if the column ``my_col`` has
-        a scalar index:
-
-        .. code-block:: python
-
-            import lancedb
-
-            db = lancedb.connect("/data/lance")
-            img_table = db.open_table("images")
-            my_df = img_table.search().where("my_col = 7", prefilter=True).to_pandas()
-
-        Scalar indices can also speed up scans containing a vector search and a
-        prefilter:
-
-        .. code-block::python
-
-            import lancedb
-
-            db = lancedb.connect("/data/lance")
-            img_table = db.open_table("images")
-            img_table.search([1, 2, 3, 4], vector_column_name="vector")
-                .where("my_col != 7", prefilter=True)
-                .to_pandas()
-
-        Scalar indices can only speed up scans for basic filters using
-        equality, comparison, range (e.g. ``my_col BETWEEN 0 AND 100``), and set
-        membership (e.g. `my_col IN (0, 1, 2)`)
-
-        Scalar indices can be used if the filter contains multiple indexed columns and
-        the filter criteria are AND'd or OR'd together
-        (e.g. ``my_col < 0 AND other_col> 100``)
-
-        Scalar indices may be used if the filter contains non-indexed columns but,
-        depending on the structure of the filter, they may not be usable.  For example,
-        if the column ``not_indexed`` does not have a scalar index then the filter
-        ``my_col = 0 OR not_indexed = 1`` will not be able to use any scalar index on
-        ``my_col``.
-
-        **Experimental API**
-
-        Parameters
-        ----------
-        column : str
-            The column to be indexed.  Must be a boolean, integer, float,
-            or string column.
-        replace : bool, default True
-            Replace the existing index if it exists.
-
-        Examples
-        --------
-
-        .. code-block:: python
-
-            import lance
-
-            dataset = lance.dataset("/tmp/images.lance")
-            dataset.create_scalar_index("category")
-        """
-        raise NotImplementedError
-
     @abstractmethod
     def add(
         self,
@@ -455,62 +381,6 @@ class Table(ABC):
         """
         raise NotImplementedError
 
-    @abstractmethod
-    def update(
-        self,
-        where: Optional[str] = None,
-        values: Optional[dict] = None,
-        *,
-        values_sql: Optional[Dict[str, str]] = None,
-    ):
-        """
-        This can be used to update zero to all rows depending on how many
-        rows match the where clause. If no where clause is provided, then
-        all rows will be updated.
-
-        Either `values` or `values_sql` must be provided. You cannot provide
-        both.
-
-        Parameters
-        ----------
-        where: str, optional
-            The SQL where clause to use when updating rows. For example, 'x = 2'
-            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
-        values: dict, optional
-            The values to update. The keys are the column names and the values
-            are the values to set.
-        values_sql: dict, optional
-            The values to update, expressed as SQL expression strings. These can
-            reference existing columns. For example, {"x": "x + 1"} will increment
-            the x column by 1.
-
-        Examples
-        --------
-        >>> import lancedb
-        >>> import pandas as pd
-        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
-        >>> db = lancedb.connect("./.lancedb")
-        >>> table = db.create_table("my_table", data)
-        >>> table.to_pandas()
-           x      vector
-        0  1  [1.0, 2.0]
-        1  2  [3.0, 4.0]
-        2  3  [5.0, 6.0]
-        >>> table.update(where="x = 2", values={"vector": [10, 10]})
-        >>> table.to_pandas()
-           x        vector
-        0  1    [1.0, 2.0]
-        1  3    [5.0, 6.0]
-        2  2  [10.0, 10.0]
-        >>> table.update(values_sql={"x": "x + 1"})
-        >>> table.to_pandas()
-           x        vector
-        0  2    [1.0, 2.0]
-        1  4    [5.0, 6.0]
-        2  3  [10.0, 10.0]
-        """
-        raise NotImplementedError
-
 
 class LanceTable(Table):
     """
@@ -679,7 +549,7 @@ class LanceTable(Table):
 
     @property
     def _dataset_uri(self) -> str:
-        return join_uri(self._conn.uri, f"{self.name}.lance")
+        return os.path.join(self._conn.uri, f"{self.name}.lance")
 
     def create_index(
         self,
@@ -705,12 +575,7 @@ class LanceTable(Table):
         self._reset_dataset()
         register_event("create_index")
 
-    def create_scalar_index(self, column: str, *, replace: bool = True):
-        self._dataset.create_scalar_index(column, index_type="BTREE", replace=replace)
-
-    def create_fts_index(
-        self, field_names: Union[str, List[str]], *, replace: bool = False
-    ):
+    def create_fts_index(self, field_names: Union[str, List[str]]):
         """Create a full-text search index on the table.
 
         Warning - this API is highly experimental and is highly likely to change
@@ -720,31 +585,17 @@ class LanceTable(Table):
         ----------
         field_names: str or list of str
             The name(s) of the field to index.
-        replace: bool, default False
-            If True, replace the existing index if it exists. Note that this is
-            not yet an atomic operation; the index will be temporarily
-            unavailable while the new index is being created.
         """
         from .fts import create_index, populate_index
 
         if isinstance(field_names, str):
             field_names = [field_names]
-
-        fs, path = fs_from_uri(self._get_fts_index_path())
-        index_exists = fs.get_file_info(path).type != pa_fs.FileType.NotFound
-        if index_exists:
-            if not replace:
-                raise ValueError(
-                    f"Index already exists. Use replace=True to overwrite."
-                )
-            fs.delete_dir(path)
-
         index = create_index(self._get_fts_index_path(), field_names)
         populate_index(index, self, field_names)
         register_event("create_fts_index")
 
     def _get_fts_index_path(self):
-        return join_uri(self._dataset_uri, "_indices", "tantivy")
+        return os.path.join(self._dataset_uri, "_indices", "tantivy")
 
     @cached_property
     def _dataset(self) -> LanceDataset:
@@ -1062,35 +913,30 @@ class LanceTable(Table):
     def delete(self, where: str):
         self._dataset.delete(where)
 
-    def update(
-        self,
-        where: Optional[str] = None,
-        values: Optional[dict] = None,
-        *,
-        values_sql: Optional[Dict[str, str]] = None,
-    ):
+    def update(self, where: str, values: dict):
         """
+        EXPERIMENTAL: Update rows in the table (not threadsafe).
+
         This can be used to update zero to all rows depending on how many
         rows match the where clause.
 
         Parameters
         ----------
-        where: str, optional
+        where: str
             The SQL where clause to use when updating rows. For example, 'x = 2'
             or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
-        values: dict, optional
+        values: dict
             The values to update. The keys are the column names and the values
             are the values to set.
-        values_sql: dict, optional
-            The values to update, expressed as SQL expression strings. These can
-            reference existing columns. For example, {"x": "x + 1"} will increment
-            the x column by 1.
 
         Examples
         --------
         >>> import lancedb
-        >>> import pandas as pd
-        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
+        >>> data = [
+        ...    {"x": 1, "vector": [1, 2]},
+        ...    {"x": 2, "vector": [3, 4]},
+        ...    {"x": 3, "vector": [5, 6]}
+        ... ]
         >>> db = lancedb.connect("./.lancedb")
         >>> table = db.create_table("my_table", data)
         >>> table.to_pandas()
@@ -1106,15 +952,18 @@ class LanceTable(Table):
         2  2  [10.0, 10.0]
 
         """
-        if values is not None and values_sql is not None:
-            raise ValueError("Only one of values or values_sql can be provided")
-        if values is None and values_sql is None:
-            raise ValueError("Either values or values_sql must be provided")
-
-        if values is not None:
-            values_sql = {k: value_to_sql(v) for k, v in values.items()}
-
-        self.to_lance().update(values_sql, where)
+        orig_data = self._dataset.to_table(filter=where).combine_chunks()
+        if len(orig_data) == 0:
+            return
+        for col, val in values.items():
+            i = orig_data.column_names.index(col)
+            if i < 0:
+                raise ValueError(f"Column {col} does not exist")
+            orig_data = orig_data.set_column(
+                i, col, pa.array([val] * len(orig_data), type=orig_data[col].type)
+            )
+        self.delete(where)
+        self.add(orig_data, mode="append")
         self._reset_dataset()
         register_event("update")
 

python/lancedb/util.py

@@ -12,13 +12,9 @@
 #  limitations under the License.
 
 import os
-from datetime import date, datetime
-from functools import singledispatch
-import pathlib
-from typing import Tuple, Union
+from typing import Tuple
 from urllib.parse import urlparse
 
-import numpy as np
 import pyarrow.fs as pa_fs
 
 
@@ -63,12 +59,6 @@ def get_uri_location(uri: str) -> str:
     str: Location part of the URL, without scheme
     """
     parsed = urlparse(uri)
-    if len(parsed.scheme) == 1:
-        # Windows drive names are parsed as the scheme
-        # e.g. "c:\path" -> ParseResult(scheme="c", netloc="", path="/path", ...)
-        # So we add special handling here for schemes that are a single character
-        return uri
-
     if not parsed.netloc:
         return parsed.path
     else:
@@ -91,29 +81,6 @@ def fs_from_uri(uri: str) -> Tuple[pa_fs.FileSystem, str]:
     return pa_fs.FileSystem.from_uri(uri)
 
 
-def join_uri(base: Union[str, pathlib.Path], *parts: str) -> str:
-    """
-    Join a URI with multiple parts, handles both local and remote paths
-
-    Parameters
-    ----------
-    base : str
-        The base URI
-    parts : str
-        The parts to join to the base URI, each separated by the
-        appropriate path separator for the URI scheme and OS
-    """
-    if isinstance(base, pathlib.Path):
-        return base.joinpath(*parts)
-    base = str(base)
-    if get_uri_scheme(base) == "file":
-        # using pathlib for local paths make this windows compatible
-        # `get_uri_scheme` returns `file` for windows drive names (e.g. `c:\path`)
-        return str(pathlib.Path(base, *parts))
-    # for remote paths, just use os.path.join
-    return "/".join([p.rstrip("/") for p in [base, *parts]])
-
-
 def safe_import_pandas():
     try:
         import pandas as pd
@@ -121,53 +88,3 @@ def safe_import_pandas():
         return pd
     except ImportError:
         return None
-
-
-@singledispatch
-def value_to_sql(value):
-    raise NotImplementedError("SQL conversion is not implemented for this type")
-
-
-@value_to_sql.register(str)
-def _(value: str):
-    return f"'{value}'"
-
-
-@value_to_sql.register(int)
-def _(value: int):
-    return str(value)
-
-
-@value_to_sql.register(float)
-def _(value: float):
-    return str(value)
-
-
-@value_to_sql.register(bool)
-def _(value: bool):
-    return str(value).upper()
-
-
-@value_to_sql.register(type(None))
-def _(value: type(None)):
-    return "NULL"
-
-
-@value_to_sql.register(datetime)
-def _(value: datetime):
-    return f"'{value.isoformat()}'"
-
-
-@value_to_sql.register(date)
-def _(value: date):
-    return f"'{value.isoformat()}'"
-
-
-@value_to_sql.register(list)
-def _(value: list):
-    return "[" + ", ".join(map(value_to_sql, value)) + "]"
-
-
-@value_to_sql.register(np.ndarray)
-def _(value: np.ndarray):
-    return value_to_sql(value.tolist())

python/pyproject.toml

@@ -1,12 +1,12 @@
 [project]
 name = "lancedb"
-version = "0.4.3"
+version = "0.3.4"
 dependencies = [
     "deprecation",
-    "pylance==0.9.2",
+    "pylance==0.8.17",
     "ratelimiter~=1.0",
     "retry>=0.9.2",
-    "tqdm>=4.27.0",
+    "tqdm>=4.1.0",
     "aiohttp",
     "pydantic>=1.10",
     "attrs>=21.3.0",
@@ -49,11 +49,11 @@ classifiers = [
 repository = "https://github.com/lancedb/lancedb"
 
 [project.optional-dependencies]
-tests = ["pandas>=1.4", "pytest", "pytest-mock", "pytest-asyncio", "requests", "duckdb", "pytz"]
+tests = ["pandas>=1.4", "pytest", "pytest-mock", "pytest-asyncio", "requests"]
 dev = ["ruff", "pre-commit", "black"]
 docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings[python]"]
 clip = ["torch", "pillow", "open-clip"]
-embeddings = ["openai>=1.6.1", "sentence-transformers", "torch", "pillow", "open-clip-torch", "cohere", "InstructorEmbedding"]
+embeddings = ["openai", "sentence-transformers", "torch", "pillow", "open-clip-torch", "cohere", "InstructorEmbedding"]
 
 [project.scripts]
 lancedb = "lancedb.cli.cli:cli"

python/tests/test_embeddings_slow.py

@@ -29,7 +29,7 @@ from lancedb.pydantic import LanceModel, Vector
 
 @pytest.mark.slow
 @pytest.mark.parametrize("alias", ["sentence-transformers", "openai"])
-def test_basic_text_embeddings(alias, tmp_path):
+def test_sentence_transformer(alias, tmp_path):
     db = lancedb.connect(tmp_path)
     registry = get_registry()
     func = registry.get(alias).create(max_retries=0)

python/tests/test_fts.py

@@ -12,7 +12,6 @@
 #  limitations under the License.
 import os
 import random
-from unittest import mock
 
 import numpy as np
 import pandas as pd
@@ -44,16 +43,7 @@ def table(tmp_path) -> ldb.table.LanceTable:
         for _ in range(100)
     ]
     table = db.create_table(
-        "test",
-        data=pd.DataFrame(
-            {
-                "vector": vectors,
-                "id": [i % 2 for i in range(100)],
-                "text": text,
-                "text2": text,
-                "nested": [{"text": t} for t in text],
-            }
-        ),
+        "test", data=pd.DataFrame({"vector": vectors, "text": text, "text2": text})
     )
     return table
 
@@ -85,25 +75,6 @@ def test_create_index_from_table(tmp_path, table):
     assert len(df) == 10
     assert "text" in df.columns
 
-    # Check whether it can be updated
-    table.add(
-        [
-            {
-                "vector": np.random.randn(128),
-                "id": 101,
-                "text": "gorilla",
-                "text2": "gorilla",
-                "nested": {"text": "gorilla"},
-            }
-        ]
-    )
-
-    with pytest.raises(ValueError, match="already exists"):
-        table.create_fts_index("text")
-
-    table.create_fts_index("text", replace=True)
-    assert len(table.search("gorilla").limit(1).to_pandas()) == 1
-
 
 def test_create_index_multiple_columns(tmp_path, table):
     table.create_fts_index(["text", "text2"])
@@ -118,32 +89,3 @@ def test_empty_rs(tmp_path, table, mocker):
     mocker.patch("lancedb.fts.search_index", return_value=([], []))
     df = table.search("puppy").limit(10).to_pandas()
     assert len(df) == 0
-
-
-def test_nested_schema(tmp_path, table):
-    table.create_fts_index("nested.text")
-    rs = table.search("puppy").limit(10).to_list()
-    assert len(rs) == 10
-
-
-def test_search_index_with_filter(table):
-    table.create_fts_index("text")
-    orig_import = __import__
-
-    def import_mock(name, *args):
-        if name == "duckdb":
-            raise ImportError
-        return orig_import(name, *args)
-
-    # no duckdb
-    with mock.patch("builtins.__import__", side_effect=import_mock):
-        rs = table.search("puppy").where("id=1").limit(10).to_list()
-        for r in rs:
-            assert r["id"] == 1
-
-    # yes duckdb
-    rs2 = table.search("puppy").where("id=1").limit(10).to_list()
-    for r in rs2:
-        assert r["id"] == 1
-
-    assert rs == rs2

python/tests/test_pydantic.py

@@ -13,10 +13,9 @@
 
 
 import json
-import pytz
 import sys
 from datetime import date, datetime
-from typing import List, Optional, Tuple
+from typing import List, Optional
 
 import pyarrow as pa
 import pydantic
@@ -39,14 +38,11 @@ def test_pydantic_to_arrow():
         id: int
         s: str
         vec: list[float]
-        li: list[int]
-        lili: list[list[float]]
-        litu: list[tuple[float, float]]
+        li: List[int]
         opt: Optional[str] = None
         st: StructModel
         dt: date
         dtt: datetime
-        dt_with_tz: datetime = Field(json_schema_extra={"tz": "Asia/Shanghai"})
         # d: dict
 
     m = TestModel(
@@ -54,12 +50,9 @@ def test_pydantic_to_arrow():
         s="hello",
         vec=[1.0, 2.0, 3.0],
         li=[2, 3, 4],
-        lili=[[2.5, 1.5], [3.5, 4.5], [5.5, 6.5]],
-        litu=[(2.5, 1.5), (3.5, 4.5), (5.5, 6.5)],
         st=StructModel(a="a", b=1.0),
         dt=date.today(),
         dtt=datetime.now(),
-        dt_with_tz=datetime.now(pytz.timezone("Asia/Shanghai")),
     )
 
     schema = pydantic_to_schema(TestModel)
@@ -70,8 +63,6 @@ def test_pydantic_to_arrow():
             pa.field("s", pa.utf8(), False),
             pa.field("vec", pa.list_(pa.float64()), False),
             pa.field("li", pa.list_(pa.int64()), False),
-            pa.field("lili", pa.list_(pa.list_(pa.float64())), False),
-            pa.field("litu", pa.list_(pa.list_(pa.float64())), False),
             pa.field("opt", pa.utf8(), True),
             pa.field(
                 "st",
@@ -82,16 +73,11 @@ def test_pydantic_to_arrow():
             ),
             pa.field("dt", pa.date32(), False),
             pa.field("dtt", pa.timestamp("us"), False),
-            pa.field("dt_with_tz", pa.timestamp("us", tz="Asia/Shanghai"), False),
         ]
     )
     assert schema == expect_schema
 
 
-@pytest.mark.skipif(
-    sys.version_info > (3, 8),
-    reason="using native type alias requires python3.9 or higher",
-)
 def test_pydantic_to_arrow_py38():
     class StructModel(pydantic.BaseModel):
         a: str
@@ -102,13 +88,10 @@ def test_pydantic_to_arrow_py38():
         s: str
         vec: List[float]
         li: List[int]
-        lili: List[List[float]]
-        litu: List[Tuple[float, float]]
         opt: Optional[str] = None
         st: StructModel
         dt: date
         dtt: datetime
-        dt_with_tz: datetime = Field(json_schema_extra={"tz": "Asia/Shanghai"})
         # d: dict
 
     m = TestModel(
@@ -116,12 +99,9 @@ def test_pydantic_to_arrow_py38():
         s="hello",
         vec=[1.0, 2.0, 3.0],
         li=[2, 3, 4],
-        lili=[[2.5, 1.5], [3.5, 4.5], [5.5, 6.5]],
-        litu=[(2.5, 1.5), (3.5, 4.5), (5.5, 6.5)],
         st=StructModel(a="a", b=1.0),
         dt=date.today(),
         dtt=datetime.now(),
-        dt_with_tz=datetime.now(pytz.timezone("Asia/Shanghai")),
     )
 
     schema = pydantic_to_schema(TestModel)
@@ -132,8 +112,6 @@ def test_pydantic_to_arrow_py38():
             pa.field("s", pa.utf8(), False),
             pa.field("vec", pa.list_(pa.float64()), False),
             pa.field("li", pa.list_(pa.int64()), False),
-            pa.field("lili", pa.list_(pa.list_(pa.float64())), False),
-            pa.field("litu", pa.list_(pa.list_(pa.float64())), False),
             pa.field("opt", pa.utf8(), True),
             pa.field(
                 "st",
@@ -144,7 +122,6 @@ def test_pydantic_to_arrow_py38():
             ),
             pa.field("dt", pa.date32(), False),
             pa.field("dtt", pa.timestamp("us"), False),
-            pa.field("dt_with_tz", pa.timestamp("us", tz="Asia/Shanghai"), False),
         ]
     )
     assert schema == expect_schema

python/tests/test_table.py

@@ -12,7 +12,7 @@
 #  limitations under the License.
 
 import functools
-from datetime import date, datetime, timedelta
+from datetime import timedelta
 from pathlib import Path
 from typing import List
 from unittest.mock import PropertyMock, patch
@@ -22,7 +22,6 @@ import numpy as np
 import pandas as pd
 import pyarrow as pa
 import pytest
-from pydantic import BaseModel
 
 from lancedb.conftest import MockTextEmbeddingFunction
 from lancedb.db import LanceDBConnection
@@ -142,44 +141,14 @@ def test_add(db):
 
 
 def test_add_pydantic_model(db):
-    # https://github.com/lancedb/lancedb/issues/562
-
-    class Metadata(BaseModel):
-        source: str
-        timestamp: datetime
-
-    class Document(BaseModel):
-        content: str
-        meta: Metadata
-
-    class LanceSchema(LanceModel):
-        id: str
-        vector: Vector(2)
+    class TestModel(LanceModel):
+        vector: Vector(16)
         li: List[int]
-        payload: Document
 
-    tbl = LanceTable.create(db, "mytable", schema=LanceSchema, mode="overwrite")
-    assert tbl.schema == LanceSchema.to_arrow_schema()
-
-    # add works
-    expected = LanceSchema(
-        id="id",
-        vector=[0.0, 0.0],
-        li=[1, 2, 3],
-        payload=Document(
-            content="foo", meta=Metadata(source="bar", timestamp=datetime.now())
-        ),
-    )
-    tbl.add([expected])
-
-    result = tbl.search([0.0, 0.0]).limit(1).to_pydantic(LanceSchema)[0]
-    assert result == expected
-
-    flattened = tbl.search([0.0, 0.0]).limit(1).to_pandas(flatten=1)
-    assert len(flattened.columns) == 6  # _distance is automatically added
-
-    really_flattened = tbl.search([0.0, 0.0]).limit(1).to_pandas(flatten=True)
-    assert len(really_flattened.columns) == 7
+    data = TestModel(vector=list(range(16)), li=[1, 2, 3])
+    table = LanceTable.create(db, "test", data=[data])
+    assert len(table) == 1
+    assert table.schema == TestModel.to_arrow_schema()
 
 
 def _add(table, schema):
@@ -379,79 +348,14 @@ def test_update(db):
     assert len(table) == 2
     assert len(table.list_versions()) == 2
     table.update(where="id=0", values={"vector": [1.1, 1.1]})
-    assert len(table.list_versions()) == 3
-    assert table.version == 3
+    assert len(table.list_versions()) == 4
+    assert table.version == 4
     assert len(table) == 2
     v = table.to_arrow()["vector"].combine_chunks()
     v = v.values.to_numpy().reshape(2, 2)
     assert np.allclose(v, np.array([[1.2, 1.9], [1.1, 1.1]]))
 
 
-def test_update_types(db):
-    table = LanceTable.create(
-        db,
-        "my_table",
-        data=[
-            {
-                "id": 0,
-                "str": "foo",
-                "float": 1.1,
-                "timestamp": datetime(2021, 1, 1),
-                "date": date(2021, 1, 1),
-                "vector1": [1.0, 0.0],
-                "vector2": [1.0, 1.0],
-            }
-        ],
-    )
-    # Update with SQL
-    table.update(
-        values_sql=dict(
-            id="1",
-            str="'bar'",
-            float="2.2",
-            timestamp="TIMESTAMP '2021-01-02 00:00:00'",
-            date="DATE '2021-01-02'",
-            vector1="[2.0, 2.0]",
-            vector2="[3.0, 3.0]",
-        )
-    )
-    actual = table.to_arrow().to_pylist()[0]
-    expected = dict(
-        id=1,
-        str="bar",
-        float=2.2,
-        timestamp=datetime(2021, 1, 2),
-        date=date(2021, 1, 2),
-        vector1=[2.0, 2.0],
-        vector2=[3.0, 3.0],
-    )
-    assert actual == expected
-
-    # Update with values
-    table.update(
-        values=dict(
-            id=2,
-            str="baz",
-            float=3.3,
-            timestamp=datetime(2021, 1, 3),
-            date=date(2021, 1, 3),
-            vector1=[3.0, 3.0],
-            vector2=np.array([4.0, 4.0]),
-        )
-    )
-    actual = table.to_arrow().to_pylist()[0]
-    expected = dict(
-        id=2,
-        str="baz",
-        float=3.3,
-        timestamp=datetime(2021, 1, 3),
-        date=date(2021, 1, 3),
-        vector1=[3.0, 3.0],
-        vector2=[4.0, 4.0],
-    )
-    assert actual == expected
-
-
 def test_create_with_embedding_function(db):
     class MyTable(LanceModel):
         text: str
@@ -532,33 +436,6 @@ def test_multiple_vector_columns(db):
     assert result1["text"].iloc[0] != result2["text"].iloc[0]
 
 
-def test_create_scalar_index(db):
-    vec_array = pa.array(
-        [[1, 1], [2, 2], [3, 3], [4, 4], [5, 5]], pa.list_(pa.float32(), 2)
-    )
-    test_data = pa.Table.from_pydict(
-        {"x": ["c", "b", "a", "e", "b"], "y": [1, 2, 3, 4, 5], "vector": vec_array}
-    )
-    table = LanceTable.create(
-        db,
-        "my_table",
-        data=test_data,
-    )
-    table.create_scalar_index("x")
-    indices = table.to_lance().list_indices()
-    assert len(indices) == 1
-    scalar_index = indices[0]
-    assert scalar_index["type"] == "Scalar"
-
-    # Confirm that prefiltering still works with the scalar index column
-    results = table.search().where("x = 'c'").to_arrow()
-    assert results == test_data.slice(0, 1)
-    results = table.search([5, 5]).to_arrow()
-    assert results["_distance"][0].as_py() == 0
-    results = table.search([5, 5]).where("x != 'b'").to_arrow()
-    assert results["_distance"][0].as_py() > 0
-
-
 def test_empty_query(db):
     table = LanceTable.create(
         db,

python/tests/test_util.py

@@ -11,12 +11,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-import os
-import pathlib
-
-import pytest
-
-from lancedb.util import get_uri_scheme, join_uri
+from lancedb.util import get_uri_scheme
 
 
 def test_normalize_uri():
@@ -33,55 +28,3 @@ def test_normalize_uri():
     for uri, expected_scheme in zip(uris, schemes):
         parsed_scheme = get_uri_scheme(uri)
         assert parsed_scheme == expected_scheme
-
-
-def test_join_uri_remote():
-    schemes = ["s3", "az", "gs"]
-    for scheme in schemes:
-        expected = f"{scheme}://bucket/path/to/table.lance"
-        base_uri = f"{scheme}://bucket/path/to/"
-        parts = ["table.lance"]
-        assert join_uri(base_uri, *parts) == expected
-
-        base_uri = f"{scheme}://bucket"
-        parts = ["path", "to", "table.lance"]
-        assert join_uri(base_uri, *parts) == expected
-
-
-# skip this test if on windows
-@pytest.mark.skipif(os.name == "nt", reason="Windows paths are not POSIX")
-def test_join_uri_posix():
-    for base in [
-        # relative path
-        "relative/path",
-        "relative/path/",
-        # an absolute path
-        "/absolute/path",
-        "/absolute/path/",
-        # a file URI
-        "file:///absolute/path",
-        "file:///absolute/path/",
-    ]:
-        joined = join_uri(base, "table.lance")
-        assert joined == str(pathlib.Path(base) / "table.lance")
-        joined = join_uri(pathlib.Path(base), "table.lance")
-        assert joined == pathlib.Path(base) / "table.lance"
-
-
-# skip this test if not on windows
-@pytest.mark.skipif(os.name != "nt", reason="Windows paths are not POSIX")
-def test_local_join_uri_windows():
-    # https://learn.microsoft.com/en-us/dotnet/standard/io/file-path-formats
-    for base in [
-        # windows relative path
-        "relative\\path",
-        "relative\\path\\",
-        # windows absolute path from current drive
-        "c:\\absolute\\path",
-        # relative path from root of current drive
-        "\\relative\\path",
-    ]:
-        joined = join_uri(base, "table.lance")
-        assert joined == str(pathlib.Path(base) / "table.lance")
-        joined = join_uri(pathlib.Path(base), "table.lance")
-        assert joined == pathlib.Path(base) / "table.lance"

rust/ffi/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb-node"
-version = "0.4.1"
+version = "0.3.9"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
 edition = "2018"

rust/ffi/node/src/error.rs

@@ -23,7 +23,7 @@ pub enum Error {
     #[snafu(display("column '{name}' is missing"))]
     MissingColumn { name: String },
     #[snafu(display("{name}: {message}"))]
-    OutOfRange { name: String, message: String },
+    RangeError { name: String, message: String },
     #[snafu(display("{index_type} is not a valid index type"))]
     InvalidIndexType { index_type: String },
 

rust/ffi/node/src/index.rs

@@ -12,5 +12,4 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub mod scalar;
 pub mod vector;

rust/ffi/node/src/index/scalar.rs

@@ -1,43 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use neon::{
-    context::{Context, FunctionContext},
-    result::JsResult,
-    types::{JsBoolean, JsBox, JsPromise, JsString},
-};
-
-use crate::{error::ResultExt, runtime, table::JsTable};
-
-pub(crate) fn table_create_scalar_index(mut cx: FunctionContext) -> JsResult<JsPromise> {
-    let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
-    let column = cx.argument::<JsString>(0)?.value(&mut cx);
-    let replace = cx.argument::<JsBoolean>(1)?.value(&mut cx);
-
-    let rt = runtime(&mut cx)?;
-
-    let (deferred, promise) = cx.promise();
-    let channel = cx.channel();
-    let mut table = js_table.table.clone();
-
-    rt.spawn(async move {
-        let idx_result = table.create_scalar_index(&column, replace).await;
-
-        deferred.settle_with(&channel, move |mut cx| {
-            idx_result.or_throw(&mut cx)?;
-            Ok(cx.undefined())
-        });
-    });
-    Ok(promise)
-}

rust/ffi/node/src/index/vector.rs

@@ -65,10 +65,12 @@ fn get_index_params_builder(
             obj.get_opt::<JsString, _, _>(cx, "index_name")?
                 .map(|s| index_builder.index_name(s.value(cx)));
 
-            if let Some(metric_type) = obj.get_opt::<JsString, _, _>(cx, "metric_type")? {
-                let metric_type = MetricType::try_from(metric_type.value(cx).as_str()).unwrap();
-                index_builder.metric_type(metric_type);
-            }
+            obj.get_opt::<JsString, _, _>(cx, "metric_type")?
+                .map(|s| MetricType::try_from(s.value(cx).as_str()))
+                .map(|mt| {
+                    let metric_type = mt.unwrap();
+                    index_builder.metric_type(metric_type);
+                });
 
             let num_partitions = obj.get_opt_usize(cx, "num_partitions")?;
             let max_iters = obj.get_opt_usize(cx, "max_iters")?;
@@ -83,29 +85,23 @@ fn get_index_params_builder(
                 index_builder.ivf_params(ivf_params)
             });
 
-            if let Some(use_opq) = obj.get_opt::<JsBoolean, _, _>(cx, "use_opq")? {
-                pq_params.use_opq = use_opq.value(cx);
-            }
+            obj.get_opt::<JsBoolean, _, _>(cx, "use_opq")?
+                .map(|s| pq_params.use_opq = s.value(cx));
 
-            if let Some(num_sub_vectors) = obj.get_opt_usize(cx, "num_sub_vectors")? {
-                pq_params.num_sub_vectors = num_sub_vectors;
-            }
+            obj.get_opt_usize(cx, "num_sub_vectors")?
+                .map(|s| pq_params.num_sub_vectors = s);
 
-            if let Some(num_bits) = obj.get_opt_usize(cx, "num_bits")? {
-                pq_params.num_bits = num_bits;
-            }
+            obj.get_opt_usize(cx, "num_bits")?
+                .map(|s| pq_params.num_bits = s);
 
-            if let Some(max_iters) = obj.get_opt_usize(cx, "max_iters")? {
-                pq_params.max_iters = max_iters;
-            }
+            obj.get_opt_usize(cx, "max_iters")?
+                .map(|s| pq_params.max_iters = s);
 
-            if let Some(max_opq_iters) = obj.get_opt_usize(cx, "max_opq_iters")? {
-                pq_params.max_opq_iters = max_opq_iters;
-            }
+            obj.get_opt_usize(cx, "max_opq_iters")?
+                .map(|s| pq_params.max_opq_iters = s);
 
-            if let Some(replace) = obj.get_opt::<JsBoolean, _, _>(cx, "replace")? {
-                index_builder.replace(replace.value(cx));
-            }
+            obj.get_opt::<JsBoolean, _, _>(cx, "replace")?
+                .map(|s| index_builder.replace(s.value(cx)));
 
             Ok(index_builder)
         }

rust/ffi/node/src/lib.rs

@@ -242,10 +242,6 @@ fn main(mut cx: ModuleContext) -> NeonResult<()> {
     cx.export_function("tableCompactFiles", JsTable::js_compact)?;
     cx.export_function("tableListIndices", JsTable::js_list_indices)?;
     cx.export_function("tableIndexStats", JsTable::js_index_stats)?;
-    cx.export_function(
-        "tableCreateScalarIndex",
-        index::scalar::table_create_scalar_index,
-    )?;
     cx.export_function(
         "tableCreateVectorIndex",
         index::vector::table_create_vector_index,

rust/ffi/node/src/neon_ext/js_object_ext.rs

@@ -47,15 +47,15 @@ fn f64_to_u32_safe(n: f64, key: &str) -> Result<u32> {
     use conv::*;
 
     n.approx_as::<u32>().map_err(|e| match e {
-        FloatError::NegOverflow(_) => Error::OutOfRange {
+        FloatError::NegOverflow(_) => Error::RangeError {
             name: key.into(),
             message: "must be > 0".to_string(),
         },
-        FloatError::PosOverflow(_) => Error::OutOfRange {
+        FloatError::PosOverflow(_) => Error::RangeError {
             name: key.into(),
             message: format!("must be < {}", u32::MAX),
         },
-        FloatError::NotANumber(_) => Error::OutOfRange {
+        FloatError::NotANumber(_) => Error::RangeError {
             name: key.into(),
             message: "not a valid number".to_string(),
         },
@@ -66,15 +66,15 @@ fn f64_to_usize_safe(n: f64, key: &str) -> Result<usize> {
     use conv::*;
 
     n.approx_as::<usize>().map_err(|e| match e {
-        FloatError::NegOverflow(_) => Error::OutOfRange {
+        FloatError::NegOverflow(_) => Error::RangeError {
             name: key.into(),
             message: "must be > 0".to_string(),
         },
-        FloatError::PosOverflow(_) => Error::OutOfRange {
+        FloatError::PosOverflow(_) => Error::RangeError {
             name: key.into(),
             message: format!("must be < {}", usize::MAX),
         },
-        FloatError::NotANumber(_) => Error::OutOfRange {
+        FloatError::NotANumber(_) => Error::RangeError {
             name: key.into(),
             message: "not a valid number".to_string(),
         },

rust/ffi/node/src/query.rs

@@ -25,11 +25,11 @@ impl JsQuery {
         let limit = query_obj
             .get_opt::<JsNumber, _, _>(&mut cx, "_limit")?
             .map(|value| {
-                let limit = value.value(&mut cx);
-                if limit <= 0.0 {
+                let limit = value.value(&mut cx) as u64;
+                if limit <= 0 {
                     panic!("Limit must be a positive integer");
                 }
-                limit as u64
+                limit
             });
         let select = query_obj
             .get_opt::<JsArray, _, _>(&mut cx, "_select")?
@@ -73,7 +73,7 @@ impl JsQuery {
 
         rt.spawn(async move {
             let mut builder = table
-                .search(query.map(Float32Array::from))
+                .search(query.map(|q| Float32Array::from(q)))
                 .refine_factor(refine_factor)
                 .nprobes(nprobes)
                 .filter(filter)

rust/ffi/node/src/table.rs

@@ -45,7 +45,7 @@ impl JsTable {
         let table_name = cx.argument::<JsString>(0)?.value(&mut cx);
         let buffer = cx.argument::<JsBuffer>(1)?;
         let (batches, schema) =
-            arrow_buffer_to_record_batch(buffer.as_slice(&cx)).or_throw(&mut cx)?;
+            arrow_buffer_to_record_batch(buffer.as_slice(&mut cx)).or_throw(&mut cx)?;
 
         // Write mode
         let mode = match cx.argument::<JsString>(2)?.value(&mut cx).as_str() {
@@ -93,7 +93,7 @@ impl JsTable {
         let buffer = cx.argument::<JsBuffer>(0)?;
         let write_mode = cx.argument::<JsString>(1)?.value(&mut cx);
         let (batches, schema) =
-            arrow_buffer_to_record_batch(buffer.as_slice(&cx)).or_throw(&mut cx)?;
+            arrow_buffer_to_record_batch(buffer.as_slice(&mut cx)).or_throw(&mut cx)?;
         let rt = runtime(&mut cx)?;
         let channel = cx.channel();
         let mut table = js_table.table.clone();
@@ -186,7 +186,7 @@ impl JsTable {
                 .downcast_or_throw::<JsString, _>(&mut cx)?;
 
             let value = updates_arg
-                .get_value(&mut cx, property)?
+                .get_value(&mut cx, property.clone())?
                 .downcast_or_throw::<JsString, _>(&mut cx)?;
 
             let property = property.value(&mut cx);
@@ -216,7 +216,7 @@ impl JsTable {
                 .map(|(k, v)| (k.as_str(), v.as_str()))
                 .collect::<Vec<_>>();
 
-            let predicate = predicate.as_deref();
+            let predicate = predicate.as_ref().map(|s| s.as_str());
 
             let update_result = table.update(predicate, updates_arg).await;
             deferred.settle_with(&channel, move |mut cx| {

rust/vectordb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb"
-version = "0.4.1"
+version = "0.3.9"
 edition = "2021"
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"

rust/vectordb/src/io/object_store.rs

@@ -26,7 +26,7 @@ use futures::{stream::BoxStream, FutureExt, StreamExt};
 use lance::io::object_store::WrappingObjectStore;
 use object_store::{
     path::Path, Error, GetOptions, GetResult, ListResult, MultipartId, ObjectMeta, ObjectStore,
-    PutOptions, PutResult, Result,
+    Result,
 };
 
 use async_trait::async_trait;
@@ -72,28 +72,13 @@ impl PrimaryOnly for Path {
 /// Note: this object store does not mirror writes to *.manifest files
 #[async_trait]
 impl ObjectStore for MirroringObjectStore {
-    async fn put(&self, location: &Path, bytes: Bytes) -> Result<PutResult> {
+    async fn put(&self, location: &Path, bytes: Bytes) -> Result<()> {
         if location.primary_only() {
             self.primary.put(location, bytes).await
         } else {
             self.secondary.put(location, bytes.clone()).await?;
-            self.primary.put(location, bytes).await
-        }
-    }
-
-    async fn put_opts(
-        &self,
-        location: &Path,
-        bytes: Bytes,
-        options: PutOptions,
-    ) -> Result<PutResult> {
-        if location.primary_only() {
-            self.primary.put_opts(location, bytes, options).await
-        } else {
-            self.secondary
-                .put_opts(location, bytes.clone(), options.clone())
-                .await?;
-            self.primary.put_opts(location, bytes, options).await
+            self.primary.put(location, bytes).await?;
+            Ok(())
         }
     }
 
@@ -144,8 +129,8 @@ impl ObjectStore for MirroringObjectStore {
         self.primary.delete(location).await
     }
 
-    fn list(&self, prefix: Option<&Path>) -> BoxStream<'_, Result<ObjectMeta>> {
-        self.primary.list(prefix)
+    async fn list(&self, prefix: Option<&Path>) -> Result<BoxStream<'_, Result<ObjectMeta>>> {
+        self.primary.list(prefix).await
     }
 
     async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {
@@ -374,9 +359,7 @@ mod test {
         assert_eq!(t.count_rows().await.unwrap(), 100);
 
         let q = t
-            .search(Some(PrimitiveArray::from_iter_values(vec![
-                0.1, 0.1, 0.1, 0.1,
-            ])))
+            .search(Some(PrimitiveArray::from_iter_values(vec![0.1, 0.1, 0.1, 0.1])))
             .limit(10)
             .execute()
             .await

rust/vectordb/src/query.rs

@@ -25,7 +25,6 @@ use crate::error::Result;
 pub struct Query {
     pub dataset: Arc<Dataset>,
     pub query_vector: Option<Float32Array>,
-    pub column: String,
     pub limit: Option<usize>,
     pub filter: Option<String>,
     pub select: Option<Vec<String>>,
@@ -51,7 +50,6 @@ impl Query {
         Query {
             dataset,
             query_vector: vector,
-            column: crate::table::VECTOR_COLUMN_NAME.to_string(),
             limit: None,
             nprobes: 20,
             refine_factor: None,
@@ -73,7 +71,7 @@ impl Query {
 
         if let Some(query) = self.query_vector.as_ref() {
             // If there is a vector query, default to limit=10 if unspecified
-            scanner.nearest(&self.column, query, self.limit.unwrap_or(10))?;
+            scanner.nearest(crate::table::VECTOR_COLUMN_NAME, query, self.limit.unwrap_or(10))?;
         } else {
             // If there is no vector query, it's ok to not have a limit
             scanner.limit(self.limit.map(|limit| limit as i64), None)?;
@@ -89,16 +87,6 @@ impl Query {
         Ok(scanner.try_into_stream().await?)
     }
 
-    /// Set the column to query
-    ///
-    /// # Arguments
-    ///
-    /// * `column` - The column name
-    pub fn column(mut self, column: &str) -> Query {
-        self.column = column.into();
-        self
-    }
-
     /// Set the maximum number of results to return.
     ///
     /// # Arguments
@@ -188,10 +176,7 @@ mod tests {
     use std::sync::Arc;
 
     use super::*;
-    use arrow_array::{
-        cast::AsArray, Float32Array, Int32Array, RecordBatch, RecordBatchIterator,
-        RecordBatchReader,
-    };
+    use arrow_array::{Float32Array, RecordBatch, RecordBatchIterator, RecordBatchReader, cast::AsArray, Int32Array};
     use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
     use futures::StreamExt;
     use lance::dataset::Dataset;
@@ -275,7 +260,7 @@ mod tests {
         let mut stream = result.expect("should have result");
         // should only have one batch
         while let Some(batch) = stream.next().await {
-            let b = batch.expect("should be Ok");
+            let b = batch.expect("should be Ok");            
             // cast arr into Int32Array
             let arr: &Int32Array = b["id"].as_primitive();
             assert!(arr.iter().all(|x| x.unwrap() % 2 == 0));

rust/vectordb/src/table.rs

@@ -14,7 +14,6 @@
 
 use chrono::Duration;
 use lance::dataset::builder::DatasetBuilder;
-use lance::index::scalar::ScalarIndexParams;
 use lance_index::IndexType;
 use std::sync::Arc;
 
@@ -263,16 +262,6 @@ impl Table {
         Ok(())
     }
 
-    /// Create a scalar index on the table
-    pub async fn create_scalar_index(&mut self, column: &str, replace: bool) -> Result<()> {
-        let mut dataset = self.dataset.as_ref().clone();
-        let params = ScalarIndexParams::default();
-        dataset
-            .create_index(&[column], IndexType::Scalar, None, &params, replace)
-            .await?;
-        Ok(())
-    }
-
     pub async fn optimize_indices(&mut self) -> Result<()> {
         let mut dataset = self.dataset.as_ref().clone();